ib_verbs.h

   1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2/*
   3 * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
   4 * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
   5 * Copyright (c) 2004, 2020 Intel Corporation.  All rights reserved.
   6 * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   7 * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
   8 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   9 * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
  10 */
  11
  12#ifndef IB_VERBS_H
  13#define IB_VERBS_H
  14
  15#include <linux/ethtool.h>
  16#include <linux/types.h>
  17#include <linux/device.h>
  18#include <linux/dma-mapping.h>
  19#include <linux/kref.h>
  20#include <linux/list.h>
  21#include <linux/rwsem.h>
  22#include <linux/workqueue.h>
  23#include <linux/irq_poll.h>
  24#include <uapi/linux/if_ether.h>
  25#include <net/ipv6.h>
  26#include <net/ip.h>
  27#include <linux/string.h>
  28#include <linux/slab.h>
  29#include <linux/netdevice.h>
  30#include <linux/refcount.h>
  31#include <linux/if_link.h>
  32#include <linux/atomic.h>
  33#include <linux/mmu_notifier.h>
  34#include <linux/uaccess.h>
  35#include <linux/cgroup_rdma.h>
  36#include <linux/irqflags.h>
  37#include <linux/preempt.h>
  38#include <linux/dim.h>
  39#include <uapi/rdma/ib_user_verbs.h>
  40#include <rdma/rdma_counter.h>
  41#include <rdma/restrack.h>
  42#include <rdma/signature.h>
  43#include <uapi/rdma/rdma_user_ioctl.h>
  44#include <uapi/rdma/ib_user_ioctl_verbs.h>
  45#include <linux/pci-tph.h>
  46
  47#define IB_FW_VERSION_NAME_MAX	ETHTOOL_FWVERS_LEN
  48
  49struct ib_umem_odp;
  50struct ib_uqp_object;
  51struct ib_usrq_object;
  52struct ib_uwq_object;
  53struct rdma_cm_id;
  54struct ib_port;
  55struct hw_stats_device_data;
  56
  57extern struct workqueue_struct *ib_wq;
  58extern struct workqueue_struct *ib_comp_wq;
  59extern struct workqueue_struct *ib_comp_unbound_wq;
  60
  61struct ib_ucq_object;
  62
  63__printf(2, 3) __cold
  64void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);
  65__printf(2, 3) __cold
  66void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);
  67__printf(2, 3) __cold
  68void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);
  69__printf(2, 3) __cold
  70void ibdev_err(const struct ib_device *ibdev, const char *format, ...);
  71__printf(2, 3) __cold
  72void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);
  73__printf(2, 3) __cold
  74void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);
  75__printf(2, 3) __cold
  76void ibdev_info(const struct ib_device *ibdev, const char *format, ...);
  77
  78#if defined(CONFIG_DYNAMIC_DEBUG) || \
  79	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
  80#define ibdev_dbg(__dev, format, args...)                       \
  81	dynamic_ibdev_dbg(__dev, format, ##args)
  82#else
  83__printf(2, 3) __cold
  84static inline
  85void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}
  86#endif
  87
  88#define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...)           \
  89do {                                                                    \
  90	static DEFINE_RATELIMIT_STATE(_rs,                              \
  91				      DEFAULT_RATELIMIT_INTERVAL,       \
  92				      DEFAULT_RATELIMIT_BURST);         \
  93	if (__ratelimit(&_rs))                                          \
  94		ibdev_level(ibdev, fmt, ##__VA_ARGS__);                 \
  95} while (0)
  96
  97#define ibdev_emerg_ratelimited(ibdev, fmt, ...) \
  98	ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__)
  99#define ibdev_alert_ratelimited(ibdev, fmt, ...) \
 100	ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__)
 101#define ibdev_crit_ratelimited(ibdev, fmt, ...) \
 102	ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__)
 103#define ibdev_err_ratelimited(ibdev, fmt, ...) \
 104	ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__)
 105#define ibdev_warn_ratelimited(ibdev, fmt, ...) \
 106	ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__)
 107#define ibdev_notice_ratelimited(ibdev, fmt, ...) \
 108	ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__)
 109#define ibdev_info_ratelimited(ibdev, fmt, ...) \
 110	ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__)
 111
 112#if defined(CONFIG_DYNAMIC_DEBUG) || \
 113	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
 114/* descriptor check is first to prevent flooding with "callbacks suppressed" */
 115#define ibdev_dbg_ratelimited(ibdev, fmt, ...)                          \
 116do {                                                                    \
 117	static DEFINE_RATELIMIT_STATE(_rs,                              \
 118				      DEFAULT_RATELIMIT_INTERVAL,       \
 119				      DEFAULT_RATELIMIT_BURST);         \
 120	DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt);                 \
 121	if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs))      \
 122		__dynamic_ibdev_dbg(&descriptor, ibdev, fmt,            \
 123				    ##__VA_ARGS__);                     \
 124} while (0)
 125#else
 126__printf(2, 3) __cold
 127static inline
 128void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {}
 129#endif
 130
 131union ib_gid {
 132	u8	raw[16];
 133	struct {
 134		__be64	subnet_prefix;
 135		__be64	interface_id;
 136	} global;
 137};
 138
 139extern union ib_gid zgid;
 140
 141enum ib_gid_type {
 142	IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB,
 143	IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1,
 144	IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2,
 145	IB_GID_TYPE_SIZE
 146};
 147
 148#define ROCE_V2_UDP_DPORT      4791
 149struct ib_gid_attr {
 150	struct net_device __rcu	*ndev;
 151	struct ib_device	*device;
 152	union ib_gid		gid;
 153	enum ib_gid_type	gid_type;
 154	u16			index;
 155	u32			port_num;
 156};
 157
 158enum {
 159	/* set the local administered indication */
 160	IB_SA_WELL_KNOWN_GUID	= BIT_ULL(57) | 2,
 161};
 162
 163enum rdma_transport_type {
 164	RDMA_TRANSPORT_IB,
 165	RDMA_TRANSPORT_IWARP,
 166	RDMA_TRANSPORT_USNIC,
 167	RDMA_TRANSPORT_USNIC_UDP,
 168	RDMA_TRANSPORT_UNSPECIFIED,
 169};
 170
 171enum rdma_protocol_type {
 172	RDMA_PROTOCOL_IB,
 173	RDMA_PROTOCOL_IBOE,
 174	RDMA_PROTOCOL_IWARP,
 175	RDMA_PROTOCOL_USNIC_UDP
 176};
 177
 178__attribute_const__ enum rdma_transport_type
 179rdma_node_get_transport(unsigned int node_type);
 180
 181enum rdma_network_type {
 182	RDMA_NETWORK_IB,
 183	RDMA_NETWORK_ROCE_V1,
 184	RDMA_NETWORK_IPV4,
 185	RDMA_NETWORK_IPV6
 186};
 187
 188static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type)
 189{
 190	if (network_type == RDMA_NETWORK_IPV4 ||
 191	    network_type == RDMA_NETWORK_IPV6)
 192		return IB_GID_TYPE_ROCE_UDP_ENCAP;
 193	else if (network_type == RDMA_NETWORK_ROCE_V1)
 194		return IB_GID_TYPE_ROCE;
 195	else
 196		return IB_GID_TYPE_IB;
 197}
 198
 199static inline enum rdma_network_type
 200rdma_gid_attr_network_type(const struct ib_gid_attr *attr)
 201{
 202	if (attr->gid_type == IB_GID_TYPE_IB)
 203		return RDMA_NETWORK_IB;
 204
 205	if (attr->gid_type == IB_GID_TYPE_ROCE)
 206		return RDMA_NETWORK_ROCE_V1;
 207
 208	if (ipv6_addr_v4mapped((struct in6_addr *)&attr->gid))
 209		return RDMA_NETWORK_IPV4;
 210	else
 211		return RDMA_NETWORK_IPV6;
 212}
 213
 214enum rdma_link_layer {
 215	IB_LINK_LAYER_UNSPECIFIED,
 216	IB_LINK_LAYER_INFINIBAND,
 217	IB_LINK_LAYER_ETHERNET,
 218};
 219
 220enum ib_device_cap_flags {
 221	IB_DEVICE_RESIZE_MAX_WR = IB_UVERBS_DEVICE_RESIZE_MAX_WR,
 222	IB_DEVICE_BAD_PKEY_CNTR = IB_UVERBS_DEVICE_BAD_PKEY_CNTR,
 223	IB_DEVICE_BAD_QKEY_CNTR = IB_UVERBS_DEVICE_BAD_QKEY_CNTR,
 224	IB_DEVICE_RAW_MULTI = IB_UVERBS_DEVICE_RAW_MULTI,
 225	IB_DEVICE_AUTO_PATH_MIG = IB_UVERBS_DEVICE_AUTO_PATH_MIG,
 226	IB_DEVICE_CHANGE_PHY_PORT = IB_UVERBS_DEVICE_CHANGE_PHY_PORT,
 227	IB_DEVICE_UD_AV_PORT_ENFORCE = IB_UVERBS_DEVICE_UD_AV_PORT_ENFORCE,
 228	IB_DEVICE_CURR_QP_STATE_MOD = IB_UVERBS_DEVICE_CURR_QP_STATE_MOD,
 229	IB_DEVICE_SHUTDOWN_PORT = IB_UVERBS_DEVICE_SHUTDOWN_PORT,
 230	/* IB_DEVICE_INIT_TYPE = IB_UVERBS_DEVICE_INIT_TYPE, (not in use) */
 231	IB_DEVICE_PORT_ACTIVE_EVENT = IB_UVERBS_DEVICE_PORT_ACTIVE_EVENT,
 232	IB_DEVICE_SYS_IMAGE_GUID = IB_UVERBS_DEVICE_SYS_IMAGE_GUID,
 233	IB_DEVICE_RC_RNR_NAK_GEN = IB_UVERBS_DEVICE_RC_RNR_NAK_GEN,
 234	IB_DEVICE_SRQ_RESIZE = IB_UVERBS_DEVICE_SRQ_RESIZE,
 235	IB_DEVICE_N_NOTIFY_CQ = IB_UVERBS_DEVICE_N_NOTIFY_CQ,
 236
 237	/* Reserved, old SEND_W_INV = 1 << 16,*/
 238	IB_DEVICE_MEM_WINDOW = IB_UVERBS_DEVICE_MEM_WINDOW,
 239	/*
 240	 * Devices should set IB_DEVICE_UD_IP_SUM if they support
 241	 * insertion of UDP and TCP checksum on outgoing UD IPoIB
 242	 * messages and can verify the validity of checksum for
 243	 * incoming messages.  Setting this flag implies that the
 244	 * IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.
 245	 */
 246	IB_DEVICE_UD_IP_CSUM = IB_UVERBS_DEVICE_UD_IP_CSUM,
 247	IB_DEVICE_XRC = IB_UVERBS_DEVICE_XRC,
 248
 249	/*
 250	 * This device supports the IB "base memory management extension",
 251	 * which includes support for fast registrations (IB_WR_REG_MR,
 252	 * IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs).  This flag should
 253	 * also be set by any iWarp device which must support FRs to comply
 254	 * to the iWarp verbs spec.  iWarp devices also support the
 255	 * IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the
 256	 * stag.
 257	 */
 258	IB_DEVICE_MEM_MGT_EXTENSIONS = IB_UVERBS_DEVICE_MEM_MGT_EXTENSIONS,
 259	IB_DEVICE_MEM_WINDOW_TYPE_2A = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2A,
 260	IB_DEVICE_MEM_WINDOW_TYPE_2B = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2B,
 261	IB_DEVICE_RC_IP_CSUM = IB_UVERBS_DEVICE_RC_IP_CSUM,
 262	/* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */
 263	IB_DEVICE_RAW_IP_CSUM = IB_UVERBS_DEVICE_RAW_IP_CSUM,
 264	IB_DEVICE_MANAGED_FLOW_STEERING =
 265		IB_UVERBS_DEVICE_MANAGED_FLOW_STEERING,
 266	/* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */
 267	IB_DEVICE_RAW_SCATTER_FCS = IB_UVERBS_DEVICE_RAW_SCATTER_FCS,
 268	/* The device supports padding incoming writes to cacheline. */
 269	IB_DEVICE_PCI_WRITE_END_PADDING =
 270		IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,
 271	/* Placement type attributes */
 272	IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
 273	IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
 274	IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
 275};
 276
 277enum ib_kernel_cap_flags {
 278	/*
 279	 * This device supports a per-device lkey or stag that can be
 280	 * used without performing a memory registration for the local
 281	 * memory.  Note that ULPs should never check this flag, but
 282	 * instead of use the local_dma_lkey flag in the ib_pd structure,
 283	 * which will always contain a usable lkey.
 284	 */
 285	IBK_LOCAL_DMA_LKEY = 1 << 0,
 286	/* IB_QP_CREATE_INTEGRITY_EN is supported to implement T10-PI */
 287	IBK_INTEGRITY_HANDOVER = 1 << 1,
 288	/* IB_ACCESS_ON_DEMAND is supported during reg_user_mr() */
 289	IBK_ON_DEMAND_PAGING = 1 << 2,
 290	/* IB_MR_TYPE_SG_GAPS is supported */
 291	IBK_SG_GAPS_REG = 1 << 3,
 292	/* Driver supports RDMA_NLDEV_CMD_DELLINK */
 293	IBK_ALLOW_USER_UNREG = 1 << 4,
 294
 295	/* ipoib will use IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK */
 296	IBK_BLOCK_MULTICAST_LOOPBACK = 1 << 5,
 297	/* iopib will use IB_QP_CREATE_IPOIB_UD_LSO for its QPs */
 298	IBK_UD_TSO = 1 << 6,
 299	/* iopib will use the device ops:
 300	 *   get_vf_config
 301	 *   get_vf_guid
 302	 *   get_vf_stats
 303	 *   set_vf_guid
 304	 *   set_vf_link_state
 305	 */
 306	IBK_VIRTUAL_FUNCTION = 1 << 7,
 307	/* ipoib will use IB_QP_CREATE_NETDEV_USE for its QPs */
 308	IBK_RDMA_NETDEV_OPA = 1 << 8,
 309};
 310
 311enum ib_atomic_cap {
 312	IB_ATOMIC_NONE,
 313	IB_ATOMIC_HCA,
 314	IB_ATOMIC_GLOB
 315};
 316
 317enum ib_odp_general_cap_bits {
 318	IB_ODP_SUPPORT		= IB_UVERBS_ODP_SUPPORT,
 319	IB_ODP_SUPPORT_IMPLICIT = IB_UVERBS_ODP_SUPPORT_IMPLICIT,
 320};
 321
 322enum ib_odp_transport_cap_bits {
 323	IB_ODP_SUPPORT_SEND	= IB_UVERBS_ODP_SUPPORT_SEND,
 324	IB_ODP_SUPPORT_RECV	= IB_UVERBS_ODP_SUPPORT_RECV,
 325	IB_ODP_SUPPORT_WRITE	= IB_UVERBS_ODP_SUPPORT_WRITE,
 326	IB_ODP_SUPPORT_READ	= IB_UVERBS_ODP_SUPPORT_READ,
 327	IB_ODP_SUPPORT_ATOMIC	= IB_UVERBS_ODP_SUPPORT_ATOMIC,
 328	IB_ODP_SUPPORT_SRQ_RECV	= IB_UVERBS_ODP_SUPPORT_SRQ_RECV,
 329	IB_ODP_SUPPORT_FLUSH	= IB_UVERBS_ODP_SUPPORT_FLUSH,
 330	IB_ODP_SUPPORT_ATOMIC_WRITE	= IB_UVERBS_ODP_SUPPORT_ATOMIC_WRITE,
 331};
 332
 333struct ib_odp_caps {
 334	uint64_t general_caps;
 335	struct {
 336		uint32_t  rc_odp_caps;
 337		uint32_t  uc_odp_caps;
 338		uint32_t  ud_odp_caps;
 339		uint32_t  xrc_odp_caps;
 340	} per_transport_caps;
 341};
 342
 343struct ib_rss_caps {
 344	/* Corresponding bit will be set if qp type from
 345	 * 'enum ib_qp_type' is supported, e.g.
 346	 * supported_qpts |= 1 << IB_QPT_UD
 347	 */
 348	u32 supported_qpts;
 349	u32 max_rwq_indirection_tables;
 350	u32 max_rwq_indirection_table_size;
 351};
 352
 353enum ib_tm_cap_flags {
 354	/*  Support tag matching with rendezvous offload for RC transport */
 355	IB_TM_CAP_RNDV_RC = 1 << 0,
 356};
 357
 358struct ib_tm_caps {
 359	/* Max size of RNDV header */
 360	u32 max_rndv_hdr_size;
 361	/* Max number of entries in tag matching list */
 362	u32 max_num_tags;
 363	/* From enum ib_tm_cap_flags */
 364	u32 flags;
 365	/* Max number of outstanding list operations */
 366	u32 max_ops;
 367	/* Max number of SGE in tag matching entry */
 368	u32 max_sge;
 369};
 370
 371struct ib_cq_init_attr {
 372	unsigned int	cqe;
 373	u32		comp_vector;
 374	u32		flags;
 375};
 376
 377enum ib_cq_attr_mask {
 378	IB_CQ_MODERATE = 1 << 0,
 379};
 380
 381struct ib_cq_caps {
 382	u16     max_cq_moderation_count;
 383	u16     max_cq_moderation_period;
 384};
 385
 386struct ib_dm_mr_attr {
 387	u64		length;
 388	u64		offset;
 389	u32		access_flags;
 390};
 391
 392struct ib_dm_alloc_attr {
 393	u64	length;
 394	u32	alignment;
 395	u32	flags;
 396};
 397
 398struct ib_device_attr {
 399	u64			fw_ver;
 400	__be64			sys_image_guid;
 401	u64			max_mr_size;
 402	u64			page_size_cap;
 403	u32			vendor_id;
 404	u32			vendor_part_id;
 405	u32			hw_ver;
 406	int			max_qp;
 407	int			max_qp_wr;
 408	u64			device_cap_flags;
 409	u64			kernel_cap_flags;
 410	int			max_send_sge;
 411	int			max_recv_sge;
 412	int			max_sge_rd;
 413	int			max_cq;
 414	int			max_cqe;
 415	int			max_mr;
 416	int			max_pd;
 417	int			max_qp_rd_atom;
 418	int			max_ee_rd_atom;
 419	int			max_res_rd_atom;
 420	int			max_qp_init_rd_atom;
 421	int			max_ee_init_rd_atom;
 422	enum ib_atomic_cap	atomic_cap;
 423	enum ib_atomic_cap	masked_atomic_cap;
 424	int			max_ee;
 425	int			max_rdd;
 426	int			max_mw;
 427	int			max_raw_ipv6_qp;
 428	int			max_raw_ethy_qp;
 429	int			max_mcast_grp;
 430	int			max_mcast_qp_attach;
 431	int			max_total_mcast_qp_attach;
 432	int			max_ah;
 433	int			max_srq;
 434	int			max_srq_wr;
 435	int			max_srq_sge;
 436	unsigned int		max_fast_reg_page_list_len;
 437	unsigned int		max_pi_fast_reg_page_list_len;
 438	u16			max_pkeys;
 439	u8			local_ca_ack_delay;
 440	int			sig_prot_cap;
 441	int			sig_guard_cap;
 442	struct ib_odp_caps	odp_caps;
 443	uint64_t		timestamp_mask;
 444	uint64_t		hca_core_clock; /* in KHZ */
 445	struct ib_rss_caps	rss_caps;
 446	u32			max_wq_type_rq;
 447	u32			raw_packet_caps; /* Use ib_raw_packet_caps enum */
 448	struct ib_tm_caps	tm_caps;
 449	struct ib_cq_caps       cq_caps;
 450	u64			max_dm_size;
 451	/* Max entries for sgl for optimized performance per READ */
 452	u32			max_sgl_rd;
 453};
 454
 455enum ib_mtu {
 456	IB_MTU_256  = 1,
 457	IB_MTU_512  = 2,
 458	IB_MTU_1024 = 3,
 459	IB_MTU_2048 = 4,
 460	IB_MTU_4096 = 5
 461};
 462
 463enum opa_mtu {
 464	OPA_MTU_8192 = 6,
 465	OPA_MTU_10240 = 7
 466};
 467
 468static inline int ib_mtu_enum_to_int(enum ib_mtu mtu)
 469{
 470	switch (mtu) {
 471	case IB_MTU_256:  return  256;
 472	case IB_MTU_512:  return  512;
 473	case IB_MTU_1024: return 1024;
 474	case IB_MTU_2048: return 2048;
 475	case IB_MTU_4096: return 4096;
 476	default: 	  return -1;
 477	}
 478}
 479
 480static inline enum ib_mtu ib_mtu_int_to_enum(int mtu)
 481{
 482	if (mtu >= 4096)
 483		return IB_MTU_4096;
 484	else if (mtu >= 2048)
 485		return IB_MTU_2048;
 486	else if (mtu >= 1024)
 487		return IB_MTU_1024;
 488	else if (mtu >= 512)
 489		return IB_MTU_512;
 490	else
 491		return IB_MTU_256;
 492}
 493
 494static inline int opa_mtu_enum_to_int(enum opa_mtu mtu)
 495{
 496	switch (mtu) {
 497	case OPA_MTU_8192:
 498		return 8192;
 499	case OPA_MTU_10240:
 500		return 10240;
 501	default:
 502		return(ib_mtu_enum_to_int((enum ib_mtu)mtu));
 503	}
 504}
 505
 506static inline enum opa_mtu opa_mtu_int_to_enum(int mtu)
 507{
 508	if (mtu >= 10240)
 509		return OPA_MTU_10240;
 510	else if (mtu >= 8192)
 511		return OPA_MTU_8192;
 512	else
 513		return ((enum opa_mtu)ib_mtu_int_to_enum(mtu));
 514}
 515
 516enum ib_port_state {
 517	IB_PORT_NOP		= 0,
 518	IB_PORT_DOWN		= 1,
 519	IB_PORT_INIT		= 2,
 520	IB_PORT_ARMED		= 3,
 521	IB_PORT_ACTIVE		= 4,
 522	IB_PORT_ACTIVE_DEFER	= 5
 523};
 524
 525static inline const char *__attribute_const__
 526ib_port_state_to_str(enum ib_port_state state)
 527{
 528	const char * const states[] = {
 529		[IB_PORT_NOP] = "NOP",
 530		[IB_PORT_DOWN] = "DOWN",
 531		[IB_PORT_INIT] = "INIT",
 532		[IB_PORT_ARMED] = "ARMED",
 533		[IB_PORT_ACTIVE] = "ACTIVE",
 534		[IB_PORT_ACTIVE_DEFER] = "ACTIVE_DEFER",
 535	};
 536
 537	if (state < ARRAY_SIZE(states))
 538		return states[state];
 539	return "UNKNOWN";
 540}
 541
 542enum ib_port_phys_state {
 543	IB_PORT_PHYS_STATE_SLEEP = 1,
 544	IB_PORT_PHYS_STATE_POLLING = 2,
 545	IB_PORT_PHYS_STATE_DISABLED = 3,
 546	IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4,
 547	IB_PORT_PHYS_STATE_LINK_UP = 5,
 548	IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6,
 549	IB_PORT_PHYS_STATE_PHY_TEST = 7,
 550};
 551
 552enum ib_port_width {
 553	IB_WIDTH_1X	= 1,
 554	IB_WIDTH_2X	= 16,
 555	IB_WIDTH_4X	= 2,
 556	IB_WIDTH_8X	= 4,
 557	IB_WIDTH_12X	= 8
 558};
 559
 560static inline int ib_width_enum_to_int(enum ib_port_width width)
 561{
 562	switch (width) {
 563	case IB_WIDTH_1X:  return  1;
 564	case IB_WIDTH_2X:  return  2;
 565	case IB_WIDTH_4X:  return  4;
 566	case IB_WIDTH_8X:  return  8;
 567	case IB_WIDTH_12X: return 12;
 568	default: 	  return -1;
 569	}
 570}
 571
 572enum ib_port_speed {
 573	IB_SPEED_SDR	= 1,
 574	IB_SPEED_DDR	= 2,
 575	IB_SPEED_QDR	= 4,
 576	IB_SPEED_FDR10	= 8,
 577	IB_SPEED_FDR	= 16,
 578	IB_SPEED_EDR	= 32,
 579	IB_SPEED_HDR	= 64,
 580	IB_SPEED_NDR	= 128,
 581	IB_SPEED_XDR	= 256,
 582};
 583
 584enum ib_stat_flag {
 585	IB_STAT_FLAG_OPTIONAL = 1 << 0,
 586};
 587
 588/**
 589 * struct rdma_stat_desc - description of one rdma stat/counter
 590 * @name: The name of the counter
 591 * @flags: Flags of the counter; For example, IB_STAT_FLAG_OPTIONAL
 592 * @priv: Driver private information; Core code should not use
 593 */
 594struct rdma_stat_desc {
 595	const char *name;
 596	unsigned int flags;
 597	const void *priv;
 598};
 599
 600/**
 601 * struct rdma_hw_stats - collection of hardware stats and their management
 602 * @lock: Mutex to protect parallel write access to lifespan and values
 603 *    of counters, which are 64bits and not guaranteed to be written
 604 *    atomicaly on 32bits systems.
 605 * @timestamp: Used by the core code to track when the last update was
 606 * @lifespan: Used by the core code to determine how old the counters
 607 *   should be before being updated again.  Stored in jiffies, defaults
 608 *   to 10 milliseconds, drivers can override the default be specifying
 609 *   their own value during their allocation routine.
 610 * @descs: Array of pointers to static descriptors used for the counters
 611 *   in directory.
 612 * @is_disabled: A bitmap to indicate each counter is currently disabled
 613 *   or not.
 614 * @num_counters: How many hardware counters there are.  If name is
 615 *   shorter than this number, a kernel oops will result.  Driver authors
 616 *   are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters)
 617 *   in their code to prevent this.
 618 * @value: Array of u64 counters that are accessed by the sysfs code and
 619 *   filled in by the drivers get_stats routine
 620 */
 621struct rdma_hw_stats {
 622	struct mutex	lock; /* Protect lifespan and values[] */
 623	unsigned long	timestamp;
 624	unsigned long	lifespan;
 625	const struct rdma_stat_desc *descs;
 626	unsigned long	*is_disabled;
 627	int		num_counters;
 628	u64		value[] __counted_by(num_counters);
 629};
 630
 631#define RDMA_HW_STATS_DEFAULT_LIFESPAN 10
 632
 633struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
 634	const struct rdma_stat_desc *descs, int num_counters,
 635	unsigned long lifespan);
 636
 637void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats);
 638
 639/* Define bits for the various functionality this port needs to be supported by
 640 * the core.
 641 */
 642/* Management                           0x00000FFF */
 643#define RDMA_CORE_CAP_IB_MAD            0x00000001
 644#define RDMA_CORE_CAP_IB_SMI            0x00000002
 645#define RDMA_CORE_CAP_IB_CM             0x00000004
 646#define RDMA_CORE_CAP_IW_CM             0x00000008
 647#define RDMA_CORE_CAP_IB_SA             0x00000010
 648#define RDMA_CORE_CAP_OPA_MAD           0x00000020
 649
 650/* Address format                       0x000FF000 */
 651#define RDMA_CORE_CAP_AF_IB             0x00001000
 652#define RDMA_CORE_CAP_ETH_AH            0x00002000
 653#define RDMA_CORE_CAP_OPA_AH            0x00004000
 654#define RDMA_CORE_CAP_IB_GRH_REQUIRED   0x00008000
 655
 656/* Protocol                             0xFFF00000 */
 657#define RDMA_CORE_CAP_PROT_IB           0x00100000
 658#define RDMA_CORE_CAP_PROT_ROCE         0x00200000
 659#define RDMA_CORE_CAP_PROT_IWARP        0x00400000
 660#define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
 661#define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000
 662#define RDMA_CORE_CAP_PROT_USNIC        0x02000000
 663
 664#define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
 665					| RDMA_CORE_CAP_PROT_ROCE     \
 666					| RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP)
 667
 668#define RDMA_CORE_PORT_IBA_IB          (RDMA_CORE_CAP_PROT_IB  \
 669					| RDMA_CORE_CAP_IB_MAD \
 670					| RDMA_CORE_CAP_IB_SMI \
 671					| RDMA_CORE_CAP_IB_CM  \
 672					| RDMA_CORE_CAP_IB_SA  \
 673					| RDMA_CORE_CAP_AF_IB)
 674#define RDMA_CORE_PORT_IBA_ROCE        (RDMA_CORE_CAP_PROT_ROCE \
 675					| RDMA_CORE_CAP_IB_MAD  \
 676					| RDMA_CORE_CAP_IB_CM   \
 677					| RDMA_CORE_CAP_AF_IB   \
 678					| RDMA_CORE_CAP_ETH_AH)
 679#define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP			\
 680					(RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \
 681					| RDMA_CORE_CAP_IB_MAD  \
 682					| RDMA_CORE_CAP_IB_CM   \
 683					| RDMA_CORE_CAP_AF_IB   \
 684					| RDMA_CORE_CAP_ETH_AH)
 685#define RDMA_CORE_PORT_IWARP           (RDMA_CORE_CAP_PROT_IWARP \
 686					| RDMA_CORE_CAP_IW_CM)
 687#define RDMA_CORE_PORT_INTEL_OPA       (RDMA_CORE_PORT_IBA_IB  \
 688					| RDMA_CORE_CAP_OPA_MAD)
 689
 690#define RDMA_CORE_PORT_RAW_PACKET	(RDMA_CORE_CAP_PROT_RAW_PACKET)
 691
 692#define RDMA_CORE_PORT_USNIC		(RDMA_CORE_CAP_PROT_USNIC)
 693
 694struct ib_port_attr {
 695	u64			subnet_prefix;
 696	enum ib_port_state	state;
 697	enum ib_mtu		max_mtu;
 698	enum ib_mtu		active_mtu;
 699	u32                     phys_mtu;
 700	int			gid_tbl_len;
 701	unsigned int		ip_gids:1;
 702	/* This is the value from PortInfo CapabilityMask, defined by IBA */
 703	u32			port_cap_flags;
 704	u32			max_msg_sz;
 705	u32			bad_pkey_cntr;
 706	u32			qkey_viol_cntr;
 707	u16			pkey_tbl_len;
 708	u32			sm_lid;
 709	u32			lid;
 710	u8			lmc;
 711	u8			max_vl_num;
 712	u8			sm_sl;
 713	u8			subnet_timeout;
 714	u8			init_type_reply;
 715	u8			active_width;
 716	u16			active_speed;
 717	u8                      phys_state;
 718	u16			port_cap_flags2;
 719};
 720
 721enum ib_device_modify_flags {
 722	IB_DEVICE_MODIFY_SYS_IMAGE_GUID	= 1 << 0,
 723	IB_DEVICE_MODIFY_NODE_DESC	= 1 << 1
 724};
 725
 726#define IB_DEVICE_NODE_DESC_MAX 64
 727
 728struct ib_device_modify {
 729	u64	sys_image_guid;
 730	char	node_desc[IB_DEVICE_NODE_DESC_MAX];
 731};
 732
 733enum ib_port_modify_flags {
 734	IB_PORT_SHUTDOWN		= 1,
 735	IB_PORT_INIT_TYPE		= (1<<2),
 736	IB_PORT_RESET_QKEY_CNTR		= (1<<3),
 737	IB_PORT_OPA_MASK_CHG		= (1<<4)
 738};
 739
 740struct ib_port_modify {
 741	u32	set_port_cap_mask;
 742	u32	clr_port_cap_mask;
 743	u8	init_type;
 744};
 745
 746enum ib_event_type {
 747	IB_EVENT_CQ_ERR,
 748	IB_EVENT_QP_FATAL,
 749	IB_EVENT_QP_REQ_ERR,
 750	IB_EVENT_QP_ACCESS_ERR,
 751	IB_EVENT_COMM_EST,
 752	IB_EVENT_SQ_DRAINED,
 753	IB_EVENT_PATH_MIG,
 754	IB_EVENT_PATH_MIG_ERR,
 755	IB_EVENT_DEVICE_FATAL,
 756	IB_EVENT_PORT_ACTIVE,
 757	IB_EVENT_PORT_ERR,
 758	IB_EVENT_LID_CHANGE,
 759	IB_EVENT_PKEY_CHANGE,
 760	IB_EVENT_SM_CHANGE,
 761	IB_EVENT_SRQ_ERR,
 762	IB_EVENT_SRQ_LIMIT_REACHED,
 763	IB_EVENT_QP_LAST_WQE_REACHED,
 764	IB_EVENT_CLIENT_REREGISTER,
 765	IB_EVENT_GID_CHANGE,
 766	IB_EVENT_WQ_FATAL,
 767};
 768
 769const char *__attribute_const__ ib_event_msg(enum ib_event_type event);
 770
 771struct ib_event {
 772	struct ib_device	*device;
 773	union {
 774		struct ib_cq	*cq;
 775		struct ib_qp	*qp;
 776		struct ib_srq	*srq;
 777		struct ib_wq	*wq;
 778		u32		port_num;
 779	} element;
 780	enum ib_event_type	event;
 781};
 782
 783struct ib_event_handler {
 784	struct ib_device *device;
 785	void            (*handler)(struct ib_event_handler *, struct ib_event *);
 786	struct list_head  list;
 787};
 788
 789#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler)		\
 790	do {							\
 791		(_ptr)->device  = _device;			\
 792		(_ptr)->handler = _handler;			\
 793		INIT_LIST_HEAD(&(_ptr)->list);			\
 794	} while (0)
 795
 796struct ib_global_route {
 797	const struct ib_gid_attr *sgid_attr;
 798	union ib_gid	dgid;
 799	u32		flow_label;
 800	u8		sgid_index;
 801	u8		hop_limit;
 802	u8		traffic_class;
 803};
 804
 805struct ib_grh {
 806	__be32		version_tclass_flow;
 807	__be16		paylen;
 808	u8		next_hdr;
 809	u8		hop_limit;
 810	union ib_gid	sgid;
 811	union ib_gid	dgid;
 812};
 813
 814union rdma_network_hdr {
 815	struct ib_grh ibgrh;
 816	struct {
 817		/* The IB spec states that if it's IPv4, the header
 818		 * is located in the last 20 bytes of the header.
 819		 */
 820		u8		reserved[20];
 821		struct iphdr	roce4grh;
 822	};
 823};
 824
 825#define IB_QPN_MASK		0xFFFFFF
 826
 827enum {
 828	IB_MULTICAST_QPN = 0xffffff
 829};
 830
 831#define IB_LID_PERMISSIVE	cpu_to_be16(0xFFFF)
 832#define IB_MULTICAST_LID_BASE	cpu_to_be16(0xC000)
 833
 834enum ib_ah_flags {
 835	IB_AH_GRH	= 1
 836};
 837
 838enum ib_rate {
 839	IB_RATE_PORT_CURRENT = 0,
 840	IB_RATE_2_5_GBPS = 2,
 841	IB_RATE_5_GBPS   = 5,
 842	IB_RATE_10_GBPS  = 3,
 843	IB_RATE_20_GBPS  = 6,
 844	IB_RATE_30_GBPS  = 4,
 845	IB_RATE_40_GBPS  = 7,
 846	IB_RATE_60_GBPS  = 8,
 847	IB_RATE_80_GBPS  = 9,
 848	IB_RATE_120_GBPS = 10,
 849	IB_RATE_14_GBPS  = 11,
 850	IB_RATE_56_GBPS  = 12,
 851	IB_RATE_112_GBPS = 13,
 852	IB_RATE_168_GBPS = 14,
 853	IB_RATE_25_GBPS  = 15,
 854	IB_RATE_100_GBPS = 16,
 855	IB_RATE_200_GBPS = 17,
 856	IB_RATE_300_GBPS = 18,
 857	IB_RATE_28_GBPS  = 19,
 858	IB_RATE_50_GBPS  = 20,
 859	IB_RATE_400_GBPS = 21,
 860	IB_RATE_600_GBPS = 22,
 861	IB_RATE_800_GBPS = 23,
 862	IB_RATE_1600_GBPS = 25,
 863};
 864
 865/**
 866 * ib_rate_to_mult - Convert the IB rate enum to a multiple of the
 867 * base rate of 2.5 Gbit/sec.  For example, IB_RATE_5_GBPS will be
 868 * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec.
 869 * @rate: rate to convert.
 870 */
 871__attribute_const__ int ib_rate_to_mult(enum ib_rate rate);
 872
 873/**
 874 * ib_rate_to_mbps - Convert the IB rate enum to Mbps.
 875 * For example, IB_RATE_2_5_GBPS will be converted to 2500.
 876 * @rate: rate to convert.
 877 */
 878__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
 879
 880
 881/**
 882 * enum ib_mr_type - memory region type
 883 * @IB_MR_TYPE_MEM_REG:       memory region that is used for
 884 *                            normal registration
 885 * @IB_MR_TYPE_SG_GAPS:       memory region that is capable to
 886 *                            register any arbitrary sg lists (without
 887 *                            the normal mr constraints - see
 888 *                            ib_map_mr_sg)
 889 * @IB_MR_TYPE_DM:            memory region that is used for device
 890 *                            memory registration
 891 * @IB_MR_TYPE_USER:          memory region that is used for the user-space
 892 *                            application
 893 * @IB_MR_TYPE_DMA:           memory region that is used for DMA operations
 894 *                            without address translations (VA=PA)
 895 * @IB_MR_TYPE_INTEGRITY:     memory region that is used for
 896 *                            data integrity operations
 897 */
 898enum ib_mr_type {
 899	IB_MR_TYPE_MEM_REG,
 900	IB_MR_TYPE_SG_GAPS,
 901	IB_MR_TYPE_DM,
 902	IB_MR_TYPE_USER,
 903	IB_MR_TYPE_DMA,
 904	IB_MR_TYPE_INTEGRITY,
 905};
 906
 907enum ib_mr_status_check {
 908	IB_MR_CHECK_SIG_STATUS = 1,
 909};
 910
 911/**
 912 * struct ib_mr_status - Memory region status container
 913 *
 914 * @fail_status: Bitmask of MR checks status. For each
 915 *     failed check a corresponding status bit is set.
 916 * @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS
 917 *     failure.
 918 */
 919struct ib_mr_status {
 920	u32		    fail_status;
 921	struct ib_sig_err   sig_err;
 922};
 923
 924/**
 925 * mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate
 926 * enum.
 927 * @mult: multiple to convert.
 928 */
 929__attribute_const__ enum ib_rate mult_to_ib_rate(int mult);
 930
 931struct rdma_ah_init_attr {
 932	struct rdma_ah_attr *ah_attr;
 933	u32 flags;
 934	struct net_device *xmit_slave;
 935};
 936
 937enum rdma_ah_attr_type {
 938	RDMA_AH_ATTR_TYPE_UNDEFINED,
 939	RDMA_AH_ATTR_TYPE_IB,
 940	RDMA_AH_ATTR_TYPE_ROCE,
 941	RDMA_AH_ATTR_TYPE_OPA,
 942};
 943
 944struct ib_ah_attr {
 945	u16			dlid;
 946	u8			src_path_bits;
 947};
 948
 949struct roce_ah_attr {
 950	u8			dmac[ETH_ALEN];
 951};
 952
 953struct opa_ah_attr {
 954	u32			dlid;
 955	u8			src_path_bits;
 956	bool			make_grd;
 957};
 958
 959struct rdma_ah_attr {
 960	struct ib_global_route	grh;
 961	u8			sl;
 962	u8			static_rate;
 963	u32			port_num;
 964	u8			ah_flags;
 965	enum rdma_ah_attr_type type;
 966	union {
 967		struct ib_ah_attr ib;
 968		struct roce_ah_attr roce;
 969		struct opa_ah_attr opa;
 970	};
 971};
 972
 973enum ib_wc_status {
 974	IB_WC_SUCCESS,
 975	IB_WC_LOC_LEN_ERR,
 976	IB_WC_LOC_QP_OP_ERR,
 977	IB_WC_LOC_EEC_OP_ERR,
 978	IB_WC_LOC_PROT_ERR,
 979	IB_WC_WR_FLUSH_ERR,
 980	IB_WC_MW_BIND_ERR,
 981	IB_WC_BAD_RESP_ERR,
 982	IB_WC_LOC_ACCESS_ERR,
 983	IB_WC_REM_INV_REQ_ERR,
 984	IB_WC_REM_ACCESS_ERR,
 985	IB_WC_REM_OP_ERR,
 986	IB_WC_RETRY_EXC_ERR,
 987	IB_WC_RNR_RETRY_EXC_ERR,
 988	IB_WC_LOC_RDD_VIOL_ERR,
 989	IB_WC_REM_INV_RD_REQ_ERR,
 990	IB_WC_REM_ABORT_ERR,
 991	IB_WC_INV_EECN_ERR,
 992	IB_WC_INV_EEC_STATE_ERR,
 993	IB_WC_FATAL_ERR,
 994	IB_WC_RESP_TIMEOUT_ERR,
 995	IB_WC_GENERAL_ERR
 996};
 997
 998const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status);
 999
1000enum ib_wc_opcode {
1001	IB_WC_SEND = IB_UVERBS_WC_SEND,
1002	IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE,
1003	IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ,
1004	IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP,
1005	IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD,
1006	IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
1007	IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
1008	IB_WC_LSO = IB_UVERBS_WC_TSO,
1009	IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,
1010	IB_WC_REG_MR,
1011	IB_WC_MASKED_COMP_SWAP,
1012	IB_WC_MASKED_FETCH_ADD,
1013	IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,
1014/*
1015 * Set value of IB_WC_RECV so consumers can test if a completion is a
1016 * receive by testing (opcode & IB_WC_RECV).
1017 */
1018	IB_WC_RECV			= 1 << 7,
1019	IB_WC_RECV_RDMA_WITH_IMM
1020};
1021
1022enum ib_wc_flags {
1023	IB_WC_GRH		= 1,
1024	IB_WC_WITH_IMM		= (1<<1),
1025	IB_WC_WITH_INVALIDATE	= (1<<2),
1026	IB_WC_IP_CSUM_OK	= (1<<3),
1027	IB_WC_WITH_SMAC		= (1<<4),
1028	IB_WC_WITH_VLAN		= (1<<5),
1029	IB_WC_WITH_NETWORK_HDR_TYPE	= (1<<6),
1030};
1031
1032struct ib_wc {
1033	union {
1034		u64		wr_id;
1035		struct ib_cqe	*wr_cqe;
1036	};
1037	enum ib_wc_status	status;
1038	enum ib_wc_opcode	opcode;
1039	u32			vendor_err;
1040	u32			byte_len;
1041	struct ib_qp	       *qp;
1042	union {
1043		__be32		imm_data;
1044		u32		invalidate_rkey;
1045	} ex;
1046	u32			src_qp;
1047	u32			slid;
1048	int			wc_flags;
1049	u16			pkey_index;
1050	u8			sl;
1051	u8			dlid_path_bits;
1052	u32 port_num; /* valid only for DR SMPs on switches */
1053	u8			smac[ETH_ALEN];
1054	u16			vlan_id;
1055	u8			network_hdr_type;
1056};
1057
1058enum ib_cq_notify_flags {
1059	IB_CQ_SOLICITED			= 1 << 0,
1060	IB_CQ_NEXT_COMP			= 1 << 1,
1061	IB_CQ_SOLICITED_MASK		= IB_CQ_SOLICITED | IB_CQ_NEXT_COMP,
1062	IB_CQ_REPORT_MISSED_EVENTS	= 1 << 2,
1063};
1064
1065enum ib_srq_type {
1066	IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC,
1067	IB_SRQT_XRC = IB_UVERBS_SRQT_XRC,
1068	IB_SRQT_TM = IB_UVERBS_SRQT_TM,
1069};
1070
1071static inline bool ib_srq_has_cq(enum ib_srq_type srq_type)
1072{
1073	return srq_type == IB_SRQT_XRC ||
1074	       srq_type == IB_SRQT_TM;
1075}
1076
1077enum ib_srq_attr_mask {
1078	IB_SRQ_MAX_WR	= 1 << 0,
1079	IB_SRQ_LIMIT	= 1 << 1,
1080};
1081
1082struct ib_srq_attr {
1083	u32	max_wr;
1084	u32	max_sge;
1085	u32	srq_limit;
1086};
1087
1088struct ib_srq_init_attr {
1089	void		      (*event_handler)(struct ib_event *, void *);
1090	void		       *srq_context;
1091	struct ib_srq_attr	attr;
1092	enum ib_srq_type	srq_type;
1093
1094	struct {
1095		struct ib_cq   *cq;
1096		union {
1097			struct {
1098				struct ib_xrcd *xrcd;
1099			} xrc;
1100
1101			struct {
1102				u32		max_num_tags;
1103			} tag_matching;
1104		};
1105	} ext;
1106};
1107
1108struct ib_qp_cap {
1109	u32	max_send_wr;
1110	u32	max_recv_wr;
1111	u32	max_send_sge;
1112	u32	max_recv_sge;
1113	u32	max_inline_data;
1114
1115	/*
1116	 * Maximum number of rdma_rw_ctx structures in flight at a time.
1117	 * ib_create_qp() will calculate the right amount of needed WRs
1118	 * and MRs based on this.
1119	 */
1120	u32	max_rdma_ctxs;
1121};
1122
1123enum ib_sig_type {
1124	IB_SIGNAL_ALL_WR,
1125	IB_SIGNAL_REQ_WR
1126};
1127
1128enum ib_qp_type {
1129	/*
1130	 * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries
1131	 * here (and in that order) since the MAD layer uses them as
1132	 * indices into a 2-entry table.
1133	 */
1134	IB_QPT_SMI,
1135	IB_QPT_GSI,
1136
1137	IB_QPT_RC = IB_UVERBS_QPT_RC,
1138	IB_QPT_UC = IB_UVERBS_QPT_UC,
1139	IB_QPT_UD = IB_UVERBS_QPT_UD,
1140	IB_QPT_RAW_IPV6,
1141	IB_QPT_RAW_ETHERTYPE,
1142	IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET,
1143	IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI,
1144	IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT,
1145	IB_QPT_MAX,
1146	IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER,
1147	/* Reserve a range for qp types internal to the low level driver.
1148	 * These qp types will not be visible at the IB core layer, so the
1149	 * IB_QPT_MAX usages should not be affected in the core layer
1150	 */
1151	IB_QPT_RESERVED1 = 0x1000,
1152	IB_QPT_RESERVED2,
1153	IB_QPT_RESERVED3,
1154	IB_QPT_RESERVED4,
1155	IB_QPT_RESERVED5,
1156	IB_QPT_RESERVED6,
1157	IB_QPT_RESERVED7,
1158	IB_QPT_RESERVED8,
1159	IB_QPT_RESERVED9,
1160	IB_QPT_RESERVED10,
1161};
1162
1163enum ib_qp_create_flags {
1164	IB_QP_CREATE_IPOIB_UD_LSO		= 1 << 0,
1165	IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK	=
1166		IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
1167	IB_QP_CREATE_CROSS_CHANNEL              = 1 << 2,
1168	IB_QP_CREATE_MANAGED_SEND               = 1 << 3,
1169	IB_QP_CREATE_MANAGED_RECV               = 1 << 4,
1170	IB_QP_CREATE_NETIF_QP			= 1 << 5,
1171	IB_QP_CREATE_INTEGRITY_EN		= 1 << 6,
1172	IB_QP_CREATE_NETDEV_USE			= 1 << 7,
1173	IB_QP_CREATE_SCATTER_FCS		=
1174		IB_UVERBS_QP_CREATE_SCATTER_FCS,
1175	IB_QP_CREATE_CVLAN_STRIPPING		=
1176		IB_UVERBS_QP_CREATE_CVLAN_STRIPPING,
1177	IB_QP_CREATE_SOURCE_QPN			= 1 << 10,
1178	IB_QP_CREATE_PCI_WRITE_END_PADDING	=
1179		IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING,
1180	/* reserve bits 26-31 for low level drivers' internal use */
1181	IB_QP_CREATE_RESERVED_START		= 1 << 26,
1182	IB_QP_CREATE_RESERVED_END		= 1 << 31,
1183};
1184
1185/*
1186 * Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler
1187 * callback to destroy the passed in QP.
1188 */
1189
1190struct ib_qp_init_attr {
1191	/* This callback occurs in workqueue context */
1192	void                  (*event_handler)(struct ib_event *, void *);
1193
1194	void		       *qp_context;
1195	struct ib_cq	       *send_cq;
1196	struct ib_cq	       *recv_cq;
1197	struct ib_srq	       *srq;
1198	struct ib_xrcd	       *xrcd;     /* XRC TGT QPs only */
1199	struct ib_qp_cap	cap;
1200	enum ib_sig_type	sq_sig_type;
1201	enum ib_qp_type		qp_type;
1202	u32			create_flags;
1203
1204	/*
1205	 * Only needed for special QP types, or when using the RW API.
1206	 */
1207	u32			port_num;
1208	struct ib_rwq_ind_table *rwq_ind_tbl;
1209	u32			source_qpn;
1210};
1211
1212struct ib_qp_open_attr {
1213	void                  (*event_handler)(struct ib_event *, void *);
1214	void		       *qp_context;
1215	u32			qp_num;
1216	enum ib_qp_type		qp_type;
1217};
1218
1219enum ib_rnr_timeout {
1220	IB_RNR_TIMER_655_36 =  0,
1221	IB_RNR_TIMER_000_01 =  1,
1222	IB_RNR_TIMER_000_02 =  2,
1223	IB_RNR_TIMER_000_03 =  3,
1224	IB_RNR_TIMER_000_04 =  4,
1225	IB_RNR_TIMER_000_06 =  5,
1226	IB_RNR_TIMER_000_08 =  6,
1227	IB_RNR_TIMER_000_12 =  7,
1228	IB_RNR_TIMER_000_16 =  8,
1229	IB_RNR_TIMER_000_24 =  9,
1230	IB_RNR_TIMER_000_32 = 10,
1231	IB_RNR_TIMER_000_48 = 11,
1232	IB_RNR_TIMER_000_64 = 12,
1233	IB_RNR_TIMER_000_96 = 13,
1234	IB_RNR_TIMER_001_28 = 14,
1235	IB_RNR_TIMER_001_92 = 15,
1236	IB_RNR_TIMER_002_56 = 16,
1237	IB_RNR_TIMER_003_84 = 17,
1238	IB_RNR_TIMER_005_12 = 18,
1239	IB_RNR_TIMER_007_68 = 19,
1240	IB_RNR_TIMER_010_24 = 20,
1241	IB_RNR_TIMER_015_36 = 21,
1242	IB_RNR_TIMER_020_48 = 22,
1243	IB_RNR_TIMER_030_72 = 23,
1244	IB_RNR_TIMER_040_96 = 24,
1245	IB_RNR_TIMER_061_44 = 25,
1246	IB_RNR_TIMER_081_92 = 26,
1247	IB_RNR_TIMER_122_88 = 27,
1248	IB_RNR_TIMER_163_84 = 28,
1249	IB_RNR_TIMER_245_76 = 29,
1250	IB_RNR_TIMER_327_68 = 30,
1251	IB_RNR_TIMER_491_52 = 31
1252};
1253
1254enum ib_qp_attr_mask {
1255	IB_QP_STATE			= 1,
1256	IB_QP_CUR_STATE			= (1<<1),
1257	IB_QP_EN_SQD_ASYNC_NOTIFY	= (1<<2),
1258	IB_QP_ACCESS_FLAGS		= (1<<3),
1259	IB_QP_PKEY_INDEX		= (1<<4),
1260	IB_QP_PORT			= (1<<5),
1261	IB_QP_QKEY			= (1<<6),
1262	IB_QP_AV			= (1<<7),
1263	IB_QP_PATH_MTU			= (1<<8),
1264	IB_QP_TIMEOUT			= (1<<9),
1265	IB_QP_RETRY_CNT			= (1<<10),
1266	IB_QP_RNR_RETRY			= (1<<11),
1267	IB_QP_RQ_PSN			= (1<<12),
1268	IB_QP_MAX_QP_RD_ATOMIC		= (1<<13),
1269	IB_QP_ALT_PATH			= (1<<14),
1270	IB_QP_MIN_RNR_TIMER		= (1<<15),
1271	IB_QP_SQ_PSN			= (1<<16),
1272	IB_QP_MAX_DEST_RD_ATOMIC	= (1<<17),
1273	IB_QP_PATH_MIG_STATE		= (1<<18),
1274	IB_QP_CAP			= (1<<19),
1275	IB_QP_DEST_QPN			= (1<<20),
1276	IB_QP_RESERVED1			= (1<<21),
1277	IB_QP_RESERVED2			= (1<<22),
1278	IB_QP_RESERVED3			= (1<<23),
1279	IB_QP_RESERVED4			= (1<<24),
1280	IB_QP_RATE_LIMIT		= (1<<25),
1281
1282	IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0),
1283};
1284
1285enum ib_qp_state {
1286	IB_QPS_RESET,
1287	IB_QPS_INIT,
1288	IB_QPS_RTR,
1289	IB_QPS_RTS,
1290	IB_QPS_SQD,
1291	IB_QPS_SQE,
1292	IB_QPS_ERR
1293};
1294
1295enum ib_mig_state {
1296	IB_MIG_MIGRATED,
1297	IB_MIG_REARM,
1298	IB_MIG_ARMED
1299};
1300
1301enum ib_mw_type {
1302	IB_MW_TYPE_1 = 1,
1303	IB_MW_TYPE_2 = 2
1304};
1305
1306struct ib_qp_attr {
1307	enum ib_qp_state	qp_state;
1308	enum ib_qp_state	cur_qp_state;
1309	enum ib_mtu		path_mtu;
1310	enum ib_mig_state	path_mig_state;
1311	u32			qkey;
1312	u32			rq_psn;
1313	u32			sq_psn;
1314	u32			dest_qp_num;
1315	int			qp_access_flags;
1316	struct ib_qp_cap	cap;
1317	struct rdma_ah_attr	ah_attr;
1318	struct rdma_ah_attr	alt_ah_attr;
1319	u16			pkey_index;
1320	u16			alt_pkey_index;
1321	u8			en_sqd_async_notify;
1322	u8			sq_draining;
1323	u8			max_rd_atomic;
1324	u8			max_dest_rd_atomic;
1325	u8			min_rnr_timer;
1326	u32			port_num;
1327	u8			timeout;
1328	u8			retry_cnt;
1329	u8			rnr_retry;
1330	u32			alt_port_num;
1331	u8			alt_timeout;
1332	u32			rate_limit;
1333	struct net_device	*xmit_slave;
1334};
1335
1336enum ib_wr_opcode {
1337	/* These are shared with userspace */
1338	IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE,
1339	IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM,
1340	IB_WR_SEND = IB_UVERBS_WR_SEND,
1341	IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM,
1342	IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ,
1343	IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP,
1344	IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD,
1345	IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW,
1346	IB_WR_LSO = IB_UVERBS_WR_TSO,
1347	IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV,
1348	IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV,
1349	IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV,
1350	IB_WR_MASKED_ATOMIC_CMP_AND_SWP =
1351		IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
1352	IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
1353		IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
1354	IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,
1355	IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,
1356
1357	/* These are kernel only and can not be issued by userspace */
1358	IB_WR_REG_MR = 0x20,
1359	IB_WR_REG_MR_INTEGRITY,
1360
1361	/* reserve values for low level drivers' internal use.
1362	 * These values will not be used at all in the ib core layer.
1363	 */
1364	IB_WR_RESERVED1 = 0xf0,
1365	IB_WR_RESERVED2,
1366	IB_WR_RESERVED3,
1367	IB_WR_RESERVED4,
1368	IB_WR_RESERVED5,
1369	IB_WR_RESERVED6,
1370	IB_WR_RESERVED7,
1371	IB_WR_RESERVED8,
1372	IB_WR_RESERVED9,
1373	IB_WR_RESERVED10,
1374};
1375
1376enum ib_send_flags {
1377	IB_SEND_FENCE		= 1,
1378	IB_SEND_SIGNALED	= (1<<1),
1379	IB_SEND_SOLICITED	= (1<<2),
1380	IB_SEND_INLINE		= (1<<3),
1381	IB_SEND_IP_CSUM		= (1<<4),
1382
1383	/* reserve bits 26-31 for low level drivers' internal use */
1384	IB_SEND_RESERVED_START	= (1 << 26),
1385	IB_SEND_RESERVED_END	= (1 << 31),
1386};
1387
1388struct ib_sge {
1389	u64	addr;
1390	u32	length;
1391	u32	lkey;
1392};
1393
1394struct ib_cqe {
1395	void (*done)(struct ib_cq *cq, struct ib_wc *wc);
1396};
1397
1398struct ib_send_wr {
1399	struct ib_send_wr      *next;
1400	union {
1401		u64		wr_id;
1402		struct ib_cqe	*wr_cqe;
1403	};
1404	struct ib_sge	       *sg_list;
1405	int			num_sge;
1406	enum ib_wr_opcode	opcode;
1407	int			send_flags;
1408	union {
1409		__be32		imm_data;
1410		u32		invalidate_rkey;
1411	} ex;
1412};
1413
1414struct ib_rdma_wr {
1415	struct ib_send_wr	wr;
1416	u64			remote_addr;
1417	u32			rkey;
1418};
1419
1420static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr)
1421{
1422	return container_of(wr, struct ib_rdma_wr, wr);
1423}
1424
1425struct ib_atomic_wr {
1426	struct ib_send_wr	wr;
1427	u64			remote_addr;
1428	u64			compare_add;
1429	u64			swap;
1430	u64			compare_add_mask;
1431	u64			swap_mask;
1432	u32			rkey;
1433};
1434
1435static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr)
1436{
1437	return container_of(wr, struct ib_atomic_wr, wr);
1438}
1439
1440struct ib_ud_wr {
1441	struct ib_send_wr	wr;
1442	struct ib_ah		*ah;
1443	void			*header;
1444	int			hlen;
1445	int			mss;
1446	u32			remote_qpn;
1447	u32			remote_qkey;
1448	u16			pkey_index; /* valid for GSI only */
1449	u32			port_num; /* valid for DR SMPs on switch only */
1450};
1451
1452static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr)
1453{
1454	return container_of(wr, struct ib_ud_wr, wr);
1455}
1456
1457struct ib_reg_wr {
1458	struct ib_send_wr	wr;
1459	struct ib_mr		*mr;
1460	u32			key;
1461	int			access;
1462};
1463
1464static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
1465{
1466	return container_of(wr, struct ib_reg_wr, wr);
1467}
1468
1469struct ib_recv_wr {
1470	struct ib_recv_wr      *next;
1471	union {
1472		u64		wr_id;
1473		struct ib_cqe	*wr_cqe;
1474	};
1475	struct ib_sge	       *sg_list;
1476	int			num_sge;
1477};
1478
1479enum ib_access_flags {
1480	IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE,
1481	IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE,
1482	IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ,
1483	IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC,
1484	IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND,
1485	IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED,
1486	IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
1487	IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
1488	IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
1489	IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,
1490	IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,
1491
1492	IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
1493	IB_ACCESS_SUPPORTED =
1494		((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL,
1495};
1496
1497/*
1498 * XXX: these are apparently used for ->rereg_user_mr, no idea why they
1499 * are hidden here instead of a uapi header!
1500 */
1501enum ib_mr_rereg_flags {
1502	IB_MR_REREG_TRANS	= 1,
1503	IB_MR_REREG_PD		= (1<<1),
1504	IB_MR_REREG_ACCESS	= (1<<2),
1505	IB_MR_REREG_SUPPORTED	= ((IB_MR_REREG_ACCESS << 1) - 1)
1506};
1507
1508struct ib_umem;
1509
1510enum rdma_remove_reason {
1511	/*
1512	 * Userspace requested uobject deletion or initial try
1513	 * to remove uobject via cleanup. Call could fail
1514	 */
1515	RDMA_REMOVE_DESTROY,
1516	/* Context deletion. This call should delete the actual object itself */
1517	RDMA_REMOVE_CLOSE,
1518	/* Driver is being hot-unplugged. This call should delete the actual object itself */
1519	RDMA_REMOVE_DRIVER_REMOVE,
1520	/* uobj is being cleaned-up before being committed */
1521	RDMA_REMOVE_ABORT,
1522	/* The driver failed to destroy the uobject and is being disconnected */
1523	RDMA_REMOVE_DRIVER_FAILURE,
1524};
1525
1526struct ib_rdmacg_object {
1527#ifdef CONFIG_CGROUP_RDMA
1528	struct rdma_cgroup	*cg;		/* owner rdma cgroup */
1529#endif
1530};
1531
1532struct ib_ucontext {
1533	struct ib_device       *device;
1534	struct ib_uverbs_file  *ufile;
1535
1536	struct ib_rdmacg_object	cg_obj;
1537	u64 enabled_caps;
1538	/*
1539	 * Implementation details of the RDMA core, don't use in drivers:
1540	 */
1541	struct rdma_restrack_entry res;
1542	struct xarray mmap_xa;
1543};
1544
1545struct ib_uobject {
1546	u64			user_handle;	/* handle given to us by userspace */
1547	/* ufile & ucontext owning this object */
1548	struct ib_uverbs_file  *ufile;
1549	/* FIXME, save memory: ufile->context == context */
1550	struct ib_ucontext     *context;	/* associated user context */
1551	void		       *object;		/* containing object */
1552	struct list_head	list;		/* link to context's list */
1553	struct ib_rdmacg_object	cg_obj;		/* rdmacg object */
1554	int			id;		/* index into kernel idr */
1555	struct kref		ref;
1556	atomic_t		usecnt;		/* protects exclusive access */
1557	struct rcu_head		rcu;		/* kfree_rcu() overhead */
1558
1559	const struct uverbs_api_object *uapi_object;
1560};
1561
1562struct ib_udata {
1563	const void __user *inbuf;
1564	void __user *outbuf;
1565	size_t       inlen;
1566	size_t       outlen;
1567};
1568
1569struct ib_pd {
1570	u32			local_dma_lkey;
1571	u32			flags;
1572	struct ib_device       *device;
1573	struct ib_uobject      *uobject;
1574	atomic_t          	usecnt; /* count all resources */
1575
1576	u32			unsafe_global_rkey;
1577
1578	/*
1579	 * Implementation details of the RDMA core, don't use in drivers:
1580	 */
1581	struct ib_mr	       *__internal_mr;
1582	struct rdma_restrack_entry res;
1583};
1584
1585struct ib_xrcd {
1586	struct ib_device       *device;
1587	atomic_t		usecnt; /* count all exposed resources */
1588	struct inode	       *inode;
1589	struct rw_semaphore	tgt_qps_rwsem;
1590	struct xarray		tgt_qps;
1591};
1592
1593struct ib_ah {
1594	struct ib_device	*device;
1595	struct ib_pd		*pd;
1596	struct ib_uobject	*uobject;
1597	const struct ib_gid_attr *sgid_attr;
1598	enum rdma_ah_attr_type	type;
1599};
1600
1601typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);
1602
1603enum ib_poll_context {
1604	IB_POLL_SOFTIRQ,	   /* poll from softirq context */
1605	IB_POLL_WORKQUEUE,	   /* poll from workqueue */
1606	IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */
1607	IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE,
1608
1609	IB_POLL_DIRECT,		   /* caller context, no hw completions */
1610};
1611
1612struct ib_cq {
1613	struct ib_device       *device;
1614	struct ib_ucq_object   *uobject;
1615	ib_comp_handler   	comp_handler;
1616	void                  (*event_handler)(struct ib_event *, void *);
1617	void                   *cq_context;
1618	int               	cqe;
1619	unsigned int		cqe_used;
1620	atomic_t          	usecnt; /* count number of work queues */
1621	enum ib_poll_context	poll_ctx;
1622	struct ib_wc		*wc;
1623	struct list_head        pool_entry;
1624	union {
1625		struct irq_poll		iop;
1626		struct work_struct	work;
1627	};
1628	struct workqueue_struct *comp_wq;
1629	struct dim *dim;
1630
1631	/* updated only by trace points */
1632	ktime_t timestamp;
1633	u8 interrupt:1;
1634	u8 shared:1;
1635	unsigned int comp_vector;
1636
1637	/*
1638	 * Implementation details of the RDMA core, don't use in drivers:
1639	 */
1640	struct rdma_restrack_entry res;
1641};
1642
1643struct ib_srq {
1644	struct ib_device       *device;
1645	struct ib_pd	       *pd;
1646	struct ib_usrq_object  *uobject;
1647	void		      (*event_handler)(struct ib_event *, void *);
1648	void		       *srq_context;
1649	enum ib_srq_type	srq_type;
1650	atomic_t		usecnt;
1651
1652	struct {
1653		struct ib_cq   *cq;
1654		union {
1655			struct {
1656				struct ib_xrcd *xrcd;
1657				u32		srq_num;
1658			} xrc;
1659		};
1660	} ext;
1661
1662	/*
1663	 * Implementation details of the RDMA core, don't use in drivers:
1664	 */
1665	struct rdma_restrack_entry res;
1666};
1667
1668enum ib_raw_packet_caps {
1669	/*
1670	 * Strip cvlan from incoming packet and report it in the matching work
1671	 * completion is supported.
1672	 */
1673	IB_RAW_PACKET_CAP_CVLAN_STRIPPING =
1674		IB_UVERBS_RAW_PACKET_CAP_CVLAN_STRIPPING,
1675	/*
1676	 * Scatter FCS field of an incoming packet to host memory is supported.
1677	 */
1678	IB_RAW_PACKET_CAP_SCATTER_FCS = IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS,
1679	/* Checksum offloads are supported (for both send and receive). */
1680	IB_RAW_PACKET_CAP_IP_CSUM = IB_UVERBS_RAW_PACKET_CAP_IP_CSUM,
1681	/*
1682	 * When a packet is received for an RQ with no receive WQEs, the
1683	 * packet processing is delayed.
1684	 */
1685	IB_RAW_PACKET_CAP_DELAY_DROP = IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP,
1686};
1687
1688enum ib_wq_type {
1689	IB_WQT_RQ = IB_UVERBS_WQT_RQ,
1690};
1691
1692enum ib_wq_state {
1693	IB_WQS_RESET,
1694	IB_WQS_RDY,
1695	IB_WQS_ERR
1696};
1697
1698struct ib_wq {
1699	struct ib_device       *device;
1700	struct ib_uwq_object   *uobject;
1701	void		    *wq_context;
1702	void		    (*event_handler)(struct ib_event *, void *);
1703	struct ib_pd	       *pd;
1704	struct ib_cq	       *cq;
1705	u32		wq_num;
1706	enum ib_wq_state       state;
1707	enum ib_wq_type	wq_type;
1708	atomic_t		usecnt;
1709};
1710
1711enum ib_wq_flags {
1712	IB_WQ_FLAGS_CVLAN_STRIPPING	= IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING,
1713	IB_WQ_FLAGS_SCATTER_FCS		= IB_UVERBS_WQ_FLAGS_SCATTER_FCS,
1714	IB_WQ_FLAGS_DELAY_DROP		= IB_UVERBS_WQ_FLAGS_DELAY_DROP,
1715	IB_WQ_FLAGS_PCI_WRITE_END_PADDING =
1716				IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING,
1717};
1718
1719struct ib_wq_init_attr {
1720	void		       *wq_context;
1721	enum ib_wq_type	wq_type;
1722	u32		max_wr;
1723	u32		max_sge;
1724	struct	ib_cq	       *cq;
1725	void		    (*event_handler)(struct ib_event *, void *);
1726	u32		create_flags; /* Use enum ib_wq_flags */
1727};
1728
1729enum ib_wq_attr_mask {
1730	IB_WQ_STATE		= 1 << 0,
1731	IB_WQ_CUR_STATE		= 1 << 1,
1732	IB_WQ_FLAGS		= 1 << 2,
1733};
1734
1735struct ib_wq_attr {
1736	enum	ib_wq_state	wq_state;
1737	enum	ib_wq_state	curr_wq_state;
1738	u32			flags; /* Use enum ib_wq_flags */
1739	u32			flags_mask; /* Use enum ib_wq_flags */
1740};
1741
1742struct ib_rwq_ind_table {
1743	struct ib_device	*device;
1744	struct ib_uobject      *uobject;
1745	atomic_t		usecnt;
1746	u32		ind_tbl_num;
1747	u32		log_ind_tbl_size;
1748	struct ib_wq	**ind_tbl;
1749};
1750
1751struct ib_rwq_ind_table_init_attr {
1752	u32		log_ind_tbl_size;
1753	/* Each entry is a pointer to Receive Work Queue */
1754	struct ib_wq	**ind_tbl;
1755};
1756
1757enum port_pkey_state {
1758	IB_PORT_PKEY_NOT_VALID = 0,
1759	IB_PORT_PKEY_VALID = 1,
1760	IB_PORT_PKEY_LISTED = 2,
1761};
1762
1763struct ib_qp_security;
1764
1765struct ib_port_pkey {
1766	enum port_pkey_state	state;
1767	u16			pkey_index;
1768	u32			port_num;
1769	struct list_head	qp_list;
1770	struct list_head	to_error_list;
1771	struct ib_qp_security  *sec;
1772};
1773
1774struct ib_ports_pkeys {
1775	struct ib_port_pkey	main;
1776	struct ib_port_pkey	alt;
1777};
1778
1779struct ib_qp_security {
1780	struct ib_qp	       *qp;
1781	struct ib_device       *dev;
1782	/* Hold this mutex when changing port and pkey settings. */
1783	struct mutex		mutex;
1784	struct ib_ports_pkeys  *ports_pkeys;
1785	/* A list of all open shared QP handles.  Required to enforce security
1786	 * properly for all users of a shared QP.
1787	 */
1788	struct list_head        shared_qp_list;
1789	void                   *security;
1790	bool			destroying;
1791	atomic_t		error_list_count;
1792	struct completion	error_complete;
1793	int			error_comps_pending;
1794};
1795
1796/*
1797 * @max_write_sge: Maximum SGE elements per RDMA WRITE request.
1798 * @max_read_sge:  Maximum SGE elements per RDMA READ request.
1799 */
1800struct ib_qp {
1801	struct ib_device       *device;
1802	struct ib_pd	       *pd;
1803	struct ib_cq	       *send_cq;
1804	struct ib_cq	       *recv_cq;
1805	spinlock_t		mr_lock;
1806	int			mrs_used;
1807	struct list_head	rdma_mrs;
1808	struct list_head	sig_mrs;
1809	struct ib_srq	       *srq;
1810	struct completion	srq_completion;
1811	struct ib_xrcd	       *xrcd; /* XRC TGT QPs only */
1812	struct list_head	xrcd_list;
1813
1814	/* count times opened, mcast attaches, flow attaches */
1815	atomic_t		usecnt;
1816	struct list_head	open_list;
1817	struct ib_qp           *real_qp;
1818	struct ib_uqp_object   *uobject;
1819	void                  (*event_handler)(struct ib_event *, void *);
1820	void                  (*registered_event_handler)(struct ib_event *, void *);
1821	void		       *qp_context;
1822	/* sgid_attrs associated with the AV's */
1823	const struct ib_gid_attr *av_sgid_attr;
1824	const struct ib_gid_attr *alt_path_sgid_attr;
1825	u32			qp_num;
1826	u32			max_write_sge;
1827	u32			max_read_sge;
1828	enum ib_qp_type		qp_type;
1829	struct ib_rwq_ind_table *rwq_ind_tbl;
1830	struct ib_qp_security  *qp_sec;
1831	u32			port;
1832
1833	bool			integrity_en;
1834	/*
1835	 * Implementation details of the RDMA core, don't use in drivers:
1836	 */
1837	struct rdma_restrack_entry     res;
1838
1839	/* The counter the qp is bind to */
1840	struct rdma_counter    *counter;
1841};
1842
1843struct ib_dm {
1844	struct ib_device  *device;
1845	u32		   length;
1846	u32		   flags;
1847	struct ib_uobject *uobject;
1848	atomic_t	   usecnt;
1849};
1850
1851/* bit values to mark existence of ib_dmah fields */
1852enum {
1853	IB_DMAH_CPU_ID_EXISTS,
1854	IB_DMAH_MEM_TYPE_EXISTS,
1855	IB_DMAH_PH_EXISTS,
1856};
1857
1858struct ib_dmah {
1859	struct ib_device *device;
1860	struct ib_uobject *uobject;
1861	/*
1862	 * Implementation details of the RDMA core, don't use in drivers:
1863	 */
1864	struct rdma_restrack_entry res;
1865	u32 cpu_id;
1866	enum tph_mem_type mem_type;
1867	atomic_t usecnt;
1868	u8 ph;
1869	u8 valid_fields; /* use IB_DMAH_XXX_EXISTS */
1870};
1871
1872struct ib_mr {
1873	struct ib_device  *device;
1874	struct ib_pd	  *pd;
1875	u32		   lkey;
1876	u32		   rkey;
1877	u64		   iova;
1878	u64		   length;
1879	unsigned int	   page_size;
1880	enum ib_mr_type	   type;
1881	bool		   need_inval;
1882	union {
1883		struct ib_uobject	*uobject;	/* user */
1884		struct list_head	qp_entry;	/* FR */
1885	};
1886
1887	struct ib_dm      *dm;
1888	struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
1889	struct ib_dmah *dmah;
1890	/*
1891	 * Implementation details of the RDMA core, don't use in drivers:
1892	 */
1893	struct rdma_restrack_entry res;
1894};
1895
1896struct ib_mw {
1897	struct ib_device	*device;
1898	struct ib_pd		*pd;
1899	struct ib_uobject	*uobject;
1900	u32			rkey;
1901	enum ib_mw_type         type;
1902};
1903
1904/* Supported steering options */
1905enum ib_flow_attr_type {
1906	/* steering according to rule specifications */
1907	IB_FLOW_ATTR_NORMAL		= 0x0,
1908	/* default unicast and multicast rule -
1909	 * receive all Eth traffic which isn't steered to any QP
1910	 */
1911	IB_FLOW_ATTR_ALL_DEFAULT	= 0x1,
1912	/* default multicast rule -
1913	 * receive all Eth multicast traffic which isn't steered to any QP
1914	 */
1915	IB_FLOW_ATTR_MC_DEFAULT		= 0x2,
1916	/* sniffer rule - receive all port traffic */
1917	IB_FLOW_ATTR_SNIFFER		= 0x3
1918};
1919
1920/* Supported steering header types */
1921enum ib_flow_spec_type {
1922	/* L2 headers*/
1923	IB_FLOW_SPEC_ETH		= 0x20,
1924	IB_FLOW_SPEC_IB			= 0x22,
1925	/* L3 header*/
1926	IB_FLOW_SPEC_IPV4		= 0x30,
1927	IB_FLOW_SPEC_IPV6		= 0x31,
1928	IB_FLOW_SPEC_ESP                = 0x34,
1929	/* L4 headers*/
1930	IB_FLOW_SPEC_TCP		= 0x40,
1931	IB_FLOW_SPEC_UDP		= 0x41,
1932	IB_FLOW_SPEC_VXLAN_TUNNEL	= 0x50,
1933	IB_FLOW_SPEC_GRE		= 0x51,
1934	IB_FLOW_SPEC_MPLS		= 0x60,
1935	IB_FLOW_SPEC_INNER		= 0x100,
1936	/* Actions */
1937	IB_FLOW_SPEC_ACTION_TAG         = 0x1000,
1938	IB_FLOW_SPEC_ACTION_DROP        = 0x1001,
1939	IB_FLOW_SPEC_ACTION_HANDLE	= 0x1002,
1940	IB_FLOW_SPEC_ACTION_COUNT       = 0x1003,
1941};
1942#define IB_FLOW_SPEC_LAYER_MASK	0xF0
1943#define IB_FLOW_SPEC_SUPPORT_LAYERS 10
1944
1945enum ib_flow_flags {
1946	IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */
1947	IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */
1948	IB_FLOW_ATTR_FLAGS_RESERVED  = 1UL << 3  /* Must be last */
1949};
1950
1951struct ib_flow_eth_filter {
1952	u8	dst_mac[6];
1953	u8	src_mac[6];
1954	__be16	ether_type;
1955	__be16	vlan_tag;
1956};
1957
1958struct ib_flow_spec_eth {
1959	u32			  type;
1960	u16			  size;
1961	struct ib_flow_eth_filter val;
1962	struct ib_flow_eth_filter mask;
1963};
1964
1965struct ib_flow_ib_filter {
1966	__be16 dlid;
1967	__u8   sl;
1968};
1969
1970struct ib_flow_spec_ib {
1971	u32			 type;
1972	u16			 size;
1973	struct ib_flow_ib_filter val;
1974	struct ib_flow_ib_filter mask;
1975};
1976
1977/* IPv4 header flags */
1978enum ib_ipv4_flags {
1979	IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */
1980	IB_IPV4_MORE_FRAG = 0X4  /* For All fragmented packets except the
1981				    last have this flag set */
1982};
1983
1984struct ib_flow_ipv4_filter {
1985	__be32	src_ip;
1986	__be32	dst_ip;
1987	u8	proto;
1988	u8	tos;
1989	u8	ttl;
1990	u8	flags;
1991};
1992
1993struct ib_flow_spec_ipv4 {
1994	u32			   type;
1995	u16			   size;
1996	struct ib_flow_ipv4_filter val;
1997	struct ib_flow_ipv4_filter mask;
1998};
1999
2000struct ib_flow_ipv6_filter {
2001	u8	src_ip[16];
2002	u8	dst_ip[16];
2003	__be32	flow_label;
2004	u8	next_hdr;
2005	u8	traffic_class;
2006	u8	hop_limit;
2007} __packed;
2008
2009struct ib_flow_spec_ipv6 {
2010	u32			   type;
2011	u16			   size;
2012	struct ib_flow_ipv6_filter val;
2013	struct ib_flow_ipv6_filter mask;
2014};
2015
2016struct ib_flow_tcp_udp_filter {
2017	__be16	dst_port;
2018	__be16	src_port;
2019};
2020
2021struct ib_flow_spec_tcp_udp {
2022	u32			      type;
2023	u16			      size;
2024	struct ib_flow_tcp_udp_filter val;
2025	struct ib_flow_tcp_udp_filter mask;
2026};
2027
2028struct ib_flow_tunnel_filter {
2029	__be32	tunnel_id;
2030};
2031
2032/* ib_flow_spec_tunnel describes the Vxlan tunnel
2033 * the tunnel_id from val has the vni value
2034 */
2035struct ib_flow_spec_tunnel {
2036	u32			      type;
2037	u16			      size;
2038	struct ib_flow_tunnel_filter  val;
2039	struct ib_flow_tunnel_filter  mask;
2040};
2041
2042struct ib_flow_esp_filter {
2043	__be32	spi;
2044	__be32  seq;
2045};
2046
2047struct ib_flow_spec_esp {
2048	u32                           type;
2049	u16			      size;
2050	struct ib_flow_esp_filter     val;
2051	struct ib_flow_esp_filter     mask;
2052};
2053
2054struct ib_flow_gre_filter {
2055	__be16 c_ks_res0_ver;
2056	__be16 protocol;
2057	__be32 key;
2058};
2059
2060struct ib_flow_spec_gre {
2061	u32                           type;
2062	u16			      size;
2063	struct ib_flow_gre_filter     val;
2064	struct ib_flow_gre_filter     mask;
2065};
2066
2067struct ib_flow_mpls_filter {
2068	__be32 tag;
2069};
2070
2071struct ib_flow_spec_mpls {
2072	u32                           type;
2073	u16			      size;
2074	struct ib_flow_mpls_filter     val;
2075	struct ib_flow_mpls_filter     mask;
2076};
2077
2078struct ib_flow_spec_action_tag {
2079	enum ib_flow_spec_type	      type;
2080	u16			      size;
2081	u32                           tag_id;
2082};
2083
2084struct ib_flow_spec_action_drop {
2085	enum ib_flow_spec_type	      type;
2086	u16			      size;
2087};
2088
2089struct ib_flow_spec_action_handle {
2090	enum ib_flow_spec_type	      type;
2091	u16			      size;
2092	struct ib_flow_action	     *act;
2093};
2094
2095enum ib_counters_description {
2096	IB_COUNTER_PACKETS,
2097	IB_COUNTER_BYTES,
2098};
2099
2100struct ib_flow_spec_action_count {
2101	enum ib_flow_spec_type type;
2102	u16 size;
2103	struct ib_counters *counters;
2104};
2105
2106union ib_flow_spec {
2107	struct {
2108		u32			type;
2109		u16			size;
2110	};
2111	struct ib_flow_spec_eth		eth;
2112	struct ib_flow_spec_ib		ib;
2113	struct ib_flow_spec_ipv4        ipv4;
2114	struct ib_flow_spec_tcp_udp	tcp_udp;
2115	struct ib_flow_spec_ipv6        ipv6;
2116	struct ib_flow_spec_tunnel      tunnel;
2117	struct ib_flow_spec_esp		esp;
2118	struct ib_flow_spec_gre		gre;
2119	struct ib_flow_spec_mpls	mpls;
2120	struct ib_flow_spec_action_tag  flow_tag;
2121	struct ib_flow_spec_action_drop drop;
2122	struct ib_flow_spec_action_handle action;
2123	struct ib_flow_spec_action_count flow_count;
2124};
2125
2126struct ib_flow_attr {
2127	enum ib_flow_attr_type type;
2128	u16	     size;
2129	u16	     priority;
2130	u32	     flags;
2131	u8	     num_of_specs;
2132	u32	     port;
2133	union ib_flow_spec flows[];
2134};
2135
2136struct ib_flow {
2137	struct ib_qp		*qp;
2138	struct ib_device	*device;
2139	struct ib_uobject	*uobject;
2140};
2141
2142enum ib_flow_action_type {
2143	IB_FLOW_ACTION_UNSPECIFIED,
2144	IB_FLOW_ACTION_ESP = 1,
2145};
2146
2147struct ib_flow_action_attrs_esp_keymats {
2148	enum ib_uverbs_flow_action_esp_keymat			protocol;
2149	union {
2150		struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm;
2151	} keymat;
2152};
2153
2154struct ib_flow_action_attrs_esp_replays {
2155	enum ib_uverbs_flow_action_esp_replay			protocol;
2156	union {
2157		struct ib_uverbs_flow_action_esp_replay_bmp	bmp;
2158	} replay;
2159};
2160
2161enum ib_flow_action_attrs_esp_flags {
2162	/* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags
2163	 * This is done in order to share the same flags between user-space and
2164	 * kernel and spare an unnecessary translation.
2165	 */
2166
2167	/* Kernel flags */
2168	IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED	= 1ULL << 32,
2169	IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS	= 1ULL << 33,
2170};
2171
2172struct ib_flow_spec_list {
2173	struct ib_flow_spec_list	*next;
2174	union ib_flow_spec		spec;
2175};
2176
2177struct ib_flow_action_attrs_esp {
2178	struct ib_flow_action_attrs_esp_keymats		*keymat;
2179	struct ib_flow_action_attrs_esp_replays		*replay;
2180	struct ib_flow_spec_list			*encap;
2181	/* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled.
2182	 * Value of 0 is a valid value.
2183	 */
2184	u32						esn;
2185	u32						spi;
2186	u32						seq;
2187	u32						tfc_pad;
2188	/* Use enum ib_flow_action_attrs_esp_flags */
2189	u64						flags;
2190	u64						hard_limit_pkts;
2191};
2192
2193struct ib_flow_action {
2194	struct ib_device		*device;
2195	struct ib_uobject		*uobject;
2196	enum ib_flow_action_type	type;
2197	atomic_t			usecnt;
2198};
2199
2200struct ib_mad;
2201
2202enum ib_process_mad_flags {
2203	IB_MAD_IGNORE_MKEY	= 1,
2204	IB_MAD_IGNORE_BKEY	= 2,
2205	IB_MAD_IGNORE_ALL	= IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY
2206};
2207
2208enum ib_mad_result {
2209	IB_MAD_RESULT_FAILURE  = 0,      /* (!SUCCESS is the important flag) */
2210	IB_MAD_RESULT_SUCCESS  = 1 << 0, /* MAD was successfully processed   */
2211	IB_MAD_RESULT_REPLY    = 1 << 1, /* Reply packet needs to be sent    */
2212	IB_MAD_RESULT_CONSUMED = 1 << 2  /* Packet consumed: stop processing */
2213};
2214
2215struct ib_port_cache {
2216	u64		      subnet_prefix;
2217	struct ib_pkey_cache  *pkey;
2218	struct ib_gid_table   *gid;
2219	u8                     lmc;
2220	enum ib_port_state     port_state;
2221	enum ib_port_state     last_port_state;
2222};
2223
2224struct ib_port_immutable {
2225	int                           pkey_tbl_len;
2226	int                           gid_tbl_len;
2227	u32                           core_cap_flags;
2228	u32                           max_mad_size;
2229};
2230
2231struct ib_port_data {
2232	struct ib_device *ib_dev;
2233
2234	struct ib_port_immutable immutable;
2235
2236	spinlock_t pkey_list_lock;
2237
2238	spinlock_t netdev_lock;
2239
2240	struct list_head pkey_list;
2241
2242	struct ib_port_cache cache;
2243
2244	struct net_device __rcu *netdev;
2245	netdevice_tracker netdev_tracker;
2246	struct hlist_node ndev_hash_link;
2247	struct rdma_port_counter port_counter;
2248	struct ib_port *sysfs;
2249};
2250
2251/* rdma netdev type - specifies protocol type */
2252enum rdma_netdev_t {
2253	RDMA_NETDEV_OPA_VNIC,
2254	RDMA_NETDEV_IPOIB,
2255};
2256
2257/**
2258 * struct rdma_netdev - rdma netdev
2259 * For cases where netstack interfacing is required.
2260 */
2261struct rdma_netdev {
2262	void              *clnt_priv;
2263	struct ib_device  *hca;
2264	u32		   port_num;
2265	int                mtu;
2266
2267	/*
2268	 * cleanup function must be specified.
2269	 * FIXME: This is only used for OPA_VNIC and that usage should be
2270	 * removed too.
2271	 */
2272	void (*free_rdma_netdev)(struct net_device *netdev);
2273
2274	/* control functions */
2275	void (*set_id)(struct net_device *netdev, int id);
2276	/* send packet */
2277	int (*send)(struct net_device *dev, struct sk_buff *skb,
2278		    struct ib_ah *address, u32 dqpn);
2279	/* multicast */
2280	int (*attach_mcast)(struct net_device *dev, struct ib_device *hca,
2281			    union ib_gid *gid, u16 mlid,
2282			    int set_qkey, u32 qkey);
2283	int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
2284			    union ib_gid *gid, u16 mlid);
2285	/* timeout */
2286	void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);
2287};
2288
2289struct rdma_netdev_alloc_params {
2290	size_t sizeof_priv;
2291	unsigned int txqs;
2292	unsigned int rxqs;
2293	void *param;
2294
2295	int (*initialize_rdma_netdev)(struct ib_device *device, u32 port_num,
2296				      struct net_device *netdev, void *param);
2297};
2298
2299struct ib_odp_counters {
2300	atomic64_t faults;
2301	atomic64_t faults_handled;
2302	atomic64_t invalidations;
2303	atomic64_t invalidations_handled;
2304	atomic64_t prefetch;
2305};
2306
2307struct ib_counters {
2308	struct ib_device	*device;
2309	struct ib_uobject	*uobject;
2310	/* num of objects attached */
2311	atomic_t	usecnt;
2312};
2313
2314struct ib_counters_read_attr {
2315	u64	*counters_buff;
2316	u32	ncounters;
2317	u32	flags; /* use enum ib_read_counters_flags */
2318};
2319
2320struct uverbs_attr_bundle;
2321struct iw_cm_id;
2322struct iw_cm_conn_param;
2323
2324#define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member)                      \
2325	.size_##ib_struct =                                                    \
2326		(sizeof(struct drv_struct) +                                   \
2327		 BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) +      \
2328		 BUILD_BUG_ON_ZERO(                                            \
2329			 !__same_type(((struct drv_struct *)NULL)->member,     \
2330				      struct ib_struct)))
2331
2332#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp)                          \
2333	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
2334					   gfp, false))
2335
2336#define rdma_zalloc_drv_obj_numa(ib_dev, ib_type)                              \
2337	((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
2338					   GFP_KERNEL, true))
2339
2340#define rdma_zalloc_drv_obj(ib_dev, ib_type)                                   \
2341	rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)
2342
2343#define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct
2344
2345struct rdma_user_mmap_entry {
2346	struct kref ref;
2347	struct ib_ucontext *ucontext;
2348	unsigned long start_pgoff;
2349	size_t npages;
2350	bool driver_removed;
2351};
2352
2353/* Return the offset (in bytes) the user should pass to libc's mmap() */
2354static inline u64
2355rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry)
2356{
2357	return (u64)entry->start_pgoff << PAGE_SHIFT;
2358}
2359
2360/**
2361 * struct ib_device_ops - InfiniBand device operations
2362 * This structure defines all the InfiniBand device operations, providers will
2363 * need to define the supported operations, otherwise they will be set to null.
2364 */
2365struct ib_device_ops {
2366	struct module *owner;
2367	enum rdma_driver_id driver_id;
2368	u32 uverbs_abi_ver;
2369	unsigned int uverbs_no_driver_id_binding:1;
2370
2371	/*
2372	 * NOTE: New drivers should not make use of device_group; instead new
2373	 * device parameter should be exposed via netlink command. This
2374	 * mechanism exists only for existing drivers.
2375	 */
2376	const struct attribute_group *device_group;
2377	const struct attribute_group **port_groups;
2378
2379	int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
2380			 const struct ib_send_wr **bad_send_wr);
2381	int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
2382			 const struct ib_recv_wr **bad_recv_wr);
2383	void (*drain_rq)(struct ib_qp *qp);
2384	void (*drain_sq)(struct ib_qp *qp);
2385	int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
2386	int (*peek_cq)(struct ib_cq *cq, int wc_cnt);
2387	int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags);
2388	int (*post_srq_recv)(struct ib_srq *srq,
2389			     const struct ib_recv_wr *recv_wr,
2390			     const struct ib_recv_wr **bad_recv_wr);
2391	int (*process_mad)(struct ib_device *device, int process_mad_flags,
2392			   u32 port_num, const struct ib_wc *in_wc,
2393			   const struct ib_grh *in_grh,
2394			   const struct ib_mad *in_mad, struct ib_mad *out_mad,
2395			   size_t *out_mad_size, u16 *out_mad_pkey_index);
2396	int (*query_device)(struct ib_device *device,
2397			    struct ib_device_attr *device_attr,
2398			    struct ib_udata *udata);
2399	int (*modify_device)(struct ib_device *device, int device_modify_mask,
2400			     struct ib_device_modify *device_modify);
2401	void (*get_dev_fw_str)(struct ib_device *device, char *str);
2402	const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,
2403						     int comp_vector);
2404	int (*query_port)(struct ib_device *device, u32 port_num,
2405			  struct ib_port_attr *port_attr);
2406	int (*modify_port)(struct ib_device *device, u32 port_num,
2407			   int port_modify_mask,
2408			   struct ib_port_modify *port_modify);
2409	/*
2410	 * The following mandatory functions are used only at device
2411	 * registration.  Keep functions such as these at the end of this
2412	 * structure to avoid cache line misses when accessing struct ib_device
2413	 * in fast paths.
2414	 */
2415	int (*get_port_immutable)(struct ib_device *device, u32 port_num,
2416				  struct ib_port_immutable *immutable);
2417	enum rdma_link_layer (*get_link_layer)(struct ib_device *device,
2418					       u32 port_num);
2419	/*
2420	 * When calling get_netdev, the HW vendor's driver should return the
2421	 * net device of device @device at port @port_num or NULL if such
2422	 * a net device doesn't exist. The vendor driver should call dev_hold
2423	 * on this net device. The HW vendor's device driver must guarantee
2424	 * that this function returns NULL before the net device has finished
2425	 * NETDEV_UNREGISTER state.
2426	 */
2427	struct net_device *(*get_netdev)(struct ib_device *device,
2428					 u32 port_num);
2429	/*
2430	 * rdma netdev operation
2431	 *
2432	 * Driver implementing alloc_rdma_netdev or rdma_netdev_get_params
2433	 * must return -EOPNOTSUPP if it doesn't support the specified type.
2434	 */
2435	struct net_device *(*alloc_rdma_netdev)(
2436		struct ib_device *device, u32 port_num, enum rdma_netdev_t type,
2437		const char *name, unsigned char name_assign_type,
2438		void (*setup)(struct net_device *));
2439
2440	int (*rdma_netdev_get_params)(struct ib_device *device, u32 port_num,
2441				      enum rdma_netdev_t type,
2442				      struct rdma_netdev_alloc_params *params);
2443	/*
2444	 * query_gid should be return GID value for @device, when @port_num
2445	 * link layer is either IB or iWarp. It is no-op if @port_num port
2446	 * is RoCE link layer.
2447	 */
2448	int (*query_gid)(struct ib_device *device, u32 port_num, int index,
2449			 union ib_gid *gid);
2450	/*
2451	 * When calling add_gid, the HW vendor's driver should add the gid
2452	 * of device of port at gid index available at @attr. Meta-info of
2453	 * that gid (for example, the network device related to this gid) is
2454	 * available at @attr. @context allows the HW vendor driver to store
2455	 * extra information together with a GID entry. The HW vendor driver may
2456	 * allocate memory to contain this information and store it in @context
2457	 * when a new GID entry is written to. Params are consistent until the
2458	 * next call of add_gid or delete_gid. The function should return 0 on
2459	 * success or error otherwise. The function could be called
2460	 * concurrently for different ports. This function is only called when
2461	 * roce_gid_table is used.
2462	 */
2463	int (*add_gid)(const struct ib_gid_attr *attr, void **context);
2464	/*
2465	 * When calling del_gid, the HW vendor's driver should delete the
2466	 * gid of device @device at gid index gid_index of port port_num
2467	 * available in @attr.
2468	 * Upon the deletion of a GID entry, the HW vendor must free any
2469	 * allocated memory. The caller will clear @context afterwards.
2470	 * This function is only called when roce_gid_table is used.
2471	 */
2472	int (*del_gid)(const struct ib_gid_attr *attr, void **context);
2473	int (*query_pkey)(struct ib_device *device, u32 port_num, u16 index,
2474			  u16 *pkey);
2475	int (*alloc_ucontext)(struct ib_ucontext *context,
2476			      struct ib_udata *udata);
2477	void (*dealloc_ucontext)(struct ib_ucontext *context);
2478	int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
2479	/*
2480	 * This will be called once refcount of an entry in mmap_xa reaches
2481	 * zero. The type of the memory that was mapped may differ between
2482	 * entries and is opaque to the rdma_user_mmap interface.
2483	 * Therefore needs to be implemented by the driver in mmap_free.
2484	 */
2485	void (*mmap_free)(struct rdma_user_mmap_entry *entry);
2486	void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);
2487	int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
2488	int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
2489	int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2490			 struct ib_udata *udata);
2491	int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
2492			      struct ib_udata *udata);
2493	int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
2494	int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
2495	int (*destroy_ah)(struct ib_ah *ah, u32 flags);
2496	int (*create_srq)(struct ib_srq *srq,
2497			  struct ib_srq_init_attr *srq_init_attr,
2498			  struct ib_udata *udata);
2499	int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
2500			  enum ib_srq_attr_mask srq_attr_mask,
2501			  struct ib_udata *udata);
2502	int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
2503	int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);
2504	int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
2505			 struct ib_udata *udata);
2506	int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2507			 int qp_attr_mask, struct ib_udata *udata);
2508	int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
2509			int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
2510	int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
2511	int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
2512			 struct uverbs_attr_bundle *attrs);
2513	int (*create_cq_umem)(struct ib_cq *cq,
2514			      const struct ib_cq_init_attr *attr,
2515			      struct ib_umem *umem,
2516			      struct uverbs_attr_bundle *attrs);
2517	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
2518	int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
2519	int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
2520	/*
2521	 * pre_destroy_cq - Prevent a cq from generating any new work
2522	 * completions, but not free any kernel resources
2523	 */
2524	int (*pre_destroy_cq)(struct ib_cq *cq);
2525	/*
2526	 * post_destroy_cq - Free all kernel resources
2527	 */
2528	void (*post_destroy_cq)(struct ib_cq *cq);
2529	struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
2530	struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
2531				     u64 virt_addr, int mr_access_flags,
2532				     struct ib_dmah *dmah,
2533				     struct ib_udata *udata);
2534	struct ib_mr *(*reg_user_mr_dmabuf)(struct ib_pd *pd, u64 offset,
2535					    u64 length, u64 virt_addr, int fd,
2536					    int mr_access_flags,
2537					    struct ib_dmah *dmah,
2538					    struct uverbs_attr_bundle *attrs);
2539	struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start,
2540				       u64 length, u64 virt_addr,
2541				       int mr_access_flags, struct ib_pd *pd,
2542				       struct ib_udata *udata);
2543	int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
2544	struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
2545				  u32 max_num_sg);
2546	struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
2547					    u32 max_num_data_sg,
2548					    u32 max_num_meta_sg);
2549	int (*advise_mr)(struct ib_pd *pd,
2550			 enum ib_uverbs_advise_mr_advice advice, u32 flags,
2551			 struct ib_sge *sg_list, u32 num_sge,
2552			 struct uverbs_attr_bundle *attrs);
2553
2554	/*
2555	 * Kernel users should universally support relaxed ordering (RO), as
2556	 * they are designed to read data only after observing the CQE and use
2557	 * the DMA API correctly.
2558	 *
2559	 * Some drivers implicitly enable RO if platform supports it.
2560	 */
2561	int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2562			 unsigned int *sg_offset);
2563	int (*check_mr_status)(struct ib_mr *mr, u32 check_mask,
2564			       struct ib_mr_status *mr_status);
2565	int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata);
2566	int (*dealloc_mw)(struct ib_mw *mw);
2567	int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
2568	int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
2569	int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
2570	int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
2571	struct ib_flow *(*create_flow)(struct ib_qp *qp,
2572				       struct ib_flow_attr *flow_attr,
2573				       struct ib_udata *udata);
2574	int (*destroy_flow)(struct ib_flow *flow_id);
2575	int (*destroy_flow_action)(struct ib_flow_action *action);
2576	int (*set_vf_link_state)(struct ib_device *device, int vf, u32 port,
2577				 int state);
2578	int (*get_vf_config)(struct ib_device *device, int vf, u32 port,
2579			     struct ifla_vf_info *ivf);
2580	int (*get_vf_stats)(struct ib_device *device, int vf, u32 port,
2581			    struct ifla_vf_stats *stats);
2582	int (*get_vf_guid)(struct ib_device *device, int vf, u32 port,
2583			    struct ifla_vf_guid *node_guid,
2584			    struct ifla_vf_guid *port_guid);
2585	int (*set_vf_guid)(struct ib_device *device, int vf, u32 port, u64 guid,
2586			   int type);
2587	struct ib_wq *(*create_wq)(struct ib_pd *pd,
2588				   struct ib_wq_init_attr *init_attr,
2589				   struct ib_udata *udata);
2590	int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
2591	int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
2592			 u32 wq_attr_mask, struct ib_udata *udata);
2593	int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table,
2594				    struct ib_rwq_ind_table_init_attr *init_attr,
2595				    struct ib_udata *udata);
2596	int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table);
2597	struct ib_dm *(*alloc_dm)(struct ib_device *device,
2598				  struct ib_ucontext *context,
2599				  struct ib_dm_alloc_attr *attr,
2600				  struct uverbs_attr_bundle *attrs);
2601	int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);
2602	int (*alloc_dmah)(struct ib_dmah *ibdmah,
2603			  struct uverbs_attr_bundle *attrs);
2604	int (*dealloc_dmah)(struct ib_dmah *dmah, struct uverbs_attr_bundle *attrs);
2605	struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
2606				   struct ib_dm_mr_attr *attr,
2607				   struct uverbs_attr_bundle *attrs);
2608	int (*create_counters)(struct ib_counters *counters,
2609			       struct uverbs_attr_bundle *attrs);
2610	int (*destroy_counters)(struct ib_counters *counters);
2611	int (*read_counters)(struct ib_counters *counters,
2612			     struct ib_counters_read_attr *counters_read_attr,
2613			     struct uverbs_attr_bundle *attrs);
2614	int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
2615			    int data_sg_nents, unsigned int *data_sg_offset,
2616			    struct scatterlist *meta_sg, int meta_sg_nents,
2617			    unsigned int *meta_sg_offset);
2618
2619	/*
2620	 * alloc_hw_[device,port]_stats - Allocate a struct rdma_hw_stats and
2621	 *   fill in the driver initialized data.  The struct is kfree()'ed by
2622	 *   the sysfs core when the device is removed.  A lifespan of -1 in the
2623	 *   return struct tells the core to set a default lifespan.
2624	 */
2625	struct rdma_hw_stats *(*alloc_hw_device_stats)(struct ib_device *device);
2626	struct rdma_hw_stats *(*alloc_hw_port_stats)(struct ib_device *device,
2627						     u32 port_num);
2628	/*
2629	 * get_hw_stats - Fill in the counter value(s) in the stats struct.
2630	 * @index - The index in the value array we wish to have updated, or
2631	 *   num_counters if we want all stats updated
2632	 * Return codes -
2633	 *   < 0 - Error, no counters updated
2634	 *   index - Updated the single counter pointed to by index
2635	 *   num_counters - Updated all counters (will reset the timestamp
2636	 *     and prevent further calls for lifespan milliseconds)
2637	 * Drivers are allowed to update all counters in leiu of just the
2638	 *   one given in index at their option
2639	 */
2640	int (*get_hw_stats)(struct ib_device *device,
2641			    struct rdma_hw_stats *stats, u32 port, int index);
2642
2643	/*
2644	 * modify_hw_stat - Modify the counter configuration
2645	 * @enable: true/false when enable/disable a counter
2646	 * Return codes - 0 on success or error code otherwise.
2647	 */
2648	int (*modify_hw_stat)(struct ib_device *device, u32 port,
2649			      unsigned int counter_index, bool enable);
2650	/*
2651	 * Allows rdma drivers to add their own restrack attributes.
2652	 */
2653	int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
2654	int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);
2655	int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);
2656	int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);
2657	int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);
2658	int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);
2659	int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);
2660	int (*fill_res_srq_entry)(struct sk_buff *msg, struct ib_srq *ib_srq);
2661	int (*fill_res_srq_entry_raw)(struct sk_buff *msg, struct ib_srq *ib_srq);
2662
2663	/* Device lifecycle callbacks */
2664	/*
2665	 * Called after the device becomes registered, before clients are
2666	 * attached
2667	 */
2668	int (*enable_driver)(struct ib_device *dev);
2669	/*
2670	 * This is called as part of ib_dealloc_device().
2671	 */
2672	void (*dealloc_driver)(struct ib_device *dev);
2673
2674	/* iWarp CM callbacks */
2675	void (*iw_add_ref)(struct ib_qp *qp);
2676	void (*iw_rem_ref)(struct ib_qp *qp);
2677	struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);
2678	int (*iw_connect)(struct iw_cm_id *cm_id,
2679			  struct iw_cm_conn_param *conn_param);
2680	int (*iw_accept)(struct iw_cm_id *cm_id,
2681			 struct iw_cm_conn_param *conn_param);
2682	int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,
2683			 u8 pdata_len);
2684	int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
2685	int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
2686	/*
2687	 * counter_bind_qp - Bind a QP to a counter.
2688	 * @counter - The counter to be bound. If counter->id is zero then
2689	 *   the driver needs to allocate a new counter and set counter->id
2690	 */
2691	int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp,
2692			       u32 port);
2693	/*
2694	 * counter_unbind_qp - Unbind the qp from the dynamically-allocated
2695	 *   counter and bind it onto the default one
2696	 */
2697	int (*counter_unbind_qp)(struct ib_qp *qp, u32 port);
2698	/*
2699	 * counter_dealloc -De-allocate the hw counter
2700	 */
2701	int (*counter_dealloc)(struct rdma_counter *counter);
2702	/*
2703	 * counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in
2704	 * the driver initialized data.
2705	 */
2706	struct rdma_hw_stats *(*counter_alloc_stats)(
2707		struct rdma_counter *counter);
2708	/*
2709	 * counter_update_stats - Query the stats value of this counter
2710	 */
2711	int (*counter_update_stats)(struct rdma_counter *counter);
2712
2713	/*
2714	 * counter_init - Initialize the driver specific rdma counter struct.
2715	 */
2716	void (*counter_init)(struct rdma_counter *counter);
2717
2718	/*
2719	 * Allows rdma drivers to add their own restrack attributes
2720	 * dumped via 'rdma stat' iproute2 command.
2721	 */
2722	int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
2723
2724	/* query driver for its ucontext properties */
2725	int (*query_ucontext)(struct ib_ucontext *context,
2726			      struct uverbs_attr_bundle *attrs);
2727
2728	/*
2729	 * Provide NUMA node. This API exists for rdmavt/hfi1 only.
2730	 * Everyone else relies on Linux memory management model.
2731	 */
2732	int (*get_numa_node)(struct ib_device *dev);
2733
2734	/*
2735	 * add_sub_dev - Add a sub IB device
2736	 */
2737	struct ib_device *(*add_sub_dev)(struct ib_device *parent,
2738					 enum rdma_nl_dev_type type,
2739					 const char *name);
2740
2741	/*
2742	 * del_sub_dev - Delete a sub IB device
2743	 */
2744	void (*del_sub_dev)(struct ib_device *sub_dev);
2745
2746	/*
2747	 * ufile_cleanup - Attempt to cleanup ubojects HW resources inside
2748	 * the ufile.
2749	 */
2750	void (*ufile_hw_cleanup)(struct ib_uverbs_file *ufile);
2751
2752	/*
2753	 * report_port_event - Drivers need to implement this if they have
2754	 * some private stuff to handle when link status changes.
2755	 */
2756	void (*report_port_event)(struct ib_device *ibdev,
2757				  struct net_device *ndev, unsigned long event);
2758
2759	DECLARE_RDMA_OBJ_SIZE(ib_ah);
2760	DECLARE_RDMA_OBJ_SIZE(ib_counters);
2761	DECLARE_RDMA_OBJ_SIZE(ib_cq);
2762	DECLARE_RDMA_OBJ_SIZE(ib_dmah);
2763	DECLARE_RDMA_OBJ_SIZE(ib_mw);
2764	DECLARE_RDMA_OBJ_SIZE(ib_pd);
2765	DECLARE_RDMA_OBJ_SIZE(ib_qp);
2766	DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table);
2767	DECLARE_RDMA_OBJ_SIZE(ib_srq);
2768	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
2769	DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
2770	DECLARE_RDMA_OBJ_SIZE(rdma_counter);
2771};
2772
2773struct ib_core_device {
2774	/* device must be the first element in structure until,
2775	 * union of ib_core_device and device exists in ib_device.
2776	 */
2777	struct device dev;
2778	possible_net_t rdma_net;
2779	struct kobject *ports_kobj;
2780	struct list_head port_list;
2781	struct ib_device *owner; /* reach back to owner ib_device */
2782};
2783
2784struct rdma_restrack_root;
2785struct ib_device {
2786	/* Do not access @dma_device directly from ULP nor from HW drivers. */
2787	struct device                *dma_device;
2788	struct ib_device_ops	     ops;
2789	char                          name[IB_DEVICE_NAME_MAX];
2790	struct rcu_head rcu_head;
2791
2792	struct list_head              event_handler_list;
2793	/* Protects event_handler_list */
2794	struct rw_semaphore event_handler_rwsem;
2795
2796	/* Protects QP's event_handler calls and open_qp list */
2797	spinlock_t qp_open_list_lock;
2798
2799	struct rw_semaphore	      client_data_rwsem;
2800	struct xarray                 client_data;
2801	struct mutex                  unregistration_lock;
2802
2803	/* Synchronize GID, Pkey cache entries, subnet prefix, LMC */
2804	rwlock_t cache_lock;
2805	/**
2806	 * port_data is indexed by port number
2807	 */
2808	struct ib_port_data *port_data;
2809
2810	int			      num_comp_vectors;
2811
2812	union {
2813		struct device		dev;
2814		struct ib_core_device	coredev;
2815	};
2816
2817	/* First group is for device attributes,
2818	 * Second group is for driver provided attributes (optional).
2819	 * Third group is for the hw_stats
2820	 * It is a NULL terminated array.
2821	 */
2822	const struct attribute_group	*groups[4];
2823	u8				hw_stats_attr_index;
2824
2825	u64			     uverbs_cmd_mask;
2826
2827	char			     node_desc[IB_DEVICE_NODE_DESC_MAX];
2828	__be64			     node_guid;
2829	u32			     local_dma_lkey;
2830	u16                          is_switch:1;
2831	/* Indicates kernel verbs support, should not be used in drivers */
2832	u16                          kverbs_provider:1;
2833	/* CQ adaptive moderation (RDMA DIM) */
2834	u16                          use_cq_dim:1;
2835	u8                           node_type;
2836	u32			     phys_port_cnt;
2837	struct ib_device_attr        attrs;
2838	struct hw_stats_device_data *hw_stats_data;
2839
2840#ifdef CONFIG_CGROUP_RDMA
2841	struct rdmacg_device         cg_device;
2842#endif
2843
2844	u32                          index;
2845
2846	spinlock_t                   cq_pools_lock;
2847	struct list_head             cq_pools[IB_POLL_LAST_POOL_TYPE + 1];
2848
2849	struct rdma_restrack_root *res;
2850
2851	const struct uapi_definition   *driver_def;
2852
2853	/*
2854	 * Positive refcount indicates that the device is currently
2855	 * registered and cannot be unregistered.
2856	 */
2857	refcount_t refcount;
2858	struct completion unreg_completion;
2859	struct work_struct unregistration_work;
2860
2861	const struct rdma_link_ops *link_ops;
2862
2863	/* Protects compat_devs xarray modifications */
2864	struct mutex compat_devs_mutex;
2865	/* Maintains compat devices for each net namespace */
2866	struct xarray compat_devs;
2867
2868	/* Used by iWarp CM */
2869	char iw_ifname[IFNAMSIZ];
2870	u32 iw_driver_flags;
2871	u32 lag_flags;
2872
2873	/* A parent device has a list of sub-devices */
2874	struct mutex subdev_lock;
2875	struct list_head subdev_list_head;
2876
2877	/* A sub device has a type and a parent */
2878	enum rdma_nl_dev_type type;
2879	struct ib_device *parent;
2880	struct list_head subdev_list;
2881
2882	enum rdma_nl_name_assign_type name_assign_type;
2883};
2884
2885static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size,
2886				    gfp_t gfp, bool is_numa_aware)
2887{
2888	if (is_numa_aware && dev->ops.get_numa_node)
2889		return kzalloc_node(size, gfp, dev->ops.get_numa_node(dev));
2890
2891	return kzalloc(size, gfp);
2892}
2893
2894struct ib_client_nl_info;
2895struct ib_client {
2896	const char *name;
2897	int (*add)(struct ib_device *ibdev);
2898	void (*remove)(struct ib_device *, void *client_data);
2899	void (*rename)(struct ib_device *dev, void *client_data);
2900	int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
2901			   struct ib_client_nl_info *res);
2902	int (*get_global_nl_info)(struct ib_client_nl_info *res);
2903
2904	/* Returns the net_dev belonging to this ib_client and matching the
2905	 * given parameters.
2906	 * @dev:	 An RDMA device that the net_dev use for communication.
2907	 * @port:	 A physical port number on the RDMA device.
2908	 * @pkey:	 P_Key that the net_dev uses if applicable.
2909	 * @gid:	 A GID that the net_dev uses to communicate.
2910	 * @addr:	 An IP address the net_dev is configured with.
2911	 * @client_data: The device's client data set by ib_set_client_data().
2912	 *
2913	 * An ib_client that implements a net_dev on top of RDMA devices
2914	 * (such as IP over IB) should implement this callback, allowing the
2915	 * rdma_cm module to find the right net_dev for a given request.
2916	 *
2917	 * The caller is responsible for calling dev_put on the returned
2918	 * netdev. */
2919	struct net_device *(*get_net_dev_by_params)(
2920			struct ib_device *dev,
2921			u32 port,
2922			u16 pkey,
2923			const union ib_gid *gid,
2924			const struct sockaddr *addr,
2925			void *client_data);
2926
2927	refcount_t uses;
2928	struct completion uses_zero;
2929	u32 client_id;
2930
2931	/* kverbs are not required by the client */
2932	u8 no_kverbs_req:1;
2933};
2934
2935/*
2936 * IB block DMA iterator
2937 *
2938 * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
2939 * to a HW supported page size.
2940 */
2941struct ib_block_iter {
2942	/* internal states */
2943	struct scatterlist *__sg;	/* sg holding the current aligned block */
2944	dma_addr_t __dma_addr;		/* unaligned DMA address of this block */
2945	size_t __sg_numblocks;		/* ib_umem_num_dma_blocks() */
2946	unsigned int __sg_nents;	/* number of SG entries */
2947	unsigned int __sg_advance;	/* number of bytes to advance in sg in next step */
2948	unsigned int __pg_bit;		/* alignment of current block */
2949};
2950
2951struct ib_device *_ib_alloc_device(size_t size, struct net *net);
2952#define ib_alloc_device(drv_struct, member)                                    \
2953	container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \
2954				      BUILD_BUG_ON_ZERO(offsetof(              \
2955					      struct drv_struct, member)),     \
2956				      &init_net),			       \
2957		     struct drv_struct, member)
2958
2959#define ib_alloc_device_with_net(drv_struct, member, net)		       \
2960	container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \
2961				      BUILD_BUG_ON_ZERO(offsetof(              \
2962					struct drv_struct, member)), net),     \
2963		     struct drv_struct, member)
2964
2965void ib_dealloc_device(struct ib_device *device);
2966
2967void ib_get_device_fw_str(struct ib_device *device, char *str);
2968
2969int ib_register_device(struct ib_device *device, const char *name,
2970		       struct device *dma_device);
2971void ib_unregister_device(struct ib_device *device);
2972void ib_unregister_driver(enum rdma_driver_id driver_id);
2973void ib_unregister_device_and_put(struct ib_device *device);
2974void ib_unregister_device_queued(struct ib_device *ib_dev);
2975
2976int ib_register_client   (struct ib_client *client);
2977void ib_unregister_client(struct ib_client *client);
2978
2979void __rdma_block_iter_start(struct ib_block_iter *biter,
2980			     struct scatterlist *sglist,
2981			     unsigned int nents,
2982			     unsigned long pgsz);
2983bool __rdma_block_iter_next(struct ib_block_iter *biter);
2984
2985/**
2986 * rdma_block_iter_dma_address - get the aligned dma address of the current
2987 * block held by the block iterator.
2988 * @biter: block iterator holding the memory block
2989 */
2990static inline dma_addr_t
2991rdma_block_iter_dma_address(struct ib_block_iter *biter)
2992{
2993	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
2994}
2995
2996/**
2997 * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
2998 * @sglist: sglist to iterate over
2999 * @biter: block iterator holding the memory block
3000 * @nents: maximum number of sg entries to iterate over
3001 * @pgsz: best HW supported page size to use
3002 *
3003 * Callers may use rdma_block_iter_dma_address() to get each
3004 * blocks aligned DMA address.
3005 */
3006#define rdma_for_each_block(sglist, biter, nents, pgsz)		\
3007	for (__rdma_block_iter_start(biter, sglist, nents,	\
3008				     pgsz);			\
3009	     __rdma_block_iter_next(biter);)
3010
3011/**
3012 * ib_get_client_data - Get IB client context
3013 * @device:Device to get context for
3014 * @client:Client to get context for
3015 *
3016 * ib_get_client_data() returns the client context data set with
3017 * ib_set_client_data(). This can only be called while the client is
3018 * registered to the device, once the ib_client remove() callback returns this
3019 * cannot be called.
3020 */
3021static inline void *ib_get_client_data(struct ib_device *device,
3022				       struct ib_client *client)
3023{
3024	return xa_load(&device->client_data, client->client_id);
3025}
3026void  ib_set_client_data(struct ib_device *device, struct ib_client *client,
3027			 void *data);
3028void ib_set_device_ops(struct ib_device *device,
3029		       const struct ib_device_ops *ops);
3030
3031int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
3032		      unsigned long pfn, unsigned long size, pgprot_t prot,
3033		      struct rdma_user_mmap_entry *entry);
3034int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
3035				struct rdma_user_mmap_entry *entry,
3036				size_t length);
3037int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
3038				      struct rdma_user_mmap_entry *entry,
3039				      size_t length, u32 min_pgoff,
3040				      u32 max_pgoff);
3041
3042#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
3043void rdma_user_mmap_disassociate(struct ib_device *device);
3044#else
3045static inline void rdma_user_mmap_disassociate(struct ib_device *device)
3046{
3047}
3048#endif
3049
3050static inline int
3051rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext,
3052				  struct rdma_user_mmap_entry *entry,
3053				  size_t length, u32 pgoff)
3054{
3055	return rdma_user_mmap_entry_insert_range(ucontext, entry, length, pgoff,
3056						 pgoff);
3057}
3058
3059struct rdma_user_mmap_entry *
3060rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
3061			       unsigned long pgoff);
3062struct rdma_user_mmap_entry *
3063rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
3064			 struct vm_area_struct *vma);
3065void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry);
3066
3067void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry);
3068
3069static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
3070{
3071	return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0;
3072}
3073
3074static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len)
3075{
3076	return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0;
3077}
3078
3079static inline bool ib_is_buffer_cleared(const void __user *p,
3080					size_t len)
3081{
3082	bool ret;
3083	u8 *buf;
3084
3085	if (len > USHRT_MAX)
3086		return false;
3087
3088	buf = memdup_user(p, len);
3089	if (IS_ERR(buf))
3090		return false;
3091
3092	ret = !memchr_inv(buf, 0, len);
3093	kfree(buf);
3094	return ret;
3095}
3096
3097static inline bool ib_is_udata_cleared(struct ib_udata *udata,
3098				       size_t offset,
3099				       size_t len)
3100{
3101	return ib_is_buffer_cleared(udata->inbuf + offset, len);
3102}
3103
3104/**
3105 * ib_modify_qp_is_ok - Check that the supplied attribute mask
3106 * contains all required attributes and no attributes not allowed for
3107 * the given QP state transition.
3108 * @cur_state: Current QP state
3109 * @next_state: Next QP state
3110 * @type: QP type
3111 * @mask: Mask of supplied QP attributes
3112 *
3113 * This function is a helper function that a low-level driver's
3114 * modify_qp method can use to validate the consumer's input.  It
3115 * checks that cur_state and next_state are valid QP states, that a
3116 * transition from cur_state to next_state is allowed by the IB spec,
3117 * and that the attribute mask supplied is allowed for the transition.
3118 */
3119bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
3120			enum ib_qp_type type, enum ib_qp_attr_mask mask);
3121
3122void ib_register_event_handler(struct ib_event_handler *event_handler);
3123void ib_unregister_event_handler(struct ib_event_handler *event_handler);
3124void ib_dispatch_event(const struct ib_event *event);
3125
3126int ib_query_port(struct ib_device *device,
3127		  u32 port_num, struct ib_port_attr *port_attr);
3128
3129enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
3130					       u32 port_num);
3131
3132/**
3133 * rdma_cap_ib_switch - Check if the device is IB switch
3134 * @device: Device to check
3135 *
3136 * Device driver is responsible for setting is_switch bit on
3137 * in ib_device structure at init time.
3138 *
3139 * Return: true if the device is IB switch.
3140 */
3141static inline bool rdma_cap_ib_switch(const struct ib_device *device)
3142{
3143	return device->is_switch;
3144}
3145
3146/**
3147 * rdma_start_port - Return the first valid port number for the device
3148 * specified
3149 *
3150 * @device: Device to be checked
3151 *
3152 * Return start port number
3153 */
3154static inline u32 rdma_start_port(const struct ib_device *device)
3155{
3156	return rdma_cap_ib_switch(device) ? 0 : 1;
3157}
3158
3159/**
3160 * rdma_for_each_port - Iterate over all valid port numbers of the IB device
3161 * @device: The struct ib_device * to iterate over
3162 * @iter: The unsigned int to store the port number
3163 */
3164#define rdma_for_each_port(device, iter)                                       \
3165	for (iter = rdma_start_port(device +				       \
3166				    BUILD_BUG_ON_ZERO(!__same_type(u32,	       \
3167								   iter)));    \
3168	     iter <= rdma_end_port(device); iter++)
3169
3170/**
3171 * rdma_end_port - Return the last valid port number for the device
3172 * specified
3173 *
3174 * @device: Device to be checked
3175 *
3176 * Return last port number
3177 */
3178static inline u32 rdma_end_port(const struct ib_device *device)
3179{
3180	return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt;
3181}
3182
3183static inline int rdma_is_port_valid(const struct ib_device *device,
3184				     unsigned int port)
3185{
3186	return (port >= rdma_start_port(device) &&
3187		port <= rdma_end_port(device));
3188}
3189
3190static inline bool rdma_is_grh_required(const struct ib_device *device,
3191					u32 port_num)
3192{
3193	return device->port_data[port_num].immutable.core_cap_flags &
3194	       RDMA_CORE_PORT_IB_GRH_REQUIRED;
3195}
3196
3197static inline bool rdma_protocol_ib(const struct ib_device *device,
3198				    u32 port_num)
3199{
3200	return device->port_data[port_num].immutable.core_cap_flags &
3201	       RDMA_CORE_CAP_PROT_IB;
3202}
3203
3204static inline bool rdma_protocol_roce(const struct ib_device *device,
3205				      u32 port_num)
3206{
3207	return device->port_data[port_num].immutable.core_cap_flags &
3208	       (RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP);
3209}
3210
3211static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device,
3212						u32 port_num)
3213{
3214	return device->port_data[port_num].immutable.core_cap_flags &
3215	       RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
3216}
3217
3218static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device,
3219						u32 port_num)
3220{
3221	return device->port_data[port_num].immutable.core_cap_flags &
3222	       RDMA_CORE_CAP_PROT_ROCE;
3223}
3224
3225static inline bool rdma_protocol_iwarp(const struct ib_device *device,
3226				       u32 port_num)
3227{
3228	return device->port_data[port_num].immutable.core_cap_flags &
3229	       RDMA_CORE_CAP_PROT_IWARP;
3230}
3231
3232static inline bool rdma_ib_or_roce(const struct ib_device *device,
3233				   u32 port_num)
3234{
3235	return rdma_protocol_ib(device, port_num) ||
3236		rdma_protocol_roce(device, port_num);
3237}
3238
3239static inline bool rdma_protocol_raw_packet(const struct ib_device *device,
3240					    u32 port_num)
3241{
3242	return device->port_data[port_num].immutable.core_cap_flags &
3243	       RDMA_CORE_CAP_PROT_RAW_PACKET;
3244}
3245
3246static inline bool rdma_protocol_usnic(const struct ib_device *device,
3247				       u32 port_num)
3248{
3249	return device->port_data[port_num].immutable.core_cap_flags &
3250	       RDMA_CORE_CAP_PROT_USNIC;
3251}
3252
3253/**
3254 * rdma_cap_ib_mad - Check if the port of a device supports Infiniband
3255 * Management Datagrams.
3256 * @device: Device to check
3257 * @port_num: Port number to check
3258 *
3259 * Management Datagrams (MAD) are a required part of the InfiniBand
3260 * specification and are supported on all InfiniBand devices.  A slightly
3261 * extended version are also supported on OPA interfaces.
3262 *
3263 * Return: true if the port supports sending/receiving of MAD packets.
3264 */
3265static inline bool rdma_cap_ib_mad(const struct ib_device *device, u32 port_num)
3266{
3267	return device->port_data[port_num].immutable.core_cap_flags &
3268	       RDMA_CORE_CAP_IB_MAD;
3269}
3270
3271/**
3272 * rdma_cap_opa_mad - Check if the port of device provides support for OPA
3273 * Management Datagrams.
3274 * @device: Device to check
3275 * @port_num: Port number to check
3276 *
3277 * Intel OmniPath devices extend and/or replace the InfiniBand Management
3278 * datagrams with their own versions.  These OPA MADs share many but not all of
3279 * the characteristics of InfiniBand MADs.
3280 *
3281 * OPA MADs differ in the following ways:
3282 *
3283 *    1) MADs are variable size up to 2K
3284 *       IBTA defined MADs remain fixed at 256 bytes
3285 *    2) OPA SMPs must carry valid PKeys
3286 *    3) OPA SMP packets are a different format
3287 *
3288 * Return: true if the port supports OPA MAD packet formats.
3289 */
3290static inline bool rdma_cap_opa_mad(struct ib_device *device, u32 port_num)
3291{
3292	return device->port_data[port_num].immutable.core_cap_flags &
3293		RDMA_CORE_CAP_OPA_MAD;
3294}
3295
3296/**
3297 * rdma_cap_ib_smi - Check if the port of a device provides an Infiniband
3298 * Subnet Management Agent (SMA) on the Subnet Management Interface (SMI).
3299 * @device: Device to check
3300 * @port_num: Port number to check
3301 *
3302 * Each InfiniBand node is required to provide a Subnet Management Agent
3303 * that the subnet manager can access.  Prior to the fabric being fully
3304 * configured by the subnet manager, the SMA is accessed via a well known
3305 * interface called the Subnet Management Interface (SMI).  This interface
3306 * uses directed route packets to communicate with the SM to get around the
3307 * chicken and egg problem of the SM needing to know what's on the fabric
3308 * in order to configure the fabric, and needing to configure the fabric in
3309 * order to send packets to the devices on the fabric.  These directed
3310 * route packets do not need the fabric fully configured in order to reach
3311 * their destination.  The SMI is the only method allowed to send
3312 * directed route packets on an InfiniBand fabric.
3313 *
3314 * Return: true if the port provides an SMI.
3315 */
3316static inline bool rdma_cap_ib_smi(const struct ib_device *device, u32 port_num)
3317{
3318	return device->port_data[port_num].immutable.core_cap_flags &
3319	       RDMA_CORE_CAP_IB_SMI;
3320}
3321
3322/**
3323 * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband
3324 * Communication Manager.
3325 * @device: Device to check
3326 * @port_num: Port number to check
3327 *
3328 * The InfiniBand Communication Manager is one of many pre-defined General
3329 * Service Agents (GSA) that are accessed via the General Service
3330 * Interface (GSI).  It's role is to facilitate establishment of connections
3331 * between nodes as well as other management related tasks for established
3332 * connections.
3333 *
3334 * Return: true if the port supports an IB CM (this does not guarantee that
3335 * a CM is actually running however).
3336 */
3337static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num)
3338{
3339	return device->port_data[port_num].immutable.core_cap_flags &
3340	       RDMA_CORE_CAP_IB_CM;
3341}
3342
3343/**
3344 * rdma_cap_iw_cm - Check if the port of device has the capability IWARP
3345 * Communication Manager.
3346 * @device: Device to check
3347 * @port_num: Port number to check
3348 *
3349 * Similar to above, but specific to iWARP connections which have a different
3350 * managment protocol than InfiniBand.
3351 *
3352 * Return: true if the port supports an iWARP CM (this does not guarantee that
3353 * a CM is actually running however).
3354 */
3355static inline bool rdma_cap_iw_cm(const struct ib_device *device, u32 port_num)
3356{
3357	return device->port_data[port_num].immutable.core_cap_flags &
3358	       RDMA_CORE_CAP_IW_CM;
3359}
3360
3361/**
3362 * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband
3363 * Subnet Administration.
3364 * @device: Device to check
3365 * @port_num: Port number to check
3366 *
3367 * An InfiniBand Subnet Administration (SA) service is a pre-defined General
3368 * Service Agent (GSA) provided by the Subnet Manager (SM).  On InfiniBand
3369 * fabrics, devices should resolve routes to other hosts by contacting the
3370 * SA to query the proper route.
3371 *
3372 * Return: true if the port should act as a client to the fabric Subnet
3373 * Administration interface.  This does not imply that the SA service is
3374 * running locally.
3375 */
3376static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num)
3377{
3378	return device->port_data[port_num].immutable.core_cap_flags &
3379	       RDMA_CORE_CAP_IB_SA;
3380}
3381
3382/**
3383 * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband
3384 * Multicast.
3385 * @device: Device to check
3386 * @port_num: Port number to check
3387 *
3388 * InfiniBand multicast registration is more complex than normal IPv4 or
3389 * IPv6 multicast registration.  Each Host Channel Adapter must register
3390 * with the Subnet Manager when it wishes to join a multicast group.  It
3391 * should do so only once regardless of how many queue pairs it subscribes
3392 * to this group.  And it should leave the group only after all queue pairs
3393 * attached to the group have been detached.
3394 *
3395 * Return: true if the port must undertake the additional adminstrative
3396 * overhead of registering/unregistering with the SM and tracking of the
3397 * total number of queue pairs attached to the multicast group.
3398 */
3399static inline bool rdma_cap_ib_mcast(const struct ib_device *device,
3400				     u32 port_num)
3401{
3402	return rdma_cap_ib_sa(device, port_num);
3403}
3404
3405/**
3406 * rdma_cap_af_ib - Check if the port of device has the capability
3407 * Native Infiniband Address.
3408 * @device: Device to check
3409 * @port_num: Port number to check
3410 *
3411 * InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default
3412 * GID.  RoCE uses a different mechanism, but still generates a GID via
3413 * a prescribed mechanism and port specific data.
3414 *
3415 * Return: true if the port uses a GID address to identify devices on the
3416 * network.
3417 */
3418static inline bool rdma_cap_af_ib(const struct ib_device *device, u32 port_num)
3419{
3420	return device->port_data[port_num].immutable.core_cap_flags &
3421	       RDMA_CORE_CAP_AF_IB;
3422}
3423
3424/**
3425 * rdma_cap_eth_ah - Check if the port of device has the capability
3426 * Ethernet Address Handle.
3427 * @device: Device to check
3428 * @port_num: Port number to check
3429 *
3430 * RoCE is InfiniBand over Ethernet, and it uses a well defined technique
3431 * to fabricate GIDs over Ethernet/IP specific addresses native to the
3432 * port.  Normally, packet headers are generated by the sending host
3433 * adapter, but when sending connectionless datagrams, we must manually
3434 * inject the proper headers for the fabric we are communicating over.
3435 *
3436 * Return: true if we are running as a RoCE port and must force the
3437 * addition of a Global Route Header built from our Ethernet Address
3438 * Handle into our header list for connectionless packets.
3439 */
3440static inline bool rdma_cap_eth_ah(const struct ib_device *device, u32 port_num)
3441{
3442	return device->port_data[port_num].immutable.core_cap_flags &
3443	       RDMA_CORE_CAP_ETH_AH;
3444}
3445
3446/**
3447 * rdma_cap_opa_ah - Check if the port of device supports
3448 * OPA Address handles
3449 * @device: Device to check
3450 * @port_num: Port number to check
3451 *
3452 * Return: true if we are running on an OPA device which supports
3453 * the extended OPA addressing.
3454 */
3455static inline bool rdma_cap_opa_ah(struct ib_device *device, u32 port_num)
3456{
3457	return (device->port_data[port_num].immutable.core_cap_flags &
3458		RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH;
3459}
3460
3461/**
3462 * rdma_max_mad_size - Return the max MAD size required by this RDMA Port.
3463 *
3464 * @device: Device
3465 * @port_num: Port number
3466 *
3467 * This MAD size includes the MAD headers and MAD payload.  No other headers
3468 * are included.
3469 *
3470 * Return the max MAD size required by the Port.  Will return 0 if the port
3471 * does not support MADs
3472 */
3473static inline size_t rdma_max_mad_size(const struct ib_device *device,
3474				       u32 port_num)
3475{
3476	return device->port_data[port_num].immutable.max_mad_size;
3477}
3478
3479/**
3480 * rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table
3481 * @device: Device to check
3482 * @port_num: Port number to check
3483 *
3484 * RoCE GID table mechanism manages the various GIDs for a device.
3485 *
3486 * NOTE: if allocating the port's GID table has failed, this call will still
3487 * return true, but any RoCE GID table API will fail.
3488 *
3489 * Return: true if the port uses RoCE GID table mechanism in order to manage
3490 * its GIDs.
3491 */
3492static inline bool rdma_cap_roce_gid_table(const struct ib_device *device,
3493					   u32 port_num)
3494{
3495	return rdma_protocol_roce(device, port_num) &&
3496		device->ops.add_gid && device->ops.del_gid;
3497}
3498
3499/*
3500 * Check if the device supports READ W/ INVALIDATE.
3501 */
3502static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)
3503{
3504	/*
3505	 * iWarp drivers must support READ W/ INVALIDATE.  No other protocol
3506	 * has support for it yet.
3507	 */
3508	return rdma_protocol_iwarp(dev, port_num);
3509}
3510
3511/**
3512 * rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not.
3513 * @device: Device
3514 * @port_num: 1 based Port number
3515 *
3516 * Return true if port is an Intel OPA port , false if not
3517 */
3518static inline bool rdma_core_cap_opa_port(struct ib_device *device,
3519					  u32 port_num)
3520{
3521	return (device->port_data[port_num].immutable.core_cap_flags &
3522		RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA;
3523}
3524
3525/**
3526 * rdma_mtu_enum_to_int - Return the mtu of the port as an integer value.
3527 * @device: Device
3528 * @port: Port number
3529 * @mtu: enum value of MTU
3530 *
3531 * Return the MTU size supported by the port as an integer value. Will return
3532 * -1 if enum value of mtu is not supported.
3533 */
3534static inline int rdma_mtu_enum_to_int(struct ib_device *device, u32 port,
3535				       int mtu)
3536{
3537	if (rdma_core_cap_opa_port(device, port))
3538		return opa_mtu_enum_to_int((enum opa_mtu)mtu);
3539	else
3540		return ib_mtu_enum_to_int((enum ib_mtu)mtu);
3541}
3542
3543/**
3544 * rdma_mtu_from_attr - Return the mtu of the port from the port attribute.
3545 * @device: Device
3546 * @port: Port number
3547 * @attr: port attribute
3548 *
3549 * Return the MTU size supported by the port as an integer value.
3550 */
3551static inline int rdma_mtu_from_attr(struct ib_device *device, u32 port,
3552				     struct ib_port_attr *attr)
3553{
3554	if (rdma_core_cap_opa_port(device, port))
3555		return attr->phys_mtu;
3556	else
3557		return ib_mtu_enum_to_int(attr->max_mtu);
3558}
3559
3560int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
3561			 int state);
3562int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
3563		     struct ifla_vf_info *info);
3564int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
3565		    struct ifla_vf_stats *stats);
3566int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
3567		    struct ifla_vf_guid *node_guid,
3568		    struct ifla_vf_guid *port_guid);
3569int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
3570		   int type);
3571
3572int ib_query_pkey(struct ib_device *device,
3573		  u32 port_num, u16 index, u16 *pkey);
3574
3575int ib_modify_device(struct ib_device *device,
3576		     int device_modify_mask,
3577		     struct ib_device_modify *device_modify);
3578
3579int ib_modify_port(struct ib_device *device,
3580		   u32 port_num, int port_modify_mask,
3581		   struct ib_port_modify *port_modify);
3582
3583int ib_find_gid(struct ib_device *device, union ib_gid *gid,
3584		u32 *port_num, u16 *index);
3585
3586int ib_find_pkey(struct ib_device *device,
3587		 u32 port_num, u16 pkey, u16 *index);
3588
3589enum ib_pd_flags {
3590	/*
3591	 * Create a memory registration for all memory in the system and place
3592	 * the rkey for it into pd->unsafe_global_rkey.  This can be used by
3593	 * ULPs to avoid the overhead of dynamic MRs.
3594	 *
3595	 * This flag is generally considered unsafe and must only be used in
3596	 * extremly trusted environments.  Every use of it will log a warning
3597	 * in the kernel log.
3598	 */
3599	IB_PD_UNSAFE_GLOBAL_RKEY	= 0x01,
3600};
3601
3602struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
3603		const char *caller);
3604
3605/**
3606 * ib_alloc_pd - Allocates an unused protection domain.
3607 * @device: The device on which to allocate the protection domain.
3608 * @flags: protection domain flags
3609 *
3610 * A protection domain object provides an association between QPs, shared
3611 * receive queues, address handles, memory regions, and memory windows.
3612 *
3613 * Every PD has a local_dma_lkey which can be used as the lkey value for local
3614 * memory operations.
3615 */
3616#define ib_alloc_pd(device, flags) \
3617	__ib_alloc_pd((device), (flags), KBUILD_MODNAME)
3618
3619int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);
3620
3621/**
3622 * ib_dealloc_pd - Deallocate kernel PD
3623 * @pd: The protection domain
3624 *
3625 * NOTE: for user PD use ib_dealloc_pd_user with valid udata!
3626 */
3627static inline void ib_dealloc_pd(struct ib_pd *pd)
3628{
3629	int ret = ib_dealloc_pd_user(pd, NULL);
3630
3631	WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail");
3632}
3633
3634enum rdma_create_ah_flags {
3635	/* In a sleepable context */
3636	RDMA_CREATE_AH_SLEEPABLE = BIT(0),
3637};
3638
3639/**
3640 * rdma_create_ah - Creates an address handle for the given address vector.
3641 * @pd: The protection domain associated with the address handle.
3642 * @ah_attr: The attributes of the address vector.
3643 * @flags: Create address handle flags (see enum rdma_create_ah_flags).
3644 *
3645 * The address handle is used to reference a local or global destination
3646 * in all UD QP post sends.
3647 */
3648struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
3649			     u32 flags);
3650
3651/**
3652 * rdma_create_user_ah - Creates an address handle for the given address vector.
3653 * It resolves destination mac address for ah attribute of RoCE type.
3654 * @pd: The protection domain associated with the address handle.
3655 * @ah_attr: The attributes of the address vector.
3656 * @udata: pointer to user's input output buffer information need by
3657 *         provider driver.
3658 *
3659 * It returns 0 on success and returns appropriate error code on error.
3660 * The address handle is used to reference a local or global destination
3661 * in all UD QP post sends.
3662 */
3663struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
3664				  struct rdma_ah_attr *ah_attr,
3665				  struct ib_udata *udata);
3666/**
3667 * ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header
3668 *   work completion.
3669 * @hdr: the L3 header to parse
3670 * @net_type: type of header to parse
3671 * @sgid: place to store source gid
3672 * @dgid: place to store destination gid
3673 */
3674int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
3675			      enum rdma_network_type net_type,
3676			      union ib_gid *sgid, union ib_gid *dgid);
3677
3678/**
3679 * ib_get_rdma_header_version - Get the header version
3680 * @hdr: the L3 header to parse
3681 */
3682int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);
3683
3684/**
3685 * ib_init_ah_attr_from_wc - Initializes address handle attributes from a
3686 *   work completion.
3687 * @device: Device on which the received message arrived.
3688 * @port_num: Port on which the received message arrived.
3689 * @wc: Work completion associated with the received message.
3690 * @grh: References the received global route header.  This parameter is
3691 *   ignored unless the work completion indicates that the GRH is valid.
3692 * @ah_attr: Returned attributes that can be used when creating an address
3693 *   handle for replying to the message.
3694 * When ib_init_ah_attr_from_wc() returns success,
3695 * (a) for IB link layer it optionally contains a reference to SGID attribute
3696 * when GRH is present for IB link layer.
3697 * (b) for RoCE link layer it contains a reference to SGID attribute.
3698 * User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID
3699 * attributes which are initialized using ib_init_ah_attr_from_wc().
3700 *
3701 */
3702int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
3703			    const struct ib_wc *wc, const struct ib_grh *grh,
3704			    struct rdma_ah_attr *ah_attr);
3705
3706/**
3707 * ib_create_ah_from_wc - Creates an address handle associated with the
3708 *   sender of the specified work completion.
3709 * @pd: The protection domain associated with the address handle.
3710 * @wc: Work completion information associated with a received message.
3711 * @grh: References the received global route header.  This parameter is
3712 *   ignored unless the work completion indicates that the GRH is valid.
3713 * @port_num: The outbound port number to associate with the address.
3714 *
3715 * The address handle is used to reference a local or global destination
3716 * in all UD QP post sends.
3717 */
3718struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
3719				   const struct ib_grh *grh, u32 port_num);
3720
3721/**
3722 * rdma_modify_ah - Modifies the address vector associated with an address
3723 *   handle.
3724 * @ah: The address handle to modify.
3725 * @ah_attr: The new address vector attributes to associate with the
3726 *   address handle.
3727 */
3728int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
3729
3730/**
3731 * rdma_query_ah - Queries the address vector associated with an address
3732 *   handle.
3733 * @ah: The address handle to query.
3734 * @ah_attr: The address vector attributes associated with the address
3735 *   handle.
3736 */
3737int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
3738
3739enum rdma_destroy_ah_flags {
3740	/* In a sleepable context */
3741	RDMA_DESTROY_AH_SLEEPABLE = BIT(0),
3742};
3743
3744/**
3745 * rdma_destroy_ah_user - Destroys an address handle.
3746 * @ah: The address handle to destroy.
3747 * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
3748 * @udata: Valid user data or NULL for kernel objects
3749 */
3750int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);
3751
3752/**
3753 * rdma_destroy_ah - Destroys an kernel address handle.
3754 * @ah: The address handle to destroy.
3755 * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
3756 *
3757 * NOTE: for user ah use rdma_destroy_ah_user with valid udata!
3758 */
3759static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags)
3760{
3761	int ret = rdma_destroy_ah_user(ah, flags, NULL);
3762
3763	WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail");
3764}
3765
3766struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
3767				  struct ib_srq_init_attr *srq_init_attr,
3768				  struct ib_usrq_object *uobject,
3769				  struct ib_udata *udata);
3770static inline struct ib_srq *
3771ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr)
3772{
3773	if (!pd->device->ops.create_srq)
3774		return ERR_PTR(-EOPNOTSUPP);
3775
3776	return ib_create_srq_user(pd, srq_init_attr, NULL, NULL);
3777}
3778
3779/**
3780 * ib_modify_srq - Modifies the attributes for the specified SRQ.
3781 * @srq: The SRQ to modify.
3782 * @srq_attr: On input, specifies the SRQ attributes to modify.  On output,
3783 *   the current values of selected SRQ attributes are returned.
3784 * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ
3785 *   are being modified.
3786 *
3787 * The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or
3788 * IB_SRQ_LIMIT to set the SRQ's limit and request notification when
3789 * the number of receives queued drops below the limit.
3790 */
3791int ib_modify_srq(struct ib_srq *srq,
3792		  struct ib_srq_attr *srq_attr,
3793		  enum ib_srq_attr_mask srq_attr_mask);
3794
3795/**
3796 * ib_query_srq - Returns the attribute list and current values for the
3797 *   specified SRQ.
3798 * @srq: The SRQ to query.
3799 * @srq_attr: The attributes of the specified SRQ.
3800 */
3801int ib_query_srq(struct ib_srq *srq,
3802		 struct ib_srq_attr *srq_attr);
3803
3804/**
3805 * ib_destroy_srq_user - Destroys the specified SRQ.
3806 * @srq: The SRQ to destroy.
3807 * @udata: Valid user data or NULL for kernel objects
3808 */
3809int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);
3810
3811/**
3812 * ib_destroy_srq - Destroys the specified kernel SRQ.
3813 * @srq: The SRQ to destroy.
3814 *
3815 * NOTE: for user srq use ib_destroy_srq_user with valid udata!
3816 */
3817static inline void ib_destroy_srq(struct ib_srq *srq)
3818{
3819	int ret = ib_destroy_srq_user(srq, NULL);
3820
3821	WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail");
3822}
3823
3824/**
3825 * ib_post_srq_recv - Posts a list of work requests to the specified SRQ.
3826 * @srq: The SRQ to post the work request on.
3827 * @recv_wr: A list of work requests to post on the receive queue.
3828 * @bad_recv_wr: On an immediate failure, this parameter will reference
3829 *   the work request that failed to be posted on the QP.
3830 */
3831static inline int ib_post_srq_recv(struct ib_srq *srq,
3832				   const struct ib_recv_wr *recv_wr,
3833				   const struct ib_recv_wr **bad_recv_wr)
3834{
3835	const struct ib_recv_wr *dummy;
3836
3837	return srq->device->ops.post_srq_recv(srq, recv_wr,
3838					      bad_recv_wr ? : &dummy);
3839}
3840
3841struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
3842				  struct ib_qp_init_attr *qp_init_attr,
3843				  const char *caller);
3844/**
3845 * ib_create_qp - Creates a kernel QP associated with the specific protection
3846 * domain.
3847 * @pd: The protection domain associated with the QP.
3848 * @init_attr: A list of initial attributes required to create the
3849 *   QP.  If QP creation succeeds, then the attributes are updated to
3850 *   the actual capabilities of the created QP.
3851 */
3852static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
3853					 struct ib_qp_init_attr *init_attr)
3854{
3855	return ib_create_qp_kernel(pd, init_attr, KBUILD_MODNAME);
3856}
3857
3858/**
3859 * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
3860 * @qp: The QP to modify.
3861 * @attr: On input, specifies the QP attributes to modify.  On output,
3862 *   the current values of selected QP attributes are returned.
3863 * @attr_mask: A bit-mask used to specify which attributes of the QP
3864 *   are being modified.
3865 * @udata: pointer to user's input output buffer information
3866 *   are being modified.
3867 * It returns 0 on success and returns appropriate error code on error.
3868 */
3869int ib_modify_qp_with_udata(struct ib_qp *qp,
3870			    struct ib_qp_attr *attr,
3871			    int attr_mask,
3872			    struct ib_udata *udata);
3873
3874/**
3875 * ib_modify_qp - Modifies the attributes for the specified QP and then
3876 *   transitions the QP to the given state.
3877 * @qp: The QP to modify.
3878 * @qp_attr: On input, specifies the QP attributes to modify.  On output,
3879 *   the current values of selected QP attributes are returned.
3880 * @qp_attr_mask: A bit-mask used to specify which attributes of the QP
3881 *   are being modified.
3882 */
3883int ib_modify_qp(struct ib_qp *qp,
3884		 struct ib_qp_attr *qp_attr,
3885		 int qp_attr_mask);
3886
3887/**
3888 * ib_query_qp - Returns the attribute list and current values for the
3889 *   specified QP.
3890 * @qp: The QP to query.
3891 * @qp_attr: The attributes of the specified QP.
3892 * @qp_attr_mask: A bit-mask used to select specific attributes to query.
3893 * @qp_init_attr: Additional attributes of the selected QP.
3894 *
3895 * The qp_attr_mask may be used to limit the query to gathering only the
3896 * selected attributes.
3897 */
3898int ib_query_qp(struct ib_qp *qp,
3899		struct ib_qp_attr *qp_attr,
3900		int qp_attr_mask,
3901		struct ib_qp_init_attr *qp_init_attr);
3902
3903/**
3904 * ib_destroy_qp - Destroys the specified QP.
3905 * @qp: The QP to destroy.
3906 * @udata: Valid udata or NULL for kernel objects
3907 */
3908int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);
3909
3910/**
3911 * ib_destroy_qp - Destroys the specified kernel QP.
3912 * @qp: The QP to destroy.
3913 *
3914 * NOTE: for user qp use ib_destroy_qp_user with valid udata!
3915 */
3916static inline int ib_destroy_qp(struct ib_qp *qp)
3917{
3918	return ib_destroy_qp_user(qp, NULL);
3919}
3920
3921/**
3922 * ib_open_qp - Obtain a reference to an existing sharable QP.
3923 * @xrcd: XRC domain
3924 * @qp_open_attr: Attributes identifying the QP to open.
3925 *
3926 * Returns a reference to a sharable QP.
3927 */
3928struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
3929			 struct ib_qp_open_attr *qp_open_attr);
3930
3931/**
3932 * ib_close_qp - Release an external reference to a QP.
3933 * @qp: The QP handle to release
3934 *
3935 * The opened QP handle is released by the caller.  The underlying
3936 * shared QP is not destroyed until all internal references are released.
3937 */
3938int ib_close_qp(struct ib_qp *qp);
3939
3940/**
3941 * ib_post_send - Posts a list of work requests to the send queue of
3942 *   the specified QP.
3943 * @qp: The QP to post the work request on.
3944 * @send_wr: A list of work requests to post on the send queue.
3945 * @bad_send_wr: On an immediate failure, this parameter will reference
3946 *   the work request that failed to be posted on the QP.
3947 *
3948 * While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate
3949 * error is returned, the QP state shall not be affected,
3950 * ib_post_send() will return an immediate error after queueing any
3951 * earlier work requests in the list.
3952 */
3953static inline int ib_post_send(struct ib_qp *qp,
3954			       const struct ib_send_wr *send_wr,
3955			       const struct ib_send_wr **bad_send_wr)
3956{
3957	const struct ib_send_wr *dummy;
3958
3959	return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy);
3960}
3961
3962/**
3963 * ib_post_recv - Posts a list of work requests to the receive queue of
3964 *   the specified QP.
3965 * @qp: The QP to post the work request on.
3966 * @recv_wr: A list of work requests to post on the receive queue.
3967 * @bad_recv_wr: On an immediate failure, this parameter will reference
3968 *   the work request that failed to be posted on the QP.
3969 */
3970static inline int ib_post_recv(struct ib_qp *qp,
3971			       const struct ib_recv_wr *recv_wr,
3972			       const struct ib_recv_wr **bad_recv_wr)
3973{
3974	const struct ib_recv_wr *dummy;
3975
3976	return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);
3977}
3978
3979struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
3980			    int comp_vector, enum ib_poll_context poll_ctx,
3981			    const char *caller);
3982static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
3983					int nr_cqe, int comp_vector,
3984					enum ib_poll_context poll_ctx)
3985{
3986	return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
3987			     KBUILD_MODNAME);
3988}
3989
3990struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
3991				int nr_cqe, enum ib_poll_context poll_ctx,
3992				const char *caller);
3993
3994/**
3995 * ib_alloc_cq_any: Allocate kernel CQ
3996 * @dev: The IB device
3997 * @private: Private data attached to the CQE
3998 * @nr_cqe: Number of CQEs in the CQ
3999 * @poll_ctx: Context used for polling the CQ
4000 */
4001static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
4002					    void *private, int nr_cqe,
4003					    enum ib_poll_context poll_ctx)
4004{
4005	return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx,
4006				 KBUILD_MODNAME);
4007}
4008
4009void ib_free_cq(struct ib_cq *cq);
4010int ib_process_cq_direct(struct ib_cq *cq, int budget);
4011
4012/**
4013 * ib_create_cq - Creates a CQ on the specified device.
4014 * @device: The device on which to create the CQ.
4015 * @comp_handler: A user-specified callback that is invoked when a
4016 *   completion event occurs on the CQ.
4017 * @event_handler: A user-specified callback that is invoked when an
4018 *   asynchronous event not associated with a completion occurs on the CQ.
4019 * @cq_context: Context associated with the CQ returned to the user via
4020 *   the associated completion and event handlers.
4021 * @cq_attr: The attributes the CQ should be created upon.
4022 *
4023 * Users can examine the cq structure to determine the actual CQ size.
4024 */
4025struct ib_cq *__ib_create_cq(struct ib_device *device,
4026			     ib_comp_handler comp_handler,
4027			     void (*event_handler)(struct ib_event *, void *),
4028			     void *cq_context,
4029			     const struct ib_cq_init_attr *cq_attr,
4030			     const char *caller);
4031#define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \
4032	__ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME)
4033
4034/**
4035 * ib_resize_cq - Modifies the capacity of the CQ.
4036 * @cq: The CQ to resize.
4037 * @cqe: The minimum size of the CQ.
4038 *
4039 * Users can examine the cq structure to determine the actual CQ size.
4040 */
4041int ib_resize_cq(struct ib_cq *cq, int cqe);
4042
4043/**
4044 * rdma_set_cq_moderation - Modifies moderation params of the CQ
4045 * @cq: The CQ to modify.
4046 * @cq_count: number of CQEs that will trigger an event
4047 * @cq_period: max period of time in usec before triggering an event
4048 *
4049 */
4050int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);
4051
4052/**
4053 * ib_destroy_cq_user - Destroys the specified CQ.
4054 * @cq: The CQ to destroy.
4055 * @udata: Valid user data or NULL for kernel objects
4056 */
4057int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
4058
4059/**
4060 * ib_destroy_cq - Destroys the specified kernel CQ.
4061 * @cq: The CQ to destroy.
4062 *
4063 * NOTE: for user cq use ib_destroy_cq_user with valid udata!
4064 */
4065static inline void ib_destroy_cq(struct ib_cq *cq)
4066{
4067	int ret = ib_destroy_cq_user(cq, NULL);
4068
4069	WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
4070}
4071
4072/**
4073 * ib_poll_cq - poll a CQ for completion(s)
4074 * @cq:the CQ being polled
4075 * @num_entries:maximum number of completions to return
4076 * @wc:array of at least @num_entries &struct ib_wc where completions
4077 *   will be returned
4078 *
4079 * Poll a CQ for (possibly multiple) completions.  If the return value
4080 * is < 0, an error occurred.  If the return value is >= 0, it is the
4081 * number of completions returned.  If the return value is
4082 * non-negative and < num_entries, then the CQ was emptied.
4083 */
4084static inline int ib_poll_cq(struct ib_cq *cq, int num_entries,
4085			     struct ib_wc *wc)
4086{
4087	return cq->device->ops.poll_cq(cq, num_entries, wc);
4088}
4089
4090/**
4091 * ib_req_notify_cq - Request completion notification on a CQ.
4092 * @cq: The CQ to generate an event for.
4093 * @flags:
4094 *   Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP
4095 *   to request an event on the next solicited event or next work
4096 *   completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS
4097 *   may also be |ed in to request a hint about missed events, as
4098 *   described below.
4099 *
4100 * Return Value:
4101 *    < 0 means an error occurred while requesting notification
4102 *   == 0 means notification was requested successfully, and if
4103 *        IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events
4104 *        were missed and it is safe to wait for another event.  In
4105 *        this case is it guaranteed that any work completions added
4106 *        to the CQ since the last CQ poll will trigger a completion
4107 *        notification event.
4108 *    > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed
4109 *        in.  It means that the consumer must poll the CQ again to
4110 *        make sure it is empty to avoid missing an event because of a
4111 *        race between requesting notification and an entry being
4112 *        added to the CQ.  This return value means it is possible
4113 *        (but not guaranteed) that a work completion has been added
4114 *        to the CQ since the last poll without triggering a
4115 *        completion notification event.
4116 */
4117static inline int ib_req_notify_cq(struct ib_cq *cq,
4118				   enum ib_cq_notify_flags flags)
4119{
4120	return cq->device->ops.req_notify_cq(cq, flags);
4121}
4122
4123struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
4124			     int comp_vector_hint,
4125			     enum ib_poll_context poll_ctx);
4126
4127void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe);
4128
4129/*
4130 * Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to
4131 * NULL. This causes the ib_dma* helpers to just stash the kernel virtual
4132 * address into the dma address.
4133 */
4134static inline bool ib_uses_virt_dma(struct ib_device *dev)
4135{
4136	return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;
4137}
4138
4139/*
4140 * Check if a IB device's underlying DMA mapping supports P2PDMA transfers.
4141 */
4142static inline bool ib_dma_pci_p2p_dma_supported(struct ib_device *dev)
4143{
4144	if (ib_uses_virt_dma(dev))
4145		return false;
4146
4147	return dma_pci_p2pdma_supported(dev->dma_device);
4148}
4149
4150/**
4151 * ib_virt_dma_to_ptr - Convert a dma_addr to a kernel pointer
4152 * @dma_addr: The DMA address
4153 *
4154 * Used by ib_uses_virt_dma() devices to get back to the kernel pointer after
4155 * going through the dma_addr marshalling.
4156 */
4157static inline void *ib_virt_dma_to_ptr(u64 dma_addr)
4158{
4159	/* virt_dma mode maps the kvs's directly into the dma addr */
4160	return (void *)(uintptr_t)dma_addr;
4161}
4162
4163/**
4164 * ib_virt_dma_to_page - Convert a dma_addr to a struct page
4165 * @dma_addr: The DMA address
4166 *
4167 * Used by ib_uses_virt_dma() device to get back to the struct page after going
4168 * through the dma_addr marshalling.
4169 */
4170static inline struct page *ib_virt_dma_to_page(u64 dma_addr)
4171{
4172	return virt_to_page(ib_virt_dma_to_ptr(dma_addr));
4173}
4174
4175/**
4176 * ib_dma_mapping_error - check a DMA addr for error
4177 * @dev: The device for which the dma_addr was created
4178 * @dma_addr: The DMA address to check
4179 */
4180static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
4181{
4182	if (ib_uses_virt_dma(dev))
4183		return 0;
4184	return dma_mapping_error(dev->dma_device, dma_addr);
4185}
4186
4187/**
4188 * ib_dma_map_single - Map a kernel virtual address to DMA address
4189 * @dev: The device for which the dma_addr is to be created
4190 * @cpu_addr: The kernel virtual address
4191 * @size: The size of the region in bytes
4192 * @direction: The direction of the DMA
4193 */
4194static inline u64 ib_dma_map_single(struct ib_device *dev,
4195				    void *cpu_addr, size_t size,
4196				    enum dma_data_direction direction)
4197{
4198	if (ib_uses_virt_dma(dev))
4199		return (uintptr_t)cpu_addr;
4200	return dma_map_single(dev->dma_device, cpu_addr, size, direction);
4201}
4202
4203/**
4204 * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
4205 * @dev: The device for which the DMA address was created
4206 * @addr: The DMA address
4207 * @size: The size of the region in bytes
4208 * @direction: The direction of the DMA
4209 */
4210static inline void ib_dma_unmap_single(struct ib_device *dev,
4211				       u64 addr, size_t size,
4212				       enum dma_data_direction direction)
4213{
4214	if (!ib_uses_virt_dma(dev))
4215		dma_unmap_single(dev->dma_device, addr, size, direction);
4216}
4217
4218/**
4219 * ib_dma_map_page - Map a physical page to DMA address
4220 * @dev: The device for which the dma_addr is to be created
4221 * @page: The page to be mapped
4222 * @offset: The offset within the page
4223 * @size: The size of the region in bytes
4224 * @direction: The direction of the DMA
4225 */
4226static inline u64 ib_dma_map_page(struct ib_device *dev,
4227				  struct page *page,
4228				  unsigned long offset,
4229				  size_t size,
4230					 enum dma_data_direction direction)
4231{
4232	if (ib_uses_virt_dma(dev))
4233		return (uintptr_t)(page_address(page) + offset);
4234	return dma_map_page(dev->dma_device, page, offset, size, direction);
4235}
4236
4237/**
4238 * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
4239 * @dev: The device for which the DMA address was created
4240 * @addr: The DMA address
4241 * @size: The size of the region in bytes
4242 * @direction: The direction of the DMA
4243 */
4244static inline void ib_dma_unmap_page(struct ib_device *dev,
4245				     u64 addr, size_t size,
4246				     enum dma_data_direction direction)
4247{
4248	if (!ib_uses_virt_dma(dev))
4249		dma_unmap_page(dev->dma_device, addr, size, direction);
4250}
4251
4252int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);
4253static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
4254				      struct scatterlist *sg, int nents,
4255				      enum dma_data_direction direction,
4256				      unsigned long dma_attrs)
4257{
4258	if (ib_uses_virt_dma(dev))
4259		return ib_dma_virt_map_sg(dev, sg, nents);
4260	return dma_map_sg_attrs(dev->dma_device, sg, nents, direction,
4261				dma_attrs);
4262}
4263
4264static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
4265					 struct scatterlist *sg, int nents,
4266					 enum dma_data_direction direction,
4267					 unsigned long dma_attrs)
4268{
4269	if (!ib_uses_virt_dma(dev))
4270		dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction,
4271				   dma_attrs);
4272}
4273
4274/**
4275 * ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses
4276 * @dev: The device for which the DMA addresses are to be created
4277 * @sgt: The sg_table object describing the buffer
4278 * @direction: The direction of the DMA
4279 * @dma_attrs: Optional DMA attributes for the map operation
4280 */
4281static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev,
4282					   struct sg_table *sgt,
4283					   enum dma_data_direction direction,
4284					   unsigned long dma_attrs)
4285{
4286	int nents;
4287
4288	if (ib_uses_virt_dma(dev)) {
4289		nents = ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents);
4290		if (!nents)
4291			return -EIO;
4292		sgt->nents = nents;
4293		return 0;
4294	}
4295	return dma_map_sgtable(dev->dma_device, sgt, direction, dma_attrs);
4296}
4297
4298static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev,
4299					      struct sg_table *sgt,
4300					      enum dma_data_direction direction,
4301					      unsigned long dma_attrs)
4302{
4303	if (!ib_uses_virt_dma(dev))
4304		dma_unmap_sgtable(dev->dma_device, sgt, direction, dma_attrs);
4305}
4306
4307/**
4308 * ib_dma_map_sg - Map a scatter/gather list to DMA addresses
4309 * @dev: The device for which the DMA addresses are to be created
4310 * @sg: The array of scatter/gather entries
4311 * @nents: The number of scatter/gather entries
4312 * @direction: The direction of the DMA
4313 */
4314static inline int ib_dma_map_sg(struct ib_device *dev,
4315				struct scatterlist *sg, int nents,
4316				enum dma_data_direction direction)
4317{
4318	return ib_dma_map_sg_attrs(dev, sg, nents, direction, 0);
4319}
4320
4321/**
4322 * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
4323 * @dev: The device for which the DMA addresses were created
4324 * @sg: The array of scatter/gather entries
4325 * @nents: The number of scatter/gather entries
4326 * @direction: The direction of the DMA
4327 */
4328static inline void ib_dma_unmap_sg(struct ib_device *dev,
4329				   struct scatterlist *sg, int nents,
4330				   enum dma_data_direction direction)
4331{
4332	ib_dma_unmap_sg_attrs(dev, sg, nents, direction, 0);
4333}
4334
4335/**
4336 * ib_dma_max_seg_size - Return the size limit of a single DMA transfer
4337 * @dev: The device to query
4338 *
4339 * The returned value represents a size in bytes.
4340 */
4341static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)
4342{
4343	if (ib_uses_virt_dma(dev))
4344		return UINT_MAX;
4345	return dma_get_max_seg_size(dev->dma_device);
4346}
4347
4348/**
4349 * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
4350 * @dev: The device for which the DMA address was created
4351 * @addr: The DMA address
4352 * @size: The size of the region in bytes
4353 * @dir: The direction of the DMA
4354 */
4355static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
4356					      u64 addr,
4357					      size_t size,
4358					      enum dma_data_direction dir)
4359{
4360	if (!ib_uses_virt_dma(dev))
4361		dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
4362}
4363
4364/**
4365 * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
4366 * @dev: The device for which the DMA address was created
4367 * @addr: The DMA address
4368 * @size: The size of the region in bytes
4369 * @dir: The direction of the DMA
4370 */
4371static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
4372						 u64 addr,
4373						 size_t size,
4374						 enum dma_data_direction dir)
4375{
4376	if (!ib_uses_virt_dma(dev))
4377		dma_sync_single_for_device(dev->dma_device, addr, size, dir);
4378}
4379
4380/* ib_reg_user_mr - register a memory region for virtual addresses from kernel
4381 * space. This function should be called when 'current' is the owning MM.
4382 */
4383struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
4384			     u64 virt_addr, int mr_access_flags);
4385
4386/* ib_advise_mr -  give an advice about an address range in a memory region */
4387int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
4388		 u32 flags, struct ib_sge *sg_list, u32 num_sge);
4389/**
4390 * ib_dereg_mr_user - Deregisters a memory region and removes it from the
4391 *   HCA translation table.
4392 * @mr: The memory region to deregister.
4393 * @udata: Valid user data or NULL for kernel object
4394 *
4395 * This function can fail, if the memory region has memory windows bound to it.
4396 */
4397int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);
4398
4399/**
4400 * ib_dereg_mr - Deregisters a kernel memory region and removes it from the
4401 *   HCA translation table.
4402 * @mr: The memory region to deregister.
4403 *
4404 * This function can fail, if the memory region has memory windows bound to it.
4405 *
4406 * NOTE: for user mr use ib_dereg_mr_user with valid udata!
4407 */
4408static inline int ib_dereg_mr(struct ib_mr *mr)
4409{
4410	return ib_dereg_mr_user(mr, NULL);
4411}
4412
4413struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
4414			  u32 max_num_sg);
4415
4416struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
4417				    u32 max_num_data_sg,
4418				    u32 max_num_meta_sg);
4419
4420/**
4421 * ib_update_fast_reg_key - updates the key portion of the fast_reg MR
4422 *   R_Key and L_Key.
4423 * @mr: struct ib_mr pointer to be updated.
4424 * @newkey: new key to be used.
4425 */
4426static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey)
4427{
4428	mr->lkey = (mr->lkey & 0xffffff00) | newkey;
4429	mr->rkey = (mr->rkey & 0xffffff00) | newkey;
4430}
4431
4432/**
4433 * ib_inc_rkey - increments the key portion of the given rkey. Can be used
4434 * for calculating a new rkey for type 2 memory windows.
4435 * @rkey: the rkey to increment.
4436 */
4437static inline u32 ib_inc_rkey(u32 rkey)
4438{
4439	const u32 mask = 0x000000ff;
4440	return ((rkey + 1) & mask) | (rkey & ~mask);
4441}
4442
4443/**
4444 * ib_attach_mcast - Attaches the specified QP to a multicast group.
4445 * @qp: QP to attach to the multicast group.  The QP must be type
4446 *   IB_QPT_UD.
4447 * @gid: Multicast group GID.
4448 * @lid: Multicast group LID in host byte order.
4449 *
4450 * In order to send and receive multicast packets, subnet
4451 * administration must have created the multicast group and configured
4452 * the fabric appropriately.  The port associated with the specified
4453 * QP must also be a member of the multicast group.
4454 */
4455int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
4456
4457/**
4458 * ib_detach_mcast - Detaches the specified QP from a multicast group.
4459 * @qp: QP to detach from the multicast group.
4460 * @gid: Multicast group GID.
4461 * @lid: Multicast group LID in host byte order.
4462 */
4463int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
4464
4465struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
4466				   struct inode *inode, struct ib_udata *udata);
4467int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);
4468
4469static inline int ib_check_mr_access(struct ib_device *ib_dev,
4470				     unsigned int flags)
4471{
4472	u64 device_cap = ib_dev->attrs.device_cap_flags;
4473
4474	/*
4475	 * Local write permission is required if remote write or
4476	 * remote atomic permission is also requested.
4477	 */
4478	if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) &&
4479	    !(flags & IB_ACCESS_LOCAL_WRITE))
4480		return -EINVAL;
4481
4482	if (flags & ~IB_ACCESS_SUPPORTED)
4483		return -EINVAL;
4484
4485	if (flags & IB_ACCESS_ON_DEMAND &&
4486	    !(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))
4487		return -EOPNOTSUPP;
4488
4489	if ((flags & IB_ACCESS_FLUSH_GLOBAL &&
4490	    !(device_cap & IB_DEVICE_FLUSH_GLOBAL)) ||
4491	    (flags & IB_ACCESS_FLUSH_PERSISTENT &&
4492	    !(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))
4493		return -EOPNOTSUPP;
4494
4495	return 0;
4496}
4497
4498static inline bool ib_access_writable(int access_flags)
4499{
4500	/*
4501	 * We have writable memory backing the MR if any of the following
4502	 * access flags are set.  "Local write" and "remote write" obviously
4503	 * require write access.  "Remote atomic" can do things like fetch and
4504	 * add, which will modify memory, and "MW bind" can change permissions
4505	 * by binding a window.
4506	 */
4507	return access_flags &
4508		(IB_ACCESS_LOCAL_WRITE   | IB_ACCESS_REMOTE_WRITE |
4509		 IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND);
4510}
4511
4512/**
4513 * ib_check_mr_status: lightweight check of MR status.
4514 *     This routine may provide status checks on a selected
4515 *     ib_mr. first use is for signature status check.
4516 *
4517 * @mr: A memory region.
4518 * @check_mask: Bitmask of which checks to perform from
4519 *     ib_mr_status_check enumeration.
4520 * @mr_status: The container of relevant status checks.
4521 *     failed checks will be indicated in the status bitmask
4522 *     and the relevant info shall be in the error item.
4523 */
4524int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
4525		       struct ib_mr_status *mr_status);
4526
4527/**
4528 * ib_device_try_get: Hold a registration lock
4529 * @dev: The device to lock
4530 *
4531 * A device under an active registration lock cannot become unregistered. It
4532 * is only possible to obtain a registration lock on a device that is fully
4533 * registered, otherwise this function returns false.
4534 *
4535 * The registration lock is only necessary for actions which require the
4536 * device to still be registered. Uses that only require the device pointer to
4537 * be valid should use get_device(&ibdev->dev) to hold the memory.
4538 *
4539 */
4540static inline bool ib_device_try_get(struct ib_device *dev)
4541{
4542	return refcount_inc_not_zero(&dev->refcount);
4543}
4544
4545void ib_device_put(struct ib_device *device);
4546struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
4547					  enum rdma_driver_id driver_id);
4548struct ib_device *ib_device_get_by_name(const char *name,
4549					enum rdma_driver_id driver_id);
4550struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u32 port,
4551					    u16 pkey, const union ib_gid *gid,
4552					    const struct sockaddr *addr);
4553int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
4554			 unsigned int port);
4555struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,
4556					u32 port);
4557int ib_query_netdev_port(struct ib_device *ibdev, struct net_device *ndev,
4558			 u32 *port);
4559
4560static inline enum ib_port_state ib_get_curr_port_state(struct net_device *net_dev)
4561{
4562	return (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
4563		IB_PORT_ACTIVE : IB_PORT_DOWN;
4564}
4565
4566void ib_dispatch_port_state_event(struct ib_device *ibdev,
4567				  struct net_device *ndev);
4568struct ib_wq *ib_create_wq(struct ib_pd *pd,
4569			   struct ib_wq_init_attr *init_attr);
4570int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata);
4571
4572int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4573		 unsigned int *sg_offset, unsigned int page_size);
4574int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
4575		    int data_sg_nents, unsigned int *data_sg_offset,
4576		    struct scatterlist *meta_sg, int meta_sg_nents,
4577		    unsigned int *meta_sg_offset, unsigned int page_size);
4578
4579static inline int
4580ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
4581		  unsigned int *sg_offset, unsigned int page_size)
4582{
4583	int n;
4584
4585	n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size);
4586	mr->iova = 0;
4587
4588	return n;
4589}
4590
4591int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
4592		unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));
4593
4594void ib_drain_rq(struct ib_qp *qp);
4595void ib_drain_sq(struct ib_qp *qp);
4596void ib_drain_qp(struct ib_qp *qp);
4597
4598int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed,
4599		     u8 *width);
4600
4601static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr)
4602{
4603	if (attr->type == RDMA_AH_ATTR_TYPE_ROCE)
4604		return attr->roce.dmac;
4605	return NULL;
4606}
4607
4608static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid)
4609{
4610	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4611		attr->ib.dlid = (u16)dlid;
4612	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4613		attr->opa.dlid = dlid;
4614}
4615
4616static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr)
4617{
4618	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4619		return attr->ib.dlid;
4620	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4621		return attr->opa.dlid;
4622	return 0;
4623}
4624
4625static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl)
4626{
4627	attr->sl = sl;
4628}
4629
4630static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr)
4631{
4632	return attr->sl;
4633}
4634
4635static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr,
4636					 u8 src_path_bits)
4637{
4638	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4639		attr->ib.src_path_bits = src_path_bits;
4640	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4641		attr->opa.src_path_bits = src_path_bits;
4642}
4643
4644static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr)
4645{
4646	if (attr->type == RDMA_AH_ATTR_TYPE_IB)
4647		return attr->ib.src_path_bits;
4648	else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4649		return attr->opa.src_path_bits;
4650	return 0;
4651}
4652
4653static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr,
4654					bool make_grd)
4655{
4656	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4657		attr->opa.make_grd = make_grd;
4658}
4659
4660static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr)
4661{
4662	if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
4663		return attr->opa.make_grd;
4664	return false;
4665}
4666
4667static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u32 port_num)
4668{
4669	attr->port_num = port_num;
4670}
4671
4672static inline u32 rdma_ah_get_port_num(const struct rdma_ah_attr *attr)
4673{
4674	return attr->port_num;
4675}
4676
4677static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr,
4678					   u8 static_rate)
4679{
4680	attr->static_rate = static_rate;
4681}
4682
4683static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr)
4684{
4685	return attr->static_rate;
4686}
4687
4688static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr,
4689					enum ib_ah_flags flag)
4690{
4691	attr->ah_flags = flag;
4692}
4693
4694static inline enum ib_ah_flags
4695		rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr)
4696{
4697	return attr->ah_flags;
4698}
4699
4700static inline const struct ib_global_route
4701		*rdma_ah_read_grh(const struct rdma_ah_attr *attr)
4702{
4703	return &attr->grh;
4704}
4705
4706/*To retrieve and modify the grh */
4707static inline struct ib_global_route
4708		*rdma_ah_retrieve_grh(struct rdma_ah_attr *attr)
4709{
4710	return &attr->grh;
4711}
4712
4713static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid)
4714{
4715	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4716
4717	memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid));
4718}
4719
4720static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr,
4721					     __be64 prefix)
4722{
4723	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4724
4725	grh->dgid.global.subnet_prefix = prefix;
4726}
4727
4728static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr,
4729					    __be64 if_id)
4730{
4731	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4732
4733	grh->dgid.global.interface_id = if_id;
4734}
4735
4736static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr,
4737				   union ib_gid *dgid, u32 flow_label,
4738				   u8 sgid_index, u8 hop_limit,
4739				   u8 traffic_class)
4740{
4741	struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
4742
4743	attr->ah_flags = IB_AH_GRH;
4744	if (dgid)
4745		grh->dgid = *dgid;
4746	grh->flow_label = flow_label;
4747	grh->sgid_index = sgid_index;
4748	grh->hop_limit = hop_limit;
4749	grh->traffic_class = traffic_class;
4750	grh->sgid_attr = NULL;
4751}
4752
4753void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr);
4754void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
4755			     u32 flow_label, u8 hop_limit, u8 traffic_class,
4756			     const struct ib_gid_attr *sgid_attr);
4757void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
4758		       const struct rdma_ah_attr *src);
4759void rdma_replace_ah_attr(struct rdma_ah_attr *old,
4760			  const struct rdma_ah_attr *new);
4761void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src);
4762
4763/**
4764 * rdma_ah_find_type - Return address handle type.
4765 *
4766 * @dev: Device to be checked
4767 * @port_num: Port number
4768 */
4769static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev,
4770						       u32 port_num)
4771{
4772	if (rdma_protocol_roce(dev, port_num))
4773		return RDMA_AH_ATTR_TYPE_ROCE;
4774	if (rdma_protocol_ib(dev, port_num)) {
4775		if (rdma_cap_opa_ah(dev, port_num))
4776			return RDMA_AH_ATTR_TYPE_OPA;
4777		return RDMA_AH_ATTR_TYPE_IB;
4778	}
4779	if (dev->type == RDMA_DEVICE_TYPE_SMI)
4780		return RDMA_AH_ATTR_TYPE_IB;
4781
4782	return RDMA_AH_ATTR_TYPE_UNDEFINED;
4783}
4784
4785/**
4786 * ib_lid_cpu16 - Return lid in 16bit CPU encoding.
4787 *     In the current implementation the only way to
4788 *     get the 32bit lid is from other sources for OPA.
4789 *     For IB, lids will always be 16bits so cast the
4790 *     value accordingly.
4791 *
4792 * @lid: A 32bit LID
4793 */
4794static inline u16 ib_lid_cpu16(u32 lid)
4795{
4796	WARN_ON_ONCE(lid & 0xFFFF0000);
4797	return (u16)lid;
4798}
4799
4800/**
4801 * ib_lid_be16 - Return lid in 16bit BE encoding.
4802 *
4803 * @lid: A 32bit LID
4804 */
4805static inline __be16 ib_lid_be16(u32 lid)
4806{
4807	WARN_ON_ONCE(lid & 0xFFFF0000);
4808	return cpu_to_be16((u16)lid);
4809}
4810
4811/**
4812 * ib_get_vector_affinity - Get the affinity mappings of a given completion
4813 *   vector
4814 * @device:         the rdma device
4815 * @comp_vector:    index of completion vector
4816 *
4817 * Returns NULL on failure, otherwise a corresponding cpu map of the
4818 * completion vector (returns all-cpus map if the device driver doesn't
4819 * implement get_vector_affinity).
4820 */
4821static inline const struct cpumask *
4822ib_get_vector_affinity(struct ib_device *device, int comp_vector)
4823{
4824	if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||
4825	    !device->ops.get_vector_affinity)
4826		return NULL;
4827
4828	return device->ops.get_vector_affinity(device, comp_vector);
4829
4830}
4831
4832/**
4833 * rdma_roce_rescan_device - Rescan all of the network devices in the system
4834 * and add their gids, as needed, to the relevant RoCE devices.
4835 *
4836 * @ibdev:         the rdma device
4837 */
4838void rdma_roce_rescan_device(struct ib_device *ibdev);
4839void rdma_roce_rescan_port(struct ib_device *ib_dev, u32 port);
4840void roce_del_all_netdev_gids(struct ib_device *ib_dev,
4841			      u32 port, struct net_device *ndev);
4842
4843struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile);
4844
4845#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
4846int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs);
4847bool rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs);
4848#else
4849static inline int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
4850{
4851	return 0;
4852}
4853static inline bool
4854rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs)
4855{
4856	return false;
4857}
4858#endif
4859
4860struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
4861				     enum rdma_netdev_t type, const char *name,
4862				     unsigned char name_assign_type,
4863				     void (*setup)(struct net_device *));
4864
4865int rdma_init_netdev(struct ib_device *device, u32 port_num,
4866		     enum rdma_netdev_t type, const char *name,
4867		     unsigned char name_assign_type,
4868		     void (*setup)(struct net_device *),
4869		     struct net_device *netdev);
4870
4871/**
4872 * rdma_device_to_ibdev - Get ib_device pointer from device pointer
4873 *
4874 * @device:	device pointer for which ib_device pointer to retrieve
4875 *
4876 * rdma_device_to_ibdev() retrieves ib_device pointer from device.
4877 *
4878 */
4879static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
4880{
4881	struct ib_core_device *coredev =
4882		container_of(device, struct ib_core_device, dev);
4883
4884	return coredev->owner;
4885}
4886
4887/**
4888 * ibdev_to_node - return the NUMA node for a given ib_device
4889 * @ibdev:	device to get the NUMA node for.
4890 */
4891static inline int ibdev_to_node(struct ib_device *ibdev)
4892{
4893	struct device *parent = ibdev->dev.parent;
4894
4895	if (!parent)
4896		return NUMA_NO_NODE;
4897	return dev_to_node(parent);
4898}
4899
4900/**
4901 * rdma_device_to_drv_device - Helper macro to reach back to driver's
4902 *			       ib_device holder structure from device pointer.
4903 *
4904 * NOTE: New drivers should not make use of this API; This API is only for
4905 * existing drivers who have exposed sysfs entries using
4906 * ops->device_group.
4907 */
4908#define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member)           \
4909	container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)
4910
4911bool rdma_dev_access_netns(const struct ib_device *device,
4912			   const struct net *net);
4913
4914bool rdma_dev_has_raw_cap(const struct ib_device *dev);
4915static inline struct net *rdma_dev_net(struct ib_device *device)
4916{
4917	return read_pnet(&device->coredev.rdma_net);
4918}
4919
4920#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
4921#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
4922#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)
4923
4924/**
4925 * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based
4926 *                               on the flow_label
4927 * @fl: flow_label value
4928 *
4929 * This function will convert the 20 bit flow_label input to a valid RoCE v2
4930 * UDP src port 14 bit value. All RoCE V2 drivers should use this same
4931 * convention.
4932 */
4933static inline u16 rdma_flow_label_to_udp_sport(u32 fl)
4934{
4935	u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;
4936
4937	fl_low ^= fl_high >> 14;
4938	return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
4939}
4940
4941/**
4942 * rdma_calc_flow_label - generate a RDMA symmetric flow label value based on
4943 *                        local and remote qpn values
4944 *
4945 * This function folded the multiplication results of two qpns, 24 bit each,
4946 * fields, and converts it to a 20 bit results.
4947 *
4948 * This function will create symmetric flow_label value based on the local
4949 * and remote qpn values. this will allow both the requester and responder
4950 * to calculate the same flow_label for a given connection.
4951 *
4952 * This helper function should be used by driver in case the upper layer
4953 * provide a zero flow_label value. This is to improve entropy of RDMA
4954 * traffic in the network.
4955 */
4956static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
4957{
4958	u64 v = (u64)lqpn * rqpn;
4959
4960	v ^= v >> 20;
4961	v ^= v >> 40;
4962
4963	return (u32)(v & IB_GRH_FLOWLABEL_MASK);
4964}
4965
4966/**
4967 * rdma_get_udp_sport - Calculate and set UDP source port based on the flow
4968 *                      label. If flow label is not defined in GRH then
4969 *                      calculate it based on lqpn/rqpn.
4970 *
4971 * @fl:                 flow label from GRH
4972 * @lqpn:               local qp number
4973 * @rqpn:               remote qp number
4974 */
4975static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
4976{
4977	if (!fl)
4978		fl = rdma_calc_flow_label(lqpn, rqpn);
4979
4980	return rdma_flow_label_to_udp_sport(fl);
4981}
4982
4983const struct ib_port_immutable*
4984ib_port_immutable_read(struct ib_device *dev, unsigned int port);
4985
4986/** ib_add_sub_device - Add a sub IB device on an existing one
4987 *
4988 * @parent: The IB device that needs to add a sub device
4989 * @type: The type of the new sub device
4990 * @name: The name of the new sub device
4991 *
4992 *
4993 * Return 0 on success, an error code otherwise
4994 */
4995int ib_add_sub_device(struct ib_device *parent,
4996		      enum rdma_nl_dev_type type,
4997		      const char *name);
4998
4999
5000/** ib_del_sub_device_and_put - Delect an IB sub device while holding a 'get'
5001 *
5002 * @sub: The sub device that is going to be deleted
5003 *
5004 * Return 0 on success, an error code otherwise
5005 */
5006int ib_del_sub_device_and_put(struct ib_device *sub);
5007
5008static inline void ib_mark_name_assigned_by_user(struct ib_device *ibdev)
5009{
5010	ibdev->name_assign_type = RDMA_NAME_ASSIGN_TYPE_USER;
5011}
5012
5013#endif /* IB_VERBS_H */