cma.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/*
   3 * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   4 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   5 * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.
   6 * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   7 */
   8
   9#include <linux/completion.h>
  10#include <linux/in.h>
  11#include <linux/in6.h>
  12#include <linux/mutex.h>
  13#include <linux/random.h>
  14#include <linux/rbtree.h>
  15#include <linux/igmp.h>
  16#include <linux/xarray.h>
  17#include <linux/inetdevice.h>
  18#include <linux/slab.h>
  19#include <linux/module.h>
  20#include <net/route.h>
  21
  22#include <net/net_namespace.h>
  23#include <net/netns/generic.h>
  24#include <net/netevent.h>
  25#include <net/tcp.h>
  26#include <net/ipv6.h>
  27#include <net/ip_fib.h>
  28#include <net/ip6_route.h>
  29
  30#include <rdma/rdma_cm.h>
  31#include <rdma/rdma_cm_ib.h>
  32#include <rdma/rdma_netlink.h>
  33#include <rdma/ib.h>
  34#include <rdma/ib_cache.h>
  35#include <rdma/ib_cm.h>
  36#include <rdma/ib_sa.h>
  37#include <rdma/iw_cm.h>
  38
  39#include "core_priv.h"
  40#include "cma_priv.h"
  41#include "cma_trace.h"
  42
  43MODULE_AUTHOR("Sean Hefty");
  44MODULE_DESCRIPTION("Generic RDMA CM Agent");
  45MODULE_LICENSE("Dual BSD/GPL");
  46
  47#define CMA_CM_RESPONSE_TIMEOUT 20
  48#define CMA_MAX_CM_RETRIES 15
  49#define CMA_IBOE_PACKET_LIFETIME 16
  50#define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
  51
  52static const char * const cma_events[] = {
  53	[RDMA_CM_EVENT_ADDR_RESOLVED]	 = "address resolved",
  54	[RDMA_CM_EVENT_ADDR_ERROR]	 = "address error",
  55	[RDMA_CM_EVENT_ROUTE_RESOLVED]	 = "route resolved ",
  56	[RDMA_CM_EVENT_ROUTE_ERROR]	 = "route error",
  57	[RDMA_CM_EVENT_CONNECT_REQUEST]	 = "connect request",
  58	[RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
  59	[RDMA_CM_EVENT_CONNECT_ERROR]	 = "connect error",
  60	[RDMA_CM_EVENT_UNREACHABLE]	 = "unreachable",
  61	[RDMA_CM_EVENT_REJECTED]	 = "rejected",
  62	[RDMA_CM_EVENT_ESTABLISHED]	 = "established",
  63	[RDMA_CM_EVENT_DISCONNECTED]	 = "disconnected",
  64	[RDMA_CM_EVENT_DEVICE_REMOVAL]	 = "device removal",
  65	[RDMA_CM_EVENT_MULTICAST_JOIN]	 = "multicast join",
  66	[RDMA_CM_EVENT_MULTICAST_ERROR]	 = "multicast error",
  67	[RDMA_CM_EVENT_ADDR_CHANGE]	 = "address change",
  68	[RDMA_CM_EVENT_TIMEWAIT_EXIT]	 = "timewait exit",
  69};
  70
  71static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
  72			      enum ib_gid_type gid_type);
  73
  74static void cma_netevent_work_handler(struct work_struct *_work);
  75
  76const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
  77{
  78	size_t index = event;
  79
  80	return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
  81			cma_events[index] : "unrecognized event";
  82}
  83EXPORT_SYMBOL(rdma_event_msg);
  84
  85const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
  86						int reason)
  87{
  88	if (rdma_ib_or_roce(id->device, id->port_num))
  89		return ibcm_reject_msg(reason);
  90
  91	if (rdma_protocol_iwarp(id->device, id->port_num))
  92		return iwcm_reject_msg(reason);
  93
  94	WARN_ON_ONCE(1);
  95	return "unrecognized transport";
  96}
  97EXPORT_SYMBOL(rdma_reject_msg);
  98
  99/**
 100 * rdma_is_consumer_reject - return true if the consumer rejected the connect
 101 *                           request.
 102 * @id: Communication identifier that received the REJECT event.
 103 * @reason: Value returned in the REJECT event status field.
 104 */
 105static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
 106{
 107	if (rdma_ib_or_roce(id->device, id->port_num))
 108		return reason == IB_CM_REJ_CONSUMER_DEFINED;
 109
 110	if (rdma_protocol_iwarp(id->device, id->port_num))
 111		return reason == -ECONNREFUSED;
 112
 113	WARN_ON_ONCE(1);
 114	return false;
 115}
 116
 117const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
 118				      struct rdma_cm_event *ev, u8 *data_len)
 119{
 120	const void *p;
 121
 122	if (rdma_is_consumer_reject(id, ev->status)) {
 123		*data_len = ev->param.conn.private_data_len;
 124		p = ev->param.conn.private_data;
 125	} else {
 126		*data_len = 0;
 127		p = NULL;
 128	}
 129	return p;
 130}
 131EXPORT_SYMBOL(rdma_consumer_reject_data);
 132
 133/**
 134 * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
 135 * @id: Communication Identifier
 136 */
 137struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
 138{
 139	struct rdma_id_private *id_priv;
 140
 141	id_priv = container_of(id, struct rdma_id_private, id);
 142	if (id->device->node_type == RDMA_NODE_RNIC)
 143		return id_priv->cm_id.iw;
 144	return NULL;
 145}
 146EXPORT_SYMBOL(rdma_iw_cm_id);
 147
 148static int cma_add_one(struct ib_device *device);
 149static void cma_remove_one(struct ib_device *device, void *client_data);
 150
 151static struct ib_client cma_client = {
 152	.name   = "cma",
 153	.add    = cma_add_one,
 154	.remove = cma_remove_one
 155};
 156
 157static struct ib_sa_client sa_client;
 158static LIST_HEAD(dev_list);
 159static LIST_HEAD(listen_any_list);
 160static DEFINE_MUTEX(lock);
 161static struct rb_root id_table = RB_ROOT;
 162/* Serialize operations of id_table tree */
 163static DEFINE_SPINLOCK(id_table_lock);
 164static struct workqueue_struct *cma_wq;
 165static unsigned int cma_pernet_id;
 166
 167struct cma_pernet {
 168	struct xarray tcp_ps;
 169	struct xarray udp_ps;
 170	struct xarray ipoib_ps;
 171	struct xarray ib_ps;
 172};
 173
 174static struct cma_pernet *cma_pernet(struct net *net)
 175{
 176	return net_generic(net, cma_pernet_id);
 177}
 178
 179static
 180struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
 181{
 182	struct cma_pernet *pernet = cma_pernet(net);
 183
 184	switch (ps) {
 185	case RDMA_PS_TCP:
 186		return &pernet->tcp_ps;
 187	case RDMA_PS_UDP:
 188		return &pernet->udp_ps;
 189	case RDMA_PS_IPOIB:
 190		return &pernet->ipoib_ps;
 191	case RDMA_PS_IB:
 192		return &pernet->ib_ps;
 193	default:
 194		return NULL;
 195	}
 196}
 197
 198struct id_table_entry {
 199	struct list_head id_list;
 200	struct rb_node rb_node;
 201};
 202
 203struct cma_device {
 204	struct list_head	list;
 205	struct ib_device	*device;
 206	struct completion	comp;
 207	refcount_t refcount;
 208	struct list_head	id_list;
 209	enum ib_gid_type	*default_gid_type;
 210	u8			*default_roce_tos;
 211};
 212
 213struct rdma_bind_list {
 214	enum rdma_ucm_port_space ps;
 215	struct hlist_head	owners;
 216	unsigned short		port;
 217};
 218
 219static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
 220			struct rdma_bind_list *bind_list, int snum)
 221{
 222	struct xarray *xa = cma_pernet_xa(net, ps);
 223
 224	return xa_insert(xa, snum, bind_list, GFP_KERNEL);
 225}
 226
 227static struct rdma_bind_list *cma_ps_find(struct net *net,
 228					  enum rdma_ucm_port_space ps, int snum)
 229{
 230	struct xarray *xa = cma_pernet_xa(net, ps);
 231
 232	return xa_load(xa, snum);
 233}
 234
 235static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
 236			  int snum)
 237{
 238	struct xarray *xa = cma_pernet_xa(net, ps);
 239
 240	xa_erase(xa, snum);
 241}
 242
 243enum {
 244	CMA_OPTION_AFONLY,
 245};
 246
 247void cma_dev_get(struct cma_device *cma_dev)
 248{
 249	refcount_inc(&cma_dev->refcount);
 250}
 251
 252void cma_dev_put(struct cma_device *cma_dev)
 253{
 254	if (refcount_dec_and_test(&cma_dev->refcount))
 255		complete(&cma_dev->comp);
 256}
 257
 258struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,
 259					     void		*cookie)
 260{
 261	struct cma_device *cma_dev;
 262	struct cma_device *found_cma_dev = NULL;
 263
 264	mutex_lock(&lock);
 265
 266	list_for_each_entry(cma_dev, &dev_list, list)
 267		if (filter(cma_dev->device, cookie)) {
 268			found_cma_dev = cma_dev;
 269			break;
 270		}
 271
 272	if (found_cma_dev)
 273		cma_dev_get(found_cma_dev);
 274	mutex_unlock(&lock);
 275	return found_cma_dev;
 276}
 277
 278int cma_get_default_gid_type(struct cma_device *cma_dev,
 279			     u32 port)
 280{
 281	if (!rdma_is_port_valid(cma_dev->device, port))
 282		return -EINVAL;
 283
 284	return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
 285}
 286
 287int cma_set_default_gid_type(struct cma_device *cma_dev,
 288			     u32 port,
 289			     enum ib_gid_type default_gid_type)
 290{
 291	unsigned long supported_gids;
 292
 293	if (!rdma_is_port_valid(cma_dev->device, port))
 294		return -EINVAL;
 295
 296	if (default_gid_type == IB_GID_TYPE_IB &&
 297	    rdma_protocol_roce_eth_encap(cma_dev->device, port))
 298		default_gid_type = IB_GID_TYPE_ROCE;
 299
 300	supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
 301
 302	if (!(supported_gids & 1 << default_gid_type))
 303		return -EINVAL;
 304
 305	cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
 306		default_gid_type;
 307
 308	return 0;
 309}
 310
 311int cma_get_default_roce_tos(struct cma_device *cma_dev, u32 port)
 312{
 313	if (!rdma_is_port_valid(cma_dev->device, port))
 314		return -EINVAL;
 315
 316	return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];
 317}
 318
 319int cma_set_default_roce_tos(struct cma_device *cma_dev, u32 port,
 320			     u8 default_roce_tos)
 321{
 322	if (!rdma_is_port_valid(cma_dev->device, port))
 323		return -EINVAL;
 324
 325	cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =
 326		 default_roce_tos;
 327
 328	return 0;
 329}
 330struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
 331{
 332	return cma_dev->device;
 333}
 334
 335/*
 336 * Device removal can occur at anytime, so we need extra handling to
 337 * serialize notifying the user of device removal with other callbacks.
 338 * We do this by disabling removal notification while a callback is in process,
 339 * and reporting it after the callback completes.
 340 */
 341
 342struct cma_multicast {
 343	struct rdma_id_private *id_priv;
 344	union {
 345		struct ib_sa_multicast *sa_mc;
 346		struct {
 347			struct work_struct work;
 348			struct rdma_cm_event event;
 349		} iboe_join;
 350	};
 351	struct list_head	list;
 352	void			*context;
 353	struct sockaddr_storage	addr;
 354	u8			join_state;
 355};
 356
 357struct cma_work {
 358	struct work_struct	work;
 359	struct rdma_id_private	*id;
 360	enum rdma_cm_state	old_state;
 361	enum rdma_cm_state	new_state;
 362	struct rdma_cm_event	event;
 363};
 364
 365union cma_ip_addr {
 366	struct in6_addr ip6;
 367	struct {
 368		__be32 pad[3];
 369		__be32 addr;
 370	} ip4;
 371};
 372
 373struct cma_hdr {
 374	u8 cma_version;
 375	u8 ip_version;	/* IP version: 7:4 */
 376	__be16 port;
 377	union cma_ip_addr src_addr;
 378	union cma_ip_addr dst_addr;
 379};
 380
 381#define CMA_VERSION 0x00
 382
 383struct cma_req_info {
 384	struct sockaddr_storage listen_addr_storage;
 385	struct sockaddr_storage src_addr_storage;
 386	struct ib_device *device;
 387	union ib_gid local_gid;
 388	__be64 service_id;
 389	int port;
 390	bool has_gid;
 391	u16 pkey;
 392};
 393
 394static int cma_comp_exch(struct rdma_id_private *id_priv,
 395			 enum rdma_cm_state comp, enum rdma_cm_state exch)
 396{
 397	unsigned long flags;
 398	int ret;
 399
 400	/*
 401	 * The FSM uses a funny double locking where state is protected by both
 402	 * the handler_mutex and the spinlock. State is not allowed to change
 403	 * to/from a handler_mutex protected value without also holding
 404	 * handler_mutex.
 405	 */
 406	if (comp == RDMA_CM_CONNECT || exch == RDMA_CM_CONNECT)
 407		lockdep_assert_held(&id_priv->handler_mutex);
 408
 409	spin_lock_irqsave(&id_priv->lock, flags);
 410	if ((ret = (id_priv->state == comp)))
 411		id_priv->state = exch;
 412	spin_unlock_irqrestore(&id_priv->lock, flags);
 413	return ret;
 414}
 415
 416static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
 417{
 418	return hdr->ip_version >> 4;
 419}
 420
 421static void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 422{
 423	hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 424}
 425
 426static struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
 427{
 428	return (struct sockaddr *)&id_priv->id.route.addr.src_addr;
 429}
 430
 431static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
 432{
 433	return (struct sockaddr *)&id_priv->id.route.addr.dst_addr;
 434}
 435
 436static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
 437{
 438	struct in_device *in_dev = NULL;
 439
 440	if (ndev) {
 441		rtnl_lock();
 442		in_dev = __in_dev_get_rtnl(ndev);
 443		if (in_dev) {
 444			if (join)
 445				ip_mc_inc_group(in_dev,
 446						*(__be32 *)(mgid->raw + 12));
 447			else
 448				ip_mc_dec_group(in_dev,
 449						*(__be32 *)(mgid->raw + 12));
 450		}
 451		rtnl_unlock();
 452	}
 453	return (in_dev) ? 0 : -ENODEV;
 454}
 455
 456static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa,
 457				 struct id_table_entry *entry_b)
 458{
 459	struct rdma_id_private *id_priv = list_first_entry(
 460		&entry_b->id_list, struct rdma_id_private, id_list_entry);
 461	int ifindex_b = id_priv->id.route.addr.dev_addr.bound_dev_if;
 462	struct sockaddr *sb = cma_dst_addr(id_priv);
 463
 464	if (ifindex_a != ifindex_b)
 465		return (ifindex_a > ifindex_b) ? 1 : -1;
 466
 467	if (sa->sa_family != sb->sa_family)
 468		return sa->sa_family - sb->sa_family;
 469
 470	if (sa->sa_family == AF_INET &&
 471	    __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in)) {
 472		return memcmp(&((struct sockaddr_in *)sa)->sin_addr,
 473			      &((struct sockaddr_in *)sb)->sin_addr,
 474			      sizeof(((struct sockaddr_in *)sa)->sin_addr));
 475	}
 476
 477	if (sa->sa_family == AF_INET6 &&
 478	    __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in6)) {
 479		return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,
 480				     &((struct sockaddr_in6 *)sb)->sin6_addr);
 481	}
 482
 483	return -1;
 484}
 485
 486static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)
 487{
 488	struct rb_node **new, *parent = NULL;
 489	struct id_table_entry *this, *node;
 490	unsigned long flags;
 491	int result;
 492
 493	node = kzalloc(sizeof(*node), GFP_KERNEL);
 494	if (!node)
 495		return -ENOMEM;
 496
 497	spin_lock_irqsave(&id_table_lock, flags);
 498	new = &id_table.rb_node;
 499	while (*new) {
 500		this = container_of(*new, struct id_table_entry, rb_node);
 501		result = compare_netdev_and_ip(
 502			node_id_priv->id.route.addr.dev_addr.bound_dev_if,
 503			cma_dst_addr(node_id_priv), this);
 504
 505		parent = *new;
 506		if (result < 0)
 507			new = &((*new)->rb_left);
 508		else if (result > 0)
 509			new = &((*new)->rb_right);
 510		else {
 511			list_add_tail(&node_id_priv->id_list_entry,
 512				      &this->id_list);
 513			kfree(node);
 514			goto unlock;
 515		}
 516	}
 517
 518	INIT_LIST_HEAD(&node->id_list);
 519	list_add_tail(&node_id_priv->id_list_entry, &node->id_list);
 520
 521	rb_link_node(&node->rb_node, parent, new);
 522	rb_insert_color(&node->rb_node, &id_table);
 523
 524unlock:
 525	spin_unlock_irqrestore(&id_table_lock, flags);
 526	return 0;
 527}
 528
 529static struct id_table_entry *
 530node_from_ndev_ip(struct rb_root *root, int ifindex, struct sockaddr *sa)
 531{
 532	struct rb_node *node = root->rb_node;
 533	struct id_table_entry *data;
 534	int result;
 535
 536	while (node) {
 537		data = container_of(node, struct id_table_entry, rb_node);
 538		result = compare_netdev_and_ip(ifindex, sa, data);
 539		if (result < 0)
 540			node = node->rb_left;
 541		else if (result > 0)
 542			node = node->rb_right;
 543		else
 544			return data;
 545	}
 546
 547	return NULL;
 548}
 549
 550static void cma_remove_id_from_tree(struct rdma_id_private *id_priv)
 551{
 552	struct id_table_entry *data;
 553	unsigned long flags;
 554
 555	spin_lock_irqsave(&id_table_lock, flags);
 556	if (list_empty(&id_priv->id_list_entry))
 557		goto out;
 558
 559	data = node_from_ndev_ip(&id_table,
 560				 id_priv->id.route.addr.dev_addr.bound_dev_if,
 561				 cma_dst_addr(id_priv));
 562	if (!data)
 563		goto out;
 564
 565	list_del_init(&id_priv->id_list_entry);
 566	if (list_empty(&data->id_list)) {
 567		rb_erase(&data->rb_node, &id_table);
 568		kfree(data);
 569	}
 570out:
 571	spin_unlock_irqrestore(&id_table_lock, flags);
 572}
 573
 574static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
 575			       struct cma_device *cma_dev)
 576{
 577	cma_dev_get(cma_dev);
 578	id_priv->cma_dev = cma_dev;
 579	id_priv->id.device = cma_dev->device;
 580	id_priv->id.route.addr.dev_addr.transport =
 581		rdma_node_get_transport(cma_dev->device->node_type);
 582	list_add_tail(&id_priv->device_item, &cma_dev->id_list);
 583
 584	trace_cm_id_attach(id_priv, cma_dev->device);
 585}
 586
 587static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 588			      struct cma_device *cma_dev)
 589{
 590	_cma_attach_to_dev(id_priv, cma_dev);
 591	id_priv->gid_type =
 592		cma_dev->default_gid_type[id_priv->id.port_num -
 593					  rdma_start_port(cma_dev->device)];
 594}
 595
 596static void cma_release_dev(struct rdma_id_private *id_priv)
 597{
 598	mutex_lock(&lock);
 599	list_del_init(&id_priv->device_item);
 600	cma_dev_put(id_priv->cma_dev);
 601	id_priv->cma_dev = NULL;
 602	id_priv->id.device = NULL;
 603	if (id_priv->id.route.addr.dev_addr.sgid_attr) {
 604		rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
 605		id_priv->id.route.addr.dev_addr.sgid_attr = NULL;
 606	}
 607	mutex_unlock(&lock);
 608}
 609
 610static inline unsigned short cma_family(struct rdma_id_private *id_priv)
 611{
 612	return id_priv->id.route.addr.src_addr.ss_family;
 613}
 614
 615static int cma_set_default_qkey(struct rdma_id_private *id_priv)
 616{
 617	struct ib_sa_mcmember_rec rec;
 618	int ret = 0;
 619
 620	switch (id_priv->id.ps) {
 621	case RDMA_PS_UDP:
 622	case RDMA_PS_IB:
 623		id_priv->qkey = RDMA_UDP_QKEY;
 624		break;
 625	case RDMA_PS_IPOIB:
 626		ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 627		ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 628					     id_priv->id.port_num, &rec.mgid,
 629					     &rec);
 630		if (!ret)
 631			id_priv->qkey = be32_to_cpu(rec.qkey);
 632		break;
 633	default:
 634		break;
 635	}
 636	return ret;
 637}
 638
 639static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
 640{
 641	if (!qkey ||
 642	    (id_priv->qkey && (id_priv->qkey != qkey)))
 643		return -EINVAL;
 644
 645	id_priv->qkey = qkey;
 646	return 0;
 647}
 648
 649static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
 650{
 651	dev_addr->dev_type = ARPHRD_INFINIBAND;
 652	rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
 653	ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
 654}
 655
 656static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 657{
 658	int ret;
 659
 660	if (addr->sa_family != AF_IB) {
 661		ret = rdma_translate_ip(addr, dev_addr);
 662	} else {
 663		cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 664		ret = 0;
 665	}
 666
 667	return ret;
 668}
 669
 670static const struct ib_gid_attr *
 671cma_validate_port(struct ib_device *device, u32 port,
 672		  enum ib_gid_type gid_type,
 673		  union ib_gid *gid,
 674		  struct rdma_id_private *id_priv)
 675{
 676	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 677	const struct ib_gid_attr *sgid_attr = ERR_PTR(-ENODEV);
 678	int bound_if_index = dev_addr->bound_dev_if;
 679	int dev_type = dev_addr->dev_type;
 680	struct net_device *ndev = NULL;
 681	struct net_device *pdev = NULL;
 682
 683	if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
 684		goto out;
 685
 686	if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
 687		goto out;
 688
 689	if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
 690		goto out;
 691
 692	/*
 693	 * For drivers that do not associate more than one net device with
 694	 * their gid tables, such as iWARP drivers, it is sufficient to
 695	 * return the first table entry.
 696	 *
 697	 * Other driver classes might be included in the future.
 698	 */
 699	if (rdma_protocol_iwarp(device, port)) {
 700		sgid_attr = rdma_get_gid_attr(device, port, 0);
 701		if (IS_ERR(sgid_attr))
 702			goto out;
 703
 704		rcu_read_lock();
 705		ndev = rcu_dereference(sgid_attr->ndev);
 706		if (ndev->ifindex != bound_if_index) {
 707			pdev = dev_get_by_index_rcu(dev_addr->net, bound_if_index);
 708			if (pdev) {
 709				if (is_vlan_dev(pdev)) {
 710					pdev = vlan_dev_real_dev(pdev);
 711					if (ndev->ifindex == pdev->ifindex)
 712						bound_if_index = pdev->ifindex;
 713				}
 714				if (is_vlan_dev(ndev)) {
 715					pdev = vlan_dev_real_dev(ndev);
 716					if (bound_if_index == pdev->ifindex)
 717						bound_if_index = ndev->ifindex;
 718				}
 719			}
 720		}
 721		if (!net_eq(dev_net(ndev), dev_addr->net) ||
 722		    ndev->ifindex != bound_if_index) {
 723			rdma_put_gid_attr(sgid_attr);
 724			sgid_attr = ERR_PTR(-ENODEV);
 725		}
 726		rcu_read_unlock();
 727		goto out;
 728	}
 729
 730	/*
 731	 * For a RXE device, it should work with TUN device and normal ethernet
 732	 * devices. Use driver_id to check if a device is a RXE device or not.
 733	 * ARPHDR_NONE means a TUN device.
 734	 */
 735	if (device->ops.driver_id == RDMA_DRIVER_RXE) {
 736		if ((dev_type == ARPHRD_NONE || dev_type == ARPHRD_ETHER)
 737			&& rdma_protocol_roce(device, port)) {
 738			ndev = dev_get_by_index(dev_addr->net, bound_if_index);
 739			if (!ndev)
 740				goto out;
 741		}
 742	} else {
 743		if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
 744			ndev = dev_get_by_index(dev_addr->net, bound_if_index);
 745			if (!ndev)
 746				goto out;
 747		} else {
 748			gid_type = IB_GID_TYPE_IB;
 749		}
 750	}
 751
 752	sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
 753	dev_put(ndev);
 754out:
 755	return sgid_attr;
 756}
 757
 758static void cma_bind_sgid_attr(struct rdma_id_private *id_priv,
 759			       const struct ib_gid_attr *sgid_attr)
 760{
 761	WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr);
 762	id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr;
 763}
 764
 765/**
 766 * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute
 767 * based on source ip address.
 768 * @id_priv:	cm_id which should be bound to cma device
 769 *
 770 * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute
 771 * based on source IP address. It returns 0 on success or error code otherwise.
 772 * It is applicable to active and passive side cm_id.
 773 */
 774static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
 775{
 776	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 777	const struct ib_gid_attr *sgid_attr;
 778	union ib_gid gid, iboe_gid, *gidp;
 779	struct cma_device *cma_dev;
 780	enum ib_gid_type gid_type;
 781	int ret = -ENODEV;
 782	u32 port;
 783
 784	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 785	    id_priv->id.ps == RDMA_PS_IPOIB)
 786		return -EINVAL;
 787
 788	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 789		    &iboe_gid);
 790
 791	memcpy(&gid, dev_addr->src_dev_addr +
 792	       rdma_addr_gid_offset(dev_addr), sizeof(gid));
 793
 794	mutex_lock(&lock);
 795	list_for_each_entry(cma_dev, &dev_list, list) {
 796		rdma_for_each_port (cma_dev->device, port) {
 797			gidp = rdma_protocol_roce(cma_dev->device, port) ?
 798			       &iboe_gid : &gid;
 799			gid_type = cma_dev->default_gid_type[port - 1];
 800			sgid_attr = cma_validate_port(cma_dev->device, port,
 801						      gid_type, gidp, id_priv);
 802			if (!IS_ERR(sgid_attr)) {
 803				id_priv->id.port_num = port;
 804				cma_bind_sgid_attr(id_priv, sgid_attr);
 805				cma_attach_to_dev(id_priv, cma_dev);
 806				ret = 0;
 807				goto out;
 808			}
 809		}
 810	}
 811out:
 812	mutex_unlock(&lock);
 813	return ret;
 814}
 815
 816/**
 817 * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute
 818 * @id_priv:		cm id to bind to cma device
 819 * @listen_id_priv:	listener cm id to match against
 820 * @req:		Pointer to req structure containaining incoming
 821 *			request information
 822 * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when
 823 * rdma device matches for listen_id and incoming request. It also verifies
 824 * that a GID table entry is present for the source address.
 825 * Returns 0 on success, or returns error code otherwise.
 826 */
 827static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,
 828			      const struct rdma_id_private *listen_id_priv,
 829			      struct cma_req_info *req)
 830{
 831	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 832	const struct ib_gid_attr *sgid_attr;
 833	enum ib_gid_type gid_type;
 834	union ib_gid gid;
 835
 836	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 837	    id_priv->id.ps == RDMA_PS_IPOIB)
 838		return -EINVAL;
 839
 840	if (rdma_protocol_roce(req->device, req->port))
 841		rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 842			    &gid);
 843	else
 844		memcpy(&gid, dev_addr->src_dev_addr +
 845		       rdma_addr_gid_offset(dev_addr), sizeof(gid));
 846
 847	gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];
 848	sgid_attr = cma_validate_port(req->device, req->port,
 849				      gid_type, &gid, id_priv);
 850	if (IS_ERR(sgid_attr))
 851		return PTR_ERR(sgid_attr);
 852
 853	id_priv->id.port_num = req->port;
 854	cma_bind_sgid_attr(id_priv, sgid_attr);
 855	/* Need to acquire lock to protect against reader
 856	 * of cma_dev->id_list such as cma_netdev_callback() and
 857	 * cma_process_remove().
 858	 */
 859	mutex_lock(&lock);
 860	cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);
 861	mutex_unlock(&lock);
 862	rdma_restrack_add(&id_priv->res);
 863	return 0;
 864}
 865
 866static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
 867			      const struct rdma_id_private *listen_id_priv)
 868{
 869	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 870	const struct ib_gid_attr *sgid_attr;
 871	struct cma_device *cma_dev;
 872	enum ib_gid_type gid_type;
 873	int ret = -ENODEV;
 874	union ib_gid gid;
 875	u32 port;
 876
 877	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 878	    id_priv->id.ps == RDMA_PS_IPOIB)
 879		return -EINVAL;
 880
 881	memcpy(&gid, dev_addr->src_dev_addr +
 882	       rdma_addr_gid_offset(dev_addr), sizeof(gid));
 883
 884	mutex_lock(&lock);
 885
 886	cma_dev = listen_id_priv->cma_dev;
 887	port = listen_id_priv->id.port_num;
 888	gid_type = listen_id_priv->gid_type;
 889	sgid_attr = cma_validate_port(cma_dev->device, port,
 890				      gid_type, &gid, id_priv);
 891	if (!IS_ERR(sgid_attr)) {
 892		id_priv->id.port_num = port;
 893		cma_bind_sgid_attr(id_priv, sgid_attr);
 894		ret = 0;
 895		goto out;
 896	}
 897
 898	list_for_each_entry(cma_dev, &dev_list, list) {
 899		rdma_for_each_port (cma_dev->device, port) {
 900			if (listen_id_priv->cma_dev == cma_dev &&
 901			    listen_id_priv->id.port_num == port)
 902				continue;
 903
 904			gid_type = cma_dev->default_gid_type[port - 1];
 905			sgid_attr = cma_validate_port(cma_dev->device, port,
 906						      gid_type, &gid, id_priv);
 907			if (!IS_ERR(sgid_attr)) {
 908				id_priv->id.port_num = port;
 909				cma_bind_sgid_attr(id_priv, sgid_attr);
 910				ret = 0;
 911				goto out;
 912			}
 913		}
 914	}
 915
 916out:
 917	if (!ret) {
 918		cma_attach_to_dev(id_priv, cma_dev);
 919		rdma_restrack_add(&id_priv->res);
 920	}
 921
 922	mutex_unlock(&lock);
 923	return ret;
 924}
 925
 926/*
 927 * Select the source IB device and address to reach the destination IB address.
 928 */
 929static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
 930{
 931	struct cma_device *cma_dev, *cur_dev;
 932	struct sockaddr_ib *addr;
 933	union ib_gid gid, sgid, *dgid;
 934	unsigned int p;
 935	u16 pkey, index;
 936	enum ib_port_state port_state;
 937	int ret;
 938	int i;
 939
 940	cma_dev = NULL;
 941	addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
 942	dgid = (union ib_gid *) &addr->sib_addr;
 943	pkey = ntohs(addr->sib_pkey);
 944
 945	mutex_lock(&lock);
 946	list_for_each_entry(cur_dev, &dev_list, list) {
 947		rdma_for_each_port (cur_dev->device, p) {
 948			if (!rdma_cap_af_ib(cur_dev->device, p))
 949				continue;
 950
 951			if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
 952				continue;
 953
 954			if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
 955				continue;
 956
 957			for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len;
 958			     ++i) {
 959				ret = rdma_query_gid(cur_dev->device, p, i,
 960						     &gid);
 961				if (ret)
 962					continue;
 963
 964				if (!memcmp(&gid, dgid, sizeof(gid))) {
 965					cma_dev = cur_dev;
 966					sgid = gid;
 967					id_priv->id.port_num = p;
 968					goto found;
 969				}
 970
 971				if (!cma_dev && (gid.global.subnet_prefix ==
 972				    dgid->global.subnet_prefix) &&
 973				    port_state == IB_PORT_ACTIVE) {
 974					cma_dev = cur_dev;
 975					sgid = gid;
 976					id_priv->id.port_num = p;
 977					goto found;
 978				}
 979			}
 980		}
 981	}
 982	mutex_unlock(&lock);
 983	return -ENODEV;
 984
 985found:
 986	cma_attach_to_dev(id_priv, cma_dev);
 987	rdma_restrack_add(&id_priv->res);
 988	mutex_unlock(&lock);
 989	addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
 990	memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
 991	cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
 992	return 0;
 993}
 994
 995static void cma_id_get(struct rdma_id_private *id_priv)
 996{
 997	refcount_inc(&id_priv->refcount);
 998}
 999
1000static void cma_id_put(struct rdma_id_private *id_priv)
1001{
1002	if (refcount_dec_and_test(&id_priv->refcount))
1003		complete(&id_priv->comp);
1004}
1005
1006static struct rdma_id_private *
1007__rdma_create_id(struct net *net, rdma_cm_event_handler event_handler,
1008		 void *context, enum rdma_ucm_port_space ps,
1009		 enum ib_qp_type qp_type, const struct rdma_id_private *parent)
1010{
1011	struct rdma_id_private *id_priv;
1012
1013	id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
1014	if (!id_priv)
1015		return ERR_PTR(-ENOMEM);
1016
1017	id_priv->state = RDMA_CM_IDLE;
1018	id_priv->id.context = context;
1019	id_priv->id.event_handler = event_handler;
1020	id_priv->id.ps = ps;
1021	id_priv->id.qp_type = qp_type;
1022	id_priv->tos_set = false;
1023	id_priv->timeout_set = false;
1024	id_priv->min_rnr_timer_set = false;
1025	id_priv->gid_type = IB_GID_TYPE_IB;
1026	spin_lock_init(&id_priv->lock);
1027	mutex_init(&id_priv->qp_mutex);
1028	init_completion(&id_priv->comp);
1029	refcount_set(&id_priv->refcount, 1);
1030	mutex_init(&id_priv->handler_mutex);
1031	INIT_LIST_HEAD(&id_priv->device_item);
1032	INIT_LIST_HEAD(&id_priv->id_list_entry);
1033	INIT_LIST_HEAD(&id_priv->listen_list);
1034	INIT_LIST_HEAD(&id_priv->mc_list);
1035	get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
1036	id_priv->id.route.addr.dev_addr.net = get_net(net);
1037	id_priv->seq_num &= 0x00ffffff;
1038	INIT_WORK(&id_priv->id.net_work, cma_netevent_work_handler);
1039
1040	rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID);
1041	if (parent)
1042		rdma_restrack_parent_name(&id_priv->res, &parent->res);
1043
1044	return id_priv;
1045}
1046
1047struct rdma_cm_id *
1048__rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler,
1049			void *context, enum rdma_ucm_port_space ps,
1050			enum ib_qp_type qp_type, const char *caller)
1051{
1052	struct rdma_id_private *ret;
1053
1054	ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL);
1055	if (IS_ERR(ret))
1056		return ERR_CAST(ret);
1057
1058	rdma_restrack_set_name(&ret->res, caller);
1059	return &ret->id;
1060}
1061EXPORT_SYMBOL(__rdma_create_kernel_id);
1062
1063struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,
1064				       void *context,
1065				       enum rdma_ucm_port_space ps,
1066				       enum ib_qp_type qp_type)
1067{
1068	struct rdma_id_private *ret;
1069
1070	ret = __rdma_create_id(current->nsproxy->net_ns, event_handler, context,
1071			       ps, qp_type, NULL);
1072	if (IS_ERR(ret))
1073		return ERR_CAST(ret);
1074
1075	rdma_restrack_set_name(&ret->res, NULL);
1076	return &ret->id;
1077}
1078EXPORT_SYMBOL(rdma_create_user_id);
1079
1080static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
1081{
1082	struct ib_qp_attr qp_attr;
1083	int qp_attr_mask, ret;
1084
1085	qp_attr.qp_state = IB_QPS_INIT;
1086	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1087	if (ret)
1088		return ret;
1089
1090	ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
1091	if (ret)
1092		return ret;
1093
1094	qp_attr.qp_state = IB_QPS_RTR;
1095	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
1096	if (ret)
1097		return ret;
1098
1099	qp_attr.qp_state = IB_QPS_RTS;
1100	qp_attr.sq_psn = 0;
1101	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
1102
1103	return ret;
1104}
1105
1106static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
1107{
1108	struct ib_qp_attr qp_attr;
1109	int qp_attr_mask, ret;
1110
1111	qp_attr.qp_state = IB_QPS_INIT;
1112	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1113	if (ret)
1114		return ret;
1115
1116	return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
1117}
1118
1119int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
1120		   struct ib_qp_init_attr *qp_init_attr)
1121{
1122	struct rdma_id_private *id_priv;
1123	struct ib_qp *qp;
1124	int ret;
1125
1126	id_priv = container_of(id, struct rdma_id_private, id);
1127	if (id->device != pd->device) {
1128		ret = -EINVAL;
1129		goto out_err;
1130	}
1131
1132	qp_init_attr->port_num = id->port_num;
1133	qp = ib_create_qp(pd, qp_init_attr);
1134	if (IS_ERR(qp)) {
1135		ret = PTR_ERR(qp);
1136		goto out_err;
1137	}
1138
1139	if (id->qp_type == IB_QPT_UD)
1140		ret = cma_init_ud_qp(id_priv, qp);
1141	else
1142		ret = cma_init_conn_qp(id_priv, qp);
1143	if (ret)
1144		goto out_destroy;
1145
1146	id->qp = qp;
1147	id_priv->qp_num = qp->qp_num;
1148	id_priv->srq = (qp->srq != NULL);
1149	trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);
1150	return 0;
1151out_destroy:
1152	ib_destroy_qp(qp);
1153out_err:
1154	trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);
1155	return ret;
1156}
1157EXPORT_SYMBOL(rdma_create_qp);
1158
1159void rdma_destroy_qp(struct rdma_cm_id *id)
1160{
1161	struct rdma_id_private *id_priv;
1162
1163	id_priv = container_of(id, struct rdma_id_private, id);
1164	trace_cm_qp_destroy(id_priv);
1165	mutex_lock(&id_priv->qp_mutex);
1166	ib_destroy_qp(id_priv->id.qp);
1167	id_priv->id.qp = NULL;
1168	mutex_unlock(&id_priv->qp_mutex);
1169}
1170EXPORT_SYMBOL(rdma_destroy_qp);
1171
1172static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
1173			     struct rdma_conn_param *conn_param)
1174{
1175	struct ib_qp_attr qp_attr;
1176	int qp_attr_mask, ret;
1177
1178	mutex_lock(&id_priv->qp_mutex);
1179	if (!id_priv->id.qp) {
1180		ret = 0;
1181		goto out;
1182	}
1183
1184	/* Need to update QP attributes from default values. */
1185	qp_attr.qp_state = IB_QPS_INIT;
1186	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1187	if (ret)
1188		goto out;
1189
1190	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1191	if (ret)
1192		goto out;
1193
1194	qp_attr.qp_state = IB_QPS_RTR;
1195	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1196	if (ret)
1197		goto out;
1198
1199	BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
1200
1201	if (conn_param)
1202		qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
1203	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1204out:
1205	mutex_unlock(&id_priv->qp_mutex);
1206	return ret;
1207}
1208
1209static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
1210			     struct rdma_conn_param *conn_param)
1211{
1212	struct ib_qp_attr qp_attr;
1213	int qp_attr_mask, ret;
1214
1215	mutex_lock(&id_priv->qp_mutex);
1216	if (!id_priv->id.qp) {
1217		ret = 0;
1218		goto out;
1219	}
1220
1221	qp_attr.qp_state = IB_QPS_RTS;
1222	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1223	if (ret)
1224		goto out;
1225
1226	if (conn_param)
1227		qp_attr.max_rd_atomic = conn_param->initiator_depth;
1228	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1229out:
1230	mutex_unlock(&id_priv->qp_mutex);
1231	return ret;
1232}
1233
1234static int cma_modify_qp_err(struct rdma_id_private *id_priv)
1235{
1236	struct ib_qp_attr qp_attr;
1237	int ret;
1238
1239	mutex_lock(&id_priv->qp_mutex);
1240	if (!id_priv->id.qp) {
1241		ret = 0;
1242		goto out;
1243	}
1244
1245	qp_attr.qp_state = IB_QPS_ERR;
1246	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
1247out:
1248	mutex_unlock(&id_priv->qp_mutex);
1249	return ret;
1250}
1251
1252static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
1253			       struct ib_qp_attr *qp_attr, int *qp_attr_mask)
1254{
1255	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1256	int ret;
1257	u16 pkey;
1258
1259	if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
1260		pkey = 0xffff;
1261	else
1262		pkey = ib_addr_get_pkey(dev_addr);
1263
1264	ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
1265				  pkey, &qp_attr->pkey_index);
1266	if (ret)
1267		return ret;
1268
1269	qp_attr->port_num = id_priv->id.port_num;
1270	*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
1271
1272	if (id_priv->id.qp_type == IB_QPT_UD) {
1273		ret = cma_set_default_qkey(id_priv);
1274		if (ret)
1275			return ret;
1276
1277		qp_attr->qkey = id_priv->qkey;
1278		*qp_attr_mask |= IB_QP_QKEY;
1279	} else {
1280		qp_attr->qp_access_flags = 0;
1281		*qp_attr_mask |= IB_QP_ACCESS_FLAGS;
1282	}
1283	return 0;
1284}
1285
1286int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
1287		       int *qp_attr_mask)
1288{
1289	struct rdma_id_private *id_priv;
1290	int ret = 0;
1291
1292	id_priv = container_of(id, struct rdma_id_private, id);
1293	if (rdma_cap_ib_cm(id->device, id->port_num)) {
1294		if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
1295			ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
1296		else
1297			ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
1298						 qp_attr_mask);
1299
1300		if (qp_attr->qp_state == IB_QPS_RTR)
1301			qp_attr->rq_psn = id_priv->seq_num;
1302	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
1303		if (!id_priv->cm_id.iw) {
1304			qp_attr->qp_access_flags = 0;
1305			*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
1306		} else
1307			ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
1308						 qp_attr_mask);
1309		qp_attr->port_num = id_priv->id.port_num;
1310		*qp_attr_mask |= IB_QP_PORT;
1311	} else {
1312		ret = -ENOSYS;
1313	}
1314
1315	if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set)
1316		qp_attr->timeout = id_priv->timeout;
1317
1318	if ((*qp_attr_mask & IB_QP_MIN_RNR_TIMER) && id_priv->min_rnr_timer_set)
1319		qp_attr->min_rnr_timer = id_priv->min_rnr_timer;
1320
1321	return ret;
1322}
1323EXPORT_SYMBOL(rdma_init_qp_attr);
1324
1325static inline bool cma_zero_addr(const struct sockaddr *addr)
1326{
1327	switch (addr->sa_family) {
1328	case AF_INET:
1329		return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
1330	case AF_INET6:
1331		return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr);
1332	case AF_IB:
1333		return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr);
1334	default:
1335		return false;
1336	}
1337}
1338
1339static inline bool cma_loopback_addr(const struct sockaddr *addr)
1340{
1341	switch (addr->sa_family) {
1342	case AF_INET:
1343		return ipv4_is_loopback(
1344			((struct sockaddr_in *)addr)->sin_addr.s_addr);
1345	case AF_INET6:
1346		return ipv6_addr_loopback(
1347			&((struct sockaddr_in6 *)addr)->sin6_addr);
1348	case AF_IB:
1349		return ib_addr_loopback(
1350			&((struct sockaddr_ib *)addr)->sib_addr);
1351	default:
1352		return false;
1353	}
1354}
1355
1356static inline bool cma_any_addr(const struct sockaddr *addr)
1357{
1358	return cma_zero_addr(addr) || cma_loopback_addr(addr);
1359}
1360
1361static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
1362{
1363	if (src->sa_family != dst->sa_family)
1364		return -1;
1365
1366	switch (src->sa_family) {
1367	case AF_INET:
1368		return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
1369		       ((struct sockaddr_in *)dst)->sin_addr.s_addr;
1370	case AF_INET6: {
1371		struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
1372		struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
1373		bool link_local;
1374
1375		if (ipv6_addr_cmp(&src_addr6->sin6_addr,
1376					  &dst_addr6->sin6_addr))
1377			return 1;
1378		link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
1379			     IPV6_ADDR_LINKLOCAL;
1380		/* Link local must match their scope_ids */
1381		return link_local ? (src_addr6->sin6_scope_id !=
1382				     dst_addr6->sin6_scope_id) :
1383				    0;
1384	}
1385
1386	default:
1387		return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
1388				   &((struct sockaddr_ib *) dst)->sib_addr);
1389	}
1390}
1391
1392static __be16 cma_port(const struct sockaddr *addr)
1393{
1394	struct sockaddr_ib *sib;
1395
1396	switch (addr->sa_family) {
1397	case AF_INET:
1398		return ((struct sockaddr_in *) addr)->sin_port;
1399	case AF_INET6:
1400		return ((struct sockaddr_in6 *) addr)->sin6_port;
1401	case AF_IB:
1402		sib = (struct sockaddr_ib *) addr;
1403		return htons((u16) (be64_to_cpu(sib->sib_sid) &
1404				    be64_to_cpu(sib->sib_sid_mask)));
1405	default:
1406		return 0;
1407	}
1408}
1409
1410static inline int cma_any_port(const struct sockaddr *addr)
1411{
1412	return !cma_port(addr);
1413}
1414
1415static void cma_save_ib_info(struct sockaddr *src_addr,
1416			     struct sockaddr *dst_addr,
1417			     const struct rdma_cm_id *listen_id,
1418			     const struct sa_path_rec *path)
1419{
1420	struct sockaddr_ib *listen_ib, *ib;
1421
1422	listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
1423	if (src_addr) {
1424		ib = (struct sockaddr_ib *)src_addr;
1425		ib->sib_family = AF_IB;
1426		if (path) {
1427			ib->sib_pkey = path->pkey;
1428			ib->sib_flowinfo = path->flow_label;
1429			memcpy(&ib->sib_addr, &path->sgid, 16);
1430			ib->sib_sid = path->service_id;
1431			ib->sib_scope_id = 0;
1432		} else {
1433			ib->sib_pkey = listen_ib->sib_pkey;
1434			ib->sib_flowinfo = listen_ib->sib_flowinfo;
1435			ib->sib_addr = listen_ib->sib_addr;
1436			ib->sib_sid = listen_ib->sib_sid;
1437			ib->sib_scope_id = listen_ib->sib_scope_id;
1438		}
1439		ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
1440	}
1441	if (dst_addr) {
1442		ib = (struct sockaddr_ib *)dst_addr;
1443		ib->sib_family = AF_IB;
1444		if (path) {
1445			ib->sib_pkey = path->pkey;
1446			ib->sib_flowinfo = path->flow_label;
1447			memcpy(&ib->sib_addr, &path->dgid, 16);
1448		}
1449	}
1450}
1451
1452static void cma_save_ip4_info(struct sockaddr_in *src_addr,
1453			      struct sockaddr_in *dst_addr,
1454			      struct cma_hdr *hdr,
1455			      __be16 local_port)
1456{
1457	if (src_addr) {
1458		*src_addr = (struct sockaddr_in) {
1459			.sin_family = AF_INET,
1460			.sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1461			.sin_port = local_port,
1462		};
1463	}
1464
1465	if (dst_addr) {
1466		*dst_addr = (struct sockaddr_in) {
1467			.sin_family = AF_INET,
1468			.sin_addr.s_addr = hdr->src_addr.ip4.addr,
1469			.sin_port = hdr->port,
1470		};
1471	}
1472}
1473
1474static void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
1475			      struct sockaddr_in6 *dst_addr,
1476			      struct cma_hdr *hdr,
1477			      __be16 local_port)
1478{
1479	if (src_addr) {
1480		*src_addr = (struct sockaddr_in6) {
1481			.sin6_family = AF_INET6,
1482			.sin6_addr = hdr->dst_addr.ip6,
1483			.sin6_port = local_port,
1484		};
1485	}
1486
1487	if (dst_addr) {
1488		*dst_addr = (struct sockaddr_in6) {
1489			.sin6_family = AF_INET6,
1490			.sin6_addr = hdr->src_addr.ip6,
1491			.sin6_port = hdr->port,
1492		};
1493	}
1494}
1495
1496static u16 cma_port_from_service_id(__be64 service_id)
1497{
1498	return (u16)be64_to_cpu(service_id);
1499}
1500
1501static int cma_save_ip_info(struct sockaddr *src_addr,
1502			    struct sockaddr *dst_addr,
1503			    const struct ib_cm_event *ib_event,
1504			    __be64 service_id)
1505{
1506	struct cma_hdr *hdr;
1507	__be16 port;
1508
1509	hdr = ib_event->private_data;
1510	if (hdr->cma_version != CMA_VERSION)
1511		return -EINVAL;
1512
1513	port = htons(cma_port_from_service_id(service_id));
1514
1515	switch (cma_get_ip_ver(hdr)) {
1516	case 4:
1517		cma_save_ip4_info((struct sockaddr_in *)src_addr,
1518				  (struct sockaddr_in *)dst_addr, hdr, port);
1519		break;
1520	case 6:
1521		cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
1522				  (struct sockaddr_in6 *)dst_addr, hdr, port);
1523		break;
1524	default:
1525		return -EAFNOSUPPORT;
1526	}
1527
1528	return 0;
1529}
1530
1531static int cma_save_net_info(struct sockaddr *src_addr,
1532			     struct sockaddr *dst_addr,
1533			     const struct rdma_cm_id *listen_id,
1534			     const struct ib_cm_event *ib_event,
1535			     sa_family_t sa_family, __be64 service_id)
1536{
1537	if (sa_family == AF_IB) {
1538		if (ib_event->event == IB_CM_REQ_RECEIVED)
1539			cma_save_ib_info(src_addr, dst_addr, listen_id,
1540					 ib_event->param.req_rcvd.primary_path);
1541		else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1542			cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
1543		return 0;
1544	}
1545
1546	return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
1547}
1548
1549static int cma_save_req_info(const struct ib_cm_event *ib_event,
1550			     struct cma_req_info *req)
1551{
1552	const struct ib_cm_req_event_param *req_param =
1553		&ib_event->param.req_rcvd;
1554	const struct ib_cm_sidr_req_event_param *sidr_param =
1555		&ib_event->param.sidr_req_rcvd;
1556
1557	switch (ib_event->event) {
1558	case IB_CM_REQ_RECEIVED:
1559		req->device	= req_param->listen_id->device;
1560		req->port	= req_param->port;
1561		memcpy(&req->local_gid, &req_param->primary_path->sgid,
1562		       sizeof(req->local_gid));
1563		req->has_gid	= true;
1564		req->service_id = req_param->primary_path->service_id;
1565		req->pkey	= be16_to_cpu(req_param->primary_path->pkey);
1566		if (req->pkey != req_param->bth_pkey)
1567			pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
1568					    "RDMA CMA: in the future this may cause the request to be dropped\n",
1569					    req_param->bth_pkey, req->pkey);
1570		break;
1571	case IB_CM_SIDR_REQ_RECEIVED:
1572		req->device	= sidr_param->listen_id->device;
1573		req->port	= sidr_param->port;
1574		req->has_gid	= false;
1575		req->service_id	= sidr_param->service_id;
1576		req->pkey	= sidr_param->pkey;
1577		if (req->pkey != sidr_param->bth_pkey)
1578			pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
1579					    "RDMA CMA: in the future this may cause the request to be dropped\n",
1580					    sidr_param->bth_pkey, req->pkey);
1581		break;
1582	default:
1583		return -EINVAL;
1584	}
1585
1586	return 0;
1587}
1588
1589static bool validate_ipv4_net_dev(struct net_device *net_dev,
1590				  const struct sockaddr_in *dst_addr,
1591				  const struct sockaddr_in *src_addr)
1592{
1593	__be32 daddr = dst_addr->sin_addr.s_addr,
1594	       saddr = src_addr->sin_addr.s_addr;
1595	struct fib_result res;
1596	struct flowi4 fl4;
1597	int err;
1598	bool ret;
1599
1600	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1601	    ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
1602	    ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
1603	    ipv4_is_loopback(saddr))
1604		return false;
1605
1606	memset(&fl4, 0, sizeof(fl4));
1607	fl4.flowi4_oif = net_dev->ifindex;
1608	fl4.daddr = daddr;
1609	fl4.saddr = saddr;
1610
1611	rcu_read_lock();
1612	err = fib_lookup(dev_net(net_dev), &fl4, &res, 0);
1613	ret = err == 0 && FIB_RES_DEV(res) == net_dev;
1614	rcu_read_unlock();
1615
1616	return ret;
1617}
1618
1619static bool validate_ipv6_net_dev(struct net_device *net_dev,
1620				  const struct sockaddr_in6 *dst_addr,
1621				  const struct sockaddr_in6 *src_addr)
1622{
1623#if IS_ENABLED(CONFIG_IPV6)
1624	const int strict = ipv6_addr_type(&dst_addr->sin6_addr) &
1625			   IPV6_ADDR_LINKLOCAL;
1626	struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr,
1627					 &src_addr->sin6_addr, net_dev->ifindex,
1628					 NULL, strict);
1629	bool ret;
1630
1631	if (!rt)
1632		return false;
1633
1634	ret = rt->rt6i_idev->dev == net_dev;
1635	ip6_rt_put(rt);
1636
1637	return ret;
1638#else
1639	return false;
1640#endif
1641}
1642
1643static bool validate_net_dev(struct net_device *net_dev,
1644			     const struct sockaddr *daddr,
1645			     const struct sockaddr *saddr)
1646{
1647	const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
1648	const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
1649	const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1650	const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
1651
1652	switch (daddr->sa_family) {
1653	case AF_INET:
1654		return saddr->sa_family == AF_INET &&
1655		       validate_ipv4_net_dev(net_dev, daddr4, saddr4);
1656
1657	case AF_INET6:
1658		return saddr->sa_family == AF_INET6 &&
1659		       validate_ipv6_net_dev(net_dev, daddr6, saddr6);
1660
1661	default:
1662		return false;
1663	}
1664}
1665
1666static struct net_device *
1667roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
1668{
1669	const struct ib_gid_attr *sgid_attr = NULL;
1670	struct net_device *ndev;
1671
1672	if (ib_event->event == IB_CM_REQ_RECEIVED)
1673		sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
1674	else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1675		sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr;
1676
1677	if (!sgid_attr)
1678		return NULL;
1679
1680	rcu_read_lock();
1681	ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
1682	if (IS_ERR(ndev))
1683		ndev = NULL;
1684	else
1685		dev_hold(ndev);
1686	rcu_read_unlock();
1687	return ndev;
1688}
1689
1690static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
1691					  struct cma_req_info *req)
1692{
1693	struct sockaddr *listen_addr =
1694			(struct sockaddr *)&req->listen_addr_storage;
1695	struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage;
1696	struct net_device *net_dev;
1697	const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
1698	int err;
1699
1700	err = cma_save_ip_info(listen_addr, src_addr, ib_event,
1701			       req->service_id);
1702	if (err)
1703		return ERR_PTR(err);
1704
1705	if (rdma_protocol_roce(req->device, req->port))
1706		net_dev = roce_get_net_dev_by_cm_event(ib_event);
1707	else
1708		net_dev = ib_get_net_dev_by_params(req->device, req->port,
1709						   req->pkey,
1710						   gid, listen_addr);
1711	if (!net_dev)
1712		return ERR_PTR(-ENODEV);
1713
1714	return net_dev;
1715}
1716
1717static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id)
1718{
1719	return (be64_to_cpu(service_id) >> 16) & 0xffff;
1720}
1721
1722static bool cma_match_private_data(struct rdma_id_private *id_priv,
1723				   const struct cma_hdr *hdr)
1724{
1725	struct sockaddr *addr = cma_src_addr(id_priv);
1726	__be32 ip4_addr;
1727	struct in6_addr ip6_addr;
1728
1729	if (cma_any_addr(addr) && !id_priv->afonly)
1730		return true;
1731
1732	switch (addr->sa_family) {
1733	case AF_INET:
1734		ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1735		if (cma_get_ip_ver(hdr) != 4)
1736			return false;
1737		if (!cma_any_addr(addr) &&
1738		    hdr->dst_addr.ip4.addr != ip4_addr)
1739			return false;
1740		break;
1741	case AF_INET6:
1742		ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1743		if (cma_get_ip_ver(hdr) != 6)
1744			return false;
1745		if (!cma_any_addr(addr) &&
1746		    memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1747			return false;
1748		break;
1749	case AF_IB:
1750		return true;
1751	default:
1752		return false;
1753	}
1754
1755	return true;
1756}
1757
1758static bool cma_protocol_roce(const struct rdma_cm_id *id)
1759{
1760	struct ib_device *device = id->device;
1761	const u32 port_num = id->port_num ?: rdma_start_port(device);
1762
1763	return rdma_protocol_roce(device, port_num);
1764}
1765
1766static bool cma_is_req_ipv6_ll(const struct cma_req_info *req)
1767{
1768	const struct sockaddr *daddr =
1769			(const struct sockaddr *)&req->listen_addr_storage;
1770	const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1771
1772	/* Returns true if the req is for IPv6 link local */
1773	return (daddr->sa_family == AF_INET6 &&
1774		(ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL));
1775}
1776
1777static bool cma_match_net_dev(const struct rdma_cm_id *id,
1778			      const struct net_device *net_dev,
1779			      const struct cma_req_info *req)
1780{
1781	const struct rdma_addr *addr = &id->route.addr;
1782
1783	if (!net_dev)
1784		/* This request is an AF_IB request */
1785		return (!id->port_num || id->port_num == req->port) &&
1786		       (addr->src_addr.ss_family == AF_IB);
1787
1788	/*
1789	 * If the request is not for IPv6 link local, allow matching
1790	 * request to any netdevice of the one or multiport rdma device.
1791	 */
1792	if (!cma_is_req_ipv6_ll(req))
1793		return true;
1794	/*
1795	 * Net namespaces must match, and if the listner is listening
1796	 * on a specific netdevice than netdevice must match as well.
1797	 */
1798	if (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
1799	    (!!addr->dev_addr.bound_dev_if ==
1800	     (addr->dev_addr.bound_dev_if == net_dev->ifindex)))
1801		return true;
1802	else
1803		return false;
1804}
1805
1806static struct rdma_id_private *cma_find_listener(
1807		const struct rdma_bind_list *bind_list,
1808		const struct ib_cm_id *cm_id,
1809		const struct ib_cm_event *ib_event,
1810		const struct cma_req_info *req,
1811		const struct net_device *net_dev)
1812{
1813	struct rdma_id_private *id_priv, *id_priv_dev;
1814
1815	lockdep_assert_held(&lock);
1816
1817	if (!bind_list)
1818		return ERR_PTR(-EINVAL);
1819
1820	hlist_for_each_entry(id_priv, &bind_list->owners, node) {
1821		if (cma_match_private_data(id_priv, ib_event->private_data)) {
1822			if (id_priv->id.device == cm_id->device &&
1823			    cma_match_net_dev(&id_priv->id, net_dev, req))
1824				return id_priv;
1825			list_for_each_entry(id_priv_dev,
1826					    &id_priv->listen_list,
1827					    listen_item) {
1828				if (id_priv_dev->id.device == cm_id->device &&
1829				    cma_match_net_dev(&id_priv_dev->id,
1830						      net_dev, req))
1831					return id_priv_dev;
1832			}
1833		}
1834	}
1835
1836	return ERR_PTR(-EINVAL);
1837}
1838
1839static struct rdma_id_private *
1840cma_ib_id_from_event(struct ib_cm_id *cm_id,
1841		     const struct ib_cm_event *ib_event,
1842		     struct cma_req_info *req,
1843		     struct net_device **net_dev)
1844{
1845	struct rdma_bind_list *bind_list;
1846	struct rdma_id_private *id_priv;
1847	int err;
1848
1849	err = cma_save_req_info(ib_event, req);
1850	if (err)
1851		return ERR_PTR(err);
1852
1853	*net_dev = cma_get_net_dev(ib_event, req);
1854	if (IS_ERR(*net_dev)) {
1855		if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
1856			/* Assuming the protocol is AF_IB */
1857			*net_dev = NULL;
1858		} else {
1859			return ERR_CAST(*net_dev);
1860		}
1861	}
1862
1863	mutex_lock(&lock);
1864	/*
1865	 * Net namespace might be getting deleted while route lookup,
1866	 * cm_id lookup is in progress. Therefore, perform netdevice
1867	 * validation, cm_id lookup under rcu lock.
1868	 * RCU lock along with netdevice state check, synchronizes with
1869	 * netdevice migrating to different net namespace and also avoids
1870	 * case where net namespace doesn't get deleted while lookup is in
1871	 * progress.
1872	 * If the device state is not IFF_UP, its properties such as ifindex
1873	 * and nd_net cannot be trusted to remain valid without rcu lock.
1874	 * net/core/dev.c change_net_namespace() ensures to synchronize with
1875	 * ongoing operations on net device after device is closed using
1876	 * synchronize_net().
1877	 */
1878	rcu_read_lock();
1879	if (*net_dev) {
1880		/*
1881		 * If netdevice is down, it is likely that it is administratively
1882		 * down or it might be migrating to different namespace.
1883		 * In that case avoid further processing, as the net namespace
1884		 * or ifindex may change.
1885		 */
1886		if (((*net_dev)->flags & IFF_UP) == 0) {
1887			id_priv = ERR_PTR(-EHOSTUNREACH);
1888			goto err;
1889		}
1890
1891		if (!validate_net_dev(*net_dev,
1892				 (struct sockaddr *)&req->src_addr_storage,
1893				 (struct sockaddr *)&req->listen_addr_storage)) {
1894			id_priv = ERR_PTR(-EHOSTUNREACH);
1895			goto err;
1896		}
1897	}
1898
1899	bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
1900				rdma_ps_from_service_id(req->service_id),
1901				cma_port_from_service_id(req->service_id));
1902	id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);
1903err:
1904	rcu_read_unlock();
1905	mutex_unlock(&lock);
1906	if (IS_ERR(id_priv) && *net_dev) {
1907		dev_put(*net_dev);
1908		*net_dev = NULL;
1909	}
1910	return id_priv;
1911}
1912
1913static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv)
1914{
1915	return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr);
1916}
1917
1918static void cma_cancel_route(struct rdma_id_private *id_priv)
1919{
1920	if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
1921		if (id_priv->query)
1922			ib_sa_cancel_query(id_priv->query_id, id_priv->query);
1923	}
1924}
1925
1926static void _cma_cancel_listens(struct rdma_id_private *id_priv)
1927{
1928	struct rdma_id_private *dev_id_priv;
1929
1930	lockdep_assert_held(&lock);
1931
1932	/*
1933	 * Remove from listen_any_list to prevent added devices from spawning
1934	 * additional listen requests.
1935	 */
1936	list_del_init(&id_priv->listen_any_item);
1937
1938	while (!list_empty(&id_priv->listen_list)) {
1939		dev_id_priv =
1940			list_first_entry(&id_priv->listen_list,
1941					 struct rdma_id_private, listen_item);
1942		/* sync with device removal to avoid duplicate destruction */
1943		list_del_init(&dev_id_priv->device_item);
1944		list_del_init(&dev_id_priv->listen_item);
1945		mutex_unlock(&lock);
1946
1947		rdma_destroy_id(&dev_id_priv->id);
1948		mutex_lock(&lock);
1949	}
1950}
1951
1952static void cma_cancel_listens(struct rdma_id_private *id_priv)
1953{
1954	mutex_lock(&lock);
1955	_cma_cancel_listens(id_priv);
1956	mutex_unlock(&lock);
1957}
1958
1959static void cma_cancel_operation(struct rdma_id_private *id_priv,
1960				 enum rdma_cm_state state)
1961{
1962	switch (state) {
1963	case RDMA_CM_ADDR_QUERY:
1964		/*
1965		 * We can avoid doing the rdma_addr_cancel() based on state,
1966		 * only RDMA_CM_ADDR_QUERY has a work that could still execute.
1967		 * Notice that the addr_handler work could still be exiting
1968		 * outside this state, however due to the interaction with the
1969		 * handler_mutex the work is guaranteed not to touch id_priv
1970		 * during exit.
1971		 */
1972		rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
1973		break;
1974	case RDMA_CM_ROUTE_QUERY:
1975		cma_cancel_route(id_priv);
1976		break;
1977	case RDMA_CM_LISTEN:
1978		if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
1979			cma_cancel_listens(id_priv);
1980		break;
1981	default:
1982		break;
1983	}
1984}
1985
1986static void cma_release_port(struct rdma_id_private *id_priv)
1987{
1988	struct rdma_bind_list *bind_list = id_priv->bind_list;
1989	struct net *net = id_priv->id.route.addr.dev_addr.net;
1990
1991	if (!bind_list)
1992		return;
1993
1994	mutex_lock(&lock);
1995	hlist_del(&id_priv->node);
1996	if (hlist_empty(&bind_list->owners)) {
1997		cma_ps_remove(net, bind_list->ps, bind_list->port);
1998		kfree(bind_list);
1999	}
2000	mutex_unlock(&lock);
2001}
2002
2003static void destroy_mc(struct rdma_id_private *id_priv,
2004		       struct cma_multicast *mc)
2005{
2006	bool send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
2007
2008	if (rdma_cap_ib_mcast(id_priv->id.device, id_priv->id.port_num))
2009		ib_sa_free_multicast(mc->sa_mc);
2010
2011	if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) {
2012		struct rdma_cm_event *event = &mc->iboe_join.event;
2013		struct rdma_dev_addr *dev_addr =
2014			&id_priv->id.route.addr.dev_addr;
2015		struct net_device *ndev = NULL;
2016
2017		if (dev_addr->bound_dev_if)
2018			ndev = dev_get_by_index(dev_addr->net,
2019						dev_addr->bound_dev_if);
2020		if (ndev && !send_only) {
2021			enum ib_gid_type gid_type;
2022			union ib_gid mgid;
2023
2024			gid_type = id_priv->cma_dev->default_gid_type
2025					   [id_priv->id.port_num -
2026					    rdma_start_port(
2027						    id_priv->cma_dev->device)];
2028			cma_iboe_set_mgid((struct sockaddr *)&mc->addr, &mgid,
2029					  gid_type);
2030			cma_igmp_send(ndev, &mgid, false);
2031		}
2032		dev_put(ndev);
2033
2034		cancel_work_sync(&mc->iboe_join.work);
2035		if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN)
2036			rdma_destroy_ah_attr(&event->param.ud.ah_attr);
2037	}
2038	kfree(mc);
2039}
2040
2041static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
2042{
2043	struct cma_multicast *mc;
2044
2045	while (!list_empty(&id_priv->mc_list)) {
2046		mc = list_first_entry(&id_priv->mc_list, struct cma_multicast,
2047				      list);
2048		list_del(&mc->list);
2049		destroy_mc(id_priv, mc);
2050	}
2051}
2052
2053static void _destroy_id(struct rdma_id_private *id_priv,
2054			enum rdma_cm_state state)
2055{
2056	cma_cancel_operation(id_priv, state);
2057
2058	rdma_restrack_del(&id_priv->res);
2059	cma_remove_id_from_tree(id_priv);
2060	if (id_priv->cma_dev) {
2061		if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
2062			if (id_priv->cm_id.ib)
2063				ib_destroy_cm_id(id_priv->cm_id.ib);
2064		} else if (rdma_cap_iw_cm(id_priv->id.device, 1)) {
2065			if (id_priv->cm_id.iw)
2066				iw_destroy_cm_id(id_priv->cm_id.iw);
2067		}
2068		cma_leave_mc_groups(id_priv);
2069		cma_release_dev(id_priv);
2070	}
2071
2072	cma_release_port(id_priv);
2073	cma_id_put(id_priv);
2074	wait_for_completion(&id_priv->comp);
2075
2076	if (id_priv->internal_id)
2077		cma_id_put(id_priv->id.context);
2078
2079	kfree(id_priv->id.route.path_rec);
2080	kfree(id_priv->id.route.path_rec_inbound);
2081	kfree(id_priv->id.route.path_rec_outbound);
2082	kfree(id_priv->id.route.service_recs);
2083
2084	put_net(id_priv->id.route.addr.dev_addr.net);
2085	kfree(id_priv);
2086}
2087
2088/*
2089 * destroy an ID from within the handler_mutex. This ensures that no other
2090 * handlers can start running concurrently.
2091 */
2092static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
2093	__releases(&idprv->handler_mutex)
2094{
2095	enum rdma_cm_state state;
2096	unsigned long flags;
2097
2098	trace_cm_id_destroy(id_priv);
2099
2100	/*
2101	 * Setting the state to destroyed under the handler mutex provides a
2102	 * fence against calling handler callbacks. If this is invoked due to
2103	 * the failure of a handler callback then it guarentees that no future
2104	 * handlers will be called.
2105	 */
2106	lockdep_assert_held(&id_priv->handler_mutex);
2107	spin_lock_irqsave(&id_priv->lock, flags);
2108	state = id_priv->state;
2109	id_priv->state = RDMA_CM_DESTROYING;
2110	spin_unlock_irqrestore(&id_priv->lock, flags);
2111	mutex_unlock(&id_priv->handler_mutex);
2112	_destroy_id(id_priv, state);
2113}
2114
2115void rdma_destroy_id(struct rdma_cm_id *id)
2116{
2117	struct rdma_id_private *id_priv =
2118		container_of(id, struct rdma_id_private, id);
2119
2120	mutex_lock(&id_priv->handler_mutex);
2121	destroy_id_handler_unlock(id_priv);
2122}
2123EXPORT_SYMBOL(rdma_destroy_id);
2124
2125static int cma_rep_recv(struct rdma_id_private *id_priv)
2126{
2127	int ret;
2128
2129	ret = cma_modify_qp_rtr(id_priv, NULL);
2130	if (ret)
2131		goto reject;
2132
2133	ret = cma_modify_qp_rts(id_priv, NULL);
2134	if (ret)
2135		goto reject;
2136
2137	trace_cm_send_rtu(id_priv);
2138	ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
2139	if (ret)
2140		goto reject;
2141
2142	return 0;
2143reject:
2144	pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);
2145	cma_modify_qp_err(id_priv);
2146	trace_cm_send_rej(id_priv);
2147	ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
2148		       NULL, 0, NULL, 0);
2149	return ret;
2150}
2151
2152static void cma_set_rep_event_data(struct rdma_cm_event *event,
2153				   const struct ib_cm_rep_event_param *rep_data,
2154				   void *private_data)
2155{
2156	event->param.conn.private_data = private_data;
2157	event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
2158	event->param.conn.responder_resources = rep_data->responder_resources;
2159	event->param.conn.initiator_depth = rep_data->initiator_depth;
2160	event->param.conn.flow_control = rep_data->flow_control;
2161	event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
2162	event->param.conn.srq = rep_data->srq;
2163	event->param.conn.qp_num = rep_data->remote_qpn;
2164
2165	event->ece.vendor_id = rep_data->ece.vendor_id;
2166	event->ece.attr_mod = rep_data->ece.attr_mod;
2167}
2168
2169static int cma_cm_event_handler(struct rdma_id_private *id_priv,
2170				struct rdma_cm_event *event)
2171{
2172	int ret;
2173
2174	lockdep_assert_held(&id_priv->handler_mutex);
2175
2176	trace_cm_event_handler(id_priv, event);
2177	ret = id_priv->id.event_handler(&id_priv->id, event);
2178	trace_cm_event_done(id_priv, event, ret);
2179	return ret;
2180}
2181
2182static int cma_ib_handler(struct ib_cm_id *cm_id,
2183			  const struct ib_cm_event *ib_event)
2184{
2185	struct rdma_id_private *id_priv = cm_id->context;
2186	struct rdma_cm_event event = {};
2187	enum rdma_cm_state state;
2188	int ret;
2189
2190	mutex_lock(&id_priv->handler_mutex);
2191	state = READ_ONCE(id_priv->state);
2192	if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
2193	     state != RDMA_CM_CONNECT) ||
2194	    (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
2195	     state != RDMA_CM_DISCONNECT))
2196		goto out;
2197
2198	switch (ib_event->event) {
2199	case IB_CM_REQ_ERROR:
2200	case IB_CM_REP_ERROR:
2201		event.event = RDMA_CM_EVENT_UNREACHABLE;
2202		event.status = -ETIMEDOUT;
2203		break;
2204	case IB_CM_REP_RECEIVED:
2205		if (state == RDMA_CM_CONNECT &&
2206		    (id_priv->id.qp_type != IB_QPT_UD)) {
2207			trace_cm_prepare_mra(id_priv);
2208			ib_prepare_cm_mra(cm_id);
2209		}
2210		if (id_priv->id.qp) {
2211			event.status = cma_rep_recv(id_priv);
2212			event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
2213						     RDMA_CM_EVENT_ESTABLISHED;
2214		} else {
2215			event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
2216		}
2217		cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
2218				       ib_event->private_data);
2219		break;
2220	case IB_CM_RTU_RECEIVED:
2221	case IB_CM_USER_ESTABLISHED:
2222		event.event = RDMA_CM_EVENT_ESTABLISHED;
2223		break;
2224	case IB_CM_DREQ_ERROR:
2225		event.status = -ETIMEDOUT;
2226		fallthrough;
2227	case IB_CM_DREQ_RECEIVED:
2228	case IB_CM_DREP_RECEIVED:
2229		if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
2230				   RDMA_CM_DISCONNECT))
2231			goto out;
2232		event.event = RDMA_CM_EVENT_DISCONNECTED;
2233		break;
2234	case IB_CM_TIMEWAIT_EXIT:
2235		event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
2236		break;
2237	case IB_CM_MRA_RECEIVED:
2238		/* ignore event */
2239		goto out;
2240	case IB_CM_REJ_RECEIVED:
2241		pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id,
2242										ib_event->param.rej_rcvd.reason));
2243		cma_modify_qp_err(id_priv);
2244		event.status = ib_event->param.rej_rcvd.reason;
2245		event.event = RDMA_CM_EVENT_REJECTED;
2246		event.param.conn.private_data = ib_event->private_data;
2247		event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
2248		break;
2249	default:
2250		pr_err("RDMA CMA: unexpected IB CM event: %d\n",
2251		       ib_event->event);
2252		goto out;
2253	}
2254
2255	ret = cma_cm_event_handler(id_priv, &event);
2256	if (ret) {
2257		/* Destroy the CM ID by returning a non-zero value. */
2258		id_priv->cm_id.ib = NULL;
2259		destroy_id_handler_unlock(id_priv);
2260		return ret;
2261	}
2262out:
2263	mutex_unlock(&id_priv->handler_mutex);
2264	return 0;
2265}
2266
2267static struct rdma_id_private *
2268cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
2269		   const struct ib_cm_event *ib_event,
2270		   struct net_device *net_dev)
2271{
2272	struct rdma_id_private *listen_id_priv;
2273	struct rdma_id_private *id_priv;
2274	struct rdma_cm_id *id;
2275	struct rdma_route *rt;
2276	const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2277	struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path;
2278	const __be64 service_id =
2279		ib_event->param.req_rcvd.primary_path->service_id;
2280	int ret;
2281
2282	listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2283	id_priv = __rdma_create_id(listen_id->route.addr.dev_addr.net,
2284				   listen_id->event_handler, listen_id->context,
2285				   listen_id->ps,
2286				   ib_event->param.req_rcvd.qp_type,
2287				   listen_id_priv);
2288	if (IS_ERR(id_priv))
2289		return NULL;
2290
2291	id = &id_priv->id;
2292	if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2293			      (struct sockaddr *)&id->route.addr.dst_addr,
2294			      listen_id, ib_event, ss_family, service_id))
2295		goto err;
2296
2297	rt = &id->route;
2298	rt->num_pri_alt_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
2299	rt->path_rec = kmalloc_array(rt->num_pri_alt_paths,
2300				     sizeof(*rt->path_rec), GFP_KERNEL);
2301	if (!rt->path_rec)
2302		goto err;
2303
2304	rt->path_rec[0] = *path;
2305	if (rt->num_pri_alt_paths == 2)
2306		rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
2307
2308	if (net_dev) {
2309		rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
2310	} else {
2311		if (!cma_protocol_roce(listen_id) &&
2312		    cma_any_addr(cma_src_addr(id_priv))) {
2313			rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
2314			rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
2315			ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
2316		} else if (!cma_any_addr(cma_src_addr(id_priv))) {
2317			ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
2318			if (ret)
2319				goto err;
2320		}
2321	}
2322	rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
2323
2324	id_priv->state = RDMA_CM_CONNECT;
2325	return id_priv;
2326
2327err:
2328	rdma_destroy_id(id);
2329	return NULL;
2330}
2331
2332static struct rdma_id_private *
2333cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
2334		  const struct ib_cm_event *ib_event,
2335		  struct net_device *net_dev)
2336{
2337	const struct rdma_id_private *listen_id_priv;
2338	struct rdma_id_private *id_priv;
2339	struct rdma_cm_id *id;
2340	const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2341	struct net *net = listen_id->route.addr.dev_addr.net;
2342	int ret;
2343
2344	listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2345	id_priv = __rdma_create_id(net, listen_id->event_handler,
2346				   listen_id->context, listen_id->ps, IB_QPT_UD,
2347				   listen_id_priv);
2348	if (IS_ERR(id_priv))
2349		return NULL;
2350
2351	id = &id_priv->id;
2352	if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2353			      (struct sockaddr *)&id->route.addr.dst_addr,
2354			      listen_id, ib_event, ss_family,
2355			      ib_event->param.sidr_req_rcvd.service_id))
2356		goto err;
2357
2358	if (net_dev) {
2359		rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
2360	} else {
2361		if (!cma_any_addr(cma_src_addr(id_priv))) {
2362			ret = cma_translate_addr(cma_src_addr(id_priv),
2363						 &id->route.addr.dev_addr);
2364			if (ret)
2365				goto err;
2366		}
2367	}
2368
2369	id_priv->state = RDMA_CM_CONNECT;
2370	return id_priv;
2371err:
2372	rdma_destroy_id(id);
2373	return NULL;
2374}
2375
2376static void cma_set_req_event_data(struct rdma_cm_event *event,
2377				   const struct ib_cm_req_event_param *req_data,
2378				   void *private_data, int offset)
2379{
2380	event->param.conn.private_data = private_data + offset;
2381	event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
2382	event->param.conn.responder_resources = req_data->responder_resources;
2383	event->param.conn.initiator_depth = req_data->initiator_depth;
2384	event->param.conn.flow_control = req_data->flow_control;
2385	event->param.conn.retry_count = req_data->retry_count;
2386	event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
2387	event->param.conn.srq = req_data->srq;
2388	event->param.conn.qp_num = req_data->remote_qpn;
2389
2390	event->ece.vendor_id = req_data->ece.vendor_id;
2391	event->ece.attr_mod = req_data->ece.attr_mod;
2392}
2393
2394static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
2395				    const struct ib_cm_event *ib_event)
2396{
2397	return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
2398		 (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
2399		((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
2400		 (id->qp_type == IB_QPT_UD)) ||
2401		(!id->qp_type));
2402}
2403
2404static int cma_ib_req_handler(struct ib_cm_id *cm_id,
2405			      const struct ib_cm_event *ib_event)
2406{
2407	struct rdma_id_private *listen_id, *conn_id = NULL;
2408	struct rdma_cm_event event = {};
2409	struct cma_req_info req = {};
2410	struct net_device *net_dev;
2411	u8 offset;
2412	int ret;
2413
2414	listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev);
2415	if (IS_ERR(listen_id))
2416		return PTR_ERR(listen_id);
2417
2418	trace_cm_req_handler(listen_id, ib_event->event);
2419	if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) {
2420		ret = -EINVAL;
2421		goto net_dev_put;
2422	}
2423
2424	mutex_lock(&listen_id->handler_mutex);
2425	if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) {
2426		ret = -ECONNABORTED;
2427		goto err_unlock;
2428	}
2429
2430	offset = cma_user_data_offset(listen_id);
2431	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2432	if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
2433		conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev);
2434		event.param.ud.private_data = ib_event->private_data + offset;
2435		event.param.ud.private_data_len =
2436				IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
2437	} else {
2438		conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev);
2439		cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
2440				       ib_event->private_data, offset);
2441	}
2442	if (!conn_id) {
2443		ret = -ENOMEM;
2444		goto err_unlock;
2445	}
2446
2447	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2448	ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2449	if (ret) {
2450		destroy_id_handler_unlock(conn_id);
2451		goto err_unlock;
2452	}
2453
2454	conn_id->cm_id.ib = cm_id;
2455	cm_id->context = conn_id;
2456	cm_id->cm_handler = cma_ib_handler;
2457
2458	ret = cma_cm_event_handler(conn_id, &event);
2459	if (ret) {
2460		/* Destroy the CM ID by returning a non-zero value. */
2461		conn_id->cm_id.ib = NULL;
2462		mutex_unlock(&listen_id->handler_mutex);
2463		destroy_id_handler_unlock(conn_id);
2464		goto net_dev_put;
2465	}
2466
2467	if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT &&
2468	    conn_id->id.qp_type != IB_QPT_UD) {
2469		trace_cm_prepare_mra(cm_id->context);
2470		ib_prepare_cm_mra(cm_id);
2471	}
2472	mutex_unlock(&conn_id->handler_mutex);
2473
2474err_unlock:
2475	mutex_unlock(&listen_id->handler_mutex);
2476
2477net_dev_put:
2478	dev_put(net_dev);
2479
2480	return ret;
2481}
2482
2483__be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
2484{
2485	if (addr->sa_family == AF_IB)
2486		return ((struct sockaddr_ib *) addr)->sib_sid;
2487
2488	return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr)));
2489}
2490EXPORT_SYMBOL(rdma_get_service_id);
2491
2492void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
2493		    union ib_gid *dgid)
2494{
2495	struct rdma_addr *addr = &cm_id->route.addr;
2496
2497	if (!cm_id->device) {
2498		if (sgid)
2499			memset(sgid, 0, sizeof(*sgid));
2500		if (dgid)
2501			memset(dgid, 0, sizeof(*dgid));
2502		return;
2503	}
2504
2505	if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
2506		if (sgid)
2507			rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
2508		if (dgid)
2509			rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
2510	} else {
2511		if (sgid)
2512			rdma_addr_get_sgid(&addr->dev_addr, sgid);
2513		if (dgid)
2514			rdma_addr_get_dgid(&addr->dev_addr, dgid);
2515	}
2516}
2517EXPORT_SYMBOL(rdma_read_gids);
2518
2519static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
2520{
2521	struct rdma_id_private *id_priv = iw_id->context;
2522	struct rdma_cm_event event = {};
2523	int ret = 0;
2524	struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2525	struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2526
2527	mutex_lock(&id_priv->handler_mutex);
2528	if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
2529		goto out;
2530
2531	switch (iw_event->event) {
2532	case IW_CM_EVENT_CLOSE:
2533		event.event = RDMA_CM_EVENT_DISCONNECTED;
2534		break;
2535	case IW_CM_EVENT_CONNECT_REPLY:
2536		memcpy(cma_src_addr(id_priv), laddr,
2537		       rdma_addr_size(laddr));
2538		memcpy(cma_dst_addr(id_priv), raddr,
2539		       rdma_addr_size(raddr));
2540		switch (iw_event->status) {
2541		case 0:
2542			event.event = RDMA_CM_EVENT_ESTABLISHED;
2543			event.param.conn.initiator_depth = iw_event->ird;
2544			event.param.conn.responder_resources = iw_event->ord;
2545			break;
2546		case -ECONNRESET:
2547		case -ECONNREFUSED:
2548			event.event = RDMA_CM_EVENT_REJECTED;
2549			break;
2550		case -ETIMEDOUT:
2551			event.event = RDMA_CM_EVENT_UNREACHABLE;
2552			break;
2553		default:
2554			event.event = RDMA_CM_EVENT_CONNECT_ERROR;
2555			break;
2556		}
2557		break;
2558	case IW_CM_EVENT_ESTABLISHED:
2559		event.event = RDMA_CM_EVENT_ESTABLISHED;
2560		event.param.conn.initiator_depth = iw_event->ird;
2561		event.param.conn.responder_resources = iw_event->ord;
2562		break;
2563	default:
2564		goto out;
2565	}
2566
2567	event.status = iw_event->status;
2568	event.param.conn.private_data = iw_event->private_data;
2569	event.param.conn.private_data_len = iw_event->private_data_len;
2570	ret = cma_cm_event_handler(id_priv, &event);
2571	if (ret) {
2572		/* Destroy the CM ID by returning a non-zero value. */
2573		id_priv->cm_id.iw = NULL;
2574		destroy_id_handler_unlock(id_priv);
2575		return ret;
2576	}
2577
2578out:
2579	mutex_unlock(&id_priv->handler_mutex);
2580	return ret;
2581}
2582
2583static int iw_conn_req_handler(struct iw_cm_id *cm_id,
2584			       struct iw_cm_event *iw_event)
2585{
2586	struct rdma_id_private *listen_id, *conn_id;
2587	struct rdma_cm_event event = {};
2588	int ret = -ECONNABORTED;
2589	struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2590	struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2591
2592	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2593	event.param.conn.private_data = iw_event->private_data;
2594	event.param.conn.private_data_len = iw_event->private_data_len;
2595	event.param.conn.initiator_depth = iw_event->ird;
2596	event.param.conn.responder_resources = iw_event->ord;
2597
2598	listen_id = cm_id->context;
2599
2600	mutex_lock(&listen_id->handler_mutex);
2601	if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN)
2602		goto out;
2603
2604	/* Create a new RDMA id for the new IW CM ID */
2605	conn_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
2606				   listen_id->id.event_handler,
2607				   listen_id->id.context, RDMA_PS_TCP,
2608				   IB_QPT_RC, listen_id);
2609	if (IS_ERR(conn_id)) {
2610		ret = -ENOMEM;
2611		goto out;
2612	}
2613	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2614	conn_id->state = RDMA_CM_CONNECT;
2615
2616	ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2617	if (ret) {
2618		mutex_unlock(&listen_id->handler_mutex);
2619		destroy_id_handler_unlock(conn_id);
2620		return ret;
2621	}
2622
2623	ret = cma_iw_acquire_dev(conn_id, listen_id);
2624	if (ret) {
2625		mutex_unlock(&listen_id->handler_mutex);
2626		destroy_id_handler_unlock(conn_id);
2627		return ret;
2628	}
2629
2630	conn_id->cm_id.iw = cm_id;
2631	cm_id->context = conn_id;
2632	cm_id->cm_handler = cma_iw_handler;
2633
2634	memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2635	memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2636
2637	ret = cma_cm_event_handler(conn_id, &event);
2638	if (ret) {
2639		/* User wants to destroy the CM ID */
2640		conn_id->cm_id.iw = NULL;
2641		mutex_unlock(&listen_id->handler_mutex);
2642		destroy_id_handler_unlock(conn_id);
2643		return ret;
2644	}
2645
2646	mutex_unlock(&conn_id->handler_mutex);
2647
2648out:
2649	mutex_unlock(&listen_id->handler_mutex);
2650	return ret;
2651}
2652
2653static int cma_ib_listen(struct rdma_id_private *id_priv)
2654{
2655	struct sockaddr *addr;
2656	struct ib_cm_id	*id;
2657	__be64 svc_id;
2658
2659	addr = cma_src_addr(id_priv);
2660	svc_id = rdma_get_service_id(&id_priv->id, addr);
2661	id = ib_cm_insert_listen(id_priv->id.device,
2662				 cma_ib_req_handler, svc_id);
2663	if (IS_ERR(id))
2664		return PTR_ERR(id);
2665	id_priv->cm_id.ib = id;
2666
2667	return 0;
2668}
2669
2670static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
2671{
2672	int ret;
2673	struct iw_cm_id	*id;
2674
2675	id = iw_create_cm_id(id_priv->id.device,
2676			     iw_conn_req_handler,
2677			     id_priv);
2678	if (IS_ERR(id))
2679		return PTR_ERR(id);
2680
2681	mutex_lock(&id_priv->qp_mutex);
2682	id->tos = id_priv->tos;
2683	id->tos_set = id_priv->tos_set;
2684	mutex_unlock(&id_priv->qp_mutex);
2685	id->afonly = id_priv->afonly;
2686	id_priv->cm_id.iw = id;
2687
2688	memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
2689	       rdma_addr_size(cma_src_addr(id_priv)));
2690
2691	ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
2692
2693	if (ret) {
2694		iw_destroy_cm_id(id_priv->cm_id.iw);
2695		id_priv->cm_id.iw = NULL;
2696	}
2697
2698	return ret;
2699}
2700
2701static int cma_listen_handler(struct rdma_cm_id *id,
2702			      struct rdma_cm_event *event)
2703{
2704	struct rdma_id_private *id_priv = id->context;
2705
2706	/* Listening IDs are always destroyed on removal */
2707	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
2708		return -1;
2709
2710	id->context = id_priv->id.context;
2711	id->event_handler = id_priv->id.event_handler;
2712	trace_cm_event_handler(id_priv, event);
2713	return id_priv->id.event_handler(id, event);
2714}
2715
2716static int cma_listen_on_dev(struct rdma_id_private *id_priv,
2717			     struct cma_device *cma_dev,
2718			     struct rdma_id_private **to_destroy)
2719{
2720	struct rdma_id_private *dev_id_priv;
2721	struct net *net = id_priv->id.route.addr.dev_addr.net;
2722	int ret;
2723
2724	lockdep_assert_held(&lock);
2725
2726	*to_destroy = NULL;
2727	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2728		return 0;
2729
2730	dev_id_priv =
2731		__rdma_create_id(net, cma_listen_handler, id_priv,
2732				 id_priv->id.ps, id_priv->id.qp_type, id_priv);
2733	if (IS_ERR(dev_id_priv))
2734		return PTR_ERR(dev_id_priv);
2735
2736	dev_id_priv->state = RDMA_CM_ADDR_BOUND;
2737	memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
2738	       rdma_addr_size(cma_src_addr(id_priv)));
2739
2740	_cma_attach_to_dev(dev_id_priv, cma_dev);
2741	rdma_restrack_add(&dev_id_priv->res);
2742	cma_id_get(id_priv);
2743	dev_id_priv->internal_id = 1;
2744	dev_id_priv->afonly = id_priv->afonly;
2745	mutex_lock(&id_priv->qp_mutex);
2746	dev_id_priv->tos_set = id_priv->tos_set;
2747	dev_id_priv->tos = id_priv->tos;
2748	mutex_unlock(&id_priv->qp_mutex);
2749
2750	ret = rdma_listen(&dev_id_priv->id, id_priv->backlog);
2751	if (ret)
2752		goto err_listen;
2753	list_add_tail(&dev_id_priv->listen_item, &id_priv->listen_list);
2754	return 0;
2755err_listen:
2756	/* Caller must destroy this after releasing lock */
2757	*to_destroy = dev_id_priv;
2758	dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret);
2759	return ret;
2760}
2761
2762static int cma_listen_on_all(struct rdma_id_private *id_priv)
2763{
2764	struct rdma_id_private *to_destroy;
2765	struct cma_device *cma_dev;
2766	int ret;
2767
2768	mutex_lock(&lock);
2769	list_add_tail(&id_priv->listen_any_item, &listen_any_list);
2770	list_for_each_entry(cma_dev, &dev_list, list) {
2771		ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
2772		if (ret) {
2773			/* Prevent racing with cma_process_remove() */
2774			if (to_destroy)
2775				list_del_init(&to_destroy->device_item);
2776			goto err_listen;
2777		}
2778	}
2779	mutex_unlock(&lock);
2780	return 0;
2781
2782err_listen:
2783	_cma_cancel_listens(id_priv);
2784	mutex_unlock(&lock);
2785	if (to_destroy)
2786		rdma_destroy_id(&to_destroy->id);
2787	return ret;
2788}
2789
2790void rdma_set_service_type(struct rdma_cm_id *id, int tos)
2791{
2792	struct rdma_id_private *id_priv;
2793
2794	id_priv = container_of(id, struct rdma_id_private, id);
2795	mutex_lock(&id_priv->qp_mutex);
2796	id_priv->tos = (u8) tos;
2797	id_priv->tos_set = true;
2798	mutex_unlock(&id_priv->qp_mutex);
2799}
2800EXPORT_SYMBOL(rdma_set_service_type);
2801
2802/**
2803 * rdma_set_ack_timeout() - Set the ack timeout of QP associated
2804 *                          with a connection identifier.
2805 * @id: Communication identifier to associated with service type.
2806 * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec.
2807 *
2808 * This function should be called before rdma_connect() on active side,
2809 * and on passive side before rdma_accept(). It is applicable to primary
2810 * path only. The timeout will affect the local side of the QP, it is not
2811 * negotiated with remote side and zero disables the timer. In case it is
2812 * set before rdma_resolve_route, the value will also be used to determine
2813 * PacketLifeTime for RoCE.
2814 *
2815 * Return: 0 for success
2816 */
2817int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
2818{
2819	struct rdma_id_private *id_priv;
2820
2821	if (id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_INI)
2822		return -EINVAL;
2823
2824	id_priv = container_of(id, struct rdma_id_private, id);
2825	mutex_lock(&id_priv->qp_mutex);
2826	id_priv->timeout = timeout;
2827	id_priv->timeout_set = true;
2828	mutex_unlock(&id_priv->qp_mutex);
2829
2830	return 0;
2831}
2832EXPORT_SYMBOL(rdma_set_ack_timeout);
2833
2834/**
2835 * rdma_set_min_rnr_timer() - Set the minimum RNR Retry timer of the
2836 *			      QP associated with a connection identifier.
2837 * @id: Communication identifier to associated with service type.
2838 * @min_rnr_timer: 5-bit value encoded as Table 45: "Encoding for RNR NAK
2839 *		   Timer Field" in the IBTA specification.
2840 *
2841 * This function should be called before rdma_connect() on active
2842 * side, and on passive side before rdma_accept(). The timer value
2843 * will be associated with the local QP. When it receives a send it is
2844 * not read to handle, typically if the receive queue is empty, an RNR
2845 * Retry NAK is returned to the requester with the min_rnr_timer
2846 * encoded. The requester will then wait at least the time specified
2847 * in the NAK before retrying. The default is zero, which translates
2848 * to a minimum RNR Timer value of 655 ms.
2849 *
2850 * Return: 0 for success
2851 */
2852int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer)
2853{
2854	struct rdma_id_private *id_priv;
2855
2856	/* It is a five-bit value */
2857	if (min_rnr_timer & 0xe0)
2858		return -EINVAL;
2859
2860	if (WARN_ON(id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_TGT))
2861		return -EINVAL;
2862
2863	id_priv = container_of(id, struct rdma_id_private, id);
2864	mutex_lock(&id_priv->qp_mutex);
2865	id_priv->min_rnr_timer = min_rnr_timer;
2866	id_priv->min_rnr_timer_set = true;
2867	mutex_unlock(&id_priv->qp_mutex);
2868
2869	return 0;
2870}
2871EXPORT_SYMBOL(rdma_set_min_rnr_timer);
2872
2873static int route_set_path_rec_inbound(struct cma_work *work,
2874				      struct sa_path_rec *path_rec)
2875{
2876	struct rdma_route *route = &work->id->id.route;
2877
2878	if (!route->path_rec_inbound) {
2879		route->path_rec_inbound =
2880			kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL);
2881		if (!route->path_rec_inbound)
2882			return -ENOMEM;
2883	}
2884
2885	*route->path_rec_inbound = *path_rec;
2886	return 0;
2887}
2888
2889static int route_set_path_rec_outbound(struct cma_work *work,
2890				       struct sa_path_rec *path_rec)
2891{
2892	struct rdma_route *route = &work->id->id.route;
2893
2894	if (!route->path_rec_outbound) {
2895		route->path_rec_outbound =
2896			kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL);
2897		if (!route->path_rec_outbound)
2898			return -ENOMEM;
2899	}
2900
2901	*route->path_rec_outbound = *path_rec;
2902	return 0;
2903}
2904
2905static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2906			      unsigned int num_prs, void *context)
2907{
2908	struct cma_work *work = context;
2909	struct rdma_route *route;
2910	int i;
2911
2912	route = &work->id->id.route;
2913
2914	if (status)
2915		goto fail;
2916
2917	for (i = 0; i < num_prs; i++) {
2918		if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP))
2919			*route->path_rec = path_rec[i];
2920		else if (path_rec[i].flags & IB_PATH_INBOUND)
2921			status = route_set_path_rec_inbound(work, &path_rec[i]);
2922		else if (path_rec[i].flags & IB_PATH_OUTBOUND)
2923			status = route_set_path_rec_outbound(work,
2924							     &path_rec[i]);
2925		else
2926			status = -EINVAL;
2927
2928		if (status)
2929			goto fail;
2930	}
2931
2932	route->num_pri_alt_paths = 1;
2933	queue_work(cma_wq, &work->work);
2934	return;
2935
2936fail:
2937	work->old_state = RDMA_CM_ROUTE_QUERY;
2938	work->new_state = RDMA_CM_ADDR_RESOLVED;
2939	work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
2940	work->event.status = status;
2941	pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",
2942			     status);
2943	queue_work(cma_wq, &work->work);
2944}
2945
2946static int cma_query_ib_route(struct rdma_id_private *id_priv,
2947			      unsigned long timeout_ms, struct cma_work *work)
2948{
2949	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2950	struct sa_path_rec path_rec;
2951	ib_sa_comp_mask comp_mask;
2952	struct sockaddr_in6 *sin6;
2953	struct sockaddr_ib *sib;
2954
2955	memset(&path_rec, 0, sizeof path_rec);
2956
2957	if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num))
2958		path_rec.rec_type = SA_PATH_REC_TYPE_OPA;
2959	else
2960		path_rec.rec_type = SA_PATH_REC_TYPE_IB;
2961	rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
2962	rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
2963	path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2964	path_rec.numb_path = 1;
2965	path_rec.reversible = 1;
2966	path_rec.service_id = rdma_get_service_id(&id_priv->id,
2967						  cma_dst_addr(id_priv));
2968
2969	comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
2970		    IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
2971		    IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
2972
2973	switch (cma_family(id_priv)) {
2974	case AF_INET:
2975		path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
2976		comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
2977		break;
2978	case AF_INET6:
2979		sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2980		path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
2981		comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2982		break;
2983	case AF_IB:
2984		sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2985		path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20);
2986		comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2987		break;
2988	}
2989
2990	id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
2991					       id_priv->id.port_num, &path_rec,
2992					       comp_mask, timeout_ms,
2993					       GFP_KERNEL, cma_query_handler,
2994					       work, &id_priv->query);
2995
2996	return (id_priv->query_id < 0) ? id_priv->query_id : 0;
2997}
2998
2999static void cma_iboe_join_work_handler(struct work_struct *work)
3000{
3001	struct cma_multicast *mc =
3002		container_of(work, struct cma_multicast, iboe_join.work);
3003	struct rdma_cm_event *event = &mc->iboe_join.event;
3004	struct rdma_id_private *id_priv = mc->id_priv;
3005	int ret;
3006
3007	mutex_lock(&id_priv->handler_mutex);
3008	if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
3009	    READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
3010		goto out_unlock;
3011
3012	ret = cma_cm_event_handler(id_priv, event);
3013	WARN_ON(ret);
3014
3015out_unlock:
3016	mutex_unlock(&id_priv->handler_mutex);
3017	if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN)
3018		rdma_destroy_ah_attr(&event->param.ud.ah_attr);
3019}
3020
3021static void cma_work_handler(struct work_struct *_work)
3022{
3023	struct cma_work *work = container_of(_work, struct cma_work, work);
3024	struct rdma_id_private *id_priv = work->id;
3025
3026	mutex_lock(&id_priv->handler_mutex);
3027	if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
3028	    READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
3029		goto out_unlock;
3030	if (work->old_state != 0 || work->new_state != 0) {
3031		if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
3032			goto out_unlock;
3033	}
3034
3035	if (cma_cm_event_handler(id_priv, &work->event)) {
3036		cma_id_put(id_priv);
3037		destroy_id_handler_unlock(id_priv);
3038		goto out_free;
3039	}
3040
3041out_unlock:
3042	mutex_unlock(&id_priv->handler_mutex);
3043	cma_id_put(id_priv);
3044out_free:
3045	if (work->event.event == RDMA_CM_EVENT_MULTICAST_JOIN)
3046		rdma_destroy_ah_attr(&work->event.param.ud.ah_attr);
3047	kfree(work);
3048}
3049
3050static void cma_init_resolve_route_work(struct cma_work *work,
3051					struct rdma_id_private *id_priv)
3052{
3053	work->id = id_priv;
3054	INIT_WORK(&work->work, cma_work_handler);
3055	work->old_state = RDMA_CM_ROUTE_QUERY;
3056	work->new_state = RDMA_CM_ROUTE_RESOLVED;
3057	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
3058}
3059
3060static void enqueue_resolve_addr_work(struct cma_work *work,
3061				      struct rdma_id_private *id_priv)
3062{
3063	/* Balances with cma_id_put() in cma_work_handler */
3064	cma_id_get(id_priv);
3065
3066	work->id = id_priv;
3067	INIT_WORK(&work->work, cma_work_handler);
3068	work->old_state = RDMA_CM_ADDR_QUERY;
3069	work->new_state = RDMA_CM_ADDR_RESOLVED;
3070	work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3071
3072	queue_work(cma_wq, &work->work);
3073}
3074
3075static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
3076				unsigned long timeout_ms)
3077{
3078	struct rdma_route *route = &id_priv->id.route;
3079	struct cma_work *work;
3080	int ret;
3081
3082	work = kzalloc(sizeof *work, GFP_KERNEL);
3083	if (!work)
3084		return -ENOMEM;
3085
3086	cma_init_resolve_route_work(work, id_priv);
3087
3088	if (!route->path_rec)
3089		route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
3090	if (!route->path_rec) {
3091		ret = -ENOMEM;
3092		goto err1;
3093	}
3094
3095	ret = cma_query_ib_route(id_priv, timeout_ms, work);
3096	if (ret)
3097		goto err2;
3098
3099	return 0;
3100err2:
3101	kfree(route->path_rec);
3102	route->path_rec = NULL;
3103err1:
3104	kfree(work);
3105	return ret;
3106}
3107
3108static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
3109					   unsigned long supported_gids,
3110					   enum ib_gid_type default_gid)
3111{
3112	if ((network_type == RDMA_NETWORK_IPV4 ||
3113	     network_type == RDMA_NETWORK_IPV6) &&
3114	    test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
3115		return IB_GID_TYPE_ROCE_UDP_ENCAP;
3116
3117	return default_gid;
3118}
3119
3120/*
3121 * cma_iboe_set_path_rec_l2_fields() is helper function which sets
3122 * path record type based on GID type.
3123 * It also sets up other L2 fields which includes destination mac address
3124 * netdev ifindex, of the path record.
3125 * It returns the netdev of the bound interface for this path record entry.
3126 */
3127static struct net_device *
3128cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
3129{
3130	struct rdma_route *route = &id_priv->id.route;
3131	enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
3132	struct rdma_addr *addr = &route->addr;
3133	unsigned long supported_gids;
3134	struct net_device *ndev;
3135
3136	if (!addr->dev_addr.bound_dev_if)
3137		return NULL;
3138
3139	ndev = dev_get_by_index(addr->dev_addr.net,
3140				addr->dev_addr.bound_dev_if);
3141	if (!ndev)
3142		return NULL;
3143
3144	supported_gids = roce_gid_type_mask_support(id_priv->id.device,
3145						    id_priv->id.port_num);
3146	gid_type = cma_route_gid_type(addr->dev_addr.network,
3147				      supported_gids,
3148				      id_priv->gid_type);
3149	/* Use the hint from IP Stack to select GID Type */
3150	if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
3151		gid_type = ib_network_to_gid_type(addr->dev_addr.network);
3152	route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
3153
3154	route->path_rec->roce.route_resolved = true;
3155	sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
3156	return ndev;
3157}
3158
3159int rdma_set_ib_path(struct rdma_cm_id *id,
3160		     struct sa_path_rec *path_rec)
3161{
3162	struct rdma_id_private *id_priv;
3163	struct net_device *ndev;
3164	int ret;
3165
3166	id_priv = container_of(id, struct rdma_id_private, id);
3167	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3168			   RDMA_CM_ROUTE_RESOLVED))
3169		return -EINVAL;
3170
3171	id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
3172				     GFP_KERNEL);
3173	if (!id->route.path_rec) {
3174		ret = -ENOMEM;
3175		goto err;
3176	}
3177
3178	if (rdma_protocol_roce(id->device, id->port_num)) {
3179		ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
3180		if (!ndev) {
3181			ret = -ENODEV;
3182			goto err_free;
3183		}
3184		dev_put(ndev);
3185	}
3186
3187	id->route.num_pri_alt_paths = 1;
3188	return 0;
3189
3190err_free:
3191	kfree(id->route.path_rec);
3192	id->route.path_rec = NULL;
3193err:
3194	cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
3195	return ret;
3196}
3197EXPORT_SYMBOL(rdma_set_ib_path);
3198
3199static int cma_resolve_iw_route(struct rdma_id_private *id_priv)
3200{
3201	struct cma_work *work;
3202
3203	work = kzalloc(sizeof *work, GFP_KERNEL);
3204	if (!work)
3205		return -ENOMEM;
3206
3207	cma_init_resolve_route_work(work, id_priv);
3208	queue_work(cma_wq, &work->work);
3209	return 0;
3210}
3211
3212static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)
3213{
3214	struct net_device *dev;
3215
3216	dev = vlan_dev_real_dev(vlan_ndev);
3217	if (dev->num_tc)
3218		return netdev_get_prio_tc_map(dev, prio);
3219
3220	return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &
3221		VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
3222}
3223
3224struct iboe_prio_tc_map {
3225	int input_prio;
3226	int output_tc;
3227	bool found;
3228};
3229
3230static int get_lower_vlan_dev_tc(struct net_device *dev,
3231				 struct netdev_nested_priv *priv)
3232{
3233	struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv->data;
3234
3235	if (is_vlan_dev(dev))
3236		map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
3237	else if (dev->num_tc)
3238		map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio);
3239	else
3240		map->output_tc = 0;
3241	/* We are interested only in first level VLAN device, so always
3242	 * return 1 to stop iterating over next level devices.
3243	 */
3244	map->found = true;
3245	return 1;
3246}
3247
3248static int iboe_tos_to_sl(struct net_device *ndev, int tos)
3249{
3250	struct iboe_prio_tc_map prio_tc_map = {};
3251	int prio = rt_tos2priority(tos);
3252	struct netdev_nested_priv priv;
3253
3254	/* If VLAN device, get it directly from the VLAN netdev */
3255	if (is_vlan_dev(ndev))
3256		return get_vlan_ndev_tc(ndev, prio);
3257
3258	prio_tc_map.input_prio = prio;
3259	priv.data = (void *)&prio_tc_map;
3260	rcu_read_lock();
3261	netdev_walk_all_lower_dev_rcu(ndev,
3262				      get_lower_vlan_dev_tc,
3263				      &priv);
3264	rcu_read_unlock();
3265	/* If map is found from lower device, use it; Otherwise
3266	 * continue with the current netdevice to get priority to tc map.
3267	 */
3268	if (prio_tc_map.found)
3269		return prio_tc_map.output_tc;
3270	else if (ndev->num_tc)
3271		return netdev_get_prio_tc_map(ndev, prio);
3272	else
3273		return 0;
3274}
3275
3276static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)
3277{
3278	struct sockaddr_in6 *addr6;
3279	u16 dport, sport;
3280	u32 hash, fl;
3281
3282	addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);
3283	fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK;
3284	if ((cma_family(id_priv) != AF_INET6) || !fl) {
3285		dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));
3286		sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));
3287		hash = (u32)sport * 31 + dport;
3288		fl = hash & IB_GRH_FLOWLABEL_MASK;
3289	}
3290
3291	return cpu_to_be32(fl);
3292}
3293
3294static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
3295{
3296	struct rdma_route *route = &id_priv->id.route;
3297	struct rdma_addr *addr = &route->addr;
3298	struct cma_work *work;
3299	int ret;
3300	struct net_device *ndev;
3301
3302	u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
3303					rdma_start_port(id_priv->cma_dev->device)];
3304	u8 tos;
3305
3306	mutex_lock(&id_priv->qp_mutex);
3307	tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
3308	mutex_unlock(&id_priv->qp_mutex);
3309
3310	work = kzalloc(sizeof *work, GFP_KERNEL);
3311	if (!work)
3312		return -ENOMEM;
3313
3314	route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
3315	if (!route->path_rec) {
3316		ret = -ENOMEM;
3317		goto err1;
3318	}
3319
3320	route->num_pri_alt_paths = 1;
3321
3322	ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
3323	if (!ndev) {
3324		ret = -ENODEV;
3325		goto err2;
3326	}
3327
3328	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
3329		    &route->path_rec->sgid);
3330	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
3331		    &route->path_rec->dgid);
3332
3333	if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
3334		/* TODO: get the hoplimit from the inet/inet6 device */
3335		route->path_rec->hop_limit = addr->dev_addr.hoplimit;
3336	else
3337		route->path_rec->hop_limit = 1;
3338	route->path_rec->reversible = 1;
3339	route->path_rec->pkey = cpu_to_be16(0xffff);
3340	route->path_rec->mtu_selector = IB_SA_EQ;
3341	route->path_rec->sl = iboe_tos_to_sl(ndev, tos);
3342	route->path_rec->traffic_class = tos;
3343	route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
3344	route->path_rec->rate_selector = IB_SA_EQ;
3345	route->path_rec->rate = IB_RATE_PORT_CURRENT;
3346	dev_put(ndev);
3347	route->path_rec->packet_life_time_selector = IB_SA_EQ;
3348	/* In case ACK timeout is set, use this value to calculate
3349	 * PacketLifeTime.  As per IBTA 12.7.34,
3350	 * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).
3351	 * Assuming a negligible local ACK delay, we can use
3352	 * PacketLifeTime = local ACK timeout/2
3353	 * as a reasonable approximation for RoCE networks.
3354	 */
3355	mutex_lock(&id_priv->qp_mutex);
3356	if (id_priv->timeout_set && id_priv->timeout)
3357		route->path_rec->packet_life_time = id_priv->timeout - 1;
3358	else
3359		route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
3360	mutex_unlock(&id_priv->qp_mutex);
3361
3362	if (!route->path_rec->mtu) {
3363		ret = -EINVAL;
3364		goto err2;
3365	}
3366
3367	if (rdma_protocol_roce_udp_encap(id_priv->id.device,
3368					 id_priv->id.port_num))
3369		route->path_rec->flow_label =
3370			cma_get_roce_udp_flow_label(id_priv);
3371
3372	cma_init_resolve_route_work(work, id_priv);
3373	queue_work(cma_wq, &work->work);
3374
3375	return 0;
3376
3377err2:
3378	kfree(route->path_rec);
3379	route->path_rec = NULL;
3380	route->num_pri_alt_paths = 0;
3381err1:
3382	kfree(work);
3383	return ret;
3384}
3385
3386int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
3387{
3388	struct rdma_id_private *id_priv;
3389	enum rdma_cm_state state;
3390	int ret;
3391
3392	if (!timeout_ms)
3393		return -EINVAL;
3394
3395	id_priv = container_of(id, struct rdma_id_private, id);
3396	state = id_priv->state;
3397	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3398			   RDMA_CM_ROUTE_QUERY) &&
3399	    !cma_comp_exch(id_priv, RDMA_CM_ADDRINFO_RESOLVED,
3400			   RDMA_CM_ROUTE_QUERY))
3401		return -EINVAL;
3402
3403	cma_id_get(id_priv);
3404	if (rdma_cap_ib_sa(id->device, id->port_num))
3405		ret = cma_resolve_ib_route(id_priv, timeout_ms);
3406	else if (rdma_protocol_roce(id->device, id->port_num)) {
3407		ret = cma_resolve_iboe_route(id_priv);
3408		if (!ret)
3409			cma_add_id_to_tree(id_priv);
3410	}
3411	else if (rdma_protocol_iwarp(id->device, id->port_num))
3412		ret = cma_resolve_iw_route(id_priv);
3413	else
3414		ret = -ENOSYS;
3415
3416	if (ret)
3417		goto err;
3418
3419	return 0;
3420err:
3421	cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, state);
3422	cma_id_put(id_priv);
3423	return ret;
3424}
3425EXPORT_SYMBOL(rdma_resolve_route);
3426
3427static void cma_set_loopback(struct sockaddr *addr)
3428{
3429	switch (addr->sa_family) {
3430	case AF_INET:
3431		((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3432		break;
3433	case AF_INET6:
3434		ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
3435			      0, 0, 0, htonl(1));
3436		break;
3437	default:
3438		ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
3439			    0, 0, 0, htonl(1));
3440		break;
3441	}
3442}
3443
3444static int cma_bind_loopback(struct rdma_id_private *id_priv)
3445{
3446	struct cma_device *cma_dev, *cur_dev;
3447	union ib_gid gid;
3448	enum ib_port_state port_state;
3449	unsigned int p;
3450	u16 pkey;
3451	int ret;
3452
3453	cma_dev = NULL;
3454	mutex_lock(&lock);
3455	list_for_each_entry(cur_dev, &dev_list, list) {
3456		if (cma_family(id_priv) == AF_IB &&
3457		    !rdma_cap_ib_cm(cur_dev->device, 1))
3458			continue;
3459
3460		if (!cma_dev)
3461			cma_dev = cur_dev;
3462
3463		rdma_for_each_port (cur_dev->device, p) {
3464			if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
3465			    port_state == IB_PORT_ACTIVE) {
3466				cma_dev = cur_dev;
3467				goto port_found;
3468			}
3469		}
3470	}
3471
3472	if (!cma_dev) {
3473		ret = -ENODEV;
3474		goto out;
3475	}
3476
3477	p = 1;
3478
3479port_found:
3480	ret = rdma_query_gid(cma_dev->device, p, 0, &gid);
3481	if (ret)
3482		goto out;
3483
3484	ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
3485	if (ret)
3486		goto out;
3487
3488	id_priv->id.route.addr.dev_addr.dev_type =
3489		(rdma_protocol_ib(cma_dev->device, p)) ?
3490		ARPHRD_INFINIBAND : ARPHRD_ETHER;
3491
3492	rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3493	ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
3494	id_priv->id.port_num = p;
3495	cma_attach_to_dev(id_priv, cma_dev);
3496	rdma_restrack_add(&id_priv->res);
3497	cma_set_loopback(cma_src_addr(id_priv));
3498out:
3499	mutex_unlock(&lock);
3500	return ret;
3501}
3502
3503static void addr_handler(int status, struct sockaddr *src_addr,
3504			 struct rdma_dev_addr *dev_addr, void *context)
3505{
3506	struct rdma_id_private *id_priv = context;
3507	struct rdma_cm_event event = {};
3508	struct sockaddr *addr;
3509	struct sockaddr_storage old_addr;
3510
3511	mutex_lock(&id_priv->handler_mutex);
3512	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
3513			   RDMA_CM_ADDR_RESOLVED))
3514		goto out;
3515
3516	/*
3517	 * Store the previous src address, so that if we fail to acquire
3518	 * matching rdma device, old address can be restored back, which helps
3519	 * to cancel the cma listen operation correctly.
3520	 */
3521	addr = cma_src_addr(id_priv);
3522	memcpy(&old_addr, addr, rdma_addr_size(addr));
3523	memcpy(addr, src_addr, rdma_addr_size(src_addr));
3524	if (!status && !id_priv->cma_dev) {
3525		status = cma_acquire_dev_by_src_ip(id_priv);
3526		if (status)
3527			pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",
3528					     status);
3529		rdma_restrack_add(&id_priv->res);
3530	} else if (status) {
3531		pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);
3532	}
3533
3534	if (status) {
3535		memcpy(addr, &old_addr,
3536		       rdma_addr_size((struct sockaddr *)&old_addr));
3537		if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3538				   RDMA_CM_ADDR_BOUND))
3539			goto out;
3540		event.event = RDMA_CM_EVENT_ADDR_ERROR;
3541		event.status = status;
3542	} else
3543		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3544
3545	if (cma_cm_event_handler(id_priv, &event)) {
3546		destroy_id_handler_unlock(id_priv);
3547		return;
3548	}
3549out:
3550	mutex_unlock(&id_priv->handler_mutex);
3551}
3552
3553static int cma_resolve_loopback(struct rdma_id_private *id_priv)
3554{
3555	struct cma_work *work;
3556	union ib_gid gid;
3557	int ret;
3558
3559	work = kzalloc(sizeof *work, GFP_KERNEL);
3560	if (!work)
3561		return -ENOMEM;
3562
3563	if (!id_priv->cma_dev) {
3564		ret = cma_bind_loopback(id_priv);
3565		if (ret)
3566			goto err;
3567	}
3568
3569	rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3570	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
3571
3572	enqueue_resolve_addr_work(work, id_priv);
3573	return 0;
3574err:
3575	kfree(work);
3576	return ret;
3577}
3578
3579static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
3580{
3581	struct cma_work *work;
3582	int ret;
3583
3584	work = kzalloc(sizeof *work, GFP_KERNEL);
3585	if (!work)
3586		return -ENOMEM;
3587
3588	if (!id_priv->cma_dev) {
3589		ret = cma_resolve_ib_dev(id_priv);
3590		if (ret)
3591			goto err;
3592	}
3593
3594	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
3595		&(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
3596
3597	enqueue_resolve_addr_work(work, id_priv);
3598	return 0;
3599err:
3600	kfree(work);
3601	return ret;
3602}
3603
3604int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3605{
3606	struct rdma_id_private *id_priv;
3607	unsigned long flags;
3608	int ret;
3609
3610	id_priv = container_of(id, struct rdma_id_private, id);
3611	spin_lock_irqsave(&id_priv->lock, flags);
3612	if ((reuse && id_priv->state != RDMA_CM_LISTEN) ||
3613	    id_priv->state == RDMA_CM_IDLE) {
3614		id_priv->reuseaddr = reuse;
3615		ret = 0;
3616	} else {
3617		ret = -EINVAL;
3618	}
3619	spin_unlock_irqrestore(&id_priv->lock, flags);
3620	return ret;
3621}
3622EXPORT_SYMBOL(rdma_set_reuseaddr);
3623
3624int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
3625{
3626	struct rdma_id_private *id_priv;
3627	unsigned long flags;
3628	int ret;
3629
3630	id_priv = container_of(id, struct rdma_id_private, id);
3631	spin_lock_irqsave(&id_priv->lock, flags);
3632	if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
3633		id_priv->options |= (1 << CMA_OPTION_AFONLY);
3634		id_priv->afonly = afonly;
3635		ret = 0;
3636	} else {
3637		ret = -EINVAL;
3638	}
3639	spin_unlock_irqrestore(&id_priv->lock, flags);
3640	return ret;
3641}
3642EXPORT_SYMBOL(rdma_set_afonly);
3643
3644static void cma_bind_port(struct rdma_bind_list *bind_list,
3645			  struct rdma_id_private *id_priv)
3646{
3647	struct sockaddr *addr;
3648	struct sockaddr_ib *sib;
3649	u64 sid, mask;
3650	__be16 port;
3651
3652	lockdep_assert_held(&lock);
3653
3654	addr = cma_src_addr(id_priv);
3655	port = htons(bind_list->port);
3656
3657	switch (addr->sa_family) {
3658	case AF_INET:
3659		((struct sockaddr_in *) addr)->sin_port = port;
3660		break;
3661	case AF_INET6:
3662		((struct sockaddr_in6 *) addr)->sin6_port = port;
3663		break;
3664	case AF_IB:
3665		sib = (struct sockaddr_ib *) addr;
3666		sid = be64_to_cpu(sib->sib_sid);
3667		mask = be64_to_cpu(sib->sib_sid_mask);
3668		sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
3669		sib->sib_sid_mask = cpu_to_be64(~0ULL);
3670		break;
3671	}
3672	id_priv->bind_list = bind_list;
3673	hlist_add_head(&id_priv->node, &bind_list->owners);
3674}
3675
3676static int cma_alloc_port(enum rdma_ucm_port_space ps,
3677			  struct rdma_id_private *id_priv, unsigned short snum)
3678{
3679	struct rdma_bind_list *bind_list;
3680	int ret;
3681
3682	lockdep_assert_held(&lock);
3683
3684	bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
3685	if (!bind_list)
3686		return -ENOMEM;
3687
3688	ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list,
3689			   snum);
3690	if (ret < 0)
3691		goto err;
3692
3693	bind_list->ps = ps;
3694	bind_list->port = snum;
3695	cma_bind_port(bind_list, id_priv);
3696	return 0;
3697err:
3698	kfree(bind_list);
3699	return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
3700}
3701
3702static int cma_port_is_unique(struct rdma_bind_list *bind_list,
3703			      struct rdma_id_private *id_priv)
3704{
3705	struct rdma_id_private *cur_id;
3706	struct sockaddr  *daddr = cma_dst_addr(id_priv);
3707	struct sockaddr  *saddr = cma_src_addr(id_priv);
3708	__be16 dport = cma_port(daddr);
3709
3710	lockdep_assert_held(&lock);
3711
3712	hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3713		struct sockaddr  *cur_daddr = cma_dst_addr(cur_id);
3714		struct sockaddr  *cur_saddr = cma_src_addr(cur_id);
3715		__be16 cur_dport = cma_port(cur_daddr);
3716
3717		if (id_priv == cur_id)
3718			continue;
3719
3720		/* different dest port -> unique */
3721		if (!cma_any_port(daddr) &&
3722		    !cma_any_port(cur_daddr) &&
3723		    (dport != cur_dport))
3724			continue;
3725
3726		/* different src address -> unique */
3727		if (!cma_any_addr(saddr) &&
3728		    !cma_any_addr(cur_saddr) &&
3729		    cma_addr_cmp(saddr, cur_saddr))
3730			continue;
3731
3732		/* different dst address -> unique */
3733		if (!cma_any_addr(daddr) &&
3734		    !cma_any_addr(cur_daddr) &&
3735		    cma_addr_cmp(daddr, cur_daddr))
3736			continue;
3737
3738		return -EADDRNOTAVAIL;
3739	}
3740	return 0;
3741}
3742
3743static int cma_alloc_any_port(enum rdma_ucm_port_space ps,
3744			      struct rdma_id_private *id_priv)
3745{
3746	static unsigned int last_used_port;
3747	int low, high, remaining;
3748	unsigned int rover;
3749	struct net *net = id_priv->id.route.addr.dev_addr.net;
3750
3751	lockdep_assert_held(&lock);
3752
3753	inet_get_local_port_range(net, &low, &high);
3754	remaining = (high - low) + 1;
3755	rover = get_random_u32_inclusive(low, remaining + low - 1);
3756retry:
3757	if (last_used_port != rover) {
3758		struct rdma_bind_list *bind_list;
3759		int ret;
3760
3761		bind_list = cma_ps_find(net, ps, (unsigned short)rover);
3762
3763		if (!bind_list) {
3764			ret = cma_alloc_port(ps, id_priv, rover);
3765		} else {
3766			ret = cma_port_is_unique(bind_list, id_priv);
3767			if (!ret)
3768				cma_bind_port(bind_list, id_priv);
3769		}
3770		/*
3771		 * Remember previously used port number in order to avoid
3772		 * re-using same port immediately after it is closed.
3773		 */
3774		if (!ret)
3775			last_used_port = rover;
3776		if (ret != -EADDRNOTAVAIL)
3777			return ret;
3778	}
3779	if (--remaining) {
3780		rover++;
3781		if ((rover < low) || (rover > high))
3782			rover = low;
3783		goto retry;
3784	}
3785	return -EADDRNOTAVAIL;
3786}
3787
3788/*
3789 * Check that the requested port is available.  This is called when trying to
3790 * bind to a specific port, or when trying to listen on a bound port.  In
3791 * the latter case, the provided id_priv may already be on the bind_list, but
3792 * we still need to check that it's okay to start listening.
3793 */
3794static int cma_check_port(struct rdma_bind_list *bind_list,
3795			  struct rdma_id_private *id_priv, uint8_t reuseaddr)
3796{
3797	struct rdma_id_private *cur_id;
3798	struct sockaddr *addr, *cur_addr;
3799
3800	lockdep_assert_held(&lock);
3801
3802	addr = cma_src_addr(id_priv);
3803	hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3804		if (id_priv == cur_id)
3805			continue;
3806
3807		if (reuseaddr && cur_id->reuseaddr)
3808			continue;
3809
3810		cur_addr = cma_src_addr(cur_id);
3811		if (id_priv->afonly && cur_id->afonly &&
3812		    (addr->sa_family != cur_addr->sa_family))
3813			continue;
3814
3815		if (cma_any_addr(addr) || cma_any_addr(cur_addr))
3816			return -EADDRNOTAVAIL;
3817
3818		if (!cma_addr_cmp(addr, cur_addr))
3819			return -EADDRINUSE;
3820	}
3821	return 0;
3822}
3823
3824static int cma_use_port(enum rdma_ucm_port_space ps,
3825			struct rdma_id_private *id_priv)
3826{
3827	struct rdma_bind_list *bind_list;
3828	unsigned short snum;
3829	int ret;
3830
3831	lockdep_assert_held(&lock);
3832
3833	snum = ntohs(cma_port(cma_src_addr(id_priv)));
3834	if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
3835		return -EACCES;
3836
3837	bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum);
3838	if (!bind_list) {
3839		ret = cma_alloc_port(ps, id_priv, snum);
3840	} else {
3841		ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
3842		if (!ret)
3843			cma_bind_port(bind_list, id_priv);
3844	}
3845	return ret;
3846}
3847
3848static enum rdma_ucm_port_space
3849cma_select_inet_ps(struct rdma_id_private *id_priv)
3850{
3851	switch (id_priv->id.ps) {
3852	case RDMA_PS_TCP:
3853	case RDMA_PS_UDP:
3854	case RDMA_PS_IPOIB:
3855	case RDMA_PS_IB:
3856		return id_priv->id.ps;
3857	default:
3858
3859		return 0;
3860	}
3861}
3862
3863static enum rdma_ucm_port_space
3864cma_select_ib_ps(struct rdma_id_private *id_priv)
3865{
3866	enum rdma_ucm_port_space ps = 0;
3867	struct sockaddr_ib *sib;
3868	u64 sid_ps, mask, sid;
3869
3870	sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
3871	mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
3872	sid = be64_to_cpu(sib->sib_sid) & mask;
3873
3874	if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
3875		sid_ps = RDMA_IB_IP_PS_IB;
3876		ps = RDMA_PS_IB;
3877	} else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
3878		   (sid == (RDMA_IB_IP_PS_TCP & mask))) {
3879		sid_ps = RDMA_IB_IP_PS_TCP;
3880		ps = RDMA_PS_TCP;
3881	} else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
3882		   (sid == (RDMA_IB_IP_PS_UDP & mask))) {
3883		sid_ps = RDMA_IB_IP_PS_UDP;
3884		ps = RDMA_PS_UDP;
3885	}
3886
3887	if (ps) {
3888		sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
3889		sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
3890						be64_to_cpu(sib->sib_sid_mask));
3891	}
3892	return ps;
3893}
3894
3895static int cma_get_port(struct rdma_id_private *id_priv)
3896{
3897	enum rdma_ucm_port_space ps;
3898	int ret;
3899
3900	if (cma_family(id_priv) != AF_IB)
3901		ps = cma_select_inet_ps(id_priv);
3902	else
3903		ps = cma_select_ib_ps(id_priv);
3904	if (!ps)
3905		return -EPROTONOSUPPORT;
3906
3907	mutex_lock(&lock);
3908	if (cma_any_port(cma_src_addr(id_priv)))
3909		ret = cma_alloc_any_port(ps, id_priv);
3910	else
3911		ret = cma_use_port(ps, id_priv);
3912	mutex_unlock(&lock);
3913
3914	return ret;
3915}
3916
3917static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
3918			       struct sockaddr *addr)
3919{
3920#if IS_ENABLED(CONFIG_IPV6)
3921	struct sockaddr_in6 *sin6;
3922
3923	if (addr->sa_family != AF_INET6)
3924		return 0;
3925
3926	sin6 = (struct sockaddr_in6 *) addr;
3927
3928	if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL))
3929		return 0;
3930
3931	if (!sin6->sin6_scope_id)
3932			return -EINVAL;
3933
3934	dev_addr->bound_dev_if = sin6->sin6_scope_id;
3935#endif
3936	return 0;
3937}
3938
3939int rdma_listen(struct rdma_cm_id *id, int backlog)
3940{
3941	struct rdma_id_private *id_priv =
3942		container_of(id, struct rdma_id_private, id);
3943	int ret;
3944
3945	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) {
3946		struct sockaddr_in any_in = {
3947			.sin_family = AF_INET,
3948			.sin_addr.s_addr = htonl(INADDR_ANY),
3949		};
3950
3951		/* For a well behaved ULP state will be RDMA_CM_IDLE */
3952		ret = rdma_bind_addr(id, (struct sockaddr *)&any_in);
3953		if (ret)
3954			return ret;
3955		if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3956					   RDMA_CM_LISTEN)))
3957			return -EINVAL;
3958	}
3959
3960	/*
3961	 * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable
3962	 * any more, and has to be unique in the bind list.
3963	 */
3964	if (id_priv->reuseaddr) {
3965		mutex_lock(&lock);
3966		ret = cma_check_port(id_priv->bind_list, id_priv, 0);
3967		if (!ret)
3968			id_priv->reuseaddr = 0;
3969		mutex_unlock(&lock);
3970		if (ret)
3971			goto err;
3972	}
3973
3974	id_priv->backlog = backlog;
3975	if (id_priv->cma_dev) {
3976		if (rdma_cap_ib_cm(id->device, 1)) {
3977			ret = cma_ib_listen(id_priv);
3978			if (ret)
3979				goto err;
3980		} else if (rdma_cap_iw_cm(id->device, 1)) {
3981			ret = cma_iw_listen(id_priv, backlog);
3982			if (ret)
3983				goto err;
3984		} else {
3985			ret = -ENOSYS;
3986			goto err;
3987		}
3988	} else {
3989		ret = cma_listen_on_all(id_priv);
3990		if (ret)
3991			goto err;
3992	}
3993
3994	return 0;
3995err:
3996	id_priv->backlog = 0;
3997	/*
3998	 * All the failure paths that lead here will not allow the req_handler's
3999	 * to have run.
4000	 */
4001	cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
4002	return ret;
4003}
4004EXPORT_SYMBOL(rdma_listen);
4005
4006static int rdma_bind_addr_dst(struct rdma_id_private *id_priv,
4007			      struct sockaddr *addr, const struct sockaddr *daddr)
4008{
4009	struct sockaddr *id_daddr;
4010	int ret;
4011
4012	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
4013	    addr->sa_family != AF_IB)
4014		return -EAFNOSUPPORT;
4015
4016	if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
4017		return -EINVAL;
4018
4019	ret = cma_check_linklocal(&id_priv->id.route.addr.dev_addr, addr);
4020	if (ret)
4021		goto err1;
4022
4023	memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
4024	if (!cma_any_addr(addr)) {
4025		ret = cma_translate_addr(addr, &id_priv->id.route.addr.dev_addr);
4026		if (ret)
4027			goto err1;
4028
4029		ret = cma_acquire_dev_by_src_ip(id_priv);
4030		if (ret)
4031			goto err1;
4032	}
4033
4034	if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
4035		if (addr->sa_family == AF_INET)
4036			id_priv->afonly = 1;
4037#if IS_ENABLED(CONFIG_IPV6)
4038		else if (addr->sa_family == AF_INET6) {
4039			struct net *net = id_priv->id.route.addr.dev_addr.net;
4040
4041			id_priv->afonly = net->ipv6.sysctl.bindv6only;
4042		}
4043#endif
4044	}
4045	id_daddr = cma_dst_addr(id_priv);
4046	if (daddr != id_daddr)
4047		memcpy(id_daddr, daddr, rdma_addr_size(addr));
4048	id_daddr->sa_family = addr->sa_family;
4049
4050	ret = cma_get_port(id_priv);
4051	if (ret)
4052		goto err2;
4053
4054	if (!cma_any_addr(addr))
4055		rdma_restrack_add(&id_priv->res);
4056	return 0;
4057err2:
4058	if (id_priv->cma_dev)
4059		cma_release_dev(id_priv);
4060err1:
4061	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
4062	return ret;
4063}
4064
4065static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4066			 const struct sockaddr *dst_addr)
4067{
4068	struct rdma_id_private *id_priv =
4069		container_of(id, struct rdma_id_private, id);
4070	struct sockaddr_storage zero_sock = {};
4071
4072	if (src_addr && src_addr->sa_family)
4073		return rdma_bind_addr_dst(id_priv, src_addr, dst_addr);
4074
4075	/*
4076	 * When the src_addr is not specified, automatically supply an any addr
4077	 */
4078	zero_sock.ss_family = dst_addr->sa_family;
4079	if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
4080		struct sockaddr_in6 *src_addr6 =
4081			(struct sockaddr_in6 *)&zero_sock;
4082		struct sockaddr_in6 *dst_addr6 =
4083			(struct sockaddr_in6 *)dst_addr;
4084
4085		src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
4086		if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
4087			id->route.addr.dev_addr.bound_dev_if =
4088				dst_addr6->sin6_scope_id;
4089	} else if (dst_addr->sa_family == AF_IB) {
4090		((struct sockaddr_ib *)&zero_sock)->sib_pkey =
4091			((struct sockaddr_ib *)dst_addr)->sib_pkey;
4092	}
4093	return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&zero_sock, dst_addr);
4094}
4095
4096/*
4097 * If required, resolve the source address for bind and leave the id_priv in
4098 * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
4099 * calls made by ULP, a previously bound ID will not be re-bound and src_addr is
4100 * ignored.
4101 */
4102static int resolve_prepare_src(struct rdma_id_private *id_priv,
4103			       struct sockaddr *src_addr,
4104			       const struct sockaddr *dst_addr)
4105{
4106	int ret;
4107
4108	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
4109		/* For a well behaved ULP state will be RDMA_CM_IDLE */
4110		ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
4111		if (ret)
4112			return ret;
4113		if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
4114					   RDMA_CM_ADDR_QUERY)))
4115			return -EINVAL;
4116
4117	} else {
4118		memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
4119	}
4120
4121	if (cma_family(id_priv) != dst_addr->sa_family) {
4122		ret = -EINVAL;
4123		goto err_state;
4124	}
4125	return 0;
4126
4127err_state:
4128	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4129	return ret;
4130}
4131
4132int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4133		      const struct sockaddr *dst_addr, unsigned long timeout_ms)
4134{
4135	struct rdma_id_private *id_priv =
4136		container_of(id, struct rdma_id_private, id);
4137	int ret;
4138
4139	ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
4140	if (ret)
4141		return ret;
4142
4143	if (cma_any_addr(dst_addr)) {
4144		ret = cma_resolve_loopback(id_priv);
4145	} else {
4146		if (dst_addr->sa_family == AF_IB) {
4147			ret = cma_resolve_ib_addr(id_priv);
4148		} else {
4149			/*
4150			 * The FSM can return back to RDMA_CM_ADDR_BOUND after
4151			 * rdma_resolve_ip() is called, eg through the error
4152			 * path in addr_handler(). If this happens the existing
4153			 * request must be canceled before issuing a new one.
4154			 * Since canceling a request is a bit slow and this
4155			 * oddball path is rare, keep track once a request has
4156			 * been issued. The track turns out to be a permanent
4157			 * state since this is the only cancel as it is
4158			 * immediately before rdma_resolve_ip().
4159			 */
4160			if (id_priv->used_resolve_ip)
4161				rdma_addr_cancel(&id->route.addr.dev_addr);
4162			else
4163				id_priv->used_resolve_ip = 1;
4164			ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
4165					      &id->route.addr.dev_addr,
4166					      timeout_ms, addr_handler,
4167					      false, id_priv);
4168		}
4169	}
4170	if (ret)
4171		goto err;
4172
4173	return 0;
4174err:
4175	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4176	return ret;
4177}
4178EXPORT_SYMBOL(rdma_resolve_addr);
4179
4180int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
4181{
4182	struct rdma_id_private *id_priv =
4183		container_of(id, struct rdma_id_private, id);
4184
4185	return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv));
4186}
4187EXPORT_SYMBOL(rdma_bind_addr);
4188
4189static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
4190{
4191	struct cma_hdr *cma_hdr;
4192
4193	cma_hdr = hdr;
4194	cma_hdr->cma_version = CMA_VERSION;
4195	if (cma_family(id_priv) == AF_INET) {
4196		struct sockaddr_in *src4, *dst4;
4197
4198		src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
4199		dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
4200
4201		cma_set_ip_ver(cma_hdr, 4);
4202		cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
4203		cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
4204		cma_hdr->port = src4->sin_port;
4205	} else if (cma_family(id_priv) == AF_INET6) {
4206		struct sockaddr_in6 *src6, *dst6;
4207
4208		src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
4209		dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
4210
4211		cma_set_ip_ver(cma_hdr, 6);
4212		cma_hdr->src_addr.ip6 = src6->sin6_addr;
4213		cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
4214		cma_hdr->port = src6->sin6_port;
4215	}
4216	return 0;
4217}
4218
4219static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
4220				const struct ib_cm_event *ib_event)
4221{
4222	struct rdma_id_private *id_priv = cm_id->context;
4223	struct rdma_cm_event event = {};
4224	const struct ib_cm_sidr_rep_event_param *rep =
4225				&ib_event->param.sidr_rep_rcvd;
4226	int ret;
4227
4228	mutex_lock(&id_priv->handler_mutex);
4229	if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4230		goto out;
4231
4232	switch (ib_event->event) {
4233	case IB_CM_SIDR_REQ_ERROR:
4234		event.event = RDMA_CM_EVENT_UNREACHABLE;
4235		event.status = -ETIMEDOUT;
4236		break;
4237	case IB_CM_SIDR_REP_RECEIVED:
4238		event.param.ud.private_data = ib_event->private_data;
4239		event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
4240		if (rep->status != IB_SIDR_SUCCESS) {
4241			event.event = RDMA_CM_EVENT_UNREACHABLE;
4242			event.status = ib_event->param.sidr_rep_rcvd.status;
4243			pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n",
4244					     event.status);
4245			break;
4246		}
4247		ret = cma_set_qkey(id_priv, rep->qkey);
4248		if (ret) {
4249			pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret);
4250			event.event = RDMA_CM_EVENT_ADDR_ERROR;
4251			event.status = ret;
4252			break;
4253		}
4254		ib_init_ah_attr_from_path(id_priv->id.device,
4255					  id_priv->id.port_num,
4256					  id_priv->id.route.path_rec,
4257					  &event.param.ud.ah_attr,
4258					  rep->sgid_attr);
4259		event.param.ud.qp_num = rep->qpn;
4260		event.param.ud.qkey = rep->qkey;
4261		event.event = RDMA_CM_EVENT_ESTABLISHED;
4262		event.status = 0;
4263		break;
4264	default:
4265		pr_err("RDMA CMA: unexpected IB CM event: %d\n",
4266		       ib_event->event);
4267		goto out;
4268	}
4269
4270	ret = cma_cm_event_handler(id_priv, &event);
4271
4272	rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4273	if (ret) {
4274		/* Destroy the CM ID by returning a non-zero value. */
4275		id_priv->cm_id.ib = NULL;
4276		destroy_id_handler_unlock(id_priv);
4277		return ret;
4278	}
4279out:
4280	mutex_unlock(&id_priv->handler_mutex);
4281	return 0;
4282}
4283
4284static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
4285			      struct rdma_conn_param *conn_param)
4286{
4287	struct ib_cm_sidr_req_param req;
4288	struct ib_cm_id	*id;
4289	void *private_data;
4290	u8 offset;
4291	int ret;
4292
4293	memset(&req, 0, sizeof req);
4294	offset = cma_user_data_offset(id_priv);
4295	if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4296		return -EINVAL;
4297
4298	if (req.private_data_len) {
4299		private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
4300		if (!private_data)
4301			return -ENOMEM;
4302	} else {
4303		private_data = NULL;
4304	}
4305
4306	if (conn_param->private_data && conn_param->private_data_len)
4307		memcpy(private_data + offset, conn_param->private_data,
4308		       conn_param->private_data_len);
4309
4310	if (private_data) {
4311		ret = cma_format_hdr(private_data, id_priv);
4312		if (ret)
4313			goto out;
4314		req.private_data = private_data;
4315	}
4316
4317	id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
4318			     id_priv);
4319	if (IS_ERR(id)) {
4320		ret = PTR_ERR(id);
4321		goto out;
4322	}
4323	id_priv->cm_id.ib = id;
4324
4325	req.path = id_priv->id.route.path_rec;
4326	req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
4327	req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
4328	req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
4329	req.max_cm_retries = CMA_MAX_CM_RETRIES;
4330
4331	trace_cm_send_sidr_req(id_priv);
4332	ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
4333	if (ret) {
4334		ib_destroy_cm_id(id_priv->cm_id.ib);
4335		id_priv->cm_id.ib = NULL;
4336	}
4337out:
4338	kfree(private_data);
4339	return ret;
4340}
4341
4342static int cma_connect_ib(struct rdma_id_private *id_priv,
4343			  struct rdma_conn_param *conn_param)
4344{
4345	struct ib_cm_req_param req;
4346	struct rdma_route *route;
4347	void *private_data;
4348	struct ib_cm_id	*id;
4349	u8 offset;
4350	int ret;
4351
4352	memset(&req, 0, sizeof req);
4353	offset = cma_user_data_offset(id_priv);
4354	if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4355		return -EINVAL;
4356
4357	if (req.private_data_len) {
4358		private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
4359		if (!private_data)
4360			return -ENOMEM;
4361	} else {
4362		private_data = NULL;
4363	}
4364
4365	if (conn_param->private_data && conn_param->private_data_len)
4366		memcpy(private_data + offset, conn_param->private_data,
4367		       conn_param->private_data_len);
4368
4369	id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
4370	if (IS_ERR(id)) {
4371		ret = PTR_ERR(id);
4372		goto out;
4373	}
4374	id_priv->cm_id.ib = id;
4375
4376	route = &id_priv->id.route;
4377	if (private_data) {
4378		ret = cma_format_hdr(private_data, id_priv);
4379		if (ret)
4380			goto out;
4381		req.private_data = private_data;
4382	}
4383
4384	req.primary_path = &route->path_rec[0];
4385	req.primary_path_inbound = route->path_rec_inbound;
4386	req.primary_path_outbound = route->path_rec_outbound;
4387	if (route->num_pri_alt_paths == 2)
4388		req.alternate_path = &route->path_rec[1];
4389
4390	req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
4391	/* Alternate path SGID attribute currently unsupported */
4392	req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
4393	req.qp_num = id_priv->qp_num;
4394	req.qp_type = id_priv->id.qp_type;
4395	req.starting_psn = id_priv->seq_num;
4396	req.responder_resources = conn_param->responder_resources;
4397	req.initiator_depth = conn_param->initiator_depth;
4398	req.flow_control = conn_param->flow_control;
4399	req.retry_count = min_t(u8, 7, conn_param->retry_count);
4400	req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4401	req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4402	req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4403	req.max_cm_retries = CMA_MAX_CM_RETRIES;
4404	req.srq = id_priv->srq ? 1 : 0;
4405	req.ece.vendor_id = id_priv->ece.vendor_id;
4406	req.ece.attr_mod = id_priv->ece.attr_mod;
4407
4408	trace_cm_send_req(id_priv);
4409	ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
4410out:
4411	if (ret && !IS_ERR(id)) {
4412		ib_destroy_cm_id(id);
4413		id_priv->cm_id.ib = NULL;
4414	}
4415
4416	kfree(private_data);
4417	return ret;
4418}
4419
4420static int cma_connect_iw(struct rdma_id_private *id_priv,
4421			  struct rdma_conn_param *conn_param)
4422{
4423	struct iw_cm_id *cm_id;
4424	int ret;
4425	struct iw_cm_conn_param iw_param;
4426
4427	cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
4428	if (IS_ERR(cm_id))
4429		return PTR_ERR(cm_id);
4430
4431	mutex_lock(&id_priv->qp_mutex);
4432	cm_id->tos = id_priv->tos;
4433	cm_id->tos_set = id_priv->tos_set;
4434	mutex_unlock(&id_priv->qp_mutex);
4435
4436	id_priv->cm_id.iw = cm_id;
4437
4438	memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
4439	       rdma_addr_size(cma_src_addr(id_priv)));
4440	memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
4441	       rdma_addr_size(cma_dst_addr(id_priv)));
4442
4443	ret = cma_modify_qp_rtr(id_priv, conn_param);
4444	if (ret)
4445		goto out;
4446
4447	if (conn_param) {
4448		iw_param.ord = conn_param->initiator_depth;
4449		iw_param.ird = conn_param->responder_resources;
4450		iw_param.private_data = conn_param->private_data;
4451		iw_param.private_data_len = conn_param->private_data_len;
4452		iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
4453	} else {
4454		memset(&iw_param, 0, sizeof iw_param);
4455		iw_param.qpn = id_priv->qp_num;
4456	}
4457	ret = iw_cm_connect(cm_id, &iw_param);
4458out:
4459	if (ret) {
4460		iw_destroy_cm_id(cm_id);
4461		id_priv->cm_id.iw = NULL;
4462	}
4463	return ret;
4464}
4465
4466/**
4467 * rdma_connect_locked - Initiate an active connection request.
4468 * @id: Connection identifier to connect.
4469 * @conn_param: Connection information used for connected QPs.
4470 *
4471 * Same as rdma_connect() but can only be called from the
4472 * RDMA_CM_EVENT_ROUTE_RESOLVED handler callback.
4473 */
4474int rdma_connect_locked(struct rdma_cm_id *id,
4475			struct rdma_conn_param *conn_param)
4476{
4477	struct rdma_id_private *id_priv =
4478		container_of(id, struct rdma_id_private, id);
4479	int ret;
4480
4481	lockdep_assert_held(&id_priv->handler_mutex);
4482
4483	if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
4484		return -EINVAL;
4485
4486	if (!id->qp) {
4487		id_priv->qp_num = conn_param->qp_num;
4488		id_priv->srq = conn_param->srq;
4489	}
4490
4491	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4492		if (id->qp_type == IB_QPT_UD)
4493			ret = cma_resolve_ib_udp(id_priv, conn_param);
4494		else
4495			ret = cma_connect_ib(id_priv, conn_param);
4496	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4497		ret = cma_connect_iw(id_priv, conn_param);
4498	} else {
4499		ret = -ENOSYS;
4500	}
4501	if (ret)
4502		goto err_state;
4503	return 0;
4504err_state:
4505	cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
4506	return ret;
4507}
4508EXPORT_SYMBOL(rdma_connect_locked);
4509
4510/**
4511 * rdma_connect - Initiate an active connection request.
4512 * @id: Connection identifier to connect.
4513 * @conn_param: Connection information used for connected QPs.
4514 *
4515 * Users must have resolved a route for the rdma_cm_id to connect with by having
4516 * called rdma_resolve_route before calling this routine.
4517 *
4518 * This call will either connect to a remote QP or obtain remote QP information
4519 * for unconnected rdma_cm_id's.  The actual operation is based on the
4520 * rdma_cm_id's port space.
4521 */
4522int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4523{
4524	struct rdma_id_private *id_priv =
4525		container_of(id, struct rdma_id_private, id);
4526	int ret;
4527
4528	mutex_lock(&id_priv->handler_mutex);
4529	ret = rdma_connect_locked(id, conn_param);
4530	mutex_unlock(&id_priv->handler_mutex);
4531	return ret;
4532}
4533EXPORT_SYMBOL(rdma_connect);
4534
4535/**
4536 * rdma_connect_ece - Initiate an active connection request with ECE data.
4537 * @id: Connection identifier to connect.
4538 * @conn_param: Connection information used for connected QPs.
4539 * @ece: ECE parameters
4540 *
4541 * See rdma_connect() explanation.
4542 */
4543int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4544		     struct rdma_ucm_ece *ece)
4545{
4546	struct rdma_id_private *id_priv =
4547		container_of(id, struct rdma_id_private, id);
4548
4549	id_priv->ece.vendor_id = ece->vendor_id;
4550	id_priv->ece.attr_mod = ece->attr_mod;
4551
4552	return rdma_connect(id, conn_param);
4553}
4554EXPORT_SYMBOL(rdma_connect_ece);
4555
4556static int cma_accept_ib(struct rdma_id_private *id_priv,
4557			 struct rdma_conn_param *conn_param)
4558{
4559	struct ib_cm_rep_param rep;
4560	int ret;
4561
4562	ret = cma_modify_qp_rtr(id_priv, conn_param);
4563	if (ret)
4564		goto out;
4565
4566	ret = cma_modify_qp_rts(id_priv, conn_param);
4567	if (ret)
4568		goto out;
4569
4570	memset(&rep, 0, sizeof rep);
4571	rep.qp_num = id_priv->qp_num;
4572	rep.starting_psn = id_priv->seq_num;
4573	rep.private_data = conn_param->private_data;
4574	rep.private_data_len = conn_param->private_data_len;
4575	rep.responder_resources = conn_param->responder_resources;
4576	rep.initiator_depth = conn_param->initiator_depth;
4577	rep.failover_accepted = 0;
4578	rep.flow_control = conn_param->flow_control;
4579	rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4580	rep.srq = id_priv->srq ? 1 : 0;
4581	rep.ece.vendor_id = id_priv->ece.vendor_id;
4582	rep.ece.attr_mod = id_priv->ece.attr_mod;
4583
4584	trace_cm_send_rep(id_priv);
4585	ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
4586out:
4587	return ret;
4588}
4589
4590static int cma_accept_iw(struct rdma_id_private *id_priv,
4591		  struct rdma_conn_param *conn_param)
4592{
4593	struct iw_cm_conn_param iw_param;
4594	int ret;
4595
4596	if (!conn_param)
4597		return -EINVAL;
4598
4599	ret = cma_modify_qp_rtr(id_priv, conn_param);
4600	if (ret)
4601		return ret;
4602
4603	iw_param.ord = conn_param->initiator_depth;
4604	iw_param.ird = conn_param->responder_resources;
4605	iw_param.private_data = conn_param->private_data;
4606	iw_param.private_data_len = conn_param->private_data_len;
4607	if (id_priv->id.qp)
4608		iw_param.qpn = id_priv->qp_num;
4609	else
4610		iw_param.qpn = conn_param->qp_num;
4611
4612	return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
4613}
4614
4615static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
4616			     enum ib_cm_sidr_status status, u32 qkey,
4617			     const void *private_data, int private_data_len)
4618{
4619	struct ib_cm_sidr_rep_param rep;
4620	int ret;
4621
4622	memset(&rep, 0, sizeof rep);
4623	rep.status = status;
4624	if (status == IB_SIDR_SUCCESS) {
4625		if (qkey)
4626			ret = cma_set_qkey(id_priv, qkey);
4627		else
4628			ret = cma_set_default_qkey(id_priv);
4629		if (ret)
4630			return ret;
4631		rep.qp_num = id_priv->qp_num;
4632		rep.qkey = id_priv->qkey;
4633
4634		rep.ece.vendor_id = id_priv->ece.vendor_id;
4635		rep.ece.attr_mod = id_priv->ece.attr_mod;
4636	}
4637
4638	rep.private_data = private_data;
4639	rep.private_data_len = private_data_len;
4640
4641	trace_cm_send_sidr_rep(id_priv);
4642	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
4643}
4644
4645/**
4646 * rdma_accept - Called to accept a connection request or response.
4647 * @id: Connection identifier associated with the request.
4648 * @conn_param: Information needed to establish the connection.  This must be
4649 *   provided if accepting a connection request.  If accepting a connection
4650 *   response, this parameter must be NULL.
4651 *
4652 * Typically, this routine is only called by the listener to accept a connection
4653 * request.  It must also be called on the active side of a connection if the
4654 * user is performing their own QP transitions.
4655 *
4656 * In the case of error, a reject message is sent to the remote side and the
4657 * state of the qp associated with the id is modified to error, such that any
4658 * previously posted receive buffers would be flushed.
4659 *
4660 * This function is for use by kernel ULPs and must be called from under the
4661 * handler callback.
4662 */
4663int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4664{
4665	struct rdma_id_private *id_priv =
4666		container_of(id, struct rdma_id_private, id);
4667	int ret;
4668
4669	lockdep_assert_held(&id_priv->handler_mutex);
4670
4671	if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4672		return -EINVAL;
4673
4674	if (!id->qp && conn_param) {
4675		id_priv->qp_num = conn_param->qp_num;
4676		id_priv->srq = conn_param->srq;
4677	}
4678
4679	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4680		if (id->qp_type == IB_QPT_UD) {
4681			if (conn_param)
4682				ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4683							conn_param->qkey,
4684							conn_param->private_data,
4685							conn_param->private_data_len);
4686			else
4687				ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4688							0, NULL, 0);
4689		} else {
4690			if (conn_param)
4691				ret = cma_accept_ib(id_priv, conn_param);
4692			else
4693				ret = cma_rep_recv(id_priv);
4694		}
4695	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4696		ret = cma_accept_iw(id_priv, conn_param);
4697	} else {
4698		ret = -ENOSYS;
4699	}
4700	if (ret)
4701		goto reject;
4702
4703	return 0;
4704reject:
4705	cma_modify_qp_err(id_priv);
4706	rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
4707	return ret;
4708}
4709EXPORT_SYMBOL(rdma_accept);
4710
4711int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4712		    struct rdma_ucm_ece *ece)
4713{
4714	struct rdma_id_private *id_priv =
4715		container_of(id, struct rdma_id_private, id);
4716
4717	id_priv->ece.vendor_id = ece->vendor_id;
4718	id_priv->ece.attr_mod = ece->attr_mod;
4719
4720	return rdma_accept(id, conn_param);
4721}
4722EXPORT_SYMBOL(rdma_accept_ece);
4723
4724void rdma_lock_handler(struct rdma_cm_id *id)
4725{
4726	struct rdma_id_private *id_priv =
4727		container_of(id, struct rdma_id_private, id);
4728
4729	mutex_lock(&id_priv->handler_mutex);
4730}
4731EXPORT_SYMBOL(rdma_lock_handler);
4732
4733void rdma_unlock_handler(struct rdma_cm_id *id)
4734{
4735	struct rdma_id_private *id_priv =
4736		container_of(id, struct rdma_id_private, id);
4737
4738	mutex_unlock(&id_priv->handler_mutex);
4739}
4740EXPORT_SYMBOL(rdma_unlock_handler);
4741
4742int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
4743{
4744	struct rdma_id_private *id_priv;
4745	int ret;
4746
4747	id_priv = container_of(id, struct rdma_id_private, id);
4748	if (!id_priv->cm_id.ib)
4749		return -EINVAL;
4750
4751	switch (id->device->node_type) {
4752	case RDMA_NODE_IB_CA:
4753		ret = ib_cm_notify(id_priv->cm_id.ib, event);
4754		break;
4755	default:
4756		ret = 0;
4757		break;
4758	}
4759	return ret;
4760}
4761EXPORT_SYMBOL(rdma_notify);
4762
4763int rdma_reject(struct rdma_cm_id *id, const void *private_data,
4764		u8 private_data_len, u8 reason)
4765{
4766	struct rdma_id_private *id_priv;
4767	int ret;
4768
4769	id_priv = container_of(id, struct rdma_id_private, id);
4770	if (!id_priv->cm_id.ib)
4771		return -EINVAL;
4772
4773	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4774		if (id->qp_type == IB_QPT_UD) {
4775			ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
4776						private_data, private_data_len);
4777		} else {
4778			trace_cm_send_rej(id_priv);
4779			ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0,
4780					     private_data, private_data_len);
4781		}
4782	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4783		ret = iw_cm_reject(id_priv->cm_id.iw,
4784				   private_data, private_data_len);
4785	} else {
4786		ret = -ENOSYS;
4787	}
4788
4789	return ret;
4790}
4791EXPORT_SYMBOL(rdma_reject);
4792
4793int rdma_disconnect(struct rdma_cm_id *id)
4794{
4795	struct rdma_id_private *id_priv;
4796	int ret;
4797
4798	id_priv = container_of(id, struct rdma_id_private, id);
4799	if (!id_priv->cm_id.ib)
4800		return -EINVAL;
4801
4802	if (rdma_cap_ib_cm(id->device, id->port_num)) {
4803		ret = cma_modify_qp_err(id_priv);
4804		if (ret)
4805			goto out;
4806		/* Initiate or respond to a disconnect. */
4807		trace_cm_disconnect(id_priv);
4808		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
4809			if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
4810				trace_cm_sent_drep(id_priv);
4811		} else {
4812			trace_cm_sent_dreq(id_priv);
4813		}
4814	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4815		ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
4816	} else
4817		ret = -EINVAL;
4818
4819out:
4820	return ret;
4821}
4822EXPORT_SYMBOL(rdma_disconnect);
4823
4824static void cma_make_mc_event(int status, struct rdma_id_private *id_priv,
4825			      struct ib_sa_multicast *multicast,
4826			      struct rdma_cm_event *event,
4827			      struct cma_multicast *mc)
4828{
4829	struct rdma_dev_addr *dev_addr;
4830	enum ib_gid_type gid_type;
4831	struct net_device *ndev;
4832
4833	if (status)
4834		pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",
4835				     status);
4836
4837	event->status = status;
4838	event->param.ud.private_data = mc->context;
4839	if (status) {
4840		event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4841		return;
4842	}
4843
4844	dev_addr = &id_priv->id.route.addr.dev_addr;
4845	ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4846	gid_type =
4847		id_priv->cma_dev
4848			->default_gid_type[id_priv->id.port_num -
4849					   rdma_start_port(
4850						   id_priv->cma_dev->device)];
4851
4852	event->event = RDMA_CM_EVENT_MULTICAST_JOIN;
4853	if (ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num,
4854				     &multicast->rec, ndev, gid_type,
4855				     &event->param.ud.ah_attr)) {
4856		event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4857		goto out;
4858	}
4859
4860	event->param.ud.qp_num = 0xFFFFFF;
4861	event->param.ud.qkey = id_priv->qkey;
4862
4863out:
4864	dev_put(ndev);
4865}
4866
4867static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
4868{
4869	struct cma_multicast *mc = multicast->context;
4870	struct rdma_id_private *id_priv = mc->id_priv;
4871	struct rdma_cm_event event = {};
4872	int ret = 0;
4873
4874	mutex_lock(&id_priv->handler_mutex);
4875	if (READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL ||
4876	    READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING)
4877		goto out;
4878
4879	ret = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
4880	if (!ret) {
4881		cma_make_mc_event(status, id_priv, multicast, &event, mc);
4882		ret = cma_cm_event_handler(id_priv, &event);
4883	}
4884	rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4885	WARN_ON(ret);
4886
4887out:
4888	mutex_unlock(&id_priv->handler_mutex);
4889	return 0;
4890}
4891
4892static void cma_set_mgid(struct rdma_id_private *id_priv,
4893			 struct sockaddr *addr, union ib_gid *mgid)
4894{
4895	unsigned char mc_map[MAX_ADDR_LEN];
4896	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4897	struct sockaddr_in *sin = (struct sockaddr_in *) addr;
4898	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
4899
4900	if (cma_any_addr(addr)) {
4901		memset(mgid, 0, sizeof *mgid);
4902	} else if ((addr->sa_family == AF_INET6) &&
4903		   ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
4904								 0xFF10A01B)) {
4905		/* IPv6 address is an SA assigned MGID. */
4906		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4907	} else if (addr->sa_family == AF_IB) {
4908		memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
4909	} else if (addr->sa_family == AF_INET6) {
4910		ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
4911		if (id_priv->id.ps == RDMA_PS_UDP)
4912			mc_map[7] = 0x01;	/* Use RDMA CM signature */
4913		*mgid = *(union ib_gid *) (mc_map + 4);
4914	} else {
4915		ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
4916		if (id_priv->id.ps == RDMA_PS_UDP)
4917			mc_map[7] = 0x01;	/* Use RDMA CM signature */
4918		*mgid = *(union ib_gid *) (mc_map + 4);
4919	}
4920}
4921
4922static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
4923				 struct cma_multicast *mc)
4924{
4925	struct ib_sa_mcmember_rec rec;
4926	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4927	ib_sa_comp_mask comp_mask;
4928	int ret;
4929
4930	ib_addr_get_mgid(dev_addr, &rec.mgid);
4931	ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
4932				     &rec.mgid, &rec);
4933	if (ret)
4934		return ret;
4935
4936	if (!id_priv->qkey) {
4937		ret = cma_set_default_qkey(id_priv);
4938		if (ret)
4939			return ret;
4940	}
4941
4942	cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
4943	rec.qkey = cpu_to_be32(id_priv->qkey);
4944	rdma_addr_get_sgid(dev_addr, &rec.port_gid);
4945	rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
4946	rec.join_state = mc->join_state;
4947
4948	comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
4949		    IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
4950		    IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
4951		    IB_SA_MCMEMBER_REC_FLOW_LABEL |
4952		    IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
4953
4954	if (id_priv->id.ps == RDMA_PS_IPOIB)
4955		comp_mask |= IB_SA_MCMEMBER_REC_RATE |
4956			     IB_SA_MCMEMBER_REC_RATE_SELECTOR |
4957			     IB_SA_MCMEMBER_REC_MTU_SELECTOR |
4958			     IB_SA_MCMEMBER_REC_MTU |
4959			     IB_SA_MCMEMBER_REC_HOP_LIMIT;
4960
4961	mc->sa_mc = ib_sa_join_multicast(&sa_client, id_priv->id.device,
4962					 id_priv->id.port_num, &rec, comp_mask,
4963					 GFP_KERNEL, cma_ib_mc_handler, mc);
4964	return PTR_ERR_OR_ZERO(mc->sa_mc);
4965}
4966
4967static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
4968			      enum ib_gid_type gid_type)
4969{
4970	struct sockaddr_in *sin = (struct sockaddr_in *)addr;
4971	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
4972
4973	if (cma_any_addr(addr)) {
4974		memset(mgid, 0, sizeof *mgid);
4975	} else if (addr->sa_family == AF_INET6) {
4976		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4977	} else {
4978		mgid->raw[0] =
4979			(gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
4980		mgid->raw[1] =
4981			(gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
4982		mgid->raw[2] = 0;
4983		mgid->raw[3] = 0;
4984		mgid->raw[4] = 0;
4985		mgid->raw[5] = 0;
4986		mgid->raw[6] = 0;
4987		mgid->raw[7] = 0;
4988		mgid->raw[8] = 0;
4989		mgid->raw[9] = 0;
4990		mgid->raw[10] = 0xff;
4991		mgid->raw[11] = 0xff;
4992		*(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
4993	}
4994}
4995
4996static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
4997				   struct cma_multicast *mc)
4998{
4999	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
5000	int err = 0;
5001	struct sockaddr *addr = (struct sockaddr *)&mc->addr;
5002	struct net_device *ndev = NULL;
5003	struct ib_sa_multicast ib = {};
5004	enum ib_gid_type gid_type;
5005	bool send_only;
5006
5007	send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
5008
5009	if (cma_zero_addr(addr))
5010		return -EINVAL;
5011
5012	gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
5013		   rdma_start_port(id_priv->cma_dev->device)];
5014	cma_iboe_set_mgid(addr, &ib.rec.mgid, gid_type);
5015
5016	ib.rec.pkey = cpu_to_be16(0xffff);
5017	if (dev_addr->bound_dev_if)
5018		ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
5019	if (!ndev)
5020		return -ENODEV;
5021
5022	ib.rec.rate = IB_RATE_PORT_CURRENT;
5023	ib.rec.hop_limit = 1;
5024	ib.rec.mtu = iboe_get_mtu(ndev->mtu);
5025
5026	if (addr->sa_family == AF_INET) {
5027		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
5028			ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
5029			if (!send_only) {
5030				err = cma_igmp_send(ndev, &ib.rec.mgid,
5031						    true);
5032			}
5033		}
5034	} else {
5035		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
5036			err = -ENOTSUPP;
5037	}
5038	dev_put(ndev);
5039	if (err || !ib.rec.mtu)
5040		return err ?: -EINVAL;
5041
5042	if (!id_priv->qkey)
5043		cma_set_default_qkey(id_priv);
5044
5045	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
5046		    &ib.rec.port_gid);
5047	INIT_WORK(&mc->iboe_join.work, cma_iboe_join_work_handler);
5048	cma_make_mc_event(0, id_priv, &ib, &mc->iboe_join.event, mc);
5049	queue_work(cma_wq, &mc->iboe_join.work);
5050	return 0;
5051}
5052
5053int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
5054			u8 join_state, void *context)
5055{
5056	struct rdma_id_private *id_priv =
5057		container_of(id, struct rdma_id_private, id);
5058	struct cma_multicast *mc;
5059	int ret;
5060
5061	/* Not supported for kernel QPs */
5062	if (WARN_ON(id->qp))
5063		return -EINVAL;
5064
5065	/* ULP is calling this wrong. */
5066	if (!id->device || (READ_ONCE(id_priv->state) != RDMA_CM_ADDR_BOUND &&
5067			    READ_ONCE(id_priv->state) != RDMA_CM_ADDR_RESOLVED))
5068		return -EINVAL;
5069
5070	if (id_priv->id.qp_type != IB_QPT_UD)
5071		return -EINVAL;
5072
5073	mc = kzalloc(sizeof(*mc), GFP_KERNEL);
5074	if (!mc)
5075		return -ENOMEM;
5076
5077	memcpy(&mc->addr, addr, rdma_addr_size(addr));
5078	mc->context = context;
5079	mc->id_priv = id_priv;
5080	mc->join_state = join_state;
5081
5082	if (rdma_protocol_roce(id->device, id->port_num)) {
5083		ret = cma_iboe_join_multicast(id_priv, mc);
5084		if (ret)
5085			goto out_err;
5086	} else if (rdma_cap_ib_mcast(id->device, id->port_num)) {
5087		ret = cma_join_ib_multicast(id_priv, mc);
5088		if (ret)
5089			goto out_err;
5090	} else {
5091		ret = -ENOSYS;
5092		goto out_err;
5093	}
5094
5095	spin_lock(&id_priv->lock);
5096	list_add(&mc->list, &id_priv->mc_list);
5097	spin_unlock(&id_priv->lock);
5098
5099	return 0;
5100out_err:
5101	kfree(mc);
5102	return ret;
5103}
5104EXPORT_SYMBOL(rdma_join_multicast);
5105
5106void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
5107{
5108	struct rdma_id_private *id_priv;
5109	struct cma_multicast *mc;
5110
5111	id_priv = container_of(id, struct rdma_id_private, id);
5112	spin_lock_irq(&id_priv->lock);
5113	list_for_each_entry(mc, &id_priv->mc_list, list) {
5114		if (memcmp(&mc->addr, addr, rdma_addr_size(addr)) != 0)
5115			continue;
5116		list_del(&mc->list);
5117		spin_unlock_irq(&id_priv->lock);
5118
5119		WARN_ON(id_priv->cma_dev->device != id->device);
5120		destroy_mc(id_priv, mc);
5121		return;
5122	}
5123	spin_unlock_irq(&id_priv->lock);
5124}
5125EXPORT_SYMBOL(rdma_leave_multicast);
5126
5127static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
5128{
5129	struct rdma_dev_addr *dev_addr;
5130	struct cma_work *work;
5131
5132	dev_addr = &id_priv->id.route.addr.dev_addr;
5133
5134	if ((dev_addr->bound_dev_if == ndev->ifindex) &&
5135	    (net_eq(dev_net(ndev), dev_addr->net)) &&
5136	    memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
5137		pr_info("RDMA CM addr change for ndev %s used by id %p\n",
5138			ndev->name, &id_priv->id);
5139		work = kzalloc(sizeof *work, GFP_KERNEL);
5140		if (!work)
5141			return -ENOMEM;
5142
5143		INIT_WORK(&work->work, cma_work_handler);
5144		work->id = id_priv;
5145		work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
5146		cma_id_get(id_priv);
5147		queue_work(cma_wq, &work->work);
5148	}
5149
5150	return 0;
5151}
5152
5153static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
5154			       void *ptr)
5155{
5156	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
5157	struct cma_device *cma_dev;
5158	struct rdma_id_private *id_priv;
5159	int ret = NOTIFY_DONE;
5160
5161	if (event != NETDEV_BONDING_FAILOVER)
5162		return NOTIFY_DONE;
5163
5164	if (!netif_is_bond_master(ndev))
5165		return NOTIFY_DONE;
5166
5167	mutex_lock(&lock);
5168	list_for_each_entry(cma_dev, &dev_list, list)
5169		list_for_each_entry(id_priv, &cma_dev->id_list, device_item) {
5170			ret = cma_netdev_change(ndev, id_priv);
5171			if (ret)
5172				goto out;
5173		}
5174
5175out:
5176	mutex_unlock(&lock);
5177	return ret;
5178}
5179
5180static void cma_netevent_work_handler(struct work_struct *_work)
5181{
5182	struct rdma_id_private *id_priv =
5183		container_of(_work, struct rdma_id_private, id.net_work);
5184	struct rdma_cm_event event = {};
5185
5186	mutex_lock(&id_priv->handler_mutex);
5187
5188	if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
5189	    READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
5190		goto out_unlock;
5191
5192	event.event = RDMA_CM_EVENT_UNREACHABLE;
5193	event.status = -ETIMEDOUT;
5194
5195	if (cma_cm_event_handler(id_priv, &event)) {
5196		__acquire(&id_priv->handler_mutex);
5197		id_priv->cm_id.ib = NULL;
5198		cma_id_put(id_priv);
5199		destroy_id_handler_unlock(id_priv);
5200		return;
5201	}
5202
5203out_unlock:
5204	mutex_unlock(&id_priv->handler_mutex);
5205	cma_id_put(id_priv);
5206}
5207
5208static int cma_netevent_callback(struct notifier_block *self,
5209				 unsigned long event, void *ctx)
5210{
5211	struct id_table_entry *ips_node = NULL;
5212	struct rdma_id_private *current_id;
5213	struct neighbour *neigh = ctx;
5214	unsigned long flags;
5215
5216	if (event != NETEVENT_NEIGH_UPDATE)
5217		return NOTIFY_DONE;
5218
5219	spin_lock_irqsave(&id_table_lock, flags);
5220	if (neigh->tbl->family == AF_INET6) {
5221		struct sockaddr_in6 neigh_sock_6;
5222
5223		neigh_sock_6.sin6_family = AF_INET6;
5224		neigh_sock_6.sin6_addr = *(struct in6_addr *)neigh->primary_key;
5225		ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex,
5226					     (struct sockaddr *)&neigh_sock_6);
5227	} else if (neigh->tbl->family == AF_INET) {
5228		struct sockaddr_in neigh_sock_4;
5229
5230		neigh_sock_4.sin_family = AF_INET;
5231		neigh_sock_4.sin_addr.s_addr = *(__be32 *)(neigh->primary_key);
5232		ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex,
5233					     (struct sockaddr *)&neigh_sock_4);
5234	} else
5235		goto out;
5236
5237	if (!ips_node)
5238		goto out;
5239
5240	list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) {
5241		if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr,
5242			   neigh->ha, ETH_ALEN))
5243			continue;
5244		cma_id_get(current_id);
5245		if (!queue_work(cma_wq, &current_id->id.net_work))
5246			cma_id_put(current_id);
5247	}
5248out:
5249	spin_unlock_irqrestore(&id_table_lock, flags);
5250	return NOTIFY_DONE;
5251}
5252
5253static struct notifier_block cma_nb = {
5254	.notifier_call = cma_netdev_callback
5255};
5256
5257static struct notifier_block cma_netevent_cb = {
5258	.notifier_call = cma_netevent_callback
5259};
5260
5261static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
5262{
5263	struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
5264	enum rdma_cm_state state;
5265	unsigned long flags;
5266
5267	mutex_lock(&id_priv->handler_mutex);
5268	/* Record that we want to remove the device */
5269	spin_lock_irqsave(&id_priv->lock, flags);
5270	state = id_priv->state;
5271	if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {
5272		spin_unlock_irqrestore(&id_priv->lock, flags);
5273		mutex_unlock(&id_priv->handler_mutex);
5274		cma_id_put(id_priv);
5275		return;
5276	}
5277	id_priv->state = RDMA_CM_DEVICE_REMOVAL;
5278	spin_unlock_irqrestore(&id_priv->lock, flags);
5279
5280	if (cma_cm_event_handler(id_priv, &event)) {
5281		/*
5282		 * At this point the ULP promises it won't call
5283		 * rdma_destroy_id() concurrently
5284		 */
5285		cma_id_put(id_priv);
5286		mutex_unlock(&id_priv->handler_mutex);
5287		trace_cm_id_destroy(id_priv);
5288		_destroy_id(id_priv, state);
5289		return;
5290	}
5291	mutex_unlock(&id_priv->handler_mutex);
5292
5293	/*
5294	 * If this races with destroy then the thread that first assigns state
5295	 * to a destroying does the cancel.
5296	 */
5297	cma_cancel_operation(id_priv, state);
5298	cma_id_put(id_priv);
5299}
5300
5301static void cma_process_remove(struct cma_device *cma_dev)
5302{
5303	mutex_lock(&lock);
5304	while (!list_empty(&cma_dev->id_list)) {
5305		struct rdma_id_private *id_priv = list_first_entry(
5306			&cma_dev->id_list, struct rdma_id_private, device_item);
5307
5308		list_del_init(&id_priv->listen_item);
5309		list_del_init(&id_priv->device_item);
5310		cma_id_get(id_priv);
5311		mutex_unlock(&lock);
5312
5313		cma_send_device_removal_put(id_priv);
5314
5315		mutex_lock(&lock);
5316	}
5317	mutex_unlock(&lock);
5318
5319	cma_dev_put(cma_dev);
5320	wait_for_completion(&cma_dev->comp);
5321}
5322
5323static bool cma_supported(struct ib_device *device)
5324{
5325	u32 i;
5326
5327	rdma_for_each_port(device, i) {
5328		if (rdma_cap_ib_cm(device, i) || rdma_cap_iw_cm(device, i))
5329			return true;
5330	}
5331	return false;
5332}
5333
5334static int cma_add_one(struct ib_device *device)
5335{
5336	struct rdma_id_private *to_destroy;
5337	struct cma_device *cma_dev;
5338	struct rdma_id_private *id_priv;
5339	unsigned long supported_gids = 0;
5340	int ret;
5341	u32 i;
5342
5343	if (!cma_supported(device))
5344		return -EOPNOTSUPP;
5345
5346	cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL);
5347	if (!cma_dev)
5348		return -ENOMEM;
5349
5350	cma_dev->device = device;
5351	cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
5352					    sizeof(*cma_dev->default_gid_type),
5353					    GFP_KERNEL);
5354	if (!cma_dev->default_gid_type) {
5355		ret = -ENOMEM;
5356		goto free_cma_dev;
5357	}
5358
5359	cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
5360					    sizeof(*cma_dev->default_roce_tos),
5361					    GFP_KERNEL);
5362	if (!cma_dev->default_roce_tos) {
5363		ret = -ENOMEM;
5364		goto free_gid_type;
5365	}
5366
5367	rdma_for_each_port (device, i) {
5368		supported_gids = roce_gid_type_mask_support(device, i);
5369		WARN_ON(!supported_gids);
5370		if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
5371			cma_dev->default_gid_type[i - rdma_start_port(device)] =
5372				CMA_PREFERRED_ROCE_GID_TYPE;
5373		else
5374			cma_dev->default_gid_type[i - rdma_start_port(device)] =
5375				find_first_bit(&supported_gids, BITS_PER_LONG);
5376		cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
5377	}
5378
5379	init_completion(&cma_dev->comp);
5380	refcount_set(&cma_dev->refcount, 1);
5381	INIT_LIST_HEAD(&cma_dev->id_list);
5382	ib_set_client_data(device, &cma_client, cma_dev);
5383
5384	mutex_lock(&lock);
5385	list_add_tail(&cma_dev->list, &dev_list);
5386	list_for_each_entry(id_priv, &listen_any_list, listen_any_item) {
5387		ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
5388		if (ret)
5389			goto free_listen;
5390	}
5391	mutex_unlock(&lock);
5392
5393	trace_cm_add_one(device);
5394	return 0;
5395
5396free_listen:
5397	list_del(&cma_dev->list);
5398	mutex_unlock(&lock);
5399
5400	/* cma_process_remove() will delete to_destroy */
5401	cma_process_remove(cma_dev);
5402	kfree(cma_dev->default_roce_tos);
5403free_gid_type:
5404	kfree(cma_dev->default_gid_type);
5405
5406free_cma_dev:
5407	kfree(cma_dev);
5408	return ret;
5409}
5410
5411static void cma_remove_one(struct ib_device *device, void *client_data)
5412{
5413	struct cma_device *cma_dev = client_data;
5414
5415	trace_cm_remove_one(device);
5416
5417	mutex_lock(&lock);
5418	list_del(&cma_dev->list);
5419	mutex_unlock(&lock);
5420
5421	cma_process_remove(cma_dev);
5422	kfree(cma_dev->default_roce_tos);
5423	kfree(cma_dev->default_gid_type);
5424	kfree(cma_dev);
5425}
5426
5427static int cma_init_net(struct net *net)
5428{
5429	struct cma_pernet *pernet = cma_pernet(net);
5430
5431	xa_init(&pernet->tcp_ps);
5432	xa_init(&pernet->udp_ps);
5433	xa_init(&pernet->ipoib_ps);
5434	xa_init(&pernet->ib_ps);
5435
5436	return 0;
5437}
5438
5439static void cma_exit_net(struct net *net)
5440{
5441	struct cma_pernet *pernet = cma_pernet(net);
5442
5443	WARN_ON(!xa_empty(&pernet->tcp_ps));
5444	WARN_ON(!xa_empty(&pernet->udp_ps));
5445	WARN_ON(!xa_empty(&pernet->ipoib_ps));
5446	WARN_ON(!xa_empty(&pernet->ib_ps));
5447}
5448
5449static struct pernet_operations cma_pernet_operations = {
5450	.init = cma_init_net,
5451	.exit = cma_exit_net,
5452	.id = &cma_pernet_id,
5453	.size = sizeof(struct cma_pernet),
5454};
5455
5456static int __init cma_init(void)
5457{
5458	int ret;
5459
5460	/*
5461	 * There is a rare lock ordering dependency in cma_netdev_callback()
5462	 * that only happens when bonding is enabled. Teach lockdep that rtnl
5463	 * must never be nested under lock so it can find these without having
5464	 * to test with bonding.
5465	 */
5466	if (IS_ENABLED(CONFIG_LOCKDEP)) {
5467		rtnl_lock();
5468		mutex_lock(&lock);
5469		mutex_unlock(&lock);
5470		rtnl_unlock();
5471	}
5472
5473	cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
5474	if (!cma_wq)
5475		return -ENOMEM;
5476
5477	ret = register_pernet_subsys(&cma_pernet_operations);
5478	if (ret)
5479		goto err_wq;
5480
5481	ib_sa_register_client(&sa_client);
5482	register_netdevice_notifier(&cma_nb);
5483	register_netevent_notifier(&cma_netevent_cb);
5484
5485	ret = ib_register_client(&cma_client);
5486	if (ret)
5487		goto err;
5488
5489	ret = cma_configfs_init();
5490	if (ret)
5491		goto err_ib;
5492
5493	return 0;
5494
5495err_ib:
5496	ib_unregister_client(&cma_client);
5497err:
5498	unregister_netevent_notifier(&cma_netevent_cb);
5499	unregister_netdevice_notifier(&cma_nb);
5500	ib_sa_unregister_client(&sa_client);
5501	unregister_pernet_subsys(&cma_pernet_operations);
5502err_wq:
5503	destroy_workqueue(cma_wq);
5504	return ret;
5505}
5506
5507static void __exit cma_cleanup(void)
5508{
5509	cma_configfs_exit();
5510	ib_unregister_client(&cma_client);
5511	unregister_netevent_notifier(&cma_netevent_cb);
5512	unregister_netdevice_notifier(&cma_nb);
5513	ib_sa_unregister_client(&sa_client);
5514	unregister_pernet_subsys(&cma_pernet_operations);
5515	destroy_workqueue(cma_wq);
5516}
5517
5518module_init(cma_init);
5519module_exit(cma_cleanup);
5520
5521static void cma_query_ib_service_handler(int status,
5522					 struct sa_service_rec *recs,
5523					 unsigned int num_recs, void *context)
5524{
5525	struct cma_work *work = context;
5526	struct rdma_id_private *id_priv = work->id;
5527	struct sockaddr_ib *addr;
5528
5529	if (status)
5530		goto fail;
5531
5532	if (!num_recs) {
5533		status = -ENOENT;
5534		goto fail;
5535	}
5536
5537	if (id_priv->id.route.service_recs) {
5538		status = -EALREADY;
5539		goto fail;
5540	}
5541
5542	id_priv->id.route.service_recs =
5543		kmalloc_array(num_recs, sizeof(*recs), GFP_KERNEL);
5544	if (!id_priv->id.route.service_recs) {
5545		status = -ENOMEM;
5546		goto fail;
5547	}
5548
5549	id_priv->id.route.num_service_recs = num_recs;
5550	memcpy(id_priv->id.route.service_recs, recs, sizeof(*recs) * num_recs);
5551
5552	addr = (struct sockaddr_ib *)&id_priv->id.route.addr.dst_addr;
5553	addr->sib_family = AF_IB;
5554	addr->sib_addr = *(struct ib_addr *)&recs->gid;
5555	addr->sib_pkey = recs->pkey;
5556	addr->sib_sid = recs->id;
5557	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr,
5558			   (union ib_gid *)&addr->sib_addr);
5559	ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr,
5560			 ntohs(addr->sib_pkey));
5561
5562	queue_work(cma_wq, &work->work);
5563	return;
5564
5565fail:
5566	work->old_state = RDMA_CM_ADDRINFO_QUERY;
5567	work->new_state = RDMA_CM_ADDR_BOUND;
5568	work->event.event = RDMA_CM_EVENT_ADDRINFO_ERROR;
5569	work->event.status = status;
5570	pr_debug_ratelimited(
5571		"RDMA CM: SERVICE_ERROR: failed to query service record. status %d\n",
5572		status);
5573	queue_work(cma_wq, &work->work);
5574}
5575
5576static int cma_resolve_ib_service(struct rdma_id_private *id_priv,
5577				  struct rdma_ucm_ib_service *ibs)
5578{
5579	struct sa_service_rec sr = {};
5580	ib_sa_comp_mask mask = 0;
5581	struct cma_work *work;
5582
5583	work = kzalloc(sizeof(*work), GFP_KERNEL);
5584	if (!work)
5585		return -ENOMEM;
5586
5587	cma_id_get(id_priv);
5588
5589	work->id = id_priv;
5590	INIT_WORK(&work->work, cma_work_handler);
5591	work->old_state = RDMA_CM_ADDRINFO_QUERY;
5592	work->new_state = RDMA_CM_ADDRINFO_RESOLVED;
5593	work->event.event = RDMA_CM_EVENT_ADDRINFO_RESOLVED;
5594
5595	if (ibs->flags & RDMA_USER_CM_IB_SERVICE_FLAG_ID) {
5596		sr.id = cpu_to_be64(ibs->service_id);
5597		mask |= IB_SA_SERVICE_REC_SERVICE_ID;
5598	}
5599	if (ibs->flags & RDMA_USER_CM_IB_SERVICE_FLAG_NAME) {
5600		strscpy(sr.name, ibs->service_name, sizeof(sr.name));
5601		mask |= IB_SA_SERVICE_REC_SERVICE_NAME;
5602	}
5603
5604	id_priv->query_id = ib_sa_service_rec_get(&sa_client,
5605						  id_priv->id.device,
5606						  id_priv->id.port_num,
5607						  &sr, mask,
5608						  2000, GFP_KERNEL,
5609						  cma_query_ib_service_handler,
5610						  work, &id_priv->query);
5611
5612	if (id_priv->query_id < 0) {
5613		cma_id_put(id_priv);
5614		kfree(work);
5615		return id_priv->query_id;
5616	}
5617
5618	return 0;
5619}
5620
5621int rdma_resolve_ib_service(struct rdma_cm_id *id,
5622			    struct rdma_ucm_ib_service *ibs)
5623{
5624	struct rdma_id_private *id_priv;
5625	int ret;
5626
5627	id_priv = container_of(id, struct rdma_id_private, id);
5628	if (!id_priv->cma_dev ||
5629	    !cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDRINFO_QUERY))
5630		return -EINVAL;
5631
5632	if (rdma_cap_ib_sa(id->device, id->port_num))
5633		ret = cma_resolve_ib_service(id_priv, ibs);
5634	else
5635		ret = -EOPNOTSUPP;
5636
5637	if (ret)
5638		goto err;
5639
5640	return 0;
5641err:
5642	cma_comp_exch(id_priv, RDMA_CM_ADDRINFO_QUERY, RDMA_CM_ADDR_BOUND);
5643	return ret;
5644}
5645EXPORT_SYMBOL(rdma_resolve_ib_service);