device.c

   1/*
   2 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenIB.org BSD license below:
  10 *
  11 *     Redistribution and use in source and binary forms, with or
  12 *     without modification, are permitted provided that the following
  13 *     conditions are met:
  14 *
  15 *      - Redistributions of source code must retain the above
  16 *        copyright notice, this list of conditions and the following
  17 *        disclaimer.
  18 *
  19 *      - Redistributions in binary form must reproduce the above
  20 *        copyright notice, this list of conditions and the following
  21 *        disclaimer in the documentation and/or other materials
  22 *        provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 */
  33
  34#include <linux/module.h>
  35#include <linux/string.h>
  36#include <linux/errno.h>
  37#include <linux/kernel.h>
  38#include <linux/slab.h>
  39#include <linux/init.h>
  40#include <linux/netdevice.h>
  41#include <net/net_namespace.h>
  42#include <linux/security.h>
  43#include <linux/notifier.h>
  44#include <linux/hashtable.h>
  45#include <rdma/rdma_netlink.h>
  46#include <rdma/ib_addr.h>
  47#include <rdma/ib_cache.h>
  48#include <rdma/rdma_counter.h>
  49
  50#include "core_priv.h"
  51#include "restrack.h"
  52
  53MODULE_AUTHOR("Roland Dreier");
  54MODULE_DESCRIPTION("core kernel InfiniBand API");
  55MODULE_LICENSE("Dual BSD/GPL");
  56
  57struct workqueue_struct *ib_comp_wq;
  58struct workqueue_struct *ib_comp_unbound_wq;
  59struct workqueue_struct *ib_wq;
  60EXPORT_SYMBOL_GPL(ib_wq);
  61static struct workqueue_struct *ib_unreg_wq;
  62
  63/*
  64 * Each of the three rwsem locks (devices, clients, client_data) protects the
  65 * xarray of the same name. Specifically it allows the caller to assert that
  66 * the MARK will/will not be changing under the lock, and for devices and
  67 * clients, that the value in the xarray is still a valid pointer. Change of
  68 * the MARK is linked to the object state, so holding the lock and testing the
  69 * MARK also asserts that the contained object is in a certain state.
  70 *
  71 * This is used to build a two stage register/unregister flow where objects
  72 * can continue to be in the xarray even though they are still in progress to
  73 * register/unregister.
  74 *
  75 * The xarray itself provides additional locking, and restartable iteration,
  76 * which is also relied on.
  77 *
  78 * Locks should not be nested, with the exception of client_data, which is
  79 * allowed to nest under the read side of the other two locks.
  80 *
  81 * The devices_rwsem also protects the device name list, any change or
  82 * assignment of device name must also hold the write side to guarantee unique
  83 * names.
  84 */
  85
  86/*
  87 * devices contains devices that have had their names assigned. The
  88 * devices may not be registered. Users that care about the registration
  89 * status need to call ib_device_try_get() on the device to ensure it is
  90 * registered, and keep it registered, for the required duration.
  91 *
  92 */
  93static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC);
  94static DECLARE_RWSEM(devices_rwsem);
  95#define DEVICE_REGISTERED XA_MARK_1
  96#define DEVICE_GID_UPDATES XA_MARK_2
  97
  98static u32 highest_client_id;
  99#define CLIENT_REGISTERED XA_MARK_1
 100static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC);
 101static DECLARE_RWSEM(clients_rwsem);
 102
 103static void ib_client_put(struct ib_client *client)
 104{
 105	if (refcount_dec_and_test(&client->uses))
 106		complete(&client->uses_zero);
 107}
 108
 109/*
 110 * If client_data is registered then the corresponding client must also still
 111 * be registered.
 112 */
 113#define CLIENT_DATA_REGISTERED XA_MARK_1
 114
 115unsigned int rdma_dev_net_id;
 116
 117/*
 118 * A list of net namespaces is maintained in an xarray. This is necessary
 119 * because we can't get the locking right using the existing net ns list. We
 120 * would require a init_net callback after the list is updated.
 121 */
 122static DEFINE_XARRAY_FLAGS(rdma_nets, XA_FLAGS_ALLOC);
 123/*
 124 * rwsem to protect accessing the rdma_nets xarray entries.
 125 */
 126static DECLARE_RWSEM(rdma_nets_rwsem);
 127
 128bool ib_devices_shared_netns = true;
 129module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444);
 130MODULE_PARM_DESC(netns_mode,
 131		 "Share device among net namespaces; default=1 (shared)");
 132/**
 133 * rdma_dev_access_netns() - Return whether an rdma device can be accessed
 134 *			     from a specified net namespace or not.
 135 * @dev:	Pointer to rdma device which needs to be checked
 136 * @net:	Pointer to net namesapce for which access to be checked
 137 *
 138 * When the rdma device is in shared mode, it ignores the net namespace.
 139 * When the rdma device is exclusive to a net namespace, rdma device net
 140 * namespace is checked against the specified one.
 141 */
 142bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net)
 143{
 144	return (ib_devices_shared_netns ||
 145		net_eq(read_pnet(&dev->coredev.rdma_net), net));
 146}
 147EXPORT_SYMBOL(rdma_dev_access_netns);
 148
 149/**
 150 * rdma_dev_has_raw_cap() - Returns whether a specified rdma device has
 151 *			    CAP_NET_RAW capability or not.
 152 *
 153 * @dev:	Pointer to rdma device whose capability to be checked
 154 *
 155 * Returns true if a rdma device's owning user namespace has CAP_NET_RAW
 156 * capability, otherwise false. When rdma subsystem is in legacy shared network,
 157 * namespace mode, the default net namespace is considered.
 158 */
 159bool rdma_dev_has_raw_cap(const struct ib_device *dev)
 160{
 161	const struct net *net;
 162
 163	/* Network namespace is the resource whose user namespace
 164	 * to be considered. When in shared mode, there is no reliable
 165	 * network namespace resource, so consider the default net namespace.
 166	 */
 167	if (ib_devices_shared_netns)
 168		net = &init_net;
 169	else
 170		net = read_pnet(&dev->coredev.rdma_net);
 171
 172	return ns_capable(net->user_ns, CAP_NET_RAW);
 173}
 174EXPORT_SYMBOL(rdma_dev_has_raw_cap);
 175
 176/*
 177 * xarray has this behavior where it won't iterate over NULL values stored in
 178 * allocated arrays.  So we need our own iterator to see all values stored in
 179 * the array. This does the same thing as xa_for_each except that it also
 180 * returns NULL valued entries if the array is allocating. Simplified to only
 181 * work on simple xarrays.
 182 */
 183static void *xan_find_marked(struct xarray *xa, unsigned long *indexp,
 184			     xa_mark_t filter)
 185{
 186	XA_STATE(xas, xa, *indexp);
 187	void *entry;
 188
 189	rcu_read_lock();
 190	do {
 191		entry = xas_find_marked(&xas, ULONG_MAX, filter);
 192		if (xa_is_zero(entry))
 193			break;
 194	} while (xas_retry(&xas, entry));
 195	rcu_read_unlock();
 196
 197	if (entry) {
 198		*indexp = xas.xa_index;
 199		if (xa_is_zero(entry))
 200			return NULL;
 201		return entry;
 202	}
 203	return XA_ERROR(-ENOENT);
 204}
 205#define xan_for_each_marked(xa, index, entry, filter)                          \
 206	for (index = 0, entry = xan_find_marked(xa, &(index), filter);         \
 207	     !xa_is_err(entry);                                                \
 208	     (index)++, entry = xan_find_marked(xa, &(index), filter))
 209
 210/* RCU hash table mapping netdevice pointers to struct ib_port_data */
 211static DEFINE_SPINLOCK(ndev_hash_lock);
 212static DECLARE_HASHTABLE(ndev_hash, 5);
 213
 214static void free_netdevs(struct ib_device *ib_dev);
 215static void ib_unregister_work(struct work_struct *work);
 216static void __ib_unregister_device(struct ib_device *device);
 217static int ib_security_change(struct notifier_block *nb, unsigned long event,
 218			      void *lsm_data);
 219static void ib_policy_change_task(struct work_struct *work);
 220static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task);
 221
 222static void __ibdev_printk(const char *level, const struct ib_device *ibdev,
 223			   struct va_format *vaf)
 224{
 225	if (ibdev && ibdev->dev.parent)
 226		dev_printk_emit(level[1] - '0',
 227				ibdev->dev.parent,
 228				"%s %s %s: %pV",
 229				dev_driver_string(ibdev->dev.parent),
 230				dev_name(ibdev->dev.parent),
 231				dev_name(&ibdev->dev),
 232				vaf);
 233	else if (ibdev)
 234		printk("%s%s: %pV",
 235		       level, dev_name(&ibdev->dev), vaf);
 236	else
 237		printk("%s(NULL ib_device): %pV", level, vaf);
 238}
 239
 240#define define_ibdev_printk_level(func, level)                  \
 241void func(const struct ib_device *ibdev, const char *fmt, ...)  \
 242{                                                               \
 243	struct va_format vaf;                                   \
 244	va_list args;                                           \
 245								\
 246	va_start(args, fmt);                                    \
 247								\
 248	vaf.fmt = fmt;                                          \
 249	vaf.va = &args;                                         \
 250								\
 251	__ibdev_printk(level, ibdev, &vaf);                     \
 252								\
 253	va_end(args);                                           \
 254}                                                               \
 255EXPORT_SYMBOL(func);
 256
 257define_ibdev_printk_level(ibdev_emerg, KERN_EMERG);
 258define_ibdev_printk_level(ibdev_alert, KERN_ALERT);
 259define_ibdev_printk_level(ibdev_crit, KERN_CRIT);
 260define_ibdev_printk_level(ibdev_err, KERN_ERR);
 261define_ibdev_printk_level(ibdev_warn, KERN_WARNING);
 262define_ibdev_printk_level(ibdev_notice, KERN_NOTICE);
 263define_ibdev_printk_level(ibdev_info, KERN_INFO);
 264
 265static struct notifier_block ibdev_lsm_nb = {
 266	.notifier_call = ib_security_change,
 267};
 268
 269static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
 270				 struct net *net);
 271
 272/* Pointer to the RCU head at the start of the ib_port_data array */
 273struct ib_port_data_rcu {
 274	struct rcu_head rcu_head;
 275	struct ib_port_data pdata[];
 276};
 277
 278static void ib_device_check_mandatory(struct ib_device *device)
 279{
 280#define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x }
 281	static const struct {
 282		size_t offset;
 283		char  *name;
 284	} mandatory_table[] = {
 285		IB_MANDATORY_FUNC(query_device),
 286		IB_MANDATORY_FUNC(query_port),
 287		IB_MANDATORY_FUNC(alloc_pd),
 288		IB_MANDATORY_FUNC(dealloc_pd),
 289		IB_MANDATORY_FUNC(create_qp),
 290		IB_MANDATORY_FUNC(modify_qp),
 291		IB_MANDATORY_FUNC(destroy_qp),
 292		IB_MANDATORY_FUNC(post_send),
 293		IB_MANDATORY_FUNC(post_recv),
 294		IB_MANDATORY_FUNC(create_cq),
 295		IB_MANDATORY_FUNC(destroy_cq),
 296		IB_MANDATORY_FUNC(poll_cq),
 297		IB_MANDATORY_FUNC(req_notify_cq),
 298		IB_MANDATORY_FUNC(get_dma_mr),
 299		IB_MANDATORY_FUNC(reg_user_mr),
 300		IB_MANDATORY_FUNC(dereg_mr),
 301		IB_MANDATORY_FUNC(get_port_immutable)
 302	};
 303	int i;
 304
 305	device->kverbs_provider = true;
 306	for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) {
 307		if (!*(void **) ((void *) &device->ops +
 308				 mandatory_table[i].offset)) {
 309			device->kverbs_provider = false;
 310			break;
 311		}
 312	}
 313}
 314
 315/*
 316 * Caller must perform ib_device_put() to return the device reference count
 317 * when ib_device_get_by_index() returns valid device pointer.
 318 */
 319struct ib_device *ib_device_get_by_index(const struct net *net, u32 index)
 320{
 321	struct ib_device *device;
 322
 323	down_read(&devices_rwsem);
 324	device = xa_load(&devices, index);
 325	if (device) {
 326		if (!rdma_dev_access_netns(device, net)) {
 327			device = NULL;
 328			goto out;
 329		}
 330
 331		if (!ib_device_try_get(device))
 332			device = NULL;
 333	}
 334out:
 335	up_read(&devices_rwsem);
 336	return device;
 337}
 338
 339/**
 340 * ib_device_put - Release IB device reference
 341 * @device: device whose reference to be released
 342 *
 343 * ib_device_put() releases reference to the IB device to allow it to be
 344 * unregistered and eventually free.
 345 */
 346void ib_device_put(struct ib_device *device)
 347{
 348	if (refcount_dec_and_test(&device->refcount))
 349		complete(&device->unreg_completion);
 350}
 351EXPORT_SYMBOL(ib_device_put);
 352
 353static struct ib_device *__ib_device_get_by_name(const char *name)
 354{
 355	struct ib_device *device;
 356	unsigned long index;
 357
 358	xa_for_each (&devices, index, device)
 359		if (!strcmp(name, dev_name(&device->dev)))
 360			return device;
 361
 362	return NULL;
 363}
 364
 365/**
 366 * ib_device_get_by_name - Find an IB device by name
 367 * @name: The name to look for
 368 * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all)
 369 *
 370 * Find and hold an ib_device by its name. The caller must call
 371 * ib_device_put() on the returned pointer.
 372 */
 373struct ib_device *ib_device_get_by_name(const char *name,
 374					enum rdma_driver_id driver_id)
 375{
 376	struct ib_device *device;
 377
 378	down_read(&devices_rwsem);
 379	device = __ib_device_get_by_name(name);
 380	if (device && driver_id != RDMA_DRIVER_UNKNOWN &&
 381	    device->ops.driver_id != driver_id)
 382		device = NULL;
 383
 384	if (device) {
 385		if (!ib_device_try_get(device))
 386			device = NULL;
 387	}
 388	up_read(&devices_rwsem);
 389	return device;
 390}
 391EXPORT_SYMBOL(ib_device_get_by_name);
 392
 393static int rename_compat_devs(struct ib_device *device)
 394{
 395	struct ib_core_device *cdev;
 396	unsigned long index;
 397	int ret = 0;
 398
 399	mutex_lock(&device->compat_devs_mutex);
 400	xa_for_each (&device->compat_devs, index, cdev) {
 401		ret = device_rename(&cdev->dev, dev_name(&device->dev));
 402		if (ret) {
 403			dev_warn(&cdev->dev,
 404				 "Fail to rename compatdev to new name %s\n",
 405				 dev_name(&device->dev));
 406			break;
 407		}
 408	}
 409	mutex_unlock(&device->compat_devs_mutex);
 410	return ret;
 411}
 412
 413int ib_device_rename(struct ib_device *ibdev, const char *name)
 414{
 415	unsigned long index;
 416	void *client_data;
 417	int ret;
 418
 419	down_write(&devices_rwsem);
 420	if (!strcmp(name, dev_name(&ibdev->dev))) {
 421		up_write(&devices_rwsem);
 422		return 0;
 423	}
 424
 425	if (__ib_device_get_by_name(name)) {
 426		up_write(&devices_rwsem);
 427		return -EEXIST;
 428	}
 429
 430	ret = device_rename(&ibdev->dev, name);
 431	if (ret) {
 432		up_write(&devices_rwsem);
 433		return ret;
 434	}
 435
 436	strscpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
 437	ret = rename_compat_devs(ibdev);
 438
 439	downgrade_write(&devices_rwsem);
 440	down_read(&ibdev->client_data_rwsem);
 441	xan_for_each_marked(&ibdev->client_data, index, client_data,
 442			    CLIENT_DATA_REGISTERED) {
 443		struct ib_client *client = xa_load(&clients, index);
 444
 445		if (!client || !client->rename)
 446			continue;
 447
 448		client->rename(ibdev, client_data);
 449	}
 450	up_read(&ibdev->client_data_rwsem);
 451	rdma_nl_notify_event(ibdev, 0, RDMA_RENAME_EVENT);
 452	up_read(&devices_rwsem);
 453	return 0;
 454}
 455
 456int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim)
 457{
 458	if (use_dim > 1)
 459		return -EINVAL;
 460	ibdev->use_cq_dim = use_dim;
 461
 462	return 0;
 463}
 464
 465static int alloc_name(struct ib_device *ibdev, const char *name)
 466{
 467	struct ib_device *device;
 468	unsigned long index;
 469	struct ida inuse;
 470	int rc;
 471	int i;
 472
 473	lockdep_assert_held_write(&devices_rwsem);
 474	ida_init(&inuse);
 475	xa_for_each (&devices, index, device) {
 476		char buf[IB_DEVICE_NAME_MAX];
 477
 478		if (sscanf(dev_name(&device->dev), name, &i) != 1)
 479			continue;
 480		if (i < 0 || i >= INT_MAX)
 481			continue;
 482		snprintf(buf, sizeof buf, name, i);
 483		if (strcmp(buf, dev_name(&device->dev)) != 0)
 484			continue;
 485
 486		rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL);
 487		if (rc < 0)
 488			goto out;
 489	}
 490
 491	rc = ida_alloc(&inuse, GFP_KERNEL);
 492	if (rc < 0)
 493		goto out;
 494
 495	rc = dev_set_name(&ibdev->dev, name, rc);
 496out:
 497	ida_destroy(&inuse);
 498	return rc;
 499}
 500
 501static void ib_device_release(struct device *device)
 502{
 503	struct ib_device *dev = container_of(device, struct ib_device, dev);
 504
 505	free_netdevs(dev);
 506	WARN_ON(refcount_read(&dev->refcount));
 507	if (dev->hw_stats_data)
 508		ib_device_release_hw_stats(dev->hw_stats_data);
 509	if (dev->port_data) {
 510		ib_cache_release_one(dev);
 511		ib_security_release_port_pkey_list(dev);
 512		rdma_counter_release(dev);
 513		kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu,
 514				       pdata[0]),
 515			  rcu_head);
 516	}
 517
 518	mutex_destroy(&dev->subdev_lock);
 519	mutex_destroy(&dev->unregistration_lock);
 520	mutex_destroy(&dev->compat_devs_mutex);
 521
 522	xa_destroy(&dev->compat_devs);
 523	xa_destroy(&dev->client_data);
 524	kfree_rcu(dev, rcu_head);
 525}
 526
 527static int ib_device_uevent(const struct device *device,
 528			    struct kobj_uevent_env *env)
 529{
 530	if (add_uevent_var(env, "NAME=%s", dev_name(device)))
 531		return -ENOMEM;
 532
 533	/*
 534	 * It would be nice to pass the node GUID with the event...
 535	 */
 536
 537	return 0;
 538}
 539
 540static const void *net_namespace(const struct device *d)
 541{
 542	const struct ib_core_device *coredev =
 543			container_of(d, struct ib_core_device, dev);
 544
 545	return read_pnet(&coredev->rdma_net);
 546}
 547
 548static struct class ib_class = {
 549	.name    = "infiniband",
 550	.dev_release = ib_device_release,
 551	.dev_uevent = ib_device_uevent,
 552	.ns_type = &net_ns_type_operations,
 553	.namespace = net_namespace,
 554};
 555
 556static void rdma_init_coredev(struct ib_core_device *coredev,
 557			      struct ib_device *dev, struct net *net)
 558{
 559	bool is_full_dev = &dev->coredev == coredev;
 560
 561	/* This BUILD_BUG_ON is intended to catch layout change
 562	 * of union of ib_core_device and device.
 563	 * dev must be the first element as ib_core and providers
 564	 * driver uses it. Adding anything in ib_core_device before
 565	 * device will break this assumption.
 566	 */
 567	BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) !=
 568		     offsetof(struct ib_device, dev));
 569
 570	coredev->dev.class = &ib_class;
 571	coredev->dev.groups = dev->groups;
 572
 573	/*
 574	 * Don't expose hw counters outside of the init namespace.
 575	 */
 576	if (!is_full_dev && dev->hw_stats_attr_index)
 577		coredev->dev.groups[dev->hw_stats_attr_index] = NULL;
 578
 579	device_initialize(&coredev->dev);
 580	coredev->owner = dev;
 581	INIT_LIST_HEAD(&coredev->port_list);
 582	write_pnet(&coredev->rdma_net, net);
 583}
 584
 585/**
 586 * _ib_alloc_device - allocate an IB device struct
 587 * @size:size of structure to allocate
 588 * @net: network namespace device should be located in, namespace
 589 *       must stay valid until ib_register_device() is completed.
 590 *
 591 * Low-level drivers should use ib_alloc_device() to allocate &struct
 592 * ib_device.  @size is the size of the structure to be allocated,
 593 * including any private data used by the low-level driver.
 594 * ib_dealloc_device() must be used to free structures allocated with
 595 * ib_alloc_device().
 596 */
 597struct ib_device *_ib_alloc_device(size_t size, struct net *net)
 598{
 599	struct ib_device *device;
 600	unsigned int i;
 601
 602	if (WARN_ON(size < sizeof(struct ib_device)))
 603		return NULL;
 604
 605	device = kzalloc(size, GFP_KERNEL);
 606	if (!device)
 607		return NULL;
 608
 609	if (rdma_restrack_init(device)) {
 610		kfree(device);
 611		return NULL;
 612	}
 613
 614	/* ib_devices_shared_netns can't change while we have active namespaces
 615	 * in the system which means either init_net is passed or the user has
 616	 * no idea what they are doing.
 617	 *
 618	 * To avoid breaking backward compatibility, when in shared mode,
 619	 * force to init the device in the init_net.
 620	 */
 621	net = ib_devices_shared_netns ? &init_net : net;
 622	rdma_init_coredev(&device->coredev, device, net);
 623
 624	INIT_LIST_HEAD(&device->event_handler_list);
 625	spin_lock_init(&device->qp_open_list_lock);
 626	init_rwsem(&device->event_handler_rwsem);
 627	mutex_init(&device->unregistration_lock);
 628	/*
 629	 * client_data needs to be alloc because we don't want our mark to be
 630	 * destroyed if the user stores NULL in the client data.
 631	 */
 632	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);
 633	init_rwsem(&device->client_data_rwsem);
 634	xa_init_flags(&device->compat_devs, XA_FLAGS_ALLOC);
 635	mutex_init(&device->compat_devs_mutex);
 636	init_completion(&device->unreg_completion);
 637	INIT_WORK(&device->unregistration_work, ib_unregister_work);
 638
 639	spin_lock_init(&device->cq_pools_lock);
 640	for (i = 0; i < ARRAY_SIZE(device->cq_pools); i++)
 641		INIT_LIST_HEAD(&device->cq_pools[i]);
 642
 643	rwlock_init(&device->cache_lock);
 644
 645	device->uverbs_cmd_mask =
 646		BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) |
 647		BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |
 648		BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) |
 649		BIT_ULL(IB_USER_VERBS_CMD_CLOSE_XRCD) |
 650		BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) |
 651		BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
 652		BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |
 653		BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) |
 654		BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ) |
 655		BIT_ULL(IB_USER_VERBS_CMD_CREATE_XSRQ) |
 656		BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) |
 657		BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |
 658		BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |
 659		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) |
 660		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) |
 661		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
 662		BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ) |
 663		BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST) |
 664		BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) |
 665		BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) |
 666		BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ) |
 667		BIT_ULL(IB_USER_VERBS_CMD_OPEN_QP) |
 668		BIT_ULL(IB_USER_VERBS_CMD_OPEN_XRCD) |
 669		BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) |
 670		BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) |
 671		BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
 672		BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ) |
 673		BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |
 674		BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |
 675		BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ);
 676
 677	mutex_init(&device->subdev_lock);
 678	INIT_LIST_HEAD(&device->subdev_list_head);
 679	INIT_LIST_HEAD(&device->subdev_list);
 680
 681	return device;
 682}
 683EXPORT_SYMBOL(_ib_alloc_device);
 684
 685/**
 686 * ib_dealloc_device - free an IB device struct
 687 * @device:structure to free
 688 *
 689 * Free a structure allocated with ib_alloc_device().
 690 */
 691void ib_dealloc_device(struct ib_device *device)
 692{
 693	if (device->ops.dealloc_driver)
 694		device->ops.dealloc_driver(device);
 695
 696	/*
 697	 * ib_unregister_driver() requires all devices to remain in the xarray
 698	 * while their ops are callable. The last op we call is dealloc_driver
 699	 * above.  This is needed to create a fence on op callbacks prior to
 700	 * allowing the driver module to unload.
 701	 */
 702	down_write(&devices_rwsem);
 703	if (xa_load(&devices, device->index) == device)
 704		xa_erase(&devices, device->index);
 705	up_write(&devices_rwsem);
 706
 707	/* Expedite releasing netdev references */
 708	free_netdevs(device);
 709
 710	WARN_ON(!xa_empty(&device->compat_devs));
 711	WARN_ON(!xa_empty(&device->client_data));
 712	WARN_ON(refcount_read(&device->refcount));
 713	rdma_restrack_clean(device);
 714	/* Balances with device_initialize */
 715	put_device(&device->dev);
 716}
 717EXPORT_SYMBOL(ib_dealloc_device);
 718
 719/*
 720 * add_client_context() and remove_client_context() must be safe against
 721 * parallel calls on the same device - registration/unregistration of both the
 722 * device and client can be occurring in parallel.
 723 *
 724 * The routines need to be a fence, any caller must not return until the add
 725 * or remove is fully completed.
 726 */
 727static int add_client_context(struct ib_device *device,
 728			      struct ib_client *client)
 729{
 730	int ret = 0;
 731
 732	if (!device->kverbs_provider && !client->no_kverbs_req)
 733		return 0;
 734
 735	down_write(&device->client_data_rwsem);
 736	/*
 737	 * So long as the client is registered hold both the client and device
 738	 * unregistration locks.
 739	 */
 740	if (!refcount_inc_not_zero(&client->uses))
 741		goto out_unlock;
 742	refcount_inc(&device->refcount);
 743
 744	/*
 745	 * Another caller to add_client_context got here first and has already
 746	 * completely initialized context.
 747	 */
 748	if (xa_get_mark(&device->client_data, client->client_id,
 749		    CLIENT_DATA_REGISTERED))
 750		goto out;
 751
 752	ret = xa_err(xa_store(&device->client_data, client->client_id, NULL,
 753			      GFP_KERNEL));
 754	if (ret)
 755		goto out;
 756	downgrade_write(&device->client_data_rwsem);
 757	if (client->add) {
 758		if (client->add(device)) {
 759			/*
 760			 * If a client fails to add then the error code is
 761			 * ignored, but we won't call any more ops on this
 762			 * client.
 763			 */
 764			xa_erase(&device->client_data, client->client_id);
 765			up_read(&device->client_data_rwsem);
 766			ib_device_put(device);
 767			ib_client_put(client);
 768			return 0;
 769		}
 770	}
 771
 772	/* Readers shall not see a client until add has been completed */
 773	xa_set_mark(&device->client_data, client->client_id,
 774		    CLIENT_DATA_REGISTERED);
 775	up_read(&device->client_data_rwsem);
 776	return 0;
 777
 778out:
 779	ib_device_put(device);
 780	ib_client_put(client);
 781out_unlock:
 782	up_write(&device->client_data_rwsem);
 783	return ret;
 784}
 785
 786static void remove_client_context(struct ib_device *device,
 787				  unsigned int client_id)
 788{
 789	struct ib_client *client;
 790	void *client_data;
 791
 792	down_write(&device->client_data_rwsem);
 793	if (!xa_get_mark(&device->client_data, client_id,
 794			 CLIENT_DATA_REGISTERED)) {
 795		up_write(&device->client_data_rwsem);
 796		return;
 797	}
 798	client_data = xa_load(&device->client_data, client_id);
 799	xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED);
 800	client = xa_load(&clients, client_id);
 801	up_write(&device->client_data_rwsem);
 802
 803	/*
 804	 * Notice we cannot be holding any exclusive locks when calling the
 805	 * remove callback as the remove callback can recurse back into any
 806	 * public functions in this module and thus try for any locks those
 807	 * functions take.
 808	 *
 809	 * For this reason clients and drivers should not call the
 810	 * unregistration functions will holdling any locks.
 811	 */
 812	if (client->remove)
 813		client->remove(device, client_data);
 814
 815	xa_erase(&device->client_data, client_id);
 816	ib_device_put(device);
 817	ib_client_put(client);
 818}
 819
 820static int alloc_port_data(struct ib_device *device)
 821{
 822	struct ib_port_data_rcu *pdata_rcu;
 823	u32 port;
 824
 825	if (device->port_data)
 826		return 0;
 827
 828	/* This can only be called once the physical port range is defined */
 829	if (WARN_ON(!device->phys_port_cnt))
 830		return -EINVAL;
 831
 832	/* Reserve U32_MAX so the logic to go over all the ports is sane */
 833	if (WARN_ON(device->phys_port_cnt == U32_MAX))
 834		return -EINVAL;
 835
 836	/*
 837	 * device->port_data is indexed directly by the port number to make
 838	 * access to this data as efficient as possible.
 839	 *
 840	 * Therefore port_data is declared as a 1 based array with potential
 841	 * empty slots at the beginning.
 842	 */
 843	pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata,
 844					size_add(rdma_end_port(device), 1)),
 845			    GFP_KERNEL);
 846	if (!pdata_rcu)
 847		return -ENOMEM;
 848	/*
 849	 * The rcu_head is put in front of the port data array and the stored
 850	 * pointer is adjusted since we never need to see that member until
 851	 * kfree_rcu.
 852	 */
 853	device->port_data = pdata_rcu->pdata;
 854
 855	rdma_for_each_port (device, port) {
 856		struct ib_port_data *pdata = &device->port_data[port];
 857
 858		pdata->ib_dev = device;
 859		spin_lock_init(&pdata->pkey_list_lock);
 860		INIT_LIST_HEAD(&pdata->pkey_list);
 861		spin_lock_init(&pdata->netdev_lock);
 862		INIT_HLIST_NODE(&pdata->ndev_hash_link);
 863	}
 864	return 0;
 865}
 866
 867static int verify_immutable(const struct ib_device *dev, u32 port)
 868{
 869	return WARN_ON(!rdma_cap_ib_mad(dev, port) &&
 870			    rdma_max_mad_size(dev, port) != 0);
 871}
 872
 873static int setup_port_data(struct ib_device *device)
 874{
 875	u32 port;
 876	int ret;
 877
 878	ret = alloc_port_data(device);
 879	if (ret)
 880		return ret;
 881
 882	rdma_for_each_port (device, port) {
 883		struct ib_port_data *pdata = &device->port_data[port];
 884
 885		ret = device->ops.get_port_immutable(device, port,
 886						     &pdata->immutable);
 887		if (ret)
 888			return ret;
 889
 890		if (verify_immutable(device, port))
 891			return -EINVAL;
 892	}
 893	return 0;
 894}
 895
 896/**
 897 * ib_port_immutable_read() - Read rdma port's immutable data
 898 * @dev: IB device
 899 * @port: port number whose immutable data to read. It starts with index 1 and
 900 *        valid upto including rdma_end_port().
 901 */
 902const struct ib_port_immutable*
 903ib_port_immutable_read(struct ib_device *dev, unsigned int port)
 904{
 905	WARN_ON(!rdma_is_port_valid(dev, port));
 906	return &dev->port_data[port].immutable;
 907}
 908EXPORT_SYMBOL(ib_port_immutable_read);
 909
 910void ib_get_device_fw_str(struct ib_device *dev, char *str)
 911{
 912	if (dev->ops.get_dev_fw_str)
 913		dev->ops.get_dev_fw_str(dev, str);
 914	else
 915		str[0] = '\0';
 916}
 917EXPORT_SYMBOL(ib_get_device_fw_str);
 918
 919static void ib_policy_change_task(struct work_struct *work)
 920{
 921	struct ib_device *dev;
 922	unsigned long index;
 923
 924	down_read(&devices_rwsem);
 925	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
 926		unsigned int i;
 927
 928		rdma_for_each_port (dev, i) {
 929			u64 sp;
 930			ib_get_cached_subnet_prefix(dev, i, &sp);
 931			ib_security_cache_change(dev, i, sp);
 932		}
 933	}
 934	up_read(&devices_rwsem);
 935}
 936
 937static int ib_security_change(struct notifier_block *nb, unsigned long event,
 938			      void *lsm_data)
 939{
 940	if (event != LSM_POLICY_CHANGE)
 941		return NOTIFY_DONE;
 942
 943	schedule_work(&ib_policy_change_work);
 944	ib_mad_agent_security_change();
 945
 946	return NOTIFY_OK;
 947}
 948
 949static void compatdev_release(struct device *dev)
 950{
 951	struct ib_core_device *cdev =
 952		container_of(dev, struct ib_core_device, dev);
 953
 954	kfree(cdev);
 955}
 956
 957static int add_one_compat_dev(struct ib_device *device,
 958			      struct rdma_dev_net *rnet)
 959{
 960	struct ib_core_device *cdev;
 961	int ret;
 962
 963	lockdep_assert_held(&rdma_nets_rwsem);
 964	if (!ib_devices_shared_netns)
 965		return 0;
 966
 967	/*
 968	 * Create and add compat device in all namespaces other than where it
 969	 * is currently bound to.
 970	 */
 971	if (net_eq(read_pnet(&rnet->net),
 972		   read_pnet(&device->coredev.rdma_net)))
 973		return 0;
 974
 975	/*
 976	 * The first of init_net() or ib_register_device() to take the
 977	 * compat_devs_mutex wins and gets to add the device. Others will wait
 978	 * for completion here.
 979	 */
 980	mutex_lock(&device->compat_devs_mutex);
 981	cdev = xa_load(&device->compat_devs, rnet->id);
 982	if (cdev) {
 983		ret = 0;
 984		goto done;
 985	}
 986	ret = xa_reserve(&device->compat_devs, rnet->id, GFP_KERNEL);
 987	if (ret)
 988		goto done;
 989
 990	cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
 991	if (!cdev) {
 992		ret = -ENOMEM;
 993		goto cdev_err;
 994	}
 995
 996	cdev->dev.parent = device->dev.parent;
 997	rdma_init_coredev(cdev, device, read_pnet(&rnet->net));
 998	cdev->dev.release = compatdev_release;
 999	ret = dev_set_name(&cdev->dev, "%s", dev_name(&device->dev));
1000	if (ret)
1001		goto add_err;
1002
1003	ret = device_add(&cdev->dev);
1004	if (ret)
1005		goto add_err;
1006	ret = ib_setup_port_attrs(cdev);
1007	if (ret)
1008		goto port_err;
1009
1010	ret = xa_err(xa_store(&device->compat_devs, rnet->id,
1011			      cdev, GFP_KERNEL));
1012	if (ret)
1013		goto insert_err;
1014
1015	mutex_unlock(&device->compat_devs_mutex);
1016	return 0;
1017
1018insert_err:
1019	ib_free_port_attrs(cdev);
1020port_err:
1021	device_del(&cdev->dev);
1022add_err:
1023	put_device(&cdev->dev);
1024cdev_err:
1025	xa_release(&device->compat_devs, rnet->id);
1026done:
1027	mutex_unlock(&device->compat_devs_mutex);
1028	return ret;
1029}
1030
1031static void remove_one_compat_dev(struct ib_device *device, u32 id)
1032{
1033	struct ib_core_device *cdev;
1034
1035	mutex_lock(&device->compat_devs_mutex);
1036	cdev = xa_erase(&device->compat_devs, id);
1037	mutex_unlock(&device->compat_devs_mutex);
1038	if (cdev) {
1039		ib_free_port_attrs(cdev);
1040		device_del(&cdev->dev);
1041		put_device(&cdev->dev);
1042	}
1043}
1044
1045static void remove_compat_devs(struct ib_device *device)
1046{
1047	struct ib_core_device *cdev;
1048	unsigned long index;
1049
1050	xa_for_each (&device->compat_devs, index, cdev)
1051		remove_one_compat_dev(device, index);
1052}
1053
1054static int add_compat_devs(struct ib_device *device)
1055{
1056	struct rdma_dev_net *rnet;
1057	unsigned long index;
1058	int ret = 0;
1059
1060	lockdep_assert_held(&devices_rwsem);
1061
1062	down_read(&rdma_nets_rwsem);
1063	xa_for_each (&rdma_nets, index, rnet) {
1064		ret = add_one_compat_dev(device, rnet);
1065		if (ret)
1066			break;
1067	}
1068	up_read(&rdma_nets_rwsem);
1069	return ret;
1070}
1071
1072static void remove_all_compat_devs(void)
1073{
1074	struct ib_compat_device *cdev;
1075	struct ib_device *dev;
1076	unsigned long index;
1077
1078	down_read(&devices_rwsem);
1079	xa_for_each (&devices, index, dev) {
1080		unsigned long c_index = 0;
1081
1082		/* Hold nets_rwsem so that any other thread modifying this
1083		 * system param can sync with this thread.
1084		 */
1085		down_read(&rdma_nets_rwsem);
1086		xa_for_each (&dev->compat_devs, c_index, cdev)
1087			remove_one_compat_dev(dev, c_index);
1088		up_read(&rdma_nets_rwsem);
1089	}
1090	up_read(&devices_rwsem);
1091}
1092
1093static int add_all_compat_devs(void)
1094{
1095	struct rdma_dev_net *rnet;
1096	struct ib_device *dev;
1097	unsigned long index;
1098	int ret = 0;
1099
1100	down_read(&devices_rwsem);
1101	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
1102		unsigned long net_index = 0;
1103
1104		/* Hold nets_rwsem so that any other thread modifying this
1105		 * system param can sync with this thread.
1106		 */
1107		down_read(&rdma_nets_rwsem);
1108		xa_for_each (&rdma_nets, net_index, rnet) {
1109			ret = add_one_compat_dev(dev, rnet);
1110			if (ret)
1111				break;
1112		}
1113		up_read(&rdma_nets_rwsem);
1114	}
1115	up_read(&devices_rwsem);
1116	if (ret)
1117		remove_all_compat_devs();
1118	return ret;
1119}
1120
1121int rdma_compatdev_set(u8 enable)
1122{
1123	struct rdma_dev_net *rnet;
1124	unsigned long index;
1125	int ret = 0;
1126
1127	down_write(&rdma_nets_rwsem);
1128	if (ib_devices_shared_netns == enable) {
1129		up_write(&rdma_nets_rwsem);
1130		return 0;
1131	}
1132
1133	/* enable/disable of compat devices is not supported
1134	 * when more than default init_net exists.
1135	 */
1136	xa_for_each (&rdma_nets, index, rnet) {
1137		ret++;
1138		break;
1139	}
1140	if (!ret)
1141		ib_devices_shared_netns = enable;
1142	up_write(&rdma_nets_rwsem);
1143	if (ret)
1144		return -EBUSY;
1145
1146	if (enable)
1147		ret = add_all_compat_devs();
1148	else
1149		remove_all_compat_devs();
1150	return ret;
1151}
1152
1153static void rdma_dev_exit_net(struct net *net)
1154{
1155	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);
1156	struct ib_device *dev;
1157	unsigned long index;
1158	int ret;
1159
1160	down_write(&rdma_nets_rwsem);
1161	/*
1162	 * Prevent the ID from being re-used and hide the id from xa_for_each.
1163	 */
1164	ret = xa_err(xa_store(&rdma_nets, rnet->id, NULL, GFP_KERNEL));
1165	WARN_ON(ret);
1166	up_write(&rdma_nets_rwsem);
1167
1168	down_read(&devices_rwsem);
1169	xa_for_each (&devices, index, dev) {
1170		get_device(&dev->dev);
1171		/*
1172		 * Release the devices_rwsem so that pontentially blocking
1173		 * device_del, doesn't hold the devices_rwsem for too long.
1174		 */
1175		up_read(&devices_rwsem);
1176
1177		remove_one_compat_dev(dev, rnet->id);
1178
1179		/*
1180		 * If the real device is in the NS then move it back to init.
1181		 */
1182		rdma_dev_change_netns(dev, net, &init_net);
1183
1184		put_device(&dev->dev);
1185		down_read(&devices_rwsem);
1186	}
1187	up_read(&devices_rwsem);
1188
1189	rdma_nl_net_exit(rnet);
1190	xa_erase(&rdma_nets, rnet->id);
1191}
1192
1193static __net_init int rdma_dev_init_net(struct net *net)
1194{
1195	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);
1196	unsigned long index;
1197	struct ib_device *dev;
1198	int ret;
1199
1200	write_pnet(&rnet->net, net);
1201
1202	ret = rdma_nl_net_init(rnet);
1203	if (ret)
1204		return ret;
1205
1206	/* No need to create any compat devices in default init_net. */
1207	if (net_eq(net, &init_net))
1208		return 0;
1209
1210	ret = xa_alloc(&rdma_nets, &rnet->id, rnet, xa_limit_32b, GFP_KERNEL);
1211	if (ret) {
1212		rdma_nl_net_exit(rnet);
1213		return ret;
1214	}
1215
1216	down_read(&devices_rwsem);
1217	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
1218		/* Hold nets_rwsem so that netlink command cannot change
1219		 * system configuration for device sharing mode.
1220		 */
1221		down_read(&rdma_nets_rwsem);
1222		ret = add_one_compat_dev(dev, rnet);
1223		up_read(&rdma_nets_rwsem);
1224		if (ret)
1225			break;
1226	}
1227	up_read(&devices_rwsem);
1228
1229	if (ret)
1230		rdma_dev_exit_net(net);
1231
1232	return ret;
1233}
1234
1235/*
1236 * Assign the unique string device name and the unique device index. This is
1237 * undone by ib_dealloc_device.
1238 */
1239static int assign_name(struct ib_device *device, const char *name)
1240{
1241	static u32 last_id;
1242	int ret;
1243
1244	down_write(&devices_rwsem);
1245	/* Assign a unique name to the device */
1246	if (strchr(name, '%'))
1247		ret = alloc_name(device, name);
1248	else
1249		ret = dev_set_name(&device->dev, name);
1250	if (ret)
1251		goto out;
1252
1253	if (__ib_device_get_by_name(dev_name(&device->dev))) {
1254		ret = -ENFILE;
1255		goto out;
1256	}
1257	strscpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);
1258
1259	ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b,
1260			&last_id, GFP_KERNEL);
1261	if (ret > 0)
1262		ret = 0;
1263
1264out:
1265	up_write(&devices_rwsem);
1266	return ret;
1267}
1268
1269/*
1270 * setup_device() allocates memory and sets up data that requires calling the
1271 * device ops, this is the only reason these actions are not done during
1272 * ib_alloc_device. It is undone by ib_dealloc_device().
1273 */
1274static int setup_device(struct ib_device *device)
1275{
1276	struct ib_udata uhw = {.outlen = 0, .inlen = 0};
1277	int ret;
1278
1279	ib_device_check_mandatory(device);
1280
1281	ret = setup_port_data(device);
1282	if (ret) {
1283		dev_warn(&device->dev, "Couldn't create per-port data\n");
1284		return ret;
1285	}
1286
1287	memset(&device->attrs, 0, sizeof(device->attrs));
1288	ret = device->ops.query_device(device, &device->attrs, &uhw);
1289	if (ret) {
1290		dev_warn(&device->dev,
1291			 "Couldn't query the device attributes\n");
1292		return ret;
1293	}
1294
1295	return 0;
1296}
1297
1298static void disable_device(struct ib_device *device)
1299{
1300	u32 cid;
1301
1302	WARN_ON(!refcount_read(&device->refcount));
1303
1304	down_write(&devices_rwsem);
1305	xa_clear_mark(&devices, device->index, DEVICE_REGISTERED);
1306	up_write(&devices_rwsem);
1307
1308	/*
1309	 * Remove clients in LIFO order, see assign_client_id. This could be
1310	 * more efficient if xarray learns to reverse iterate. Since no new
1311	 * clients can be added to this ib_device past this point we only need
1312	 * the maximum possible client_id value here.
1313	 */
1314	down_read(&clients_rwsem);
1315	cid = highest_client_id;
1316	up_read(&clients_rwsem);
1317	while (cid) {
1318		cid--;
1319		remove_client_context(device, cid);
1320	}
1321
1322	ib_cq_pool_cleanup(device);
1323
1324	/* Pairs with refcount_set in enable_device */
1325	ib_device_put(device);
1326	wait_for_completion(&device->unreg_completion);
1327
1328	/*
1329	 * compat devices must be removed after device refcount drops to zero.
1330	 * Otherwise init_net() may add more compatdevs after removing compat
1331	 * devices and before device is disabled.
1332	 */
1333	remove_compat_devs(device);
1334}
1335
1336/*
1337 * An enabled device is visible to all clients and to all the public facing
1338 * APIs that return a device pointer. This always returns with a new get, even
1339 * if it fails.
1340 */
1341static int enable_device_and_get(struct ib_device *device)
1342{
1343	struct ib_client *client;
1344	unsigned long index;
1345	int ret = 0;
1346
1347	/*
1348	 * One ref belongs to the xa and the other belongs to this
1349	 * thread. This is needed to guard against parallel unregistration.
1350	 */
1351	refcount_set(&device->refcount, 2);
1352	down_write(&devices_rwsem);
1353	xa_set_mark(&devices, device->index, DEVICE_REGISTERED);
1354
1355	/*
1356	 * By using downgrade_write() we ensure that no other thread can clear
1357	 * DEVICE_REGISTERED while we are completing the client setup.
1358	 */
1359	downgrade_write(&devices_rwsem);
1360
1361	if (device->ops.enable_driver) {
1362		ret = device->ops.enable_driver(device);
1363		if (ret)
1364			goto out;
1365	}
1366
1367	down_read(&clients_rwsem);
1368	xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
1369		ret = add_client_context(device, client);
1370		if (ret)
1371			break;
1372	}
1373	up_read(&clients_rwsem);
1374	if (!ret)
1375		ret = add_compat_devs(device);
1376out:
1377	up_read(&devices_rwsem);
1378	return ret;
1379}
1380
1381static void prevent_dealloc_device(struct ib_device *ib_dev)
1382{
1383}
1384
1385static void ib_device_notify_register(struct ib_device *device)
1386{
1387	struct net_device *netdev;
1388	u32 port;
1389	int ret;
1390
1391	down_read(&devices_rwsem);
1392
1393	/* Mark for userspace that device is ready */
1394	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
1395
1396	ret = rdma_nl_notify_event(device, 0, RDMA_REGISTER_EVENT);
1397	if (ret)
1398		goto out;
1399
1400	rdma_for_each_port(device, port) {
1401		netdev = ib_device_get_netdev(device, port);
1402		if (!netdev)
1403			continue;
1404
1405		ret = rdma_nl_notify_event(device, port,
1406					   RDMA_NETDEV_ATTACH_EVENT);
1407		dev_put(netdev);
1408		if (ret)
1409			goto out;
1410	}
1411
1412out:
1413	up_read(&devices_rwsem);
1414}
1415
1416/**
1417 * ib_register_device - Register an IB device with IB core
1418 * @device: Device to register
1419 * @name: unique string device name. This may include a '%' which will
1420 * 	  cause a unique index to be added to the passed device name.
1421 * @dma_device: pointer to a DMA-capable device. If %NULL, then the IB
1422 *	        device will be used. In this case the caller should fully
1423 *		setup the ibdev for DMA. This usually means using dma_virt_ops.
1424 *
1425 * Low-level drivers use ib_register_device() to register their
1426 * devices with the IB core.  All registered clients will receive a
1427 * callback for each device that is added. @device must be allocated
1428 * with ib_alloc_device().
1429 *
1430 * If the driver uses ops.dealloc_driver and calls any ib_unregister_device()
1431 * asynchronously then the device pointer may become freed as soon as this
1432 * function returns.
1433 */
1434int ib_register_device(struct ib_device *device, const char *name,
1435		       struct device *dma_device)
1436{
1437	int ret;
1438
1439	ret = assign_name(device, name);
1440	if (ret)
1441		return ret;
1442
1443	/*
1444	 * If the caller does not provide a DMA capable device then the IB core
1445	 * will set up ib_sge and scatterlist structures that stash the kernel
1446	 * virtual address into the address field.
1447	 */
1448	WARN_ON(dma_device && !dma_device->dma_parms);
1449	device->dma_device = dma_device;
1450
1451	ret = setup_device(device);
1452	if (ret)
1453		return ret;
1454
1455	ret = ib_cache_setup_one(device);
1456	if (ret) {
1457		dev_warn(&device->dev,
1458			 "Couldn't set up InfiniBand P_Key/GID cache\n");
1459		return ret;
1460	}
1461
1462	device->groups[0] = &ib_dev_attr_group;
1463	device->groups[1] = device->ops.device_group;
1464	ret = ib_setup_device_attrs(device);
1465	if (ret)
1466		goto cache_cleanup;
1467
1468	ib_device_register_rdmacg(device);
1469
1470	rdma_counter_init(device);
1471
1472	/*
1473	 * Ensure that ADD uevent is not fired because it
1474	 * is too early amd device is not initialized yet.
1475	 */
1476	dev_set_uevent_suppress(&device->dev, true);
1477	ret = device_add(&device->dev);
1478	if (ret)
1479		goto cg_cleanup;
1480
1481	ret = ib_setup_port_attrs(&device->coredev);
1482	if (ret) {
1483		dev_warn(&device->dev,
1484			 "Couldn't register device with driver model\n");
1485		goto dev_cleanup;
1486	}
1487
1488	ret = enable_device_and_get(device);
1489	if (ret) {
1490		void (*dealloc_fn)(struct ib_device *);
1491
1492		/*
1493		 * If we hit this error flow then we don't want to
1494		 * automatically dealloc the device since the caller is
1495		 * expected to call ib_dealloc_device() after
1496		 * ib_register_device() fails. This is tricky due to the
1497		 * possibility for a parallel unregistration along with this
1498		 * error flow. Since we have a refcount here we know any
1499		 * parallel flow is stopped in disable_device and will see the
1500		 * special dealloc_driver pointer, causing the responsibility to
1501		 * ib_dealloc_device() to revert back to this thread.
1502		 */
1503		dealloc_fn = device->ops.dealloc_driver;
1504		device->ops.dealloc_driver = prevent_dealloc_device;
1505		ib_device_put(device);
1506		__ib_unregister_device(device);
1507		device->ops.dealloc_driver = dealloc_fn;
1508		dev_set_uevent_suppress(&device->dev, false);
1509		return ret;
1510	}
1511	dev_set_uevent_suppress(&device->dev, false);
1512
1513	ib_device_notify_register(device);
1514
1515	ib_device_put(device);
1516
1517	return 0;
1518
1519dev_cleanup:
1520	device_del(&device->dev);
1521cg_cleanup:
1522	dev_set_uevent_suppress(&device->dev, false);
1523	ib_device_unregister_rdmacg(device);
1524cache_cleanup:
1525	ib_cache_cleanup_one(device);
1526	return ret;
1527}
1528EXPORT_SYMBOL(ib_register_device);
1529
1530/* Callers must hold a get on the device. */
1531static void __ib_unregister_device(struct ib_device *ib_dev)
1532{
1533	struct ib_device *sub, *tmp;
1534
1535	mutex_lock(&ib_dev->subdev_lock);
1536	list_for_each_entry_safe_reverse(sub, tmp,
1537					 &ib_dev->subdev_list_head,
1538					 subdev_list) {
1539		list_del(&sub->subdev_list);
1540		ib_dev->ops.del_sub_dev(sub);
1541		ib_device_put(ib_dev);
1542	}
1543	mutex_unlock(&ib_dev->subdev_lock);
1544
1545	/*
1546	 * We have a registration lock so that all the calls to unregister are
1547	 * fully fenced, once any unregister returns the device is truly
1548	 * unregistered even if multiple callers are unregistering it at the
1549	 * same time. This also interacts with the registration flow and
1550	 * provides sane semantics if register and unregister are racing.
1551	 */
1552	mutex_lock(&ib_dev->unregistration_lock);
1553	if (!refcount_read(&ib_dev->refcount))
1554		goto out;
1555
1556	disable_device(ib_dev);
1557	rdma_nl_notify_event(ib_dev, 0, RDMA_UNREGISTER_EVENT);
1558
1559	/* Expedite removing unregistered pointers from the hash table */
1560	free_netdevs(ib_dev);
1561
1562	ib_free_port_attrs(&ib_dev->coredev);
1563	device_del(&ib_dev->dev);
1564	ib_device_unregister_rdmacg(ib_dev);
1565	ib_cache_cleanup_one(ib_dev);
1566
1567	/*
1568	 * Drivers using the new flow may not call ib_dealloc_device except
1569	 * in error unwind prior to registration success.
1570	 */
1571	if (ib_dev->ops.dealloc_driver &&
1572	    ib_dev->ops.dealloc_driver != prevent_dealloc_device) {
1573		WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1);
1574		ib_dealloc_device(ib_dev);
1575	}
1576out:
1577	mutex_unlock(&ib_dev->unregistration_lock);
1578}
1579
1580/**
1581 * ib_unregister_device - Unregister an IB device
1582 * @ib_dev: The device to unregister
1583 *
1584 * Unregister an IB device.  All clients will receive a remove callback.
1585 *
1586 * Callers should call this routine only once, and protect against races with
1587 * registration. Typically it should only be called as part of a remove
1588 * callback in an implementation of driver core's struct device_driver and
1589 * related.
1590 *
1591 * If ops.dealloc_driver is used then ib_dev will be freed upon return from
1592 * this function.
1593 */
1594void ib_unregister_device(struct ib_device *ib_dev)
1595{
1596	get_device(&ib_dev->dev);
1597	__ib_unregister_device(ib_dev);
1598	put_device(&ib_dev->dev);
1599}
1600EXPORT_SYMBOL(ib_unregister_device);
1601
1602/**
1603 * ib_unregister_device_and_put - Unregister a device while holding a 'get'
1604 * @ib_dev: The device to unregister
1605 *
1606 * This is the same as ib_unregister_device(), except it includes an internal
1607 * ib_device_put() that should match a 'get' obtained by the caller.
1608 *
1609 * It is safe to call this routine concurrently from multiple threads while
1610 * holding the 'get'. When the function returns the device is fully
1611 * unregistered.
1612 *
1613 * Drivers using this flow MUST use the driver_unregister callback to clean up
1614 * their resources associated with the device and dealloc it.
1615 */
1616void ib_unregister_device_and_put(struct ib_device *ib_dev)
1617{
1618	WARN_ON(!ib_dev->ops.dealloc_driver);
1619	get_device(&ib_dev->dev);
1620	ib_device_put(ib_dev);
1621	__ib_unregister_device(ib_dev);
1622	put_device(&ib_dev->dev);
1623}
1624EXPORT_SYMBOL(ib_unregister_device_and_put);
1625
1626/**
1627 * ib_unregister_driver - Unregister all IB devices for a driver
1628 * @driver_id: The driver to unregister
1629 *
1630 * This implements a fence for device unregistration. It only returns once all
1631 * devices associated with the driver_id have fully completed their
1632 * unregistration and returned from ib_unregister_device*().
1633 *
1634 * If device's are not yet unregistered it goes ahead and starts unregistering
1635 * them.
1636 *
1637 * This does not block creation of new devices with the given driver_id, that
1638 * is the responsibility of the caller.
1639 */
1640void ib_unregister_driver(enum rdma_driver_id driver_id)
1641{
1642	struct ib_device *ib_dev;
1643	unsigned long index;
1644
1645	down_read(&devices_rwsem);
1646	xa_for_each (&devices, index, ib_dev) {
1647		if (ib_dev->ops.driver_id != driver_id)
1648			continue;
1649
1650		get_device(&ib_dev->dev);
1651		up_read(&devices_rwsem);
1652
1653		WARN_ON(!ib_dev->ops.dealloc_driver);
1654		__ib_unregister_device(ib_dev);
1655
1656		put_device(&ib_dev->dev);
1657		down_read(&devices_rwsem);
1658	}
1659	up_read(&devices_rwsem);
1660}
1661EXPORT_SYMBOL(ib_unregister_driver);
1662
1663static void ib_unregister_work(struct work_struct *work)
1664{
1665	struct ib_device *ib_dev =
1666		container_of(work, struct ib_device, unregistration_work);
1667
1668	__ib_unregister_device(ib_dev);
1669	put_device(&ib_dev->dev);
1670}
1671
1672/**
1673 * ib_unregister_device_queued - Unregister a device using a work queue
1674 * @ib_dev: The device to unregister
1675 *
1676 * This schedules an asynchronous unregistration using a WQ for the device. A
1677 * driver should use this to avoid holding locks while doing unregistration,
1678 * such as holding the RTNL lock.
1679 *
1680 * Drivers using this API must use ib_unregister_driver before module unload
1681 * to ensure that all scheduled unregistrations have completed.
1682 */
1683void ib_unregister_device_queued(struct ib_device *ib_dev)
1684{
1685	WARN_ON(!refcount_read(&ib_dev->refcount));
1686	WARN_ON(!ib_dev->ops.dealloc_driver);
1687	get_device(&ib_dev->dev);
1688	if (!queue_work(ib_unreg_wq, &ib_dev->unregistration_work))
1689		put_device(&ib_dev->dev);
1690}
1691EXPORT_SYMBOL(ib_unregister_device_queued);
1692
1693/*
1694 * The caller must pass in a device that has the kref held and the refcount
1695 * released. If the device is in cur_net and still registered then it is moved
1696 * into net.
1697 */
1698static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
1699				 struct net *net)
1700{
1701	int ret2 = -EINVAL;
1702	int ret;
1703
1704	mutex_lock(&device->unregistration_lock);
1705
1706	/*
1707	 * If a device not under ib_device_get() or if the unregistration_lock
1708	 * is not held, the namespace can be changed, or it can be unregistered.
1709	 * Check again under the lock.
1710	 */
1711	if (refcount_read(&device->refcount) == 0 ||
1712	    !net_eq(cur_net, read_pnet(&device->coredev.rdma_net))) {
1713		ret = -ENODEV;
1714		goto out;
1715	}
1716
1717	kobject_uevent(&device->dev.kobj, KOBJ_REMOVE);
1718	disable_device(device);
1719
1720	/*
1721	 * At this point no one can be using the device, so it is safe to
1722	 * change the namespace.
1723	 */
1724	write_pnet(&device->coredev.rdma_net, net);
1725
1726	down_read(&devices_rwsem);
1727	/*
1728	 * Currently rdma devices are system wide unique. So the device name
1729	 * is guaranteed free in the new namespace. Publish the new namespace
1730	 * at the sysfs level.
1731	 */
1732	ret = device_rename(&device->dev, dev_name(&device->dev));
1733	up_read(&devices_rwsem);
1734	if (ret) {
1735		dev_warn(&device->dev,
1736			 "%s: Couldn't rename device after namespace change\n",
1737			 __func__);
1738		/* Try and put things back and re-enable the device */
1739		write_pnet(&device->coredev.rdma_net, cur_net);
1740	}
1741
1742	ret2 = enable_device_and_get(device);
1743	if (ret2) {
1744		/*
1745		 * This shouldn't really happen, but if it does, let the user
1746		 * retry at later point. So don't disable the device.
1747		 */
1748		dev_warn(&device->dev,
1749			 "%s: Couldn't re-enable device after namespace change\n",
1750			 __func__);
1751	}
1752	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
1753
1754	ib_device_put(device);
1755out:
1756	mutex_unlock(&device->unregistration_lock);
1757	if (ret)
1758		return ret;
1759	return ret2;
1760}
1761
1762int ib_device_set_netns_put(struct sk_buff *skb,
1763			    struct ib_device *dev, u32 ns_fd)
1764{
1765	struct net *net;
1766	int ret;
1767
1768	net = get_net_ns_by_fd(ns_fd);
1769	if (IS_ERR(net)) {
1770		ret = PTR_ERR(net);
1771		goto net_err;
1772	}
1773
1774	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
1775		ret = -EPERM;
1776		goto ns_err;
1777	}
1778
1779	/*
1780	 * All the ib_clients, including uverbs, are reset when the namespace is
1781	 * changed and this cannot be blocked waiting for userspace to do
1782	 * something, so disassociation is mandatory.
1783	 */
1784	if (!dev->ops.disassociate_ucontext || ib_devices_shared_netns) {
1785		ret = -EOPNOTSUPP;
1786		goto ns_err;
1787	}
1788
1789	get_device(&dev->dev);
1790	ib_device_put(dev);
1791	ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net);
1792	put_device(&dev->dev);
1793
1794	put_net(net);
1795	return ret;
1796
1797ns_err:
1798	put_net(net);
1799net_err:
1800	ib_device_put(dev);
1801	return ret;
1802}
1803
1804static struct pernet_operations rdma_dev_net_ops = {
1805	.init = rdma_dev_init_net,
1806	.exit = rdma_dev_exit_net,
1807	.id = &rdma_dev_net_id,
1808	.size = sizeof(struct rdma_dev_net),
1809};
1810
1811static int assign_client_id(struct ib_client *client)
1812{
1813	int ret;
1814
1815	lockdep_assert_held(&clients_rwsem);
1816	/*
1817	 * The add/remove callbacks must be called in FIFO/LIFO order. To
1818	 * achieve this we assign client_ids so they are sorted in
1819	 * registration order.
1820	 */
1821	client->client_id = highest_client_id;
1822	ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL);
1823	if (ret)
1824		return ret;
1825
1826	highest_client_id++;
1827	xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED);
1828	return 0;
1829}
1830
1831static void remove_client_id(struct ib_client *client)
1832{
1833	down_write(&clients_rwsem);
1834	xa_erase(&clients, client->client_id);
1835	for (; highest_client_id; highest_client_id--)
1836		if (xa_load(&clients, highest_client_id - 1))
1837			break;
1838	up_write(&clients_rwsem);
1839}
1840
1841/**
1842 * ib_register_client - Register an IB client
1843 * @client:Client to register
1844 *
1845 * Upper level users of the IB drivers can use ib_register_client() to
1846 * register callbacks for IB device addition and removal.  When an IB
1847 * device is added, each registered client's add method will be called
1848 * (in the order the clients were registered), and when a device is
1849 * removed, each client's remove method will be called (in the reverse
1850 * order that clients were registered).  In addition, when
1851 * ib_register_client() is called, the client will receive an add
1852 * callback for all devices already registered.
1853 */
1854int ib_register_client(struct ib_client *client)
1855{
1856	struct ib_device *device;
1857	unsigned long index;
1858	bool need_unreg = false;
1859	int ret;
1860
1861	refcount_set(&client->uses, 1);
1862	init_completion(&client->uses_zero);
1863
1864	/*
1865	 * The devices_rwsem is held in write mode to ensure that a racing
1866	 * ib_register_device() sees a consisent view of clients and devices.
1867	 */
1868	down_write(&devices_rwsem);
1869	down_write(&clients_rwsem);
1870	ret = assign_client_id(client);
1871	if (ret)
1872		goto out;
1873
1874	need_unreg = true;
1875	xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) {
1876		ret = add_client_context(device, client);
1877		if (ret)
1878			goto out;
1879	}
1880	ret = 0;
1881out:
1882	up_write(&clients_rwsem);
1883	up_write(&devices_rwsem);
1884	if (need_unreg && ret)
1885		ib_unregister_client(client);
1886	return ret;
1887}
1888EXPORT_SYMBOL(ib_register_client);
1889
1890/**
1891 * ib_unregister_client - Unregister an IB client
1892 * @client:Client to unregister
1893 *
1894 * Upper level users use ib_unregister_client() to remove their client
1895 * registration.  When ib_unregister_client() is called, the client
1896 * will receive a remove callback for each IB device still registered.
1897 *
1898 * This is a full fence, once it returns no client callbacks will be called,
1899 * or are running in another thread.
1900 */
1901void ib_unregister_client(struct ib_client *client)
1902{
1903	struct ib_device *device;
1904	unsigned long index;
1905
1906	down_write(&clients_rwsem);
1907	ib_client_put(client);
1908	xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED);
1909	up_write(&clients_rwsem);
1910
1911	/* We do not want to have locks while calling client->remove() */
1912	rcu_read_lock();
1913	xa_for_each (&devices, index, device) {
1914		if (!ib_device_try_get(device))
1915			continue;
1916		rcu_read_unlock();
1917
1918		remove_client_context(device, client->client_id);
1919
1920		ib_device_put(device);
1921		rcu_read_lock();
1922	}
1923	rcu_read_unlock();
1924
1925	/*
1926	 * remove_client_context() is not a fence, it can return even though a
1927	 * removal is ongoing. Wait until all removals are completed.
1928	 */
1929	wait_for_completion(&client->uses_zero);
1930	remove_client_id(client);
1931}
1932EXPORT_SYMBOL(ib_unregister_client);
1933
1934static int __ib_get_global_client_nl_info(const char *client_name,
1935					  struct ib_client_nl_info *res)
1936{
1937	struct ib_client *client;
1938	unsigned long index;
1939	int ret = -ENOENT;
1940
1941	down_read(&clients_rwsem);
1942	xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
1943		if (strcmp(client->name, client_name) != 0)
1944			continue;
1945		if (!client->get_global_nl_info) {
1946			ret = -EOPNOTSUPP;
1947			break;
1948		}
1949		ret = client->get_global_nl_info(res);
1950		if (WARN_ON(ret == -ENOENT))
1951			ret = -EINVAL;
1952		if (!ret && res->cdev)
1953			get_device(res->cdev);
1954		break;
1955	}
1956	up_read(&clients_rwsem);
1957	return ret;
1958}
1959
1960static int __ib_get_client_nl_info(struct ib_device *ibdev,
1961				   const char *client_name,
1962				   struct ib_client_nl_info *res)
1963{
1964	unsigned long index;
1965	void *client_data;
1966	int ret = -ENOENT;
1967
1968	down_read(&ibdev->client_data_rwsem);
1969	xan_for_each_marked (&ibdev->client_data, index, client_data,
1970			     CLIENT_DATA_REGISTERED) {
1971		struct ib_client *client = xa_load(&clients, index);
1972
1973		if (!client || strcmp(client->name, client_name) != 0)
1974			continue;
1975		if (!client->get_nl_info) {
1976			ret = -EOPNOTSUPP;
1977			break;
1978		}
1979		ret = client->get_nl_info(ibdev, client_data, res);
1980		if (WARN_ON(ret == -ENOENT))
1981			ret = -EINVAL;
1982
1983		/*
1984		 * The cdev is guaranteed valid as long as we are inside the
1985		 * client_data_rwsem as remove_one can't be called. Keep it
1986		 * valid for the caller.
1987		 */
1988		if (!ret && res->cdev)
1989			get_device(res->cdev);
1990		break;
1991	}
1992	up_read(&ibdev->client_data_rwsem);
1993
1994	return ret;
1995}
1996
1997/**
1998 * ib_get_client_nl_info - Fetch the nl_info from a client
1999 * @ibdev: IB device
2000 * @client_name: Name of the client
2001 * @res: Result of the query
2002 */
2003int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,
2004			  struct ib_client_nl_info *res)
2005{
2006	int ret;
2007
2008	if (ibdev)
2009		ret = __ib_get_client_nl_info(ibdev, client_name, res);
2010	else
2011		ret = __ib_get_global_client_nl_info(client_name, res);
2012#ifdef CONFIG_MODULES
2013	if (ret == -ENOENT) {
2014		request_module("rdma-client-%s", client_name);
2015		if (ibdev)
2016			ret = __ib_get_client_nl_info(ibdev, client_name, res);
2017		else
2018			ret = __ib_get_global_client_nl_info(client_name, res);
2019	}
2020#endif
2021	if (ret) {
2022		if (ret == -ENOENT)
2023			return -EOPNOTSUPP;
2024		return ret;
2025	}
2026
2027	if (WARN_ON(!res->cdev))
2028		return -EINVAL;
2029	return 0;
2030}
2031
2032/**
2033 * ib_set_client_data - Set IB client context
2034 * @device:Device to set context for
2035 * @client:Client to set context for
2036 * @data:Context to set
2037 *
2038 * ib_set_client_data() sets client context data that can be retrieved with
2039 * ib_get_client_data(). This can only be called while the client is
2040 * registered to the device, once the ib_client remove() callback returns this
2041 * cannot be called.
2042 */
2043void ib_set_client_data(struct ib_device *device, struct ib_client *client,
2044			void *data)
2045{
2046	void *rc;
2047
2048	if (WARN_ON(IS_ERR(data)))
2049		data = NULL;
2050
2051	rc = xa_store(&device->client_data, client->client_id, data,
2052		      GFP_KERNEL);
2053	WARN_ON(xa_is_err(rc));
2054}
2055EXPORT_SYMBOL(ib_set_client_data);
2056
2057/**
2058 * ib_register_event_handler - Register an IB event handler
2059 * @event_handler:Handler to register
2060 *
2061 * ib_register_event_handler() registers an event handler that will be
2062 * called back when asynchronous IB events occur (as defined in
2063 * chapter 11 of the InfiniBand Architecture Specification). This
2064 * callback occurs in workqueue context.
2065 */
2066void ib_register_event_handler(struct ib_event_handler *event_handler)
2067{
2068	down_write(&event_handler->device->event_handler_rwsem);
2069	list_add_tail(&event_handler->list,
2070		      &event_handler->device->event_handler_list);
2071	up_write(&event_handler->device->event_handler_rwsem);
2072}
2073EXPORT_SYMBOL(ib_register_event_handler);
2074
2075/**
2076 * ib_unregister_event_handler - Unregister an event handler
2077 * @event_handler:Handler to unregister
2078 *
2079 * Unregister an event handler registered with
2080 * ib_register_event_handler().
2081 */
2082void ib_unregister_event_handler(struct ib_event_handler *event_handler)
2083{
2084	down_write(&event_handler->device->event_handler_rwsem);
2085	list_del(&event_handler->list);
2086	up_write(&event_handler->device->event_handler_rwsem);
2087}
2088EXPORT_SYMBOL(ib_unregister_event_handler);
2089
2090void ib_dispatch_event_clients(struct ib_event *event)
2091{
2092	struct ib_event_handler *handler;
2093
2094	down_read(&event->device->event_handler_rwsem);
2095
2096	list_for_each_entry(handler, &event->device->event_handler_list, list)
2097		handler->handler(handler, event);
2098
2099	up_read(&event->device->event_handler_rwsem);
2100}
2101
2102static int iw_query_port(struct ib_device *device,
2103			   u32 port_num,
2104			   struct ib_port_attr *port_attr)
2105{
2106	struct in_device *inetdev;
2107	struct net_device *netdev;
2108
2109	memset(port_attr, 0, sizeof(*port_attr));
2110
2111	netdev = ib_device_get_netdev(device, port_num);
2112	if (!netdev)
2113		return -ENODEV;
2114
2115	port_attr->max_mtu = IB_MTU_4096;
2116	port_attr->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
2117
2118	if (!netif_carrier_ok(netdev)) {
2119		port_attr->state = IB_PORT_DOWN;
2120		port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
2121	} else {
2122		rcu_read_lock();
2123		inetdev = __in_dev_get_rcu(netdev);
2124
2125		if (inetdev && inetdev->ifa_list) {
2126			port_attr->state = IB_PORT_ACTIVE;
2127			port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
2128		} else {
2129			port_attr->state = IB_PORT_INIT;
2130			port_attr->phys_state =
2131				IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING;
2132		}
2133
2134		rcu_read_unlock();
2135	}
2136
2137	dev_put(netdev);
2138	return device->ops.query_port(device, port_num, port_attr);
2139}
2140
2141static int __ib_query_port(struct ib_device *device,
2142			   u32 port_num,
2143			   struct ib_port_attr *port_attr)
2144{
2145	int err;
2146
2147	memset(port_attr, 0, sizeof(*port_attr));
2148
2149	err = device->ops.query_port(device, port_num, port_attr);
2150	if (err || port_attr->subnet_prefix)
2151		return err;
2152
2153	if (rdma_port_get_link_layer(device, port_num) !=
2154	    IB_LINK_LAYER_INFINIBAND)
2155		return 0;
2156
2157	ib_get_cached_subnet_prefix(device, port_num,
2158				    &port_attr->subnet_prefix);
2159	return 0;
2160}
2161
2162/**
2163 * ib_query_port - Query IB port attributes
2164 * @device:Device to query
2165 * @port_num:Port number to query
2166 * @port_attr:Port attributes
2167 *
2168 * ib_query_port() returns the attributes of a port through the
2169 * @port_attr pointer.
2170 */
2171int ib_query_port(struct ib_device *device,
2172		  u32 port_num,
2173		  struct ib_port_attr *port_attr)
2174{
2175	if (!rdma_is_port_valid(device, port_num))
2176		return -EINVAL;
2177
2178	if (rdma_protocol_iwarp(device, port_num))
2179		return iw_query_port(device, port_num, port_attr);
2180	else
2181		return __ib_query_port(device, port_num, port_attr);
2182}
2183EXPORT_SYMBOL(ib_query_port);
2184
2185static void add_ndev_hash(struct ib_port_data *pdata)
2186{
2187	unsigned long flags;
2188
2189	might_sleep();
2190
2191	spin_lock_irqsave(&ndev_hash_lock, flags);
2192	if (hash_hashed(&pdata->ndev_hash_link)) {
2193		hash_del_rcu(&pdata->ndev_hash_link);
2194		spin_unlock_irqrestore(&ndev_hash_lock, flags);
2195		/*
2196		 * We cannot do hash_add_rcu after a hash_del_rcu until the
2197		 * grace period
2198		 */
2199		synchronize_rcu();
2200		spin_lock_irqsave(&ndev_hash_lock, flags);
2201	}
2202	if (pdata->netdev)
2203		hash_add_rcu(ndev_hash, &pdata->ndev_hash_link,
2204			     (uintptr_t)pdata->netdev);
2205	spin_unlock_irqrestore(&ndev_hash_lock, flags);
2206}
2207
2208/**
2209 * ib_device_set_netdev - Associate the ib_dev with an underlying net_device
2210 * @ib_dev: Device to modify
2211 * @ndev: net_device to affiliate, may be NULL
2212 * @port: IB port the net_device is connected to
2213 *
2214 * Drivers should use this to link the ib_device to a netdev so the netdev
2215 * shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be
2216 * affiliated with any port.
2217 *
2218 * The caller must ensure that the given ndev is not unregistered or
2219 * unregistering, and that either the ib_device is unregistered or
2220 * ib_device_set_netdev() is called with NULL when the ndev sends a
2221 * NETDEV_UNREGISTER event.
2222 */
2223int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
2224			 u32 port)
2225{
2226	enum rdma_nl_notify_event_type etype;
2227	struct net_device *old_ndev;
2228	struct ib_port_data *pdata;
2229	unsigned long flags;
2230	int ret;
2231
2232	if (!rdma_is_port_valid(ib_dev, port))
2233		return -EINVAL;
2234
2235	/*
2236	 * Drivers wish to call this before ib_register_driver, so we have to
2237	 * setup the port data early.
2238	 */
2239	ret = alloc_port_data(ib_dev);
2240	if (ret)
2241		return ret;
2242
2243	pdata = &ib_dev->port_data[port];
2244	spin_lock_irqsave(&pdata->netdev_lock, flags);
2245	old_ndev = rcu_dereference_protected(
2246		pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2247	if (old_ndev == ndev) {
2248		spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2249		return 0;
2250	}
2251
2252	rcu_assign_pointer(pdata->netdev, ndev);
2253	netdev_put(old_ndev, &pdata->netdev_tracker);
2254	netdev_hold(ndev, &pdata->netdev_tracker, GFP_ATOMIC);
2255	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2256
2257	add_ndev_hash(pdata);
2258
2259	/* Make sure that the device is registered before we send events */
2260	if (xa_load(&devices, ib_dev->index) != ib_dev)
2261		return 0;
2262
2263	etype = ndev ? RDMA_NETDEV_ATTACH_EVENT : RDMA_NETDEV_DETACH_EVENT;
2264	rdma_nl_notify_event(ib_dev, port, etype);
2265
2266	return 0;
2267}
2268EXPORT_SYMBOL(ib_device_set_netdev);
2269
2270static void free_netdevs(struct ib_device *ib_dev)
2271{
2272	unsigned long flags;
2273	u32 port;
2274
2275	if (!ib_dev->port_data)
2276		return;
2277
2278	rdma_for_each_port (ib_dev, port) {
2279		struct ib_port_data *pdata = &ib_dev->port_data[port];
2280		struct net_device *ndev;
2281
2282		spin_lock_irqsave(&pdata->netdev_lock, flags);
2283		ndev = rcu_dereference_protected(
2284			pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2285		if (ndev) {
2286			spin_lock(&ndev_hash_lock);
2287			hash_del_rcu(&pdata->ndev_hash_link);
2288			spin_unlock(&ndev_hash_lock);
2289
2290			/*
2291			 * If this is the last dev_put there is still a
2292			 * synchronize_rcu before the netdev is kfreed, so we
2293			 * can continue to rely on unlocked pointer
2294			 * comparisons after the put
2295			 */
2296			rcu_assign_pointer(pdata->netdev, NULL);
2297			netdev_put(ndev, &pdata->netdev_tracker);
2298		}
2299		spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2300	}
2301}
2302
2303struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,
2304					u32 port)
2305{
2306	struct ib_port_data *pdata;
2307	struct net_device *res;
2308
2309	if (!rdma_is_port_valid(ib_dev, port))
2310		return NULL;
2311
2312	if (!ib_dev->port_data)
2313		return NULL;
2314
2315	pdata = &ib_dev->port_data[port];
2316
2317	/*
2318	 * New drivers should use ib_device_set_netdev() not the legacy
2319	 * get_netdev().
2320	 */
2321	if (ib_dev->ops.get_netdev)
2322		res = ib_dev->ops.get_netdev(ib_dev, port);
2323	else {
2324		spin_lock(&pdata->netdev_lock);
2325		res = rcu_dereference_protected(
2326			pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2327		dev_hold(res);
2328		spin_unlock(&pdata->netdev_lock);
2329	}
2330
2331	return res;
2332}
2333EXPORT_SYMBOL(ib_device_get_netdev);
2334
2335/**
2336 * ib_query_netdev_port - Query the port number of a net_device
2337 * associated with an ibdev
2338 * @ibdev: IB device
2339 * @ndev: Network device
2340 * @port: IB port the net_device is connected to
2341 */
2342int ib_query_netdev_port(struct ib_device *ibdev, struct net_device *ndev,
2343			 u32 *port)
2344{
2345	struct net_device *ib_ndev;
2346	u32 port_num;
2347
2348	rdma_for_each_port(ibdev, port_num) {
2349		ib_ndev = ib_device_get_netdev(ibdev, port_num);
2350		if (ndev == ib_ndev) {
2351			*port = port_num;
2352			dev_put(ib_ndev);
2353			return 0;
2354		}
2355		dev_put(ib_ndev);
2356	}
2357
2358	return -ENOENT;
2359}
2360EXPORT_SYMBOL(ib_query_netdev_port);
2361
2362/**
2363 * ib_device_get_by_netdev - Find an IB device associated with a netdev
2364 * @ndev: netdev to locate
2365 * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all)
2366 *
2367 * Find and hold an ib_device that is associated with a netdev via
2368 * ib_device_set_netdev(). The caller must call ib_device_put() on the
2369 * returned pointer.
2370 */
2371struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
2372					  enum rdma_driver_id driver_id)
2373{
2374	struct ib_device *res = NULL;
2375	struct ib_port_data *cur;
2376
2377	rcu_read_lock();
2378	hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link,
2379				    (uintptr_t)ndev) {
2380		if (rcu_access_pointer(cur->netdev) == ndev &&
2381		    (driver_id == RDMA_DRIVER_UNKNOWN ||
2382		     cur->ib_dev->ops.driver_id == driver_id) &&
2383		    ib_device_try_get(cur->ib_dev)) {
2384			res = cur->ib_dev;
2385			break;
2386		}
2387	}
2388	rcu_read_unlock();
2389
2390	return res;
2391}
2392EXPORT_SYMBOL(ib_device_get_by_netdev);
2393
2394/**
2395 * ib_enum_roce_netdev - enumerate all RoCE ports
2396 * @ib_dev : IB device we want to query
2397 * @filter: Should we call the callback?
2398 * @filter_cookie: Cookie passed to filter
2399 * @cb: Callback to call for each found RoCE ports
2400 * @cookie: Cookie passed back to the callback
2401 *
2402 * Enumerates all of the physical RoCE ports of ib_dev
2403 * which are related to netdevice and calls callback() on each
2404 * device for which filter() function returns non zero.
2405 */
2406void ib_enum_roce_netdev(struct ib_device *ib_dev,
2407			 roce_netdev_filter filter,
2408			 void *filter_cookie,
2409			 roce_netdev_callback cb,
2410			 void *cookie)
2411{
2412	u32 port;
2413
2414	rdma_for_each_port (ib_dev, port)
2415		if (rdma_protocol_roce(ib_dev, port)) {
2416			struct net_device *idev =
2417				ib_device_get_netdev(ib_dev, port);
2418
2419			if (filter(ib_dev, port, idev, filter_cookie))
2420				cb(ib_dev, port, idev, cookie);
2421			dev_put(idev);
2422		}
2423}
2424
2425/**
2426 * ib_enum_all_roce_netdevs - enumerate all RoCE devices
2427 * @filter: Should we call the callback?
2428 * @filter_cookie: Cookie passed to filter
2429 * @cb: Callback to call for each found RoCE ports
2430 * @cookie: Cookie passed back to the callback
2431 *
2432 * Enumerates all RoCE devices' physical ports which are related
2433 * to netdevices and calls callback() on each device for which
2434 * filter() function returns non zero.
2435 */
2436void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
2437			      void *filter_cookie,
2438			      roce_netdev_callback cb,
2439			      void *cookie)
2440{
2441	struct ib_device *dev;
2442	unsigned long index;
2443
2444	down_read(&devices_rwsem);
2445	xa_for_each_marked(&devices, index, dev, DEVICE_GID_UPDATES)
2446		ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie);
2447	up_read(&devices_rwsem);
2448}
2449
2450/**
2451 * ib_device_enable_gid_updates - Mark device as ready for GID cache updates
2452 * @device: Device to mark
2453 *
2454 * Called after GID table is allocated and initialized. After this mark is set,
2455 * netdevice event handlers can update the device's GID cache. This allows
2456 * events that arrive during device registration to be processed, avoiding
2457 * stale GID entries when netdev properties change during the device
2458 * registration process.
2459 */
2460void ib_device_enable_gid_updates(struct ib_device *device)
2461{
2462	down_write(&devices_rwsem);
2463	xa_set_mark(&devices, device->index, DEVICE_GID_UPDATES);
2464	up_write(&devices_rwsem);
2465}
2466
2467/**
2468 * ib_device_disable_gid_updates - Clear the GID updates mark
2469 * @device: Device to unmark
2470 *
2471 * Called before GID table cleanup to prevent event handlers from accessing
2472 * the device while it's being torn down.
2473 */
2474void ib_device_disable_gid_updates(struct ib_device *device)
2475{
2476	down_write(&devices_rwsem);
2477	xa_clear_mark(&devices, device->index, DEVICE_GID_UPDATES);
2478	up_write(&devices_rwsem);
2479}
2480
2481/*
2482 * ib_enum_all_devs - enumerate all ib_devices
2483 * @cb: Callback to call for each found ib_device
2484 *
2485 * Enumerates all ib_devices and calls callback() on each device.
2486 */
2487int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
2488		     struct netlink_callback *cb)
2489{
2490	unsigned long index;
2491	struct ib_device *dev;
2492	unsigned int idx = 0;
2493	int ret = 0;
2494
2495	down_read(&devices_rwsem);
2496	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
2497		if (!rdma_dev_access_netns(dev, sock_net(skb->sk)))
2498			continue;
2499
2500		ret = nldev_cb(dev, skb, cb, idx);
2501		if (ret)
2502			break;
2503		idx++;
2504	}
2505	up_read(&devices_rwsem);
2506	return ret;
2507}
2508
2509/**
2510 * ib_query_pkey - Get P_Key table entry
2511 * @device:Device to query
2512 * @port_num:Port number to query
2513 * @index:P_Key table index to query
2514 * @pkey:Returned P_Key
2515 *
2516 * ib_query_pkey() fetches the specified P_Key table entry.
2517 */
2518int ib_query_pkey(struct ib_device *device,
2519		  u32 port_num, u16 index, u16 *pkey)
2520{
2521	if (!rdma_is_port_valid(device, port_num))
2522		return -EINVAL;
2523
2524	if (!device->ops.query_pkey)
2525		return -EOPNOTSUPP;
2526
2527	return device->ops.query_pkey(device, port_num, index, pkey);
2528}
2529EXPORT_SYMBOL(ib_query_pkey);
2530
2531/**
2532 * ib_modify_device - Change IB device attributes
2533 * @device:Device to modify
2534 * @device_modify_mask:Mask of attributes to change
2535 * @device_modify:New attribute values
2536 *
2537 * ib_modify_device() changes a device's attributes as specified by
2538 * the @device_modify_mask and @device_modify structure.
2539 */
2540int ib_modify_device(struct ib_device *device,
2541		     int device_modify_mask,
2542		     struct ib_device_modify *device_modify)
2543{
2544	if (!device->ops.modify_device)
2545		return -EOPNOTSUPP;
2546
2547	return device->ops.modify_device(device, device_modify_mask,
2548					 device_modify);
2549}
2550EXPORT_SYMBOL(ib_modify_device);
2551
2552/**
2553 * ib_modify_port - Modifies the attributes for the specified port.
2554 * @device: The device to modify.
2555 * @port_num: The number of the port to modify.
2556 * @port_modify_mask: Mask used to specify which attributes of the port
2557 *   to change.
2558 * @port_modify: New attribute values for the port.
2559 *
2560 * ib_modify_port() changes a port's attributes as specified by the
2561 * @port_modify_mask and @port_modify structure.
2562 */
2563int ib_modify_port(struct ib_device *device,
2564		   u32 port_num, int port_modify_mask,
2565		   struct ib_port_modify *port_modify)
2566{
2567	int rc;
2568
2569	if (!rdma_is_port_valid(device, port_num))
2570		return -EINVAL;
2571
2572	if (device->ops.modify_port)
2573		rc = device->ops.modify_port(device, port_num,
2574					     port_modify_mask,
2575					     port_modify);
2576	else if (rdma_protocol_roce(device, port_num) &&
2577		 ((port_modify->set_port_cap_mask & ~IB_PORT_CM_SUP) == 0 ||
2578		  (port_modify->clr_port_cap_mask & ~IB_PORT_CM_SUP) == 0))
2579		rc = 0;
2580	else
2581		rc = -EOPNOTSUPP;
2582	return rc;
2583}
2584EXPORT_SYMBOL(ib_modify_port);
2585
2586/**
2587 * ib_find_gid - Returns the port number and GID table index where
2588 *   a specified GID value occurs. Its searches only for IB link layer.
2589 * @device: The device to query.
2590 * @gid: The GID value to search for.
2591 * @port_num: The port number of the device where the GID value was found.
2592 * @index: The index into the GID table where the GID was found.  This
2593 *   parameter may be NULL.
2594 */
2595int ib_find_gid(struct ib_device *device, union ib_gid *gid,
2596		u32 *port_num, u16 *index)
2597{
2598	union ib_gid tmp_gid;
2599	u32 port;
2600	int ret, i;
2601
2602	rdma_for_each_port (device, port) {
2603		if (!rdma_protocol_ib(device, port))
2604			continue;
2605
2606		for (i = 0; i < device->port_data[port].immutable.gid_tbl_len;
2607		     ++i) {
2608			ret = rdma_query_gid(device, port, i, &tmp_gid);
2609			if (ret)
2610				continue;
2611
2612			if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
2613				*port_num = port;
2614				if (index)
2615					*index = i;
2616				return 0;
2617			}
2618		}
2619	}
2620
2621	return -ENOENT;
2622}
2623EXPORT_SYMBOL(ib_find_gid);
2624
2625/**
2626 * ib_find_pkey - Returns the PKey table index where a specified
2627 *   PKey value occurs.
2628 * @device: The device to query.
2629 * @port_num: The port number of the device to search for the PKey.
2630 * @pkey: The PKey value to search for.
2631 * @index: The index into the PKey table where the PKey was found.
2632 */
2633int ib_find_pkey(struct ib_device *device,
2634		 u32 port_num, u16 pkey, u16 *index)
2635{
2636	int ret, i;
2637	u16 tmp_pkey;
2638	int partial_ix = -1;
2639
2640	for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len;
2641	     ++i) {
2642		ret = ib_query_pkey(device, port_num, i, &tmp_pkey);
2643		if (ret)
2644			return ret;
2645		if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) {
2646			/* if there is full-member pkey take it.*/
2647			if (tmp_pkey & 0x8000) {
2648				*index = i;
2649				return 0;
2650			}
2651			if (partial_ix < 0)
2652				partial_ix = i;
2653		}
2654	}
2655
2656	/*no full-member, if exists take the limited*/
2657	if (partial_ix >= 0) {
2658		*index = partial_ix;
2659		return 0;
2660	}
2661	return -ENOENT;
2662}
2663EXPORT_SYMBOL(ib_find_pkey);
2664
2665/**
2666 * ib_get_net_dev_by_params() - Return the appropriate net_dev
2667 * for a received CM request
2668 * @dev:	An RDMA device on which the request has been received.
2669 * @port:	Port number on the RDMA device.
2670 * @pkey:	The Pkey the request came on.
2671 * @gid:	A GID that the net_dev uses to communicate.
2672 * @addr:	Contains the IP address that the request specified as its
2673 *		destination.
2674 *
2675 */
2676struct net_device *ib_get_net_dev_by_params(struct ib_device *dev,
2677					    u32 port,
2678					    u16 pkey,
2679					    const union ib_gid *gid,
2680					    const struct sockaddr *addr)
2681{
2682	struct net_device *net_dev = NULL;
2683	unsigned long index;
2684	void *client_data;
2685
2686	if (!rdma_protocol_ib(dev, port))
2687		return NULL;
2688
2689	/*
2690	 * Holding the read side guarantees that the client will not become
2691	 * unregistered while we are calling get_net_dev_by_params()
2692	 */
2693	down_read(&dev->client_data_rwsem);
2694	xan_for_each_marked (&dev->client_data, index, client_data,
2695			     CLIENT_DATA_REGISTERED) {
2696		struct ib_client *client = xa_load(&clients, index);
2697
2698		if (!client || !client->get_net_dev_by_params)
2699			continue;
2700
2701		net_dev = client->get_net_dev_by_params(dev, port, pkey, gid,
2702							addr, client_data);
2703		if (net_dev)
2704			break;
2705	}
2706	up_read(&dev->client_data_rwsem);
2707
2708	return net_dev;
2709}
2710EXPORT_SYMBOL(ib_get_net_dev_by_params);
2711
2712void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
2713{
2714	struct ib_device_ops *dev_ops = &dev->ops;
2715#define SET_DEVICE_OP(ptr, name)                                               \
2716	do {                                                                   \
2717		if (ops->name)                                                 \
2718			if (!((ptr)->name))				       \
2719				(ptr)->name = ops->name;                       \
2720	} while (0)
2721
2722#define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name)
2723
2724	if (ops->driver_id != RDMA_DRIVER_UNKNOWN) {
2725		WARN_ON(dev_ops->driver_id != RDMA_DRIVER_UNKNOWN &&
2726			dev_ops->driver_id != ops->driver_id);
2727		dev_ops->driver_id = ops->driver_id;
2728	}
2729	if (ops->owner) {
2730		WARN_ON(dev_ops->owner && dev_ops->owner != ops->owner);
2731		dev_ops->owner = ops->owner;
2732	}
2733	if (ops->uverbs_abi_ver)
2734		dev_ops->uverbs_abi_ver = ops->uverbs_abi_ver;
2735
2736	dev_ops->uverbs_no_driver_id_binding |=
2737		ops->uverbs_no_driver_id_binding;
2738
2739	SET_DEVICE_OP(dev_ops, add_gid);
2740	SET_DEVICE_OP(dev_ops, add_sub_dev);
2741	SET_DEVICE_OP(dev_ops, advise_mr);
2742	SET_DEVICE_OP(dev_ops, alloc_dm);
2743	SET_DEVICE_OP(dev_ops, alloc_dmah);
2744	SET_DEVICE_OP(dev_ops, alloc_hw_device_stats);
2745	SET_DEVICE_OP(dev_ops, alloc_hw_port_stats);
2746	SET_DEVICE_OP(dev_ops, alloc_mr);
2747	SET_DEVICE_OP(dev_ops, alloc_mr_integrity);
2748	SET_DEVICE_OP(dev_ops, alloc_mw);
2749	SET_DEVICE_OP(dev_ops, alloc_pd);
2750	SET_DEVICE_OP(dev_ops, alloc_rdma_netdev);
2751	SET_DEVICE_OP(dev_ops, alloc_ucontext);
2752	SET_DEVICE_OP(dev_ops, alloc_xrcd);
2753	SET_DEVICE_OP(dev_ops, attach_mcast);
2754	SET_DEVICE_OP(dev_ops, check_mr_status);
2755	SET_DEVICE_OP(dev_ops, counter_alloc_stats);
2756	SET_DEVICE_OP(dev_ops, counter_bind_qp);
2757	SET_DEVICE_OP(dev_ops, counter_dealloc);
2758	SET_DEVICE_OP(dev_ops, counter_init);
2759	SET_DEVICE_OP(dev_ops, counter_unbind_qp);
2760	SET_DEVICE_OP(dev_ops, counter_update_stats);
2761	SET_DEVICE_OP(dev_ops, create_ah);
2762	SET_DEVICE_OP(dev_ops, create_counters);
2763	SET_DEVICE_OP(dev_ops, create_cq);
2764	SET_DEVICE_OP(dev_ops, create_cq_umem);
2765	SET_DEVICE_OP(dev_ops, create_flow);
2766	SET_DEVICE_OP(dev_ops, create_qp);
2767	SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
2768	SET_DEVICE_OP(dev_ops, create_srq);
2769	SET_DEVICE_OP(dev_ops, create_user_ah);
2770	SET_DEVICE_OP(dev_ops, create_wq);
2771	SET_DEVICE_OP(dev_ops, dealloc_dm);
2772	SET_DEVICE_OP(dev_ops, dealloc_dmah);
2773	SET_DEVICE_OP(dev_ops, dealloc_driver);
2774	SET_DEVICE_OP(dev_ops, dealloc_mw);
2775	SET_DEVICE_OP(dev_ops, dealloc_pd);
2776	SET_DEVICE_OP(dev_ops, dealloc_ucontext);
2777	SET_DEVICE_OP(dev_ops, dealloc_xrcd);
2778	SET_DEVICE_OP(dev_ops, del_gid);
2779	SET_DEVICE_OP(dev_ops, del_sub_dev);
2780	SET_DEVICE_OP(dev_ops, dereg_mr);
2781	SET_DEVICE_OP(dev_ops, destroy_ah);
2782	SET_DEVICE_OP(dev_ops, destroy_counters);
2783	SET_DEVICE_OP(dev_ops, destroy_cq);
2784	SET_DEVICE_OP(dev_ops, destroy_flow);
2785	SET_DEVICE_OP(dev_ops, destroy_flow_action);
2786	SET_DEVICE_OP(dev_ops, destroy_qp);
2787	SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table);
2788	SET_DEVICE_OP(dev_ops, destroy_srq);
2789	SET_DEVICE_OP(dev_ops, destroy_wq);
2790	SET_DEVICE_OP(dev_ops, device_group);
2791	SET_DEVICE_OP(dev_ops, detach_mcast);
2792	SET_DEVICE_OP(dev_ops, disassociate_ucontext);
2793	SET_DEVICE_OP(dev_ops, drain_rq);
2794	SET_DEVICE_OP(dev_ops, drain_sq);
2795	SET_DEVICE_OP(dev_ops, enable_driver);
2796	SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry);
2797	SET_DEVICE_OP(dev_ops, fill_res_cq_entry);
2798	SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw);
2799	SET_DEVICE_OP(dev_ops, fill_res_mr_entry);
2800	SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw);
2801	SET_DEVICE_OP(dev_ops, fill_res_qp_entry);
2802	SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw);
2803	SET_DEVICE_OP(dev_ops, fill_res_srq_entry);
2804	SET_DEVICE_OP(dev_ops, fill_res_srq_entry_raw);
2805	SET_DEVICE_OP(dev_ops, fill_stat_mr_entry);
2806	SET_DEVICE_OP(dev_ops, get_dev_fw_str);
2807	SET_DEVICE_OP(dev_ops, get_dma_mr);
2808	SET_DEVICE_OP(dev_ops, get_hw_stats);
2809	SET_DEVICE_OP(dev_ops, get_link_layer);
2810	SET_DEVICE_OP(dev_ops, get_netdev);
2811	SET_DEVICE_OP(dev_ops, get_numa_node);
2812	SET_DEVICE_OP(dev_ops, get_port_immutable);
2813	SET_DEVICE_OP(dev_ops, get_vector_affinity);
2814	SET_DEVICE_OP(dev_ops, get_vf_config);
2815	SET_DEVICE_OP(dev_ops, get_vf_guid);
2816	SET_DEVICE_OP(dev_ops, get_vf_stats);
2817	SET_DEVICE_OP(dev_ops, iw_accept);
2818	SET_DEVICE_OP(dev_ops, iw_add_ref);
2819	SET_DEVICE_OP(dev_ops, iw_connect);
2820	SET_DEVICE_OP(dev_ops, iw_create_listen);
2821	SET_DEVICE_OP(dev_ops, iw_destroy_listen);
2822	SET_DEVICE_OP(dev_ops, iw_get_qp);
2823	SET_DEVICE_OP(dev_ops, iw_reject);
2824	SET_DEVICE_OP(dev_ops, iw_rem_ref);
2825	SET_DEVICE_OP(dev_ops, map_mr_sg);
2826	SET_DEVICE_OP(dev_ops, map_mr_sg_pi);
2827	SET_DEVICE_OP(dev_ops, mmap);
2828	SET_DEVICE_OP(dev_ops, mmap_free);
2829	SET_DEVICE_OP(dev_ops, modify_ah);
2830	SET_DEVICE_OP(dev_ops, modify_cq);
2831	SET_DEVICE_OP(dev_ops, modify_device);
2832	SET_DEVICE_OP(dev_ops, modify_hw_stat);
2833	SET_DEVICE_OP(dev_ops, modify_port);
2834	SET_DEVICE_OP(dev_ops, modify_qp);
2835	SET_DEVICE_OP(dev_ops, modify_srq);
2836	SET_DEVICE_OP(dev_ops, modify_wq);
2837	SET_DEVICE_OP(dev_ops, peek_cq);
2838	SET_DEVICE_OP(dev_ops, pre_destroy_cq);
2839	SET_DEVICE_OP(dev_ops, poll_cq);
2840	SET_DEVICE_OP(dev_ops, port_groups);
2841	SET_DEVICE_OP(dev_ops, post_destroy_cq);
2842	SET_DEVICE_OP(dev_ops, post_recv);
2843	SET_DEVICE_OP(dev_ops, post_send);
2844	SET_DEVICE_OP(dev_ops, post_srq_recv);
2845	SET_DEVICE_OP(dev_ops, process_mad);
2846	SET_DEVICE_OP(dev_ops, query_ah);
2847	SET_DEVICE_OP(dev_ops, query_device);
2848	SET_DEVICE_OP(dev_ops, query_gid);
2849	SET_DEVICE_OP(dev_ops, query_pkey);
2850	SET_DEVICE_OP(dev_ops, query_port);
2851	SET_DEVICE_OP(dev_ops, query_qp);
2852	SET_DEVICE_OP(dev_ops, query_srq);
2853	SET_DEVICE_OP(dev_ops, query_ucontext);
2854	SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);
2855	SET_DEVICE_OP(dev_ops, read_counters);
2856	SET_DEVICE_OP(dev_ops, reg_dm_mr);
2857	SET_DEVICE_OP(dev_ops, reg_user_mr);
2858	SET_DEVICE_OP(dev_ops, reg_user_mr_dmabuf);
2859	SET_DEVICE_OP(dev_ops, req_notify_cq);
2860	SET_DEVICE_OP(dev_ops, rereg_user_mr);
2861	SET_DEVICE_OP(dev_ops, resize_cq);
2862	SET_DEVICE_OP(dev_ops, set_vf_guid);
2863	SET_DEVICE_OP(dev_ops, set_vf_link_state);
2864	SET_DEVICE_OP(dev_ops, ufile_hw_cleanup);
2865	SET_DEVICE_OP(dev_ops, report_port_event);
2866
2867	SET_OBJ_SIZE(dev_ops, ib_ah);
2868	SET_OBJ_SIZE(dev_ops, ib_counters);
2869	SET_OBJ_SIZE(dev_ops, ib_cq);
2870	SET_OBJ_SIZE(dev_ops, ib_dmah);
2871	SET_OBJ_SIZE(dev_ops, ib_mw);
2872	SET_OBJ_SIZE(dev_ops, ib_pd);
2873	SET_OBJ_SIZE(dev_ops, ib_qp);
2874	SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table);
2875	SET_OBJ_SIZE(dev_ops, ib_srq);
2876	SET_OBJ_SIZE(dev_ops, ib_ucontext);
2877	SET_OBJ_SIZE(dev_ops, ib_xrcd);
2878	SET_OBJ_SIZE(dev_ops, rdma_counter);
2879}
2880EXPORT_SYMBOL(ib_set_device_ops);
2881
2882int ib_add_sub_device(struct ib_device *parent,
2883		      enum rdma_nl_dev_type type,
2884		      const char *name)
2885{
2886	struct ib_device *sub;
2887	int ret = 0;
2888
2889	if (!parent->ops.add_sub_dev || !parent->ops.del_sub_dev)
2890		return -EOPNOTSUPP;
2891
2892	if (!ib_device_try_get(parent))
2893		return -EINVAL;
2894
2895	sub = parent->ops.add_sub_dev(parent, type, name);
2896	if (IS_ERR(sub)) {
2897		ib_device_put(parent);
2898		return PTR_ERR(sub);
2899	}
2900
2901	sub->type = type;
2902	sub->parent = parent;
2903
2904	mutex_lock(&parent->subdev_lock);
2905	list_add_tail(&parent->subdev_list_head, &sub->subdev_list);
2906	mutex_unlock(&parent->subdev_lock);
2907
2908	return ret;
2909}
2910EXPORT_SYMBOL(ib_add_sub_device);
2911
2912int ib_del_sub_device_and_put(struct ib_device *sub)
2913{
2914	struct ib_device *parent = sub->parent;
2915
2916	if (!parent) {
2917		ib_device_put(sub);
2918		return -EOPNOTSUPP;
2919	}
2920
2921	mutex_lock(&parent->subdev_lock);
2922	list_del(&sub->subdev_list);
2923	mutex_unlock(&parent->subdev_lock);
2924
2925	ib_device_put(sub);
2926	parent->ops.del_sub_dev(sub);
2927	ib_device_put(parent);
2928
2929	return 0;
2930}
2931EXPORT_SYMBOL(ib_del_sub_device_and_put);
2932
2933#ifdef CONFIG_INFINIBAND_VIRT_DMA
2934int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents)
2935{
2936	struct scatterlist *s;
2937	int i;
2938
2939	for_each_sg(sg, s, nents, i) {
2940		sg_dma_address(s) = (uintptr_t)sg_virt(s);
2941		sg_dma_len(s) = s->length;
2942	}
2943	return nents;
2944}
2945EXPORT_SYMBOL(ib_dma_virt_map_sg);
2946#endif /* CONFIG_INFINIBAND_VIRT_DMA */
2947
2948static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {
2949	[RDMA_NL_LS_OP_RESOLVE] = {
2950		.doit = ib_nl_handle_resolve_resp,
2951		.flags = RDMA_NL_ADMIN_PERM,
2952	},
2953	[RDMA_NL_LS_OP_SET_TIMEOUT] = {
2954		.doit = ib_nl_handle_set_timeout,
2955		.flags = RDMA_NL_ADMIN_PERM,
2956	},
2957	[RDMA_NL_LS_OP_IP_RESOLVE] = {
2958		.doit = ib_nl_handle_ip_res_resp,
2959		.flags = RDMA_NL_ADMIN_PERM,
2960	},
2961};
2962
2963void ib_dispatch_port_state_event(struct ib_device *ibdev, struct net_device *ndev)
2964{
2965	enum ib_port_state curr_state;
2966	struct ib_event ibevent = {};
2967	u32 port;
2968
2969	if (ib_query_netdev_port(ibdev, ndev, &port))
2970		return;
2971
2972	curr_state = ib_get_curr_port_state(ndev);
2973
2974	write_lock_irq(&ibdev->cache_lock);
2975	if (ibdev->port_data[port].cache.last_port_state == curr_state) {
2976		write_unlock_irq(&ibdev->cache_lock);
2977		return;
2978	}
2979	ibdev->port_data[port].cache.last_port_state = curr_state;
2980	write_unlock_irq(&ibdev->cache_lock);
2981
2982	ibevent.event = (curr_state == IB_PORT_DOWN) ?
2983					IB_EVENT_PORT_ERR : IB_EVENT_PORT_ACTIVE;
2984	ibevent.device = ibdev;
2985	ibevent.element.port_num = port;
2986	ib_dispatch_event(&ibevent);
2987}
2988EXPORT_SYMBOL(ib_dispatch_port_state_event);
2989
2990static void handle_port_event(struct net_device *ndev, unsigned long event)
2991{
2992	struct ib_device *ibdev;
2993
2994	/* Currently, link events in bonding scenarios are still
2995	 * reported by drivers that support bonding.
2996	 */
2997	if (netif_is_lag_master(ndev) || netif_is_lag_port(ndev))
2998		return;
2999
3000	ibdev = ib_device_get_by_netdev(ndev, RDMA_DRIVER_UNKNOWN);
3001	if (!ibdev)
3002		return;
3003
3004	if (ibdev->ops.report_port_event) {
3005		ibdev->ops.report_port_event(ibdev, ndev, event);
3006		goto put_ibdev;
3007	}
3008
3009	ib_dispatch_port_state_event(ibdev, ndev);
3010
3011put_ibdev:
3012	ib_device_put(ibdev);
3013};
3014
3015static int ib_netdevice_event(struct notifier_block *this,
3016			      unsigned long event, void *ptr)
3017{
3018	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3019	struct ib_device *ibdev;
3020	u32 port;
3021
3022	switch (event) {
3023	case NETDEV_CHANGENAME:
3024		ibdev = ib_device_get_by_netdev(ndev, RDMA_DRIVER_UNKNOWN);
3025		if (!ibdev)
3026			return NOTIFY_DONE;
3027
3028		if (ib_query_netdev_port(ibdev, ndev, &port)) {
3029			ib_device_put(ibdev);
3030			break;
3031		}
3032
3033		rdma_nl_notify_event(ibdev, port, RDMA_NETDEV_RENAME_EVENT);
3034		ib_device_put(ibdev);
3035		break;
3036
3037	case NETDEV_UP:
3038	case NETDEV_CHANGE:
3039	case NETDEV_DOWN:
3040		handle_port_event(ndev, event);
3041		break;
3042
3043	default:
3044		break;
3045	}
3046
3047	return NOTIFY_DONE;
3048}
3049
3050static struct notifier_block nb_netdevice = {
3051	.notifier_call = ib_netdevice_event,
3052};
3053
3054static int __init ib_core_init(void)
3055{
3056	int ret = -ENOMEM;
3057
3058	ib_wq = alloc_workqueue("infiniband", WQ_PERCPU, 0);
3059	if (!ib_wq)
3060		return -ENOMEM;
3061
3062	ib_unreg_wq = alloc_workqueue("ib-unreg-wq", WQ_UNBOUND,
3063				      WQ_UNBOUND_MAX_ACTIVE);
3064	if (!ib_unreg_wq)
3065		goto err;
3066
3067	ib_comp_wq = alloc_workqueue("ib-comp-wq",
3068			WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS | WQ_PERCPU, 0);
3069	if (!ib_comp_wq)
3070		goto err_unbound;
3071
3072	ib_comp_unbound_wq =
3073		alloc_workqueue("ib-comp-unb-wq",
3074				WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM |
3075				WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE);
3076	if (!ib_comp_unbound_wq)
3077		goto err_comp;
3078
3079	ret = class_register(&ib_class);
3080	if (ret) {
3081		pr_warn("Couldn't create InfiniBand device class\n");
3082		goto err_comp_unbound;
3083	}
3084
3085	rdma_nl_init();
3086
3087	ret = addr_init();
3088	if (ret) {
3089		pr_warn("Couldn't init IB address resolution\n");
3090		goto err_ibnl;
3091	}
3092
3093	ret = ib_mad_init();
3094	if (ret) {
3095		pr_warn("Couldn't init IB MAD\n");
3096		goto err_addr;
3097	}
3098
3099	ret = ib_sa_init();
3100	if (ret) {
3101		pr_warn("Couldn't init SA\n");
3102		goto err_mad;
3103	}
3104
3105	ret = register_blocking_lsm_notifier(&ibdev_lsm_nb);
3106	if (ret) {
3107		pr_warn("Couldn't register LSM notifier. ret %d\n", ret);
3108		goto err_sa;
3109	}
3110
3111	ret = register_pernet_device(&rdma_dev_net_ops);
3112	if (ret) {
3113		pr_warn("Couldn't init compat dev. ret %d\n", ret);
3114		goto err_compat;
3115	}
3116
3117	nldev_init();
3118	rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
3119	ret = roce_gid_mgmt_init();
3120	if (ret) {
3121		pr_warn("Couldn't init RoCE GID management\n");
3122		goto err_parent;
3123	}
3124
3125	register_netdevice_notifier(&nb_netdevice);
3126
3127	return 0;
3128
3129err_parent:
3130	rdma_nl_unregister(RDMA_NL_LS);
3131	nldev_exit();
3132	unregister_pernet_device(&rdma_dev_net_ops);
3133err_compat:
3134	unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
3135err_sa:
3136	ib_sa_cleanup();
3137err_mad:
3138	ib_mad_cleanup();
3139err_addr:
3140	addr_cleanup();
3141err_ibnl:
3142	class_unregister(&ib_class);
3143err_comp_unbound:
3144	destroy_workqueue(ib_comp_unbound_wq);
3145err_comp:
3146	destroy_workqueue(ib_comp_wq);
3147err_unbound:
3148	destroy_workqueue(ib_unreg_wq);
3149err:
3150	destroy_workqueue(ib_wq);
3151	return ret;
3152}
3153
3154static void __exit ib_core_cleanup(void)
3155{
3156	unregister_netdevice_notifier(&nb_netdevice);
3157	roce_gid_mgmt_cleanup();
3158	rdma_nl_unregister(RDMA_NL_LS);
3159	nldev_exit();
3160	unregister_pernet_device(&rdma_dev_net_ops);
3161	unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
3162	ib_sa_cleanup();
3163	ib_mad_cleanup();
3164	addr_cleanup();
3165	rdma_nl_exit();
3166	class_unregister(&ib_class);
3167	destroy_workqueue(ib_comp_unbound_wq);
3168	destroy_workqueue(ib_comp_wq);
3169	/* Make sure that any pending umem accounting work is done. */
3170	destroy_workqueue(ib_wq);
3171	destroy_workqueue(ib_unreg_wq);
3172	WARN_ON(!xa_empty(&clients));
3173	WARN_ON(!xa_empty(&devices));
3174}
3175
3176MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4);
3177
3178/* ib core relies on netdev stack to first register net_ns_type_operations
3179 * ns kobject type before ib_core initialization.
3180 */
3181fs_initcall(ib_core_init);
3182module_exit(ib_core_cleanup);