ipoib_main.c

   1/*
   2 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include "ipoib.h"
  36
  37#include <linux/module.h>
  38
  39#include <linux/init.h>
  40#include <linux/slab.h>
  41#include <linux/kernel.h>
  42#include <linux/vmalloc.h>
  43
  44#include <linux/if_arp.h>	/* For ARPHRD_xxx */
  45
  46#include <linux/ip.h>
  47#include <linux/in.h>
  48
  49#include <linux/jhash.h>
  50#include <net/arp.h>
  51#include <net/addrconf.h>
  52#include <net/netdev_lock.h>
  53#include <net/pkt_sched.h>
  54#include <linux/inetdevice.h>
  55#include <rdma/ib_cache.h>
  56
  57MODULE_AUTHOR("Roland Dreier");
  58MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
  59MODULE_LICENSE("Dual BSD/GPL");
  60
  61int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
  62int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;
  63
  64module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);
  65MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
  66module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
  67MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");
  68
  69#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
  70int ipoib_debug_level;
  71
  72module_param_named(debug_level, ipoib_debug_level, int, 0644);
  73MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
  74#endif
  75
  76struct ipoib_path_iter {
  77	struct net_device *dev;
  78	struct ipoib_path  path;
  79};
  80
  81static const u8 ipv4_bcast_addr[] = {
  82	0x00, 0xff, 0xff, 0xff,
  83	0xff, 0x12, 0x40, 0x1b,	0x00, 0x00, 0x00, 0x00,
  84	0x00, 0x00, 0x00, 0x00,	0xff, 0xff, 0xff, 0xff
  85};
  86
  87struct workqueue_struct *ipoib_workqueue;
  88
  89struct ib_sa_client ipoib_sa_client;
  90
  91static int ipoib_add_one(struct ib_device *device);
  92static void ipoib_remove_one(struct ib_device *device, void *client_data);
  93static void ipoib_neigh_reclaim(struct rcu_head *rp);
  94static struct net_device *ipoib_get_net_dev_by_params(
  95		struct ib_device *dev, u32 port, u16 pkey,
  96		const union ib_gid *gid, const struct sockaddr *addr,
  97		void *client_data);
  98static int ipoib_set_mac(struct net_device *dev, void *addr);
  99static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
 100		       int cmd);
 101
 102static struct ib_client ipoib_client = {
 103	.name   = "ipoib",
 104	.add    = ipoib_add_one,
 105	.remove = ipoib_remove_one,
 106	.get_net_dev_by_params = ipoib_get_net_dev_by_params,
 107};
 108
 109#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
 110static int ipoib_netdev_event(struct notifier_block *this,
 111			      unsigned long event, void *ptr)
 112{
 113	struct netdev_notifier_info *ni = ptr;
 114	struct net_device *dev = ni->dev;
 115
 116	if (dev->netdev_ops->ndo_open != ipoib_open)
 117		return NOTIFY_DONE;
 118
 119	switch (event) {
 120	case NETDEV_REGISTER:
 121		ipoib_create_debug_files(dev);
 122		break;
 123	case NETDEV_CHANGENAME:
 124		ipoib_delete_debug_files(dev);
 125		ipoib_create_debug_files(dev);
 126		break;
 127	case NETDEV_UNREGISTER:
 128		ipoib_delete_debug_files(dev);
 129		break;
 130	}
 131
 132	return NOTIFY_DONE;
 133}
 134#endif
 135
 136struct ipoib_ifupdown_work {
 137	struct work_struct work;
 138	struct net_device *dev;
 139	netdevice_tracker dev_tracker;
 140	bool up;
 141};
 142
 143static void ipoib_ifupdown_task(struct work_struct *work)
 144{
 145	struct ipoib_ifupdown_work *pwork =
 146		container_of(work, struct ipoib_ifupdown_work, work);
 147	struct net_device *dev = pwork->dev;
 148	unsigned int flags;
 149
 150	rtnl_lock();
 151	flags = dev->flags;
 152	if (pwork->up)
 153		flags |= IFF_UP;
 154	else
 155		flags &= ~IFF_UP;
 156
 157	if (dev->flags != flags)
 158		dev_change_flags(dev, flags, NULL);
 159	rtnl_unlock();
 160	netdev_put(dev, &pwork->dev_tracker);
 161	kfree(pwork);
 162}
 163
 164static void ipoib_schedule_ifupdown_task(struct net_device *dev, bool up)
 165{
 166	struct ipoib_ifupdown_work *work;
 167
 168	if ((up && (dev->flags & IFF_UP)) ||
 169	    (!up && !(dev->flags & IFF_UP)))
 170		return;
 171
 172	work = kmalloc(sizeof(*work), GFP_KERNEL);
 173	if (!work)
 174		return;
 175	work->dev = dev;
 176	netdev_hold(dev, &work->dev_tracker, GFP_KERNEL);
 177	work->up = up;
 178	INIT_WORK(&work->work, ipoib_ifupdown_task);
 179	queue_work(ipoib_workqueue, &work->work);
 180}
 181
 182int ipoib_open(struct net_device *dev)
 183{
 184	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 185
 186	ipoib_dbg(priv, "bringing up interface\n");
 187
 188	netif_carrier_off(dev);
 189
 190	set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
 191
 192	if (ipoib_ib_dev_open(dev)) {
 193		if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
 194			return 0;
 195		goto err_disable;
 196	}
 197
 198	ipoib_ib_dev_up(dev);
 199
 200	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
 201		struct ipoib_dev_priv *cpriv;
 202
 203		/* Bring up any child interfaces too */
 204		netdev_lock_ops_to_full(dev);
 205		list_for_each_entry(cpriv, &priv->child_intfs, list)
 206			ipoib_schedule_ifupdown_task(cpriv->dev, true);
 207		netdev_unlock_full_to_ops(dev);
 208	} else if (priv->parent) {
 209		struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
 210
 211		if (!test_bit(IPOIB_FLAG_ADMIN_UP, &ppriv->flags))
 212			ipoib_dbg(priv, "parent device %s is not up, so child device may be not functioning.\n",
 213				  ppriv->dev->name);
 214	}
 215	netif_start_queue(dev);
 216
 217	return 0;
 218
 219err_disable:
 220	clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
 221
 222	return -EINVAL;
 223}
 224
 225static int ipoib_stop(struct net_device *dev)
 226{
 227	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 228
 229	ipoib_dbg(priv, "stopping interface\n");
 230
 231	clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
 232
 233	netif_stop_queue(dev);
 234
 235	ipoib_ib_dev_down(dev);
 236	ipoib_ib_dev_stop(dev);
 237
 238	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
 239		struct ipoib_dev_priv *cpriv;
 240
 241		/* Bring down any child interfaces too */
 242		netdev_lock_ops_to_full(dev);
 243		list_for_each_entry(cpriv, &priv->child_intfs, list)
 244			ipoib_schedule_ifupdown_task(cpriv->dev, false);
 245		netdev_unlock_full_to_ops(dev);
 246	}
 247
 248	return 0;
 249}
 250
 251static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)
 252{
 253	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 254
 255	if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
 256		features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);
 257
 258	return features;
 259}
 260
 261static int ipoib_change_mtu(struct net_device *dev, int new_mtu)
 262{
 263	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 264	int ret = 0;
 265
 266	/* dev->mtu > 2K ==> connected mode */
 267	if (ipoib_cm_admin_enabled(dev)) {
 268		if (new_mtu > ipoib_cm_max_mtu(dev))
 269			return -EINVAL;
 270
 271		if (new_mtu > priv->mcast_mtu)
 272			ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
 273				   priv->mcast_mtu);
 274
 275		WRITE_ONCE(dev->mtu, new_mtu);
 276		return 0;
 277	}
 278
 279	if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||
 280	    new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
 281		return -EINVAL;
 282
 283	priv->admin_mtu = new_mtu;
 284
 285	if (priv->mcast_mtu < priv->admin_mtu)
 286		ipoib_dbg(priv, "MTU must be smaller than the underlying "
 287				"link layer MTU - 4 (%u)\n", priv->mcast_mtu);
 288
 289	new_mtu = min(priv->mcast_mtu, priv->admin_mtu);
 290
 291	if (priv->rn_ops->ndo_change_mtu) {
 292		bool carrier_status = netif_carrier_ok(dev);
 293
 294		netif_carrier_off(dev);
 295
 296		/* notify lower level on the real mtu */
 297		ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);
 298
 299		if (carrier_status)
 300			netif_carrier_on(dev);
 301	} else {
 302		WRITE_ONCE(dev->mtu, new_mtu);
 303	}
 304
 305	return ret;
 306}
 307
 308static void ipoib_get_stats(struct net_device *dev,
 309			    struct rtnl_link_stats64 *stats)
 310{
 311	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 312
 313	if (priv->rn_ops->ndo_get_stats64)
 314		priv->rn_ops->ndo_get_stats64(dev, stats);
 315	else
 316		netdev_stats_to_stats64(stats, &dev->stats);
 317}
 318
 319/* Called with an RCU read lock taken */
 320static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,
 321					struct net_device *dev)
 322{
 323	struct net *net = dev_net(dev);
 324	struct in_device *in_dev;
 325	struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;
 326	struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;
 327	__be32 ret_addr;
 328
 329	switch (addr->sa_family) {
 330	case AF_INET:
 331		in_dev = in_dev_get(dev);
 332		if (!in_dev)
 333			return false;
 334
 335		ret_addr = inet_confirm_addr(net, in_dev, 0,
 336					     addr_in->sin_addr.s_addr,
 337					     RT_SCOPE_HOST);
 338		in_dev_put(in_dev);
 339		if (ret_addr)
 340			return true;
 341
 342		break;
 343	case AF_INET6:
 344		if (IS_ENABLED(CONFIG_IPV6) &&
 345		    ipv6_chk_addr(net, &addr_in6->sin6_addr, dev, 1))
 346			return true;
 347
 348		break;
 349	}
 350	return false;
 351}
 352
 353/*
 354 * Find the L2 master net_device on top of the given net_device.
 355 * @dev: base IPoIB net_device
 356 *
 357 * Returns the L2 master net_device with reference held if the L2 master
 358 * exists (such as bond netdevice), or returns same netdev with reference
 359 * held when master does not exist or when L3 master (such as VRF netdev).
 360 */
 361static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)
 362{
 363	struct net_device *master;
 364
 365	rcu_read_lock();
 366
 367	master = netdev_master_upper_dev_get_rcu(dev);
 368	if (!master || netif_is_l3_master(master))
 369		master = dev;
 370
 371	dev_hold(master);
 372	rcu_read_unlock();
 373
 374	return master;
 375}
 376
 377struct ipoib_walk_data {
 378	const struct sockaddr *addr;
 379	struct net_device *result;
 380};
 381
 382static int ipoib_upper_walk(struct net_device *upper,
 383			    struct netdev_nested_priv *priv)
 384{
 385	struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;
 386	int ret = 0;
 387
 388	if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {
 389		dev_hold(upper);
 390		data->result = upper;
 391		ret = 1;
 392	}
 393
 394	return ret;
 395}
 396
 397/**
 398 * ipoib_get_net_dev_match_addr - Find a net_device matching
 399 * the given address, which is an upper device of the given net_device.
 400 *
 401 * @addr: IP address to look for.
 402 * @dev: base IPoIB net_device
 403 *
 404 * If found, returns the net_device with a reference held. Otherwise return
 405 * NULL.
 406 */
 407static struct net_device *ipoib_get_net_dev_match_addr(
 408		const struct sockaddr *addr, struct net_device *dev)
 409{
 410	struct netdev_nested_priv priv;
 411	struct ipoib_walk_data data = {
 412		.addr = addr,
 413	};
 414
 415	priv.data = (void *)&data;
 416	rcu_read_lock();
 417	if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
 418		dev_hold(dev);
 419		data.result = dev;
 420		goto out;
 421	}
 422
 423	netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &priv);
 424out:
 425	rcu_read_unlock();
 426	return data.result;
 427}
 428
 429/* returns the number of IPoIB netdevs on top a given ipoib device matching a
 430 * pkey_index and address, if one exists.
 431 *
 432 * @found_net_dev: contains a matching net_device if the return value >= 1,
 433 * with a reference held. */
 434static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,
 435				     const union ib_gid *gid,
 436				     u16 pkey_index,
 437				     const struct sockaddr *addr,
 438				     int nesting,
 439				     struct net_device **found_net_dev)
 440{
 441	struct ipoib_dev_priv *child_priv;
 442	struct net_device *net_dev = NULL;
 443	int matches = 0;
 444
 445	if (priv->pkey_index == pkey_index &&
 446	    (!gid || !memcmp(gid, &priv->local_gid, sizeof(*gid)))) {
 447		if (!addr) {
 448			net_dev = ipoib_get_master_net_dev(priv->dev);
 449		} else {
 450			/* Verify the net_device matches the IP address, as
 451			 * IPoIB child devices currently share a GID. */
 452			net_dev = ipoib_get_net_dev_match_addr(addr, priv->dev);
 453		}
 454		if (net_dev) {
 455			if (!*found_net_dev)
 456				*found_net_dev = net_dev;
 457			else
 458				dev_put(net_dev);
 459			++matches;
 460		}
 461	}
 462
 463	if (test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
 464		return matches;
 465
 466	/* Check child interfaces */
 467	netdev_lock(priv->dev);
 468	list_for_each_entry(child_priv, &priv->child_intfs, list) {
 469		matches += ipoib_match_gid_pkey_addr(child_priv, gid,
 470						     pkey_index, addr,
 471						     nesting + 1,
 472						     found_net_dev);
 473		if (matches > 1)
 474			break;
 475	}
 476	netdev_unlock(priv->dev);
 477
 478	return matches;
 479}
 480
 481/* Returns the number of matching net_devs found (between 0 and 2). Also
 482 * return the matching net_device in the @net_dev parameter, holding a
 483 * reference to the net_device, if the number of matches >= 1 */
 484static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u32 port,
 485					 u16 pkey_index,
 486					 const union ib_gid *gid,
 487					 const struct sockaddr *addr,
 488					 struct net_device **net_dev)
 489{
 490	struct ipoib_dev_priv *priv;
 491	int matches = 0;
 492
 493	*net_dev = NULL;
 494
 495	list_for_each_entry(priv, dev_list, list) {
 496		if (priv->port != port)
 497			continue;
 498
 499		matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,
 500						     addr, 0, net_dev);
 501		if (matches > 1)
 502			break;
 503	}
 504
 505	return matches;
 506}
 507
 508static struct net_device *ipoib_get_net_dev_by_params(
 509		struct ib_device *dev, u32 port, u16 pkey,
 510		const union ib_gid *gid, const struct sockaddr *addr,
 511		void *client_data)
 512{
 513	struct net_device *net_dev;
 514	struct list_head *dev_list = client_data;
 515	u16 pkey_index;
 516	int matches;
 517	int ret;
 518
 519	if (!rdma_protocol_ib(dev, port))
 520		return NULL;
 521
 522	ret = ib_find_cached_pkey(dev, port, pkey, &pkey_index);
 523	if (ret)
 524		return NULL;
 525
 526	/* See if we can find a unique device matching the pkey and GID */
 527	matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
 528						gid, NULL, &net_dev);
 529
 530	switch (matches) {
 531	case 0:
 532		return NULL;
 533	case 1:
 534		return net_dev;
 535	}
 536
 537	dev_put(net_dev);
 538
 539	/* Couldn't find a unique device with pkey and GID only. Use L3
 540	 * address to uniquely match the net device */
 541	matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
 542						gid, addr, &net_dev);
 543	switch (matches) {
 544	case 0:
 545		return NULL;
 546	default:
 547		dev_warn_ratelimited(&dev->dev,
 548				     "duplicate IP address detected\n");
 549		fallthrough;
 550	case 1:
 551		return net_dev;
 552	}
 553}
 554
 555int ipoib_set_mode(struct net_device *dev, const char *buf)
 556{
 557	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 558
 559	if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
 560	     !strcmp(buf, "connected\n")) ||
 561	     (!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
 562	     !strcmp(buf, "datagram\n"))) {
 563		return 0;
 564	}
 565
 566	/* flush paths if we switch modes so that connections are restarted */
 567	if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {
 568		set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
 569		ipoib_warn(priv, "enabling connected mode "
 570			   "will cause multicast packet drops\n");
 571		netdev_lock_ops(dev);
 572		netdev_update_features(dev);
 573		netif_set_mtu(dev, ipoib_cm_max_mtu(dev));
 574		netif_set_real_num_tx_queues(dev, 1);
 575		netdev_unlock_ops(dev);
 576		rtnl_unlock();
 577		priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
 578
 579		ipoib_flush_paths(dev);
 580		return (!rtnl_trylock()) ? -EBUSY : 0;
 581	}
 582
 583	if (!strcmp(buf, "datagram\n")) {
 584		clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
 585		netdev_lock_ops(dev);
 586		netdev_update_features(dev);
 587		netif_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
 588		netif_set_real_num_tx_queues(dev, dev->num_tx_queues);
 589		netdev_unlock_ops(dev);
 590		rtnl_unlock();
 591		ipoib_flush_paths(dev);
 592		return (!rtnl_trylock()) ? -EBUSY : 0;
 593	}
 594
 595	return -EINVAL;
 596}
 597
 598struct ipoib_path *__path_find(struct net_device *dev, void *gid)
 599{
 600	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 601	struct rb_node *n = priv->path_tree.rb_node;
 602	struct ipoib_path *path;
 603	int ret;
 604
 605	while (n) {
 606		path = rb_entry(n, struct ipoib_path, rb_node);
 607
 608		ret = memcmp(gid, path->pathrec.dgid.raw,
 609			     sizeof (union ib_gid));
 610
 611		if (ret < 0)
 612			n = n->rb_left;
 613		else if (ret > 0)
 614			n = n->rb_right;
 615		else
 616			return path;
 617	}
 618
 619	return NULL;
 620}
 621
 622static int __path_add(struct net_device *dev, struct ipoib_path *path)
 623{
 624	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 625	struct rb_node **n = &priv->path_tree.rb_node;
 626	struct rb_node *pn = NULL;
 627	struct ipoib_path *tpath;
 628	int ret;
 629
 630	while (*n) {
 631		pn = *n;
 632		tpath = rb_entry(pn, struct ipoib_path, rb_node);
 633
 634		ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw,
 635			     sizeof (union ib_gid));
 636		if (ret < 0)
 637			n = &pn->rb_left;
 638		else if (ret > 0)
 639			n = &pn->rb_right;
 640		else
 641			return -EEXIST;
 642	}
 643
 644	rb_link_node(&path->rb_node, pn, n);
 645	rb_insert_color(&path->rb_node, &priv->path_tree);
 646
 647	list_add_tail(&path->list, &priv->path_list);
 648
 649	return 0;
 650}
 651
 652static void path_free(struct net_device *dev, struct ipoib_path *path)
 653{
 654	struct sk_buff *skb;
 655
 656	while ((skb = __skb_dequeue(&path->queue)))
 657		dev_kfree_skb_irq(skb);
 658
 659	ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);
 660
 661	/* remove all neigh connected to this path */
 662	ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);
 663
 664	if (path->ah)
 665		ipoib_put_ah(path->ah);
 666
 667	kfree(path);
 668}
 669
 670#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
 671
 672struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)
 673{
 674	struct ipoib_path_iter *iter;
 675
 676	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
 677	if (!iter)
 678		return NULL;
 679
 680	iter->dev = dev;
 681	memset(iter->path.pathrec.dgid.raw, 0, 16);
 682
 683	if (ipoib_path_iter_next(iter)) {
 684		kfree(iter);
 685		return NULL;
 686	}
 687
 688	return iter;
 689}
 690
 691int ipoib_path_iter_next(struct ipoib_path_iter *iter)
 692{
 693	struct ipoib_dev_priv *priv = ipoib_priv(iter->dev);
 694	struct rb_node *n;
 695	struct ipoib_path *path;
 696	int ret = 1;
 697
 698	spin_lock_irq(&priv->lock);
 699
 700	n = rb_first(&priv->path_tree);
 701
 702	while (n) {
 703		path = rb_entry(n, struct ipoib_path, rb_node);
 704
 705		if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw,
 706			   sizeof (union ib_gid)) < 0) {
 707			iter->path = *path;
 708			ret = 0;
 709			break;
 710		}
 711
 712		n = rb_next(n);
 713	}
 714
 715	spin_unlock_irq(&priv->lock);
 716
 717	return ret;
 718}
 719
 720void ipoib_path_iter_read(struct ipoib_path_iter *iter,
 721			  struct ipoib_path *path)
 722{
 723	*path = iter->path;
 724}
 725
 726#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */
 727
 728void ipoib_mark_paths_invalid(struct net_device *dev)
 729{
 730	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 731	struct ipoib_path *path, *tp;
 732
 733	spin_lock_irq(&priv->lock);
 734
 735	list_for_each_entry_safe(path, tp, &priv->path_list, list) {
 736		ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
 737			  be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
 738			  path->pathrec.dgid.raw);
 739		if (path->ah)
 740			path->ah->valid = 0;
 741	}
 742
 743	spin_unlock_irq(&priv->lock);
 744}
 745
 746static void push_pseudo_header(struct sk_buff *skb, const char *daddr)
 747{
 748	struct ipoib_pseudo_header *phdr;
 749
 750	phdr = skb_push(skb, sizeof(*phdr));
 751	memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);
 752}
 753
 754void ipoib_flush_paths(struct net_device *dev)
 755{
 756	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 757	struct ipoib_path *path, *tp;
 758	LIST_HEAD(remove_list);
 759	unsigned long flags;
 760
 761	netif_tx_lock_bh(dev);
 762	spin_lock_irqsave(&priv->lock, flags);
 763
 764	list_splice_init(&priv->path_list, &remove_list);
 765
 766	list_for_each_entry(path, &remove_list, list)
 767		rb_erase(&path->rb_node, &priv->path_tree);
 768
 769	list_for_each_entry_safe(path, tp, &remove_list, list) {
 770		if (path->query)
 771			ib_sa_cancel_query(path->query_id, path->query);
 772		spin_unlock_irqrestore(&priv->lock, flags);
 773		netif_tx_unlock_bh(dev);
 774		wait_for_completion(&path->done);
 775		path_free(dev, path);
 776		netif_tx_lock_bh(dev);
 777		spin_lock_irqsave(&priv->lock, flags);
 778	}
 779
 780	spin_unlock_irqrestore(&priv->lock, flags);
 781	netif_tx_unlock_bh(dev);
 782}
 783
 784static void path_rec_completion(int status,
 785				struct sa_path_rec *pathrec,
 786				unsigned int num_prs, void *path_ptr)
 787{
 788	struct ipoib_path *path = path_ptr;
 789	struct net_device *dev = path->dev;
 790	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 791	struct ipoib_ah *ah = NULL;
 792	struct ipoib_ah *old_ah = NULL;
 793	struct ipoib_neigh *neigh, *tn;
 794	struct sk_buff_head skqueue;
 795	struct sk_buff *skb;
 796	unsigned long flags;
 797
 798	if (!status)
 799		ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",
 800			  be32_to_cpu(sa_path_get_dlid(pathrec)),
 801			  pathrec->dgid.raw);
 802	else
 803		ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",
 804			  status, path->pathrec.dgid.raw);
 805
 806	skb_queue_head_init(&skqueue);
 807
 808	if (!status) {
 809		struct rdma_ah_attr av;
 810
 811		if (!ib_init_ah_attr_from_path(priv->ca, priv->port,
 812					       pathrec, &av, NULL)) {
 813			ah = ipoib_create_ah(dev, priv->pd, &av);
 814			rdma_destroy_ah_attr(&av);
 815		}
 816	}
 817
 818	spin_lock_irqsave(&priv->lock, flags);
 819
 820	if (!IS_ERR_OR_NULL(ah)) {
 821		/*
 822		 * pathrec.dgid is used as the database key from the LLADDR,
 823		 * it must remain unchanged even if the SA returns a different
 824		 * GID to use in the AH.
 825		 */
 826		if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,
 827			   sizeof(union ib_gid))) {
 828			ipoib_dbg(
 829				priv,
 830				"%s got PathRec for gid %pI6 while asked for %pI6\n",
 831				dev->name, pathrec->dgid.raw,
 832				path->pathrec.dgid.raw);
 833			memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
 834			       sizeof(union ib_gid));
 835		}
 836
 837		path->pathrec = *pathrec;
 838
 839		old_ah   = path->ah;
 840		path->ah = ah;
 841
 842		ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",
 843			  ah, be32_to_cpu(sa_path_get_dlid(pathrec)),
 844			  pathrec->sl);
 845
 846		while ((skb = __skb_dequeue(&path->queue)))
 847			__skb_queue_tail(&skqueue, skb);
 848
 849		list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
 850			if (neigh->ah) {
 851				WARN_ON(neigh->ah != old_ah);
 852				/*
 853				 * Dropping the ah reference inside
 854				 * priv->lock is safe here, because we
 855				 * will hold one more reference from
 856				 * the original value of path->ah (ie
 857				 * old_ah).
 858				 */
 859				ipoib_put_ah(neigh->ah);
 860			}
 861			kref_get(&path->ah->ref);
 862			neigh->ah = path->ah;
 863
 864			if (ipoib_cm_enabled(dev, neigh->daddr)) {
 865				if (!ipoib_cm_get(neigh))
 866					ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,
 867									       path,
 868									       neigh));
 869				if (!ipoib_cm_get(neigh)) {
 870					ipoib_neigh_free(neigh);
 871					continue;
 872				}
 873			}
 874
 875			while ((skb = __skb_dequeue(&neigh->queue)))
 876				__skb_queue_tail(&skqueue, skb);
 877		}
 878		path->ah->valid = 1;
 879	}
 880
 881	path->query = NULL;
 882	complete(&path->done);
 883
 884	spin_unlock_irqrestore(&priv->lock, flags);
 885
 886	if (IS_ERR_OR_NULL(ah))
 887		ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);
 888
 889	if (old_ah)
 890		ipoib_put_ah(old_ah);
 891
 892	while ((skb = __skb_dequeue(&skqueue))) {
 893		int ret;
 894		skb->dev = dev;
 895		ret = dev_queue_xmit(skb);
 896		if (ret)
 897			ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",
 898				   __func__, ret);
 899	}
 900}
 901
 902static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
 903			  void *gid)
 904{
 905	path->dev = priv->dev;
 906
 907	if (rdma_cap_opa_ah(priv->ca, priv->port))
 908		path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
 909	else
 910		path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;
 911
 912	memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
 913	path->pathrec.sgid	    = priv->local_gid;
 914	path->pathrec.pkey	    = cpu_to_be16(priv->pkey);
 915	path->pathrec.numb_path     = 1;
 916	path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
 917}
 918
 919static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
 920{
 921	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 922	struct ipoib_path *path;
 923
 924	if (!priv->broadcast)
 925		return NULL;
 926
 927	path = kzalloc(sizeof(*path), GFP_ATOMIC);
 928	if (!path)
 929		return NULL;
 930
 931	skb_queue_head_init(&path->queue);
 932
 933	INIT_LIST_HEAD(&path->neigh_list);
 934
 935	init_path_rec(priv, path, gid);
 936
 937	return path;
 938}
 939
 940static int path_rec_start(struct net_device *dev,
 941			  struct ipoib_path *path)
 942{
 943	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 944
 945	ipoib_dbg(priv, "Start path record lookup for %pI6\n",
 946		  path->pathrec.dgid.raw);
 947
 948	init_completion(&path->done);
 949
 950	path->query_id =
 951		ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
 952				   &path->pathrec,
 953				   IB_SA_PATH_REC_DGID		|
 954				   IB_SA_PATH_REC_SGID		|
 955				   IB_SA_PATH_REC_NUMB_PATH	|
 956				   IB_SA_PATH_REC_TRAFFIC_CLASS |
 957				   IB_SA_PATH_REC_PKEY,
 958				   1000, GFP_ATOMIC,
 959				   path_rec_completion,
 960				   path, &path->query);
 961	if (path->query_id < 0) {
 962		ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);
 963		path->query = NULL;
 964		complete(&path->done);
 965		return path->query_id;
 966	}
 967
 968	return 0;
 969}
 970
 971static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
 972			       struct net_device *dev)
 973{
 974	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 975	struct ipoib_path *path;
 976	unsigned long flags;
 977
 978	spin_lock_irqsave(&priv->lock, flags);
 979
 980	path = __path_find(dev, daddr + 4);
 981	if (!path)
 982		goto out;
 983	if (!path->query)
 984		path_rec_start(dev, path);
 985out:
 986	spin_unlock_irqrestore(&priv->lock, flags);
 987}
 988
 989static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
 990					  struct net_device *dev)
 991{
 992	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 993	struct rdma_netdev *rn = netdev_priv(dev);
 994	struct ipoib_path *path;
 995	struct ipoib_neigh *neigh;
 996	unsigned long flags;
 997
 998	spin_lock_irqsave(&priv->lock, flags);
 999	neigh = ipoib_neigh_alloc(daddr, dev);
1000	if (!neigh) {
1001		spin_unlock_irqrestore(&priv->lock, flags);
1002		++dev->stats.tx_dropped;
1003		dev_kfree_skb_any(skb);
1004		return NULL;
1005	}
1006
1007	/* To avoid race condition, make sure that the
1008	 * neigh will be added only once.
1009	 */
1010	if (unlikely(!list_empty(&neigh->list))) {
1011		spin_unlock_irqrestore(&priv->lock, flags);
1012		return neigh;
1013	}
1014
1015	path = __path_find(dev, daddr + 4);
1016	if (!path) {
1017		path = path_rec_create(dev, daddr + 4);
1018		if (!path)
1019			goto err_path;
1020
1021		__path_add(dev, path);
1022	}
1023
1024	list_add_tail(&neigh->list, &path->neigh_list);
1025
1026	if (path->ah && path->ah->valid) {
1027		kref_get(&path->ah->ref);
1028		neigh->ah = path->ah;
1029
1030		if (ipoib_cm_enabled(dev, neigh->daddr)) {
1031			if (!ipoib_cm_get(neigh))
1032				ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));
1033			if (!ipoib_cm_get(neigh)) {
1034				ipoib_neigh_free(neigh);
1035				goto err_drop;
1036			}
1037			if (skb_queue_len(&neigh->queue) <
1038			    IPOIB_MAX_PATH_REC_QUEUE) {
1039				push_pseudo_header(skb, neigh->daddr);
1040				__skb_queue_tail(&neigh->queue, skb);
1041			} else {
1042				ipoib_warn(priv, "queue length limit %d. Packet drop.\n",
1043					   skb_queue_len(&neigh->queue));
1044				goto err_drop;
1045			}
1046		} else {
1047			spin_unlock_irqrestore(&priv->lock, flags);
1048			path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1049						       IPOIB_QPN(daddr));
1050			ipoib_neigh_put(neigh);
1051			return NULL;
1052		}
1053	} else {
1054		neigh->ah  = NULL;
1055
1056		if (!path->query && path_rec_start(dev, path))
1057			goto err_path;
1058		if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1059			push_pseudo_header(skb, neigh->daddr);
1060			__skb_queue_tail(&neigh->queue, skb);
1061		} else {
1062			goto err_drop;
1063		}
1064	}
1065
1066	spin_unlock_irqrestore(&priv->lock, flags);
1067	ipoib_neigh_put(neigh);
1068	return NULL;
1069
1070err_path:
1071	ipoib_neigh_free(neigh);
1072err_drop:
1073	++dev->stats.tx_dropped;
1074	dev_kfree_skb_any(skb);
1075
1076	spin_unlock_irqrestore(&priv->lock, flags);
1077	ipoib_neigh_put(neigh);
1078
1079	return NULL;
1080}
1081
1082static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
1083			     struct ipoib_pseudo_header *phdr)
1084{
1085	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1086	struct rdma_netdev *rn = netdev_priv(dev);
1087	struct ipoib_path *path;
1088	unsigned long flags;
1089
1090	spin_lock_irqsave(&priv->lock, flags);
1091
1092	/* no broadcast means that all paths are (going to be) not valid */
1093	if (!priv->broadcast)
1094		goto drop_and_unlock;
1095
1096	path = __path_find(dev, phdr->hwaddr + 4);
1097	if (!path || !path->ah || !path->ah->valid) {
1098		if (!path) {
1099			path = path_rec_create(dev, phdr->hwaddr + 4);
1100			if (!path)
1101				goto drop_and_unlock;
1102			__path_add(dev, path);
1103		} else {
1104			/*
1105			 * make sure there are no changes in the existing
1106			 * path record
1107			 */
1108			init_path_rec(priv, path, phdr->hwaddr + 4);
1109		}
1110		if (!path->query && path_rec_start(dev, path)) {
1111			goto drop_and_unlock;
1112		}
1113
1114		if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1115			push_pseudo_header(skb, phdr->hwaddr);
1116			__skb_queue_tail(&path->queue, skb);
1117			goto unlock;
1118		} else {
1119			goto drop_and_unlock;
1120		}
1121	}
1122
1123	spin_unlock_irqrestore(&priv->lock, flags);
1124	ipoib_dbg(priv, "Send unicast ARP to %08x\n",
1125		  be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
1126	path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1127				       IPOIB_QPN(phdr->hwaddr));
1128	return;
1129
1130drop_and_unlock:
1131	++dev->stats.tx_dropped;
1132	dev_kfree_skb_any(skb);
1133unlock:
1134	spin_unlock_irqrestore(&priv->lock, flags);
1135}
1136
1137static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
1138{
1139	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1140	struct rdma_netdev *rn = netdev_priv(dev);
1141	struct ipoib_neigh *neigh;
1142	struct ipoib_pseudo_header *phdr;
1143	struct ipoib_header *header;
1144	unsigned long flags;
1145
1146	phdr = (struct ipoib_pseudo_header *) skb->data;
1147	skb_pull(skb, sizeof(*phdr));
1148	header = (struct ipoib_header *) skb->data;
1149
1150	if (unlikely(phdr->hwaddr[4] == 0xff)) {
1151		/* multicast, arrange "if" according to probability */
1152		if ((header->proto != htons(ETH_P_IP)) &&
1153		    (header->proto != htons(ETH_P_IPV6)) &&
1154		    (header->proto != htons(ETH_P_ARP)) &&
1155		    (header->proto != htons(ETH_P_RARP)) &&
1156		    (header->proto != htons(ETH_P_TIPC))) {
1157			/* ethertype not supported by IPoIB */
1158			++dev->stats.tx_dropped;
1159			dev_kfree_skb_any(skb);
1160			return NETDEV_TX_OK;
1161		}
1162		/* Add in the P_Key for multicast*/
1163		phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;
1164		phdr->hwaddr[9] = priv->pkey & 0xff;
1165
1166		neigh = ipoib_neigh_get(dev, phdr->hwaddr);
1167		if (likely(neigh))
1168			goto send_using_neigh;
1169		ipoib_mcast_send(dev, phdr->hwaddr, skb);
1170		return NETDEV_TX_OK;
1171	}
1172
1173	/* unicast, arrange "switch" according to probability */
1174	switch (header->proto) {
1175	case htons(ETH_P_IP):
1176	case htons(ETH_P_IPV6):
1177	case htons(ETH_P_TIPC):
1178		neigh = ipoib_neigh_get(dev, phdr->hwaddr);
1179		if (unlikely(!neigh)) {
1180			neigh = neigh_add_path(skb, phdr->hwaddr, dev);
1181			if (likely(!neigh))
1182				return NETDEV_TX_OK;
1183		}
1184		break;
1185	case htons(ETH_P_ARP):
1186	case htons(ETH_P_RARP):
1187		/* for unicast ARP and RARP should always perform path find */
1188		unicast_arp_send(skb, dev, phdr);
1189		return NETDEV_TX_OK;
1190	default:
1191		/* ethertype not supported by IPoIB */
1192		++dev->stats.tx_dropped;
1193		dev_kfree_skb_any(skb);
1194		return NETDEV_TX_OK;
1195	}
1196
1197send_using_neigh:
1198	/* note we now hold a ref to neigh */
1199	if (ipoib_cm_get(neigh)) {
1200		if (ipoib_cm_up(neigh)) {
1201			ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));
1202			goto unref;
1203		}
1204	} else if (neigh->ah && neigh->ah->valid) {
1205		neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
1206						IPOIB_QPN(phdr->hwaddr));
1207		goto unref;
1208	} else if (neigh->ah) {
1209		neigh_refresh_path(neigh, phdr->hwaddr, dev);
1210	}
1211
1212	if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1213		push_pseudo_header(skb, phdr->hwaddr);
1214		spin_lock_irqsave(&priv->lock, flags);
1215		__skb_queue_tail(&neigh->queue, skb);
1216		spin_unlock_irqrestore(&priv->lock, flags);
1217	} else {
1218		++dev->stats.tx_dropped;
1219		dev_kfree_skb_any(skb);
1220	}
1221
1222unref:
1223	ipoib_neigh_put(neigh);
1224
1225	return NETDEV_TX_OK;
1226}
1227
1228static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
1229{
1230	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1231	struct rdma_netdev *rn = netdev_priv(dev);
1232
1233	if (rn->tx_timeout) {
1234		rn->tx_timeout(dev, txqueue);
1235		return;
1236	}
1237	ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
1238		   jiffies_to_msecs(jiffies - dev_trans_start(dev)));
1239	ipoib_warn(priv,
1240		   "queue stopped %d, tx_head %u, tx_tail %u, global_tx_head %u, global_tx_tail %u\n",
1241		   netif_queue_stopped(dev), priv->tx_head, priv->tx_tail,
1242		   priv->global_tx_head, priv->global_tx_tail);
1243
1244
1245	schedule_work(&priv->tx_timeout_work);
1246}
1247
1248void ipoib_ib_tx_timeout_work(struct work_struct *work)
1249{
1250	struct ipoib_dev_priv *priv = container_of(work,
1251						   struct ipoib_dev_priv,
1252						   tx_timeout_work);
1253	int err;
1254
1255	rtnl_lock();
1256	netdev_lock_ops(priv->dev);
1257
1258	if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
1259		goto unlock;
1260
1261	ipoib_stop(priv->dev);
1262	err = ipoib_open(priv->dev);
1263	if (err) {
1264		ipoib_warn(priv, "ipoib_open failed recovering from a tx_timeout, err(%d).\n",
1265				err);
1266		goto unlock;
1267	}
1268
1269	netif_tx_wake_all_queues(priv->dev);
1270unlock:
1271	netdev_unlock_ops(priv->dev);
1272	rtnl_unlock();
1273
1274}
1275
1276static int ipoib_hard_header(struct sk_buff *skb,
1277			     struct net_device *dev,
1278			     unsigned short type,
1279			     const void *daddr,
1280			     const void *saddr,
1281			     unsigned int len)
1282{
1283	struct ipoib_header *header;
1284
1285	header = skb_push(skb, sizeof(*header));
1286
1287	header->proto = htons(type);
1288	header->reserved = 0;
1289
1290	/*
1291	 * we don't rely on dst_entry structure,  always stuff the
1292	 * destination address into skb hard header so we can figure out where
1293	 * to send the packet later.
1294	 */
1295	push_pseudo_header(skb, daddr);
1296
1297	return IPOIB_HARD_LEN;
1298}
1299
1300static void ipoib_set_mcast_list(struct net_device *dev)
1301{
1302	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1303
1304	if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
1305		ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
1306		return;
1307	}
1308
1309	queue_work(priv->wq, &priv->restart_task);
1310}
1311
1312static int ipoib_get_iflink(const struct net_device *dev)
1313{
1314	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1315
1316	/* parent interface */
1317	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
1318		return READ_ONCE(dev->ifindex);
1319
1320	/* child/vlan interface */
1321	return READ_ONCE(priv->parent->ifindex);
1322}
1323
1324static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)
1325{
1326	/*
1327	 * Use only the address parts that contributes to spreading
1328	 * The subnet prefix is not used as one can not connect to
1329	 * same remote port (GUID) using the same remote QPN via two
1330	 * different subnets.
1331	 */
1332	 /* qpn octets[1:4) & port GUID octets[12:20) */
1333	u32 *d32 = (u32 *) daddr;
1334	u32 hv;
1335
1336	hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0);
1337	return hv & htbl->mask;
1338}
1339
1340struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
1341{
1342	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1343	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1344	struct ipoib_neigh_hash *htbl;
1345	struct ipoib_neigh *neigh = NULL;
1346	u32 hash_val;
1347
1348	rcu_read_lock_bh();
1349
1350	htbl = rcu_dereference_bh(ntbl->htbl);
1351
1352	if (!htbl)
1353		goto out_unlock;
1354
1355	hash_val = ipoib_addr_hash(htbl, daddr);
1356	for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);
1357	     neigh != NULL;
1358	     neigh = rcu_dereference_bh(neigh->hnext)) {
1359		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
1360			/* found, take one ref on behalf of the caller */
1361			if (!refcount_inc_not_zero(&neigh->refcnt)) {
1362				/* deleted */
1363				neigh = NULL;
1364				goto out_unlock;
1365			}
1366
1367			if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))
1368				neigh->alive = jiffies;
1369			goto out_unlock;
1370		}
1371	}
1372
1373out_unlock:
1374	rcu_read_unlock_bh();
1375	return neigh;
1376}
1377
1378static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
1379{
1380	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1381	struct ipoib_neigh_hash *htbl;
1382	unsigned long neigh_obsolete;
1383	unsigned long dt;
1384	unsigned long flags;
1385	int i;
1386	LIST_HEAD(remove_list);
1387
1388	spin_lock_irqsave(&priv->lock, flags);
1389
1390	htbl = rcu_dereference_protected(ntbl->htbl,
1391					 lockdep_is_held(&priv->lock));
1392
1393	if (!htbl)
1394		goto out_unlock;
1395
1396	/* neigh is obsolete if it was idle for two GC periods */
1397	dt = 2 * arp_tbl.gc_interval;
1398	neigh_obsolete = jiffies - dt;
1399
1400	for (i = 0; i < htbl->size; i++) {
1401		struct ipoib_neigh *neigh;
1402		struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1403
1404		while ((neigh = rcu_dereference_protected(*np,
1405							  lockdep_is_held(&priv->lock))) != NULL) {
1406			/* was the neigh idle for two GC periods */
1407			if (time_after(neigh_obsolete, neigh->alive)) {
1408
1409				ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);
1410
1411				rcu_assign_pointer(*np,
1412						   rcu_dereference_protected(neigh->hnext,
1413									     lockdep_is_held(&priv->lock)));
1414				/* remove from path/mc list */
1415				list_del_init(&neigh->list);
1416				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1417			} else {
1418				np = &neigh->hnext;
1419			}
1420
1421		}
1422	}
1423
1424out_unlock:
1425	spin_unlock_irqrestore(&priv->lock, flags);
1426	ipoib_mcast_remove_list(&remove_list);
1427}
1428
1429static void ipoib_reap_neigh(struct work_struct *work)
1430{
1431	struct ipoib_dev_priv *priv =
1432		container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);
1433
1434	__ipoib_reap_neigh(priv);
1435
1436	queue_delayed_work(priv->wq, &priv->neigh_reap_task,
1437			   arp_tbl.gc_interval);
1438}
1439
1440
1441static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,
1442				      struct net_device *dev)
1443{
1444	struct ipoib_neigh *neigh;
1445
1446	neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);
1447	if (!neigh)
1448		return NULL;
1449
1450	neigh->dev = dev;
1451	memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));
1452	skb_queue_head_init(&neigh->queue);
1453	INIT_LIST_HEAD(&neigh->list);
1454	ipoib_cm_set(neigh, NULL);
1455	/* one ref on behalf of the caller */
1456	refcount_set(&neigh->refcnt, 1);
1457
1458	return neigh;
1459}
1460
1461struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,
1462				      struct net_device *dev)
1463{
1464	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1465	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1466	struct ipoib_neigh_hash *htbl;
1467	struct ipoib_neigh *neigh;
1468	u32 hash_val;
1469
1470	htbl = rcu_dereference_protected(ntbl->htbl,
1471					 lockdep_is_held(&priv->lock));
1472	if (!htbl) {
1473		neigh = NULL;
1474		goto out_unlock;
1475	}
1476
1477	/* need to add a new neigh, but maybe some other thread succeeded?
1478	 * recalc hash, maybe hash resize took place so we do a search
1479	 */
1480	hash_val = ipoib_addr_hash(htbl, daddr);
1481	for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],
1482					       lockdep_is_held(&priv->lock));
1483	     neigh != NULL;
1484	     neigh = rcu_dereference_protected(neigh->hnext,
1485					       lockdep_is_held(&priv->lock))) {
1486		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
1487			/* found, take one ref on behalf of the caller */
1488			if (!refcount_inc_not_zero(&neigh->refcnt)) {
1489				/* deleted */
1490				neigh = NULL;
1491				break;
1492			}
1493			neigh->alive = jiffies;
1494			goto out_unlock;
1495		}
1496	}
1497
1498	neigh = ipoib_neigh_ctor(daddr, dev);
1499	if (!neigh)
1500		goto out_unlock;
1501
1502	/* one ref on behalf of the hash table */
1503	refcount_inc(&neigh->refcnt);
1504	neigh->alive = jiffies;
1505	/* put in hash */
1506	rcu_assign_pointer(neigh->hnext,
1507			   rcu_dereference_protected(htbl->buckets[hash_val],
1508						     lockdep_is_held(&priv->lock)));
1509	rcu_assign_pointer(htbl->buckets[hash_val], neigh);
1510	atomic_inc(&ntbl->entries);
1511
1512out_unlock:
1513
1514	return neigh;
1515}
1516
1517void ipoib_neigh_dtor(struct ipoib_neigh *neigh)
1518{
1519	/* neigh reference count was dropprd to zero */
1520	struct net_device *dev = neigh->dev;
1521	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1522	struct sk_buff *skb;
1523	if (neigh->ah)
1524		ipoib_put_ah(neigh->ah);
1525	while ((skb = __skb_dequeue(&neigh->queue))) {
1526		++dev->stats.tx_dropped;
1527		dev_kfree_skb_any(skb);
1528	}
1529	if (ipoib_cm_get(neigh))
1530		ipoib_cm_destroy_tx(ipoib_cm_get(neigh));
1531	ipoib_dbg(ipoib_priv(dev),
1532		  "neigh free for %06x %pI6\n",
1533		  IPOIB_QPN(neigh->daddr),
1534		  neigh->daddr + 4);
1535	kfree(neigh);
1536	if (atomic_dec_and_test(&priv->ntbl.entries)) {
1537		if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))
1538			complete(&priv->ntbl.flushed);
1539	}
1540}
1541
1542static void ipoib_neigh_reclaim(struct rcu_head *rp)
1543{
1544	/* Called as a result of removal from hash table */
1545	struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);
1546	/* note TX context may hold another ref */
1547	ipoib_neigh_put(neigh);
1548}
1549
1550void ipoib_neigh_free(struct ipoib_neigh *neigh)
1551{
1552	struct net_device *dev = neigh->dev;
1553	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1554	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1555	struct ipoib_neigh_hash *htbl;
1556	struct ipoib_neigh __rcu **np;
1557	struct ipoib_neigh *n;
1558	u32 hash_val;
1559
1560	htbl = rcu_dereference_protected(ntbl->htbl,
1561					lockdep_is_held(&priv->lock));
1562	if (!htbl)
1563		return;
1564
1565	hash_val = ipoib_addr_hash(htbl, neigh->daddr);
1566	np = &htbl->buckets[hash_val];
1567	for (n = rcu_dereference_protected(*np,
1568					    lockdep_is_held(&priv->lock));
1569	     n != NULL;
1570	     n = rcu_dereference_protected(*np,
1571					lockdep_is_held(&priv->lock))) {
1572		if (n == neigh) {
1573			/* found */
1574			rcu_assign_pointer(*np,
1575					   rcu_dereference_protected(neigh->hnext,
1576								     lockdep_is_held(&priv->lock)));
1577			/* remove from parent list */
1578			list_del_init(&neigh->list);
1579			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1580			return;
1581		} else {
1582			np = &n->hnext;
1583		}
1584	}
1585}
1586
1587static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
1588{
1589	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1590	struct ipoib_neigh_hash *htbl;
1591	struct ipoib_neigh __rcu **buckets;
1592	u32 size;
1593
1594	clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
1595	ntbl->htbl = NULL;
1596	htbl = kzalloc(sizeof(*htbl), GFP_KERNEL);
1597	if (!htbl)
1598		return -ENOMEM;
1599	size = roundup_pow_of_two(arp_tbl.gc_thresh3);
1600	buckets = kvcalloc(size, sizeof(*buckets), GFP_KERNEL);
1601	if (!buckets) {
1602		kfree(htbl);
1603		return -ENOMEM;
1604	}
1605	htbl->size = size;
1606	htbl->mask = (size - 1);
1607	htbl->buckets = buckets;
1608	RCU_INIT_POINTER(ntbl->htbl, htbl);
1609	htbl->ntbl = ntbl;
1610	atomic_set(&ntbl->entries, 0);
1611
1612	/* start garbage collection */
1613	queue_delayed_work(priv->wq, &priv->neigh_reap_task,
1614			   arp_tbl.gc_interval);
1615
1616	return 0;
1617}
1618
1619static void neigh_hash_free_rcu(struct rcu_head *head)
1620{
1621	struct ipoib_neigh_hash *htbl = container_of(head,
1622						    struct ipoib_neigh_hash,
1623						    rcu);
1624	struct ipoib_neigh __rcu **buckets = htbl->buckets;
1625	struct ipoib_neigh_table *ntbl = htbl->ntbl;
1626
1627	kvfree(buckets);
1628	kfree(htbl);
1629	complete(&ntbl->deleted);
1630}
1631
1632void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)
1633{
1634	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1635	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1636	struct ipoib_neigh_hash *htbl;
1637	unsigned long flags;
1638	int i;
1639
1640	/* remove all neigh connected to a given path or mcast */
1641	spin_lock_irqsave(&priv->lock, flags);
1642
1643	htbl = rcu_dereference_protected(ntbl->htbl,
1644					 lockdep_is_held(&priv->lock));
1645
1646	if (!htbl)
1647		goto out_unlock;
1648
1649	for (i = 0; i < htbl->size; i++) {
1650		struct ipoib_neigh *neigh;
1651		struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1652
1653		while ((neigh = rcu_dereference_protected(*np,
1654							  lockdep_is_held(&priv->lock))) != NULL) {
1655			/* delete neighs belong to this parent */
1656			if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {
1657				rcu_assign_pointer(*np,
1658						   rcu_dereference_protected(neigh->hnext,
1659									     lockdep_is_held(&priv->lock)));
1660				/* remove from parent list */
1661				list_del_init(&neigh->list);
1662				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1663			} else {
1664				np = &neigh->hnext;
1665			}
1666
1667		}
1668	}
1669out_unlock:
1670	spin_unlock_irqrestore(&priv->lock, flags);
1671}
1672
1673static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)
1674{
1675	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1676	struct ipoib_neigh_hash *htbl;
1677	unsigned long flags;
1678	int i, wait_flushed = 0;
1679
1680	init_completion(&priv->ntbl.flushed);
1681	set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
1682
1683	spin_lock_irqsave(&priv->lock, flags);
1684
1685	htbl = rcu_dereference_protected(ntbl->htbl,
1686					lockdep_is_held(&priv->lock));
1687	if (!htbl)
1688		goto out_unlock;
1689
1690	wait_flushed = atomic_read(&priv->ntbl.entries);
1691	if (!wait_flushed)
1692		goto free_htbl;
1693
1694	for (i = 0; i < htbl->size; i++) {
1695		struct ipoib_neigh *neigh;
1696		struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1697
1698		while ((neigh = rcu_dereference_protected(*np,
1699				       lockdep_is_held(&priv->lock))) != NULL) {
1700			rcu_assign_pointer(*np,
1701					   rcu_dereference_protected(neigh->hnext,
1702								     lockdep_is_held(&priv->lock)));
1703			/* remove from path/mc list */
1704			list_del_init(&neigh->list);
1705			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1706		}
1707	}
1708
1709free_htbl:
1710	rcu_assign_pointer(ntbl->htbl, NULL);
1711	call_rcu(&htbl->rcu, neigh_hash_free_rcu);
1712
1713out_unlock:
1714	spin_unlock_irqrestore(&priv->lock, flags);
1715	if (wait_flushed)
1716		wait_for_completion(&priv->ntbl.flushed);
1717}
1718
1719static void ipoib_neigh_hash_uninit(struct net_device *dev)
1720{
1721	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1722
1723	ipoib_dbg(priv, "%s\n", __func__);
1724	init_completion(&priv->ntbl.deleted);
1725
1726	cancel_delayed_work_sync(&priv->neigh_reap_task);
1727
1728	ipoib_flush_neighs(priv);
1729
1730	wait_for_completion(&priv->ntbl.deleted);
1731}
1732
1733static void ipoib_napi_add(struct net_device *dev)
1734{
1735	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1736
1737	netif_napi_add_weight(dev, &priv->recv_napi, ipoib_rx_poll,
1738			      IPOIB_NUM_WC);
1739	netif_napi_add_weight(dev, &priv->send_napi, ipoib_tx_poll,
1740			      MAX_SEND_CQE);
1741}
1742
1743static void ipoib_napi_del(struct net_device *dev)
1744{
1745	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1746
1747	netif_napi_del(&priv->recv_napi);
1748	netif_napi_del(&priv->send_napi);
1749}
1750
1751static void ipoib_dev_uninit_default(struct net_device *dev)
1752{
1753	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1754
1755	ipoib_transport_dev_cleanup(dev);
1756
1757	ipoib_napi_del(dev);
1758
1759	ipoib_cm_dev_cleanup(dev);
1760
1761	kfree(priv->rx_ring);
1762	vfree(priv->tx_ring);
1763
1764	priv->rx_ring = NULL;
1765	priv->tx_ring = NULL;
1766}
1767
1768static int ipoib_dev_init_default(struct net_device *dev)
1769{
1770	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1771	u8 addr_mod[3];
1772
1773	ipoib_napi_add(dev);
1774
1775	/* Allocate RX/TX "rings" to hold queued skbs */
1776	priv->rx_ring =	kcalloc(ipoib_recvq_size,
1777				       sizeof(*priv->rx_ring),
1778				       GFP_KERNEL);
1779	if (!priv->rx_ring)
1780		goto out;
1781
1782	priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,
1783					   sizeof(*priv->tx_ring)));
1784	if (!priv->tx_ring) {
1785		pr_warn("%s: failed to allocate TX ring (%d entries)\n",
1786			priv->ca->name, ipoib_sendq_size);
1787		goto out_rx_ring_cleanup;
1788	}
1789
1790	/* priv->tx_head, tx_tail and global_tx_tail/head are already 0 */
1791
1792	if (ipoib_transport_dev_init(dev, priv->ca)) {
1793		pr_warn("%s: ipoib_transport_dev_init failed\n",
1794			priv->ca->name);
1795		goto out_tx_ring_cleanup;
1796	}
1797
1798	/* after qp created set dev address */
1799	addr_mod[0] = (priv->qp->qp_num >> 16) & 0xff;
1800	addr_mod[1] = (priv->qp->qp_num >>  8) & 0xff;
1801	addr_mod[2] = (priv->qp->qp_num) & 0xff;
1802	dev_addr_mod(priv->dev, 1, addr_mod, sizeof(addr_mod));
1803
1804	return 0;
1805
1806out_tx_ring_cleanup:
1807	vfree(priv->tx_ring);
1808
1809out_rx_ring_cleanup:
1810	kfree(priv->rx_ring);
1811
1812out:
1813	ipoib_napi_del(dev);
1814	return -ENOMEM;
1815}
1816
1817static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
1818		       int cmd)
1819{
1820	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1821
1822	if (!priv->rn_ops->ndo_eth_ioctl)
1823		return -EOPNOTSUPP;
1824
1825	return priv->rn_ops->ndo_eth_ioctl(dev, ifr, cmd);
1826}
1827
1828static int ipoib_hwtstamp_get(struct net_device *dev,
1829			      struct kernel_hwtstamp_config *config)
1830{
1831	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1832
1833	if (!priv->rn_ops->ndo_hwtstamp_get)
1834		/* legacy */
1835		return dev_eth_ioctl(dev, config->ifr, SIOCGHWTSTAMP);
1836
1837	return priv->rn_ops->ndo_hwtstamp_get(dev, config);
1838}
1839
1840static int ipoib_hwtstamp_set(struct net_device *dev,
1841			      struct kernel_hwtstamp_config *config,
1842			      struct netlink_ext_ack *extack)
1843{
1844	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1845
1846	if (!priv->rn_ops->ndo_hwtstamp_set)
1847		/* legacy */
1848		return dev_eth_ioctl(dev, config->ifr, SIOCSHWTSTAMP);
1849
1850	return priv->rn_ops->ndo_hwtstamp_set(dev, config, extack);
1851}
1852
1853static int ipoib_dev_init(struct net_device *dev)
1854{
1855	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1856	int ret = -ENOMEM;
1857
1858	priv->qp = NULL;
1859
1860	/*
1861	 * the various IPoIB tasks assume they will never race against
1862	 * themselves, so always use a single thread workqueue
1863	 */
1864	priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
1865	if (!priv->wq) {
1866		pr_warn("%s: failed to allocate device WQ\n", dev->name);
1867		goto out;
1868	}
1869
1870	/* create pd, which used both for control and datapath*/
1871	priv->pd = ib_alloc_pd(priv->ca, 0);
1872	if (IS_ERR(priv->pd)) {
1873		pr_warn("%s: failed to allocate PD\n", priv->ca->name);
1874		goto clean_wq;
1875	}
1876
1877	ret = priv->rn_ops->ndo_init(dev);
1878	if (ret) {
1879		pr_warn("%s failed to init HW resource\n", dev->name);
1880		goto out_free_pd;
1881	}
1882
1883	ret = ipoib_neigh_hash_init(priv);
1884	if (ret) {
1885		pr_warn("%s failed to init neigh hash\n", dev->name);
1886		goto out_dev_uninit;
1887	}
1888
1889	if (dev->flags & IFF_UP) {
1890		if (ipoib_ib_dev_open(dev)) {
1891			pr_warn("%s failed to open device\n", dev->name);
1892			ret = -ENODEV;
1893			goto out_hash_uninit;
1894		}
1895	}
1896
1897	return 0;
1898
1899out_hash_uninit:
1900	ipoib_neigh_hash_uninit(dev);
1901
1902out_dev_uninit:
1903	ipoib_ib_dev_cleanup(dev);
1904
1905out_free_pd:
1906	if (priv->pd) {
1907		ib_dealloc_pd(priv->pd);
1908		priv->pd = NULL;
1909	}
1910
1911clean_wq:
1912	if (priv->wq) {
1913		destroy_workqueue(priv->wq);
1914		priv->wq = NULL;
1915	}
1916
1917out:
1918	return ret;
1919}
1920
1921/*
1922 * This must be called before doing an unregister_netdev on a parent device to
1923 * shutdown the IB event handler.
1924 */
1925static void ipoib_parent_unregister_pre(struct net_device *ndev)
1926{
1927	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1928
1929	/*
1930	 * ipoib_set_mac checks netif_running before pushing work, clearing
1931	 * running ensures the it will not add more work.
1932	 */
1933	rtnl_lock();
1934	dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP, NULL);
1935	rtnl_unlock();
1936
1937	/* ipoib_event() cannot be running once this returns */
1938	ib_unregister_event_handler(&priv->event_handler);
1939
1940	/*
1941	 * Work on the queue grabs the rtnl lock, so this cannot be done while
1942	 * also holding it.
1943	 */
1944	flush_workqueue(ipoib_workqueue);
1945}
1946
1947static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)
1948{
1949	priv->hca_caps = priv->ca->attrs.device_cap_flags;
1950	priv->kernel_caps = priv->ca->attrs.kernel_cap_flags;
1951
1952	if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
1953		priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
1954
1955		if (priv->kernel_caps & IBK_UD_TSO)
1956			priv->dev->hw_features |= NETIF_F_TSO;
1957
1958		priv->dev->features |= priv->dev->hw_features;
1959	}
1960}
1961
1962static int ipoib_parent_init(struct net_device *ndev)
1963{
1964	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1965	struct ib_port_attr attr;
1966	int result;
1967
1968	result = ib_query_port(priv->ca, priv->port, &attr);
1969	if (result) {
1970		pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,
1971			priv->port);
1972		return result;
1973	}
1974	priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);
1975
1976	result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
1977	if (result) {
1978		pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
1979			priv->ca->name, priv->port, result);
1980		return result;
1981	}
1982
1983	result = rdma_query_gid(priv->ca, priv->port, 0, &priv->local_gid);
1984	if (result) {
1985		pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",
1986			priv->ca->name, priv->port, result);
1987		return result;
1988	}
1989	dev_addr_mod(priv->dev, 4, priv->local_gid.raw, sizeof(union ib_gid));
1990
1991	SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);
1992	priv->dev->dev_port = priv->port - 1;
1993	/* Let's set this one too for backwards compatibility. */
1994	priv->dev->dev_id = priv->port - 1;
1995
1996	return 0;
1997}
1998
1999static void ipoib_child_init(struct net_device *ndev)
2000{
2001	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2002	struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
2003
2004	priv->max_ib_mtu = ppriv->max_ib_mtu;
2005	set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
2006	if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))
2007		memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
2008		       sizeof(priv->local_gid));
2009	else {
2010		__dev_addr_set(priv->dev, ppriv->dev->dev_addr,
2011			       INFINIBAND_ALEN);
2012		memcpy(&priv->local_gid, &ppriv->local_gid,
2013		       sizeof(priv->local_gid));
2014	}
2015}
2016
2017static int ipoib_ndo_init(struct net_device *ndev)
2018{
2019	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2020	int rc;
2021	struct rdma_netdev *rn = netdev_priv(ndev);
2022
2023	if (priv->parent) {
2024		ipoib_child_init(ndev);
2025	} else {
2026		rc = ipoib_parent_init(ndev);
2027		if (rc)
2028			return rc;
2029	}
2030
2031	/* MTU will be reset when mcast join happens */
2032	ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
2033	priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
2034	rn->mtu = priv->mcast_mtu;
2035	ndev->max_mtu = IPOIB_CM_MTU;
2036
2037	ndev->neigh_priv_len = sizeof(struct ipoib_neigh);
2038
2039	/*
2040	 * Set the full membership bit, so that we join the right
2041	 * broadcast group, etc.
2042	 */
2043	priv->pkey |= 0x8000;
2044
2045	ndev->broadcast[8] = priv->pkey >> 8;
2046	ndev->broadcast[9] = priv->pkey & 0xff;
2047	set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
2048
2049	ipoib_set_dev_features(priv);
2050
2051	rc = ipoib_dev_init(ndev);
2052	if (rc) {
2053		pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",
2054			priv->ca->name, priv->dev->name, priv->port, rc);
2055		return rc;
2056	}
2057
2058	if (priv->parent) {
2059		struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
2060
2061		dev_hold(priv->parent);
2062
2063		netdev_lock(priv->parent);
2064		list_add_tail(&priv->list, &ppriv->child_intfs);
2065		netdev_unlock(priv->parent);
2066	}
2067
2068	return 0;
2069}
2070
2071static void ipoib_ndo_uninit(struct net_device *dev)
2072{
2073	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2074
2075	/*
2076	 * ipoib_remove_one guarantees the children are removed before the
2077	 * parent, and that is the only place where a parent can be removed.
2078	 */
2079	WARN_ON(!list_empty(&priv->child_intfs));
2080
2081	if (priv->parent) {
2082		struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
2083
2084		netdev_lock(ppriv->dev);
2085		list_del(&priv->list);
2086		netdev_unlock(ppriv->dev);
2087	}
2088
2089	ipoib_neigh_hash_uninit(dev);
2090
2091	ipoib_ib_dev_cleanup(dev);
2092
2093	/* no more works over the priv->wq */
2094	if (priv->wq) {
2095		/* See ipoib_mcast_carrier_on_task() */
2096		WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
2097		destroy_workqueue(priv->wq);
2098		priv->wq = NULL;
2099	}
2100
2101	dev_put(priv->parent);
2102}
2103
2104static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)
2105{
2106	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2107
2108	return ib_set_vf_link_state(priv->ca, vf, priv->port, link_state);
2109}
2110
2111static int ipoib_get_vf_config(struct net_device *dev, int vf,
2112			       struct ifla_vf_info *ivf)
2113{
2114	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2115	int err;
2116
2117	err = ib_get_vf_config(priv->ca, vf, priv->port, ivf);
2118	if (err)
2119		return err;
2120
2121	ivf->vf = vf;
2122	memcpy(ivf->mac, dev->dev_addr, dev->addr_len);
2123
2124	return 0;
2125}
2126
2127static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)
2128{
2129	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2130
2131	if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)
2132		return -EINVAL;
2133
2134	return ib_set_vf_guid(priv->ca, vf, priv->port, guid, type);
2135}
2136
2137static int ipoib_get_vf_guid(struct net_device *dev, int vf,
2138			     struct ifla_vf_guid *node_guid,
2139			     struct ifla_vf_guid *port_guid)
2140{
2141	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2142
2143	return ib_get_vf_guid(priv->ca, vf, priv->port, node_guid, port_guid);
2144}
2145
2146static int ipoib_get_vf_stats(struct net_device *dev, int vf,
2147			      struct ifla_vf_stats *vf_stats)
2148{
2149	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2150
2151	return ib_get_vf_stats(priv->ca, vf, priv->port, vf_stats);
2152}
2153
2154static const struct header_ops ipoib_header_ops = {
2155	.create	= ipoib_hard_header,
2156};
2157
2158static const struct net_device_ops ipoib_netdev_ops_pf = {
2159	.ndo_init		 = ipoib_ndo_init,
2160	.ndo_uninit		 = ipoib_ndo_uninit,
2161	.ndo_open		 = ipoib_open,
2162	.ndo_stop		 = ipoib_stop,
2163	.ndo_change_mtu		 = ipoib_change_mtu,
2164	.ndo_fix_features	 = ipoib_fix_features,
2165	.ndo_start_xmit		 = ipoib_start_xmit,
2166	.ndo_tx_timeout		 = ipoib_timeout,
2167	.ndo_set_rx_mode	 = ipoib_set_mcast_list,
2168	.ndo_get_iflink		 = ipoib_get_iflink,
2169	.ndo_set_vf_link_state	 = ipoib_set_vf_link_state,
2170	.ndo_get_vf_config	 = ipoib_get_vf_config,
2171	.ndo_get_vf_stats	 = ipoib_get_vf_stats,
2172	.ndo_get_vf_guid	 = ipoib_get_vf_guid,
2173	.ndo_set_vf_guid	 = ipoib_set_vf_guid,
2174	.ndo_set_mac_address	 = ipoib_set_mac,
2175	.ndo_get_stats64	 = ipoib_get_stats,
2176	.ndo_eth_ioctl		 = ipoib_ioctl,
2177	.ndo_hwtstamp_get	 = ipoib_hwtstamp_get,
2178	.ndo_hwtstamp_set	 = ipoib_hwtstamp_set,
2179};
2180
2181static const struct net_device_ops ipoib_netdev_ops_vf = {
2182	.ndo_init		 = ipoib_ndo_init,
2183	.ndo_uninit		 = ipoib_ndo_uninit,
2184	.ndo_open		 = ipoib_open,
2185	.ndo_stop		 = ipoib_stop,
2186	.ndo_change_mtu		 = ipoib_change_mtu,
2187	.ndo_fix_features	 = ipoib_fix_features,
2188	.ndo_start_xmit	 	 = ipoib_start_xmit,
2189	.ndo_tx_timeout		 = ipoib_timeout,
2190	.ndo_set_rx_mode	 = ipoib_set_mcast_list,
2191	.ndo_get_iflink		 = ipoib_get_iflink,
2192	.ndo_get_stats64	 = ipoib_get_stats,
2193	.ndo_eth_ioctl		 = ipoib_ioctl,
2194	.ndo_hwtstamp_get	 = ipoib_hwtstamp_get,
2195	.ndo_hwtstamp_set	 = ipoib_hwtstamp_set,
2196};
2197
2198static const struct net_device_ops ipoib_netdev_default_pf = {
2199	.ndo_init		 = ipoib_dev_init_default,
2200	.ndo_uninit		 = ipoib_dev_uninit_default,
2201	.ndo_open		 = ipoib_ib_dev_open_default,
2202	.ndo_stop		 = ipoib_ib_dev_stop_default,
2203};
2204
2205void ipoib_setup_common(struct net_device *dev)
2206{
2207	dev->header_ops		 = &ipoib_header_ops;
2208	dev->netdev_ops          = &ipoib_netdev_default_pf;
2209
2210	ipoib_set_ethtool_ops(dev);
2211
2212	dev->watchdog_timeo	 = 10 * HZ;
2213
2214	dev->flags		|= IFF_BROADCAST | IFF_MULTICAST;
2215
2216	dev->hard_header_len	 = IPOIB_HARD_LEN;
2217	dev->addr_len		 = INFINIBAND_ALEN;
2218	dev->type		 = ARPHRD_INFINIBAND;
2219	dev->tx_queue_len	 = DEFAULT_TX_QUEUE_LEN;
2220	dev->features		 = (NETIF_F_VLAN_CHALLENGED	|
2221				    NETIF_F_HIGHDMA);
2222	netif_keep_dst(dev);
2223
2224	memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);
2225
2226	/*
2227	 * unregister_netdev always frees the netdev, we use this mode
2228	 * consistently to unify all the various unregister paths, including
2229	 * those connected to rtnl_link_ops which require it.
2230	 */
2231	dev->needs_free_netdev = true;
2232}
2233
2234static void ipoib_build_priv(struct net_device *dev)
2235{
2236	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2237
2238	priv->dev = dev;
2239	spin_lock_init(&priv->lock);
2240	mutex_init(&priv->mcast_mutex);
2241
2242	INIT_LIST_HEAD(&priv->path_list);
2243	INIT_LIST_HEAD(&priv->child_intfs);
2244	INIT_LIST_HEAD(&priv->dead_ahs);
2245	INIT_LIST_HEAD(&priv->multicast_list);
2246
2247	INIT_DELAYED_WORK(&priv->mcast_task,   ipoib_mcast_join_task);
2248	INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);
2249	INIT_WORK(&priv->reschedule_napi_work, ipoib_napi_schedule_work);
2250	INIT_WORK(&priv->flush_light,   ipoib_ib_dev_flush_light);
2251	INIT_WORK(&priv->flush_normal,   ipoib_ib_dev_flush_normal);
2252	INIT_WORK(&priv->flush_heavy,   ipoib_ib_dev_flush_heavy);
2253	INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
2254	INIT_WORK(&priv->tx_timeout_work, ipoib_ib_tx_timeout_work);
2255	INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
2256	INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
2257}
2258
2259static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u32 port,
2260					     const char *name)
2261{
2262	struct net_device *dev;
2263
2264	dev = rdma_alloc_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
2265				NET_NAME_UNKNOWN, ipoib_setup_common);
2266	if (!IS_ERR(dev) || PTR_ERR(dev) != -EOPNOTSUPP)
2267		return dev;
2268
2269	dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,
2270			   ipoib_setup_common);
2271	if (!dev)
2272		return ERR_PTR(-ENOMEM);
2273	return dev;
2274}
2275
2276int ipoib_intf_init(struct ib_device *hca, u32 port, const char *name,
2277		    struct net_device *dev)
2278{
2279	struct rdma_netdev *rn = netdev_priv(dev);
2280	struct ipoib_dev_priv *priv;
2281	int rc;
2282
2283	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2284	if (!priv)
2285		return -ENOMEM;
2286
2287	priv->ca = hca;
2288	priv->port = port;
2289
2290	rc = rdma_init_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
2291			      NET_NAME_UNKNOWN, ipoib_setup_common, dev);
2292	if (rc) {
2293		if (rc != -EOPNOTSUPP)
2294			goto out;
2295
2296		rn->send = ipoib_send;
2297		rn->attach_mcast = ipoib_mcast_attach;
2298		rn->detach_mcast = ipoib_mcast_detach;
2299		rn->hca = hca;
2300
2301		rc = netif_set_real_num_tx_queues(dev, 1);
2302		if (rc)
2303			goto out;
2304
2305		rc = netif_set_real_num_rx_queues(dev, 1);
2306		if (rc)
2307			goto out;
2308	}
2309
2310	priv->rn_ops = dev->netdev_ops;
2311
2312	if (hca->attrs.kernel_cap_flags & IBK_VIRTUAL_FUNCTION)
2313		dev->netdev_ops	= &ipoib_netdev_ops_vf;
2314	else
2315		dev->netdev_ops	= &ipoib_netdev_ops_pf;
2316
2317	rn->clnt_priv = priv;
2318	/*
2319	 * Only the child register_netdev flows can handle priv_destructor
2320	 * being set, so we force it to NULL here and handle manually until it
2321	 * is safe to turn on.
2322	 */
2323	priv->next_priv_destructor = dev->priv_destructor;
2324	dev->priv_destructor = NULL;
2325
2326	ipoib_build_priv(dev);
2327
2328	return 0;
2329
2330out:
2331	kfree(priv);
2332	return rc;
2333}
2334
2335struct net_device *ipoib_intf_alloc(struct ib_device *hca, u32 port,
2336				    const char *name)
2337{
2338	struct net_device *dev;
2339	int rc;
2340
2341	dev = ipoib_alloc_netdev(hca, port, name);
2342	if (IS_ERR(dev))
2343		return dev;
2344
2345	rc = ipoib_intf_init(hca, port, name, dev);
2346	if (rc) {
2347		free_netdev(dev);
2348		return ERR_PTR(rc);
2349	}
2350
2351	/*
2352	 * Upon success the caller must ensure ipoib_intf_free is called or
2353	 * register_netdevice succeed'd and priv_destructor is set to
2354	 * ipoib_intf_free.
2355	 */
2356	return dev;
2357}
2358
2359void ipoib_intf_free(struct net_device *dev)
2360{
2361	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2362	struct rdma_netdev *rn = netdev_priv(dev);
2363
2364	dev->priv_destructor = priv->next_priv_destructor;
2365	if (dev->priv_destructor)
2366		dev->priv_destructor(dev);
2367
2368	/*
2369	 * There are some error flows around register_netdev failing that may
2370	 * attempt to call priv_destructor twice, prevent that from happening.
2371	 */
2372	dev->priv_destructor = NULL;
2373
2374	/* unregister/destroy is very complicated. Make bugs more obvious. */
2375	rn->clnt_priv = NULL;
2376
2377	kfree(priv);
2378}
2379
2380static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,
2381			 char *buf)
2382{
2383	struct net_device *ndev = to_net_dev(dev);
2384	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2385
2386	return sysfs_emit(buf, "0x%04x\n", priv->pkey);
2387}
2388static DEVICE_ATTR_RO(pkey);
2389
2390static ssize_t umcast_show(struct device *dev, struct device_attribute *attr,
2391			   char *buf)
2392{
2393	struct net_device *ndev = to_net_dev(dev);
2394	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2395
2396	return sysfs_emit(buf, "%d\n",
2397			  test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
2398}
2399
2400void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
2401{
2402	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
2403
2404	if (umcast_val > 0) {
2405		set_bit(IPOIB_FLAG_UMCAST, &priv->flags);
2406		ipoib_warn(priv, "ignoring multicast groups joined directly "
2407				"by userspace\n");
2408	} else
2409		clear_bit(IPOIB_FLAG_UMCAST, &priv->flags);
2410}
2411
2412static ssize_t umcast_store(struct device *dev, struct device_attribute *attr,
2413			    const char *buf, size_t count)
2414{
2415	unsigned long umcast_val = simple_strtoul(buf, NULL, 0);
2416
2417	ipoib_set_umcast(to_net_dev(dev), umcast_val);
2418
2419	return count;
2420}
2421static DEVICE_ATTR_RW(umcast);
2422
2423int ipoib_add_umcast_attr(struct net_device *dev)
2424{
2425	return device_create_file(&dev->dev, &dev_attr_umcast);
2426}
2427
2428static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)
2429{
2430	struct ipoib_dev_priv *child_priv;
2431	struct net_device *netdev = priv->dev;
2432
2433	netif_addr_lock_bh(netdev);
2434
2435	memcpy(&priv->local_gid.global.interface_id,
2436	       &gid->global.interface_id,
2437	       sizeof(gid->global.interface_id));
2438	dev_addr_mod(netdev, 4, (u8 *)&priv->local_gid, sizeof(priv->local_gid));
2439	clear_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
2440
2441	netif_addr_unlock_bh(netdev);
2442
2443	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
2444		netdev_lock_ops_to_full(priv->dev);
2445		list_for_each_entry(child_priv, &priv->child_intfs, list)
2446			set_base_guid(child_priv, gid);
2447		netdev_unlock_full_to_ops(priv->dev);
2448	}
2449}
2450
2451static int ipoib_check_lladdr(struct net_device *dev,
2452			      struct sockaddr_storage *ss)
2453{
2454	union ib_gid *gid = (union ib_gid *)(ss->__data + 4);
2455	int ret = 0;
2456
2457	netif_addr_lock_bh(dev);
2458
2459	/* Make sure the QPN, reserved and subnet prefix match the current
2460	 * lladdr, it also makes sure the lladdr is unicast.
2461	 */
2462	if (memcmp(dev->dev_addr, ss->__data,
2463		   4 + sizeof(gid->global.subnet_prefix)) ||
2464	    gid->global.interface_id == 0)
2465		ret = -EINVAL;
2466
2467	netif_addr_unlock_bh(dev);
2468
2469	return ret;
2470}
2471
2472static int ipoib_set_mac(struct net_device *dev, void *addr)
2473{
2474	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2475	struct sockaddr_storage *ss = addr;
2476	int ret;
2477
2478	if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
2479		return -EBUSY;
2480
2481	ret = ipoib_check_lladdr(dev, ss);
2482	if (ret)
2483		return ret;
2484
2485	set_base_guid(priv, (union ib_gid *)(ss->__data + 4));
2486
2487	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
2488		struct ipoib_dev_priv *cpriv;
2489
2490		netdev_lock_ops_to_full(dev);
2491		list_for_each_entry(cpriv, &priv->child_intfs, list)
2492			queue_work(ipoib_workqueue, &cpriv->flush_light);
2493		netdev_unlock_full_to_ops(dev);
2494	}
2495	queue_work(ipoib_workqueue, &priv->flush_light);
2496
2497	return 0;
2498}
2499
2500static ssize_t create_child_store(struct device *dev,
2501				  struct device_attribute *attr,
2502				  const char *buf, size_t count)
2503{
2504	int pkey;
2505	int ret;
2506
2507	if (sscanf(buf, "%i", &pkey) != 1)
2508		return -EINVAL;
2509
2510	if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)
2511		return -EINVAL;
2512
2513	ret = ipoib_vlan_add(to_net_dev(dev), pkey);
2514
2515	return ret ? ret : count;
2516}
2517static DEVICE_ATTR_WO(create_child);
2518
2519static ssize_t delete_child_store(struct device *dev,
2520				  struct device_attribute *attr,
2521				  const char *buf, size_t count)
2522{
2523	int pkey;
2524	int ret;
2525
2526	if (sscanf(buf, "%i", &pkey) != 1)
2527		return -EINVAL;
2528
2529	if (pkey < 0 || pkey > 0xffff)
2530		return -EINVAL;
2531
2532	ret = ipoib_vlan_delete(to_net_dev(dev), pkey);
2533
2534	return ret ? ret : count;
2535
2536}
2537static DEVICE_ATTR_WO(delete_child);
2538
2539int ipoib_add_pkey_attr(struct net_device *dev)
2540{
2541	return device_create_file(&dev->dev, &dev_attr_pkey);
2542}
2543
2544/*
2545 * We erroneously exposed the iface's port number in the dev_id
2546 * sysfs field long after dev_port was introduced for that purpose[1],
2547 * and we need to stop everyone from relying on that.
2548 * Let's overload the shower routine for the dev_id file here
2549 * to gently bring the issue up.
2550 *
2551 * [1] https://www.spinics.net/lists/netdev/msg272123.html
2552 */
2553static ssize_t dev_id_show(struct device *dev,
2554			   struct device_attribute *attr, char *buf)
2555{
2556	struct net_device *ndev = to_net_dev(dev);
2557
2558	/*
2559	 * ndev->dev_port will be equal to 0 in old kernel prior to commit
2560	 * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
2561	 * port numbers") Zero was chosen as special case for user space
2562	 * applications to fallback and query dev_id to check if it has
2563	 * different value or not.
2564	 *
2565	 * Don't print warning in such scenario.
2566	 *
2567	 * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
2568	 */
2569	if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
2570		netdev_info_once(ndev,
2571			"\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
2572			current->comm);
2573
2574	return sysfs_emit(buf, "%#x\n", ndev->dev_id);
2575}
2576static DEVICE_ATTR_RO(dev_id);
2577
2578static int ipoib_intercept_dev_id_attr(struct net_device *dev)
2579{
2580	device_remove_file(&dev->dev, &dev_attr_dev_id);
2581	return device_create_file(&dev->dev, &dev_attr_dev_id);
2582}
2583
2584static struct net_device *ipoib_add_port(const char *format,
2585					 struct ib_device *hca, u32 port)
2586{
2587	struct rtnl_link_ops *ops = ipoib_get_link_ops();
2588	struct rdma_netdev_alloc_params params;
2589	struct ipoib_dev_priv *priv;
2590	struct net_device *ndev;
2591	int result;
2592
2593	ndev = ipoib_intf_alloc(hca, port, format);
2594	if (IS_ERR(ndev)) {
2595		pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,
2596			PTR_ERR(ndev));
2597		return ndev;
2598	}
2599	priv = ipoib_priv(ndev);
2600
2601	INIT_IB_EVENT_HANDLER(&priv->event_handler,
2602			      priv->ca, ipoib_event);
2603	ib_register_event_handler(&priv->event_handler);
2604
2605	/* call event handler to ensure pkey in sync */
2606	ipoib_queue_work(priv, IPOIB_FLUSH_HEAVY);
2607
2608	ndev->rtnl_link_ops = ipoib_get_link_ops();
2609
2610	dev_net_set(ndev, rdma_dev_net(hca));
2611
2612	result = register_netdev(ndev);
2613	if (result) {
2614		pr_warn("%s: couldn't register ipoib port %d; error %d\n",
2615			hca->name, port, result);
2616
2617		ipoib_parent_unregister_pre(ndev);
2618		ipoib_intf_free(ndev);
2619		free_netdev(ndev);
2620
2621		return ERR_PTR(result);
2622	}
2623
2624	if (hca->ops.rdma_netdev_get_params) {
2625		int rc = hca->ops.rdma_netdev_get_params(hca, port,
2626						     RDMA_NETDEV_IPOIB,
2627						     &params);
2628
2629		if (!rc && ops->priv_size < params.sizeof_priv)
2630			ops->priv_size = params.sizeof_priv;
2631	}
2632	/*
2633	 * We cannot set priv_destructor before register_netdev because we
2634	 * need priv to be always valid during the error flow to execute
2635	 * ipoib_parent_unregister_pre(). Instead handle it manually and only
2636	 * enter priv_destructor mode once we are completely registered.
2637	 */
2638	ndev->priv_destructor = ipoib_intf_free;
2639
2640	if (ipoib_intercept_dev_id_attr(ndev))
2641		goto sysfs_failed;
2642	if (ipoib_cm_add_mode_attr(ndev))
2643		goto sysfs_failed;
2644	if (ipoib_add_pkey_attr(ndev))
2645		goto sysfs_failed;
2646	if (ipoib_add_umcast_attr(ndev))
2647		goto sysfs_failed;
2648	if (device_create_file(&ndev->dev, &dev_attr_create_child))
2649		goto sysfs_failed;
2650	if (device_create_file(&ndev->dev, &dev_attr_delete_child))
2651		goto sysfs_failed;
2652
2653	return ndev;
2654
2655sysfs_failed:
2656	ipoib_parent_unregister_pre(ndev);
2657	unregister_netdev(ndev);
2658	return ERR_PTR(-ENOMEM);
2659}
2660
2661static int ipoib_add_one(struct ib_device *device)
2662{
2663	struct list_head *dev_list;
2664	struct net_device *dev;
2665	struct ipoib_dev_priv *priv;
2666	unsigned int p;
2667	int count = 0;
2668
2669	dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
2670	if (!dev_list)
2671		return -ENOMEM;
2672
2673	INIT_LIST_HEAD(dev_list);
2674
2675	rdma_for_each_port (device, p) {
2676		if (!rdma_protocol_ib(device, p))
2677			continue;
2678		dev = ipoib_add_port("ib%d", device, p);
2679		if (!IS_ERR(dev)) {
2680			priv = ipoib_priv(dev);
2681			list_add_tail(&priv->list, dev_list);
2682			count++;
2683		}
2684	}
2685
2686	if (!count) {
2687		kfree(dev_list);
2688		return -EOPNOTSUPP;
2689	}
2690
2691	ib_set_client_data(device, &ipoib_client, dev_list);
2692	return 0;
2693}
2694
2695static void ipoib_remove_one(struct ib_device *device, void *client_data)
2696{
2697	struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
2698	struct list_head *dev_list = client_data;
2699
2700	list_for_each_entry_safe(priv, tmp, dev_list, list) {
2701		LIST_HEAD(head);
2702		ipoib_parent_unregister_pre(priv->dev);
2703
2704		rtnl_lock();
2705
2706		netdev_lock(priv->dev);
2707		list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,
2708					 list)
2709			unregister_netdevice_queue(cpriv->dev, &head);
2710		netdev_unlock(priv->dev);
2711		unregister_netdevice_queue(priv->dev, &head);
2712		unregister_netdevice_many(&head);
2713
2714		rtnl_unlock();
2715	}
2716
2717	kfree(dev_list);
2718}
2719
2720#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2721static struct notifier_block ipoib_netdev_notifier = {
2722	.notifier_call = ipoib_netdev_event,
2723};
2724#endif
2725
2726static int __init ipoib_init_module(void)
2727{
2728	int ret;
2729
2730	ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);
2731	ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);
2732	ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);
2733
2734	ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);
2735	ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);
2736	ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);
2737#ifdef CONFIG_INFINIBAND_IPOIB_CM
2738	ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
2739	ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);
2740#endif
2741
2742	/*
2743	 * When copying small received packets, we only copy from the
2744	 * linear data part of the SKB, so we rely on this condition.
2745	 */
2746	BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);
2747
2748	ipoib_register_debugfs();
2749
2750	/*
2751	 * We create a global workqueue here that is used for all flush
2752	 * operations.  However, if you attempt to flush a workqueue
2753	 * from a task on that same workqueue, it deadlocks the system.
2754	 * We want to be able to flush the tasks associated with a
2755	 * specific net device, so we also create a workqueue for each
2756	 * netdevice.  We queue up the tasks for that device only on
2757	 * its private workqueue, and we only queue up flush events
2758	 * on our global flush workqueue.  This avoids the deadlocks.
2759	 */
2760	ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);
2761	if (!ipoib_workqueue) {
2762		ret = -ENOMEM;
2763		goto err_fs;
2764	}
2765
2766	ib_sa_register_client(&ipoib_sa_client);
2767
2768	ret = ib_register_client(&ipoib_client);
2769	if (ret)
2770		goto err_sa;
2771
2772	ret = ipoib_netlink_init();
2773	if (ret)
2774		goto err_client;
2775
2776#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2777	register_netdevice_notifier(&ipoib_netdev_notifier);
2778#endif
2779	return 0;
2780
2781err_client:
2782	ib_unregister_client(&ipoib_client);
2783
2784err_sa:
2785	ib_sa_unregister_client(&ipoib_sa_client);
2786	destroy_workqueue(ipoib_workqueue);
2787
2788err_fs:
2789	ipoib_unregister_debugfs();
2790
2791	return ret;
2792}
2793
2794static void __exit ipoib_cleanup_module(void)
2795{
2796#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2797	unregister_netdevice_notifier(&ipoib_netdev_notifier);
2798#endif
2799	ipoib_netlink_fini();
2800	ib_unregister_client(&ipoib_client);
2801	ib_sa_unregister_client(&ipoib_sa_client);
2802	ipoib_unregister_debugfs();
2803	destroy_workqueue(ipoib_workqueue);
2804}
2805
2806module_init(ipoib_init_module);
2807module_exit(ipoib_cleanup_module);