ipoib_ib.c

   1/*
   2 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
   6 *
   7 * This software is available to you under a choice of one of two
   8 * licenses.  You may choose to be licensed under the terms of the GNU
   9 * General Public License (GPL) Version 2, available from the file
  10 * COPYING in the main directory of this source tree, or the
  11 * OpenIB.org BSD license below:
  12 *
  13 *     Redistribution and use in source and binary forms, with or
  14 *     without modification, are permitted provided that the following
  15 *     conditions are met:
  16 *
  17 *      - Redistributions of source code must retain the above
  18 *        copyright notice, this list of conditions and the following
  19 *        disclaimer.
  20 *
  21 *      - Redistributions in binary form must reproduce the above
  22 *        copyright notice, this list of conditions and the following
  23 *        disclaimer in the documentation and/or other materials
  24 *        provided with the distribution.
  25 *
  26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33 * SOFTWARE.
  34 */
  35
  36#include <linux/delay.h>
  37#include <linux/moduleparam.h>
  38#include <linux/dma-mapping.h>
  39#include <linux/slab.h>
  40
  41#include <linux/ip.h>
  42#include <linux/tcp.h>
  43#include <net/netdev_lock.h>
  44#include <rdma/ib_cache.h>
  45
  46#include "ipoib.h"
  47
  48#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
  49static int data_debug_level;
  50
  51module_param(data_debug_level, int, 0644);
  52MODULE_PARM_DESC(data_debug_level,
  53		 "Enable data path debug tracing if > 0");
  54#endif
  55
  56struct ipoib_ah *ipoib_create_ah(struct net_device *dev,
  57				 struct ib_pd *pd, struct rdma_ah_attr *attr)
  58{
  59	struct ipoib_ah *ah;
  60	struct ib_ah *vah;
  61
  62	ah = kmalloc(sizeof(*ah), GFP_KERNEL);
  63	if (!ah)
  64		return ERR_PTR(-ENOMEM);
  65
  66	ah->dev       = dev;
  67	ah->last_send = 0;
  68	kref_init(&ah->ref);
  69
  70	vah = rdma_create_ah(pd, attr, RDMA_CREATE_AH_SLEEPABLE);
  71	if (IS_ERR(vah)) {
  72		kfree(ah);
  73		ah = (struct ipoib_ah *)vah;
  74	} else {
  75		ah->ah = vah;
  76		ipoib_dbg(ipoib_priv(dev), "Created ah %p\n", ah->ah);
  77	}
  78
  79	return ah;
  80}
  81
  82void ipoib_free_ah(struct kref *kref)
  83{
  84	struct ipoib_ah *ah = container_of(kref, struct ipoib_ah, ref);
  85	struct ipoib_dev_priv *priv = ipoib_priv(ah->dev);
  86
  87	unsigned long flags;
  88
  89	spin_lock_irqsave(&priv->lock, flags);
  90	list_add_tail(&ah->list, &priv->dead_ahs);
  91	spin_unlock_irqrestore(&priv->lock, flags);
  92}
  93
  94static void ipoib_ud_dma_unmap_rx(struct ipoib_dev_priv *priv,
  95				  u64 mapping[IPOIB_UD_RX_SG])
  96{
  97	ib_dma_unmap_single(priv->ca, mapping[0],
  98			    IPOIB_UD_BUF_SIZE(priv->max_ib_mtu),
  99			    DMA_FROM_DEVICE);
 100}
 101
 102static int ipoib_ib_post_receive(struct net_device *dev, int id)
 103{
 104	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 105	int ret;
 106
 107	priv->rx_wr.wr_id   = id | IPOIB_OP_RECV;
 108	priv->rx_sge[0].addr = priv->rx_ring[id].mapping[0];
 109	priv->rx_sge[1].addr = priv->rx_ring[id].mapping[1];
 110
 111
 112	ret = ib_post_recv(priv->qp, &priv->rx_wr, NULL);
 113	if (unlikely(ret)) {
 114		ipoib_warn(priv, "receive failed for buf %d (%d)\n", id, ret);
 115		ipoib_ud_dma_unmap_rx(priv, priv->rx_ring[id].mapping);
 116		dev_kfree_skb_any(priv->rx_ring[id].skb);
 117		priv->rx_ring[id].skb = NULL;
 118	}
 119
 120	return ret;
 121}
 122
 123static struct sk_buff *ipoib_alloc_rx_skb(struct net_device *dev, int id)
 124{
 125	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 126	struct sk_buff *skb;
 127	int buf_size;
 128	u64 *mapping;
 129
 130	buf_size = IPOIB_UD_BUF_SIZE(priv->max_ib_mtu);
 131
 132	skb = dev_alloc_skb(buf_size + IPOIB_HARD_LEN);
 133	if (unlikely(!skb))
 134		return NULL;
 135
 136	/*
 137	 * the IP header will be at IPOIP_HARD_LEN + IB_GRH_BYTES, that is
 138	 * 64 bytes aligned
 139	 */
 140	skb_reserve(skb, sizeof(struct ipoib_pseudo_header));
 141
 142	mapping = priv->rx_ring[id].mapping;
 143	mapping[0] = ib_dma_map_single(priv->ca, skb->data, buf_size,
 144				       DMA_FROM_DEVICE);
 145	if (unlikely(ib_dma_mapping_error(priv->ca, mapping[0])))
 146		goto error;
 147
 148	priv->rx_ring[id].skb = skb;
 149	return skb;
 150error:
 151	dev_kfree_skb_any(skb);
 152	return NULL;
 153}
 154
 155static int ipoib_ib_post_receives(struct net_device *dev)
 156{
 157	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 158	int i;
 159
 160	for (i = 0; i < ipoib_recvq_size; ++i) {
 161		if (!ipoib_alloc_rx_skb(dev, i)) {
 162			ipoib_warn(priv, "failed to allocate receive buffer %d\n", i);
 163			return -ENOMEM;
 164		}
 165		if (ipoib_ib_post_receive(dev, i)) {
 166			ipoib_warn(priv, "ipoib_ib_post_receive failed for buf %d\n", i);
 167			return -EIO;
 168		}
 169	}
 170
 171	return 0;
 172}
 173
 174static void ipoib_ib_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
 175{
 176	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 177	unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV;
 178	struct sk_buff *skb;
 179	u64 mapping[IPOIB_UD_RX_SG];
 180	union ib_gid *dgid;
 181	union ib_gid *sgid;
 182
 183	ipoib_dbg_data(priv, "recv completion: id %d, status: %d\n",
 184		       wr_id, wc->status);
 185
 186	if (unlikely(wr_id >= ipoib_recvq_size)) {
 187		ipoib_warn(priv, "recv completion event with wrid %d (> %d)\n",
 188			   wr_id, ipoib_recvq_size);
 189		return;
 190	}
 191
 192	skb  = priv->rx_ring[wr_id].skb;
 193
 194	if (unlikely(wc->status != IB_WC_SUCCESS)) {
 195		if (wc->status != IB_WC_WR_FLUSH_ERR)
 196			ipoib_warn(priv,
 197				   "failed recv event (status=%d, wrid=%d vend_err %#x)\n",
 198				   wc->status, wr_id, wc->vendor_err);
 199		ipoib_ud_dma_unmap_rx(priv, priv->rx_ring[wr_id].mapping);
 200		dev_kfree_skb_any(skb);
 201		priv->rx_ring[wr_id].skb = NULL;
 202		return;
 203	}
 204
 205	memcpy(mapping, priv->rx_ring[wr_id].mapping,
 206	       IPOIB_UD_RX_SG * sizeof(*mapping));
 207
 208	/*
 209	 * If we can't allocate a new RX buffer, dump
 210	 * this packet and reuse the old buffer.
 211	 */
 212	if (unlikely(!ipoib_alloc_rx_skb(dev, wr_id))) {
 213		++dev->stats.rx_dropped;
 214		goto repost;
 215	}
 216
 217	ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
 218		       wc->byte_len, wc->slid);
 219
 220	ipoib_ud_dma_unmap_rx(priv, mapping);
 221
 222	skb_put(skb, wc->byte_len);
 223
 224	/* First byte of dgid signals multicast when 0xff */
 225	dgid = &((struct ib_grh *)skb->data)->dgid;
 226
 227	if (!(wc->wc_flags & IB_WC_GRH) || dgid->raw[0] != 0xff)
 228		skb->pkt_type = PACKET_HOST;
 229	else if (memcmp(dgid, dev->broadcast + 4, sizeof(union ib_gid)) == 0)
 230		skb->pkt_type = PACKET_BROADCAST;
 231	else
 232		skb->pkt_type = PACKET_MULTICAST;
 233
 234	sgid = &((struct ib_grh *)skb->data)->sgid;
 235
 236	/*
 237	 * Drop packets that this interface sent, ie multicast packets
 238	 * that the HCA has replicated.
 239	 */
 240	if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num) {
 241		int need_repost = 1;
 242
 243		if ((wc->wc_flags & IB_WC_GRH) &&
 244		    sgid->global.interface_id != priv->local_gid.global.interface_id)
 245			need_repost = 0;
 246
 247		if (need_repost) {
 248			dev_kfree_skb_any(skb);
 249			goto repost;
 250		}
 251	}
 252
 253	skb_pull(skb, IB_GRH_BYTES);
 254
 255	skb->protocol = ((struct ipoib_header *) skb->data)->proto;
 256	skb_add_pseudo_hdr(skb);
 257
 258	++dev->stats.rx_packets;
 259	dev->stats.rx_bytes += skb->len;
 260	if (skb->pkt_type == PACKET_MULTICAST)
 261		dev->stats.multicast++;
 262
 263	skb->dev = dev;
 264	if ((dev->features & NETIF_F_RXCSUM) &&
 265			likely(wc->wc_flags & IB_WC_IP_CSUM_OK))
 266		skb->ip_summed = CHECKSUM_UNNECESSARY;
 267
 268	napi_gro_receive(&priv->recv_napi, skb);
 269
 270repost:
 271	if (unlikely(ipoib_ib_post_receive(dev, wr_id)))
 272		ipoib_warn(priv, "ipoib_ib_post_receive failed "
 273			   "for buf %d\n", wr_id);
 274}
 275
 276int ipoib_dma_map_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req)
 277{
 278	struct sk_buff *skb = tx_req->skb;
 279	u64 *mapping = tx_req->mapping;
 280	int i;
 281	int off;
 282
 283	if (skb_headlen(skb)) {
 284		mapping[0] = ib_dma_map_single(ca, skb->data, skb_headlen(skb),
 285					       DMA_TO_DEVICE);
 286		if (unlikely(ib_dma_mapping_error(ca, mapping[0])))
 287			return -EIO;
 288
 289		off = 1;
 290	} else
 291		off = 0;
 292
 293	for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
 294		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 295		mapping[i + off] = ib_dma_map_page(ca,
 296						 skb_frag_page(frag),
 297						 skb_frag_off(frag),
 298						 skb_frag_size(frag),
 299						 DMA_TO_DEVICE);
 300		if (unlikely(ib_dma_mapping_error(ca, mapping[i + off])))
 301			goto partial_error;
 302	}
 303	return 0;
 304
 305partial_error:
 306	for (; i > 0; --i) {
 307		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
 308
 309		ib_dma_unmap_page(ca, mapping[i - !off], skb_frag_size(frag), DMA_TO_DEVICE);
 310	}
 311
 312	if (off)
 313		ib_dma_unmap_single(ca, mapping[0], skb_headlen(skb), DMA_TO_DEVICE);
 314
 315	return -EIO;
 316}
 317
 318void ipoib_dma_unmap_tx(struct ipoib_dev_priv *priv,
 319			struct ipoib_tx_buf *tx_req)
 320{
 321	struct sk_buff *skb = tx_req->skb;
 322	u64 *mapping = tx_req->mapping;
 323	int i;
 324	int off;
 325
 326	if (skb_headlen(skb)) {
 327		ib_dma_unmap_single(priv->ca, mapping[0], skb_headlen(skb),
 328				    DMA_TO_DEVICE);
 329		off = 1;
 330	} else
 331		off = 0;
 332
 333	for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
 334		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 335
 336		ib_dma_unmap_page(priv->ca, mapping[i + off],
 337				  skb_frag_size(frag), DMA_TO_DEVICE);
 338	}
 339}
 340
 341/*
 342 * As the result of a completion error the QP Can be transferred to SQE states.
 343 * The function checks if the (send)QP is in SQE state and
 344 * moves it back to RTS state, that in order to have it functional again.
 345 */
 346static void ipoib_qp_state_validate_work(struct work_struct *work)
 347{
 348	struct ipoib_qp_state_validate *qp_work =
 349		container_of(work, struct ipoib_qp_state_validate, work);
 350
 351	struct ipoib_dev_priv *priv = qp_work->priv;
 352	struct ib_qp_attr qp_attr;
 353	struct ib_qp_init_attr query_init_attr;
 354	int ret;
 355
 356	ret = ib_query_qp(priv->qp, &qp_attr, IB_QP_STATE, &query_init_attr);
 357	if (ret) {
 358		ipoib_warn(priv, "%s: Failed to query QP ret: %d\n",
 359			   __func__, ret);
 360		goto free_res;
 361	}
 362	pr_info("%s: QP: 0x%x is in state: %d\n",
 363		__func__, priv->qp->qp_num, qp_attr.qp_state);
 364
 365	/* currently support only in SQE->RTS transition*/
 366	if (qp_attr.qp_state == IB_QPS_SQE) {
 367		qp_attr.qp_state = IB_QPS_RTS;
 368
 369		ret = ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE);
 370		if (ret) {
 371			pr_warn("failed(%d) modify QP:0x%x SQE->RTS\n",
 372				ret, priv->qp->qp_num);
 373			goto free_res;
 374		}
 375		pr_info("%s: QP: 0x%x moved from IB_QPS_SQE to IB_QPS_RTS\n",
 376			__func__, priv->qp->qp_num);
 377	} else {
 378		pr_warn("QP (%d) will stay in state: %d\n",
 379			priv->qp->qp_num, qp_attr.qp_state);
 380	}
 381
 382free_res:
 383	kfree(qp_work);
 384}
 385
 386static void ipoib_ib_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
 387{
 388	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 389	unsigned int wr_id = wc->wr_id;
 390	struct ipoib_tx_buf *tx_req;
 391
 392	ipoib_dbg_data(priv, "send completion: id %d, status: %d\n",
 393		       wr_id, wc->status);
 394
 395	if (unlikely(wr_id >= ipoib_sendq_size)) {
 396		ipoib_warn(priv, "send completion event with wrid %d (> %d)\n",
 397			   wr_id, ipoib_sendq_size);
 398		return;
 399	}
 400
 401	tx_req = &priv->tx_ring[wr_id];
 402
 403	ipoib_dma_unmap_tx(priv, tx_req);
 404
 405	++dev->stats.tx_packets;
 406	dev->stats.tx_bytes += tx_req->skb->len;
 407
 408	dev_kfree_skb_any(tx_req->skb);
 409
 410	++priv->tx_tail;
 411	++priv->global_tx_tail;
 412
 413	if (unlikely(netif_queue_stopped(dev) &&
 414		     ((priv->global_tx_head - priv->global_tx_tail) <=
 415		      ipoib_sendq_size >> 1) &&
 416		     test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)))
 417		netif_wake_queue(dev);
 418
 419	if (wc->status != IB_WC_SUCCESS &&
 420	    wc->status != IB_WC_WR_FLUSH_ERR) {
 421		struct ipoib_qp_state_validate *qp_work;
 422		ipoib_warn(priv,
 423			   "failed send event (status=%d, wrid=%d vend_err %#x)\n",
 424			   wc->status, wr_id, wc->vendor_err);
 425		qp_work = kzalloc(sizeof(*qp_work), GFP_ATOMIC);
 426		if (!qp_work)
 427			return;
 428
 429		INIT_WORK(&qp_work->work, ipoib_qp_state_validate_work);
 430		qp_work->priv = priv;
 431		queue_work(priv->wq, &qp_work->work);
 432	}
 433}
 434
 435static int poll_tx(struct ipoib_dev_priv *priv)
 436{
 437	int n, i;
 438	struct ib_wc *wc;
 439
 440	n = ib_poll_cq(priv->send_cq, MAX_SEND_CQE, priv->send_wc);
 441	for (i = 0; i < n; ++i) {
 442		wc = priv->send_wc + i;
 443		if (wc->wr_id & IPOIB_OP_CM)
 444			ipoib_cm_handle_tx_wc(priv->dev, priv->send_wc + i);
 445		else
 446			ipoib_ib_handle_tx_wc(priv->dev, priv->send_wc + i);
 447	}
 448	return n == MAX_SEND_CQE;
 449}
 450
 451int ipoib_rx_poll(struct napi_struct *napi, int budget)
 452{
 453	struct ipoib_dev_priv *priv =
 454		container_of(napi, struct ipoib_dev_priv, recv_napi);
 455	struct net_device *dev = priv->dev;
 456	int done;
 457	int t;
 458	int n, i;
 459
 460	done  = 0;
 461
 462poll_more:
 463	while (done < budget) {
 464		int max = (budget - done);
 465
 466		t = min(IPOIB_NUM_WC, max);
 467		n = ib_poll_cq(priv->recv_cq, t, priv->ibwc);
 468
 469		for (i = 0; i < n; i++) {
 470			struct ib_wc *wc = priv->ibwc + i;
 471
 472			if (wc->wr_id & IPOIB_OP_RECV) {
 473				++done;
 474				if (wc->wr_id & IPOIB_OP_CM)
 475					ipoib_cm_handle_rx_wc(dev, wc);
 476				else
 477					ipoib_ib_handle_rx_wc(dev, wc);
 478			} else {
 479				pr_warn("%s: Got unexpected wqe id\n", __func__);
 480			}
 481		}
 482
 483		if (n != t)
 484			break;
 485	}
 486
 487	if (done < budget) {
 488		napi_complete(napi);
 489		if (unlikely(ib_req_notify_cq(priv->recv_cq,
 490					      IB_CQ_NEXT_COMP |
 491					      IB_CQ_REPORT_MISSED_EVENTS)) &&
 492		    napi_schedule(napi))
 493			goto poll_more;
 494	}
 495
 496	return done;
 497}
 498
 499int ipoib_tx_poll(struct napi_struct *napi, int budget)
 500{
 501	struct ipoib_dev_priv *priv = container_of(napi, struct ipoib_dev_priv,
 502						   send_napi);
 503	struct net_device *dev = priv->dev;
 504	int n, i;
 505	struct ib_wc *wc;
 506
 507poll_more:
 508	n = ib_poll_cq(priv->send_cq, MAX_SEND_CQE, priv->send_wc);
 509
 510	for (i = 0; i < n; i++) {
 511		wc = priv->send_wc + i;
 512		if (wc->wr_id & IPOIB_OP_CM)
 513			ipoib_cm_handle_tx_wc(dev, wc);
 514		else
 515			ipoib_ib_handle_tx_wc(dev, wc);
 516	}
 517
 518	if (n < budget) {
 519		napi_complete(napi);
 520		if (unlikely(ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP |
 521					      IB_CQ_REPORT_MISSED_EVENTS)) &&
 522		    napi_schedule(napi))
 523			goto poll_more;
 524	}
 525	return n < 0 ? 0 : n;
 526}
 527
 528void ipoib_ib_rx_completion(struct ib_cq *cq, void *ctx_ptr)
 529{
 530	struct ipoib_dev_priv *priv = ctx_ptr;
 531
 532	napi_schedule(&priv->recv_napi);
 533}
 534
 535/* The function will force napi_schedule */
 536void ipoib_napi_schedule_work(struct work_struct *work)
 537{
 538	struct ipoib_dev_priv *priv =
 539		container_of(work, struct ipoib_dev_priv, reschedule_napi_work);
 540	bool ret;
 541
 542	do {
 543		ret = napi_schedule(&priv->send_napi);
 544		if (!ret)
 545			msleep(3);
 546	} while (!ret && netif_queue_stopped(priv->dev) &&
 547		 test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags));
 548}
 549
 550void ipoib_ib_tx_completion(struct ib_cq *cq, void *ctx_ptr)
 551{
 552	struct ipoib_dev_priv *priv = ctx_ptr;
 553	bool ret;
 554
 555	ret = napi_schedule(&priv->send_napi);
 556	/*
 557	 * if the queue is closed the driver must be able to schedule napi,
 558	 * otherwise we can end with closed queue forever, because no new
 559	 * packets to send and napi callback might not get new event after
 560	 * its re-arm of the napi.
 561	 */
 562	if (!ret && netif_queue_stopped(priv->dev))
 563		schedule_work(&priv->reschedule_napi_work);
 564}
 565
 566static inline int post_send(struct ipoib_dev_priv *priv,
 567			    unsigned int wr_id,
 568			    struct ib_ah *address, u32 dqpn,
 569			    struct ipoib_tx_buf *tx_req,
 570			    void *head, int hlen)
 571{
 572	struct sk_buff *skb = tx_req->skb;
 573
 574	ipoib_build_sge(priv, tx_req);
 575
 576	priv->tx_wr.wr.wr_id	= wr_id;
 577	priv->tx_wr.remote_qpn	= dqpn;
 578	priv->tx_wr.ah		= address;
 579
 580	if (head) {
 581		priv->tx_wr.mss		= skb_shinfo(skb)->gso_size;
 582		priv->tx_wr.header	= head;
 583		priv->tx_wr.hlen	= hlen;
 584		priv->tx_wr.wr.opcode	= IB_WR_LSO;
 585	} else
 586		priv->tx_wr.wr.opcode	= IB_WR_SEND;
 587
 588	return ib_post_send(priv->qp, &priv->tx_wr.wr, NULL);
 589}
 590
 591int ipoib_send(struct net_device *dev, struct sk_buff *skb,
 592	       struct ib_ah *address, u32 dqpn)
 593{
 594	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 595	struct ipoib_tx_buf *tx_req;
 596	int hlen, rc;
 597	void *phead;
 598	unsigned int usable_sge = priv->max_send_sge - !!skb_headlen(skb);
 599
 600	if (skb_is_gso(skb)) {
 601		hlen = skb_tcp_all_headers(skb);
 602		phead = skb->data;
 603		if (unlikely(!skb_pull(skb, hlen))) {
 604			ipoib_warn(priv, "linear data too small\n");
 605			++dev->stats.tx_dropped;
 606			++dev->stats.tx_errors;
 607			dev_kfree_skb_any(skb);
 608			return -1;
 609		}
 610	} else {
 611		if (unlikely(skb->len > priv->mcast_mtu + IPOIB_ENCAP_LEN)) {
 612			ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n",
 613				   skb->len, priv->mcast_mtu + IPOIB_ENCAP_LEN);
 614			++dev->stats.tx_dropped;
 615			++dev->stats.tx_errors;
 616			ipoib_cm_skb_too_long(dev, skb, priv->mcast_mtu);
 617			return -1;
 618		}
 619		phead = NULL;
 620		hlen  = 0;
 621	}
 622	if (skb_shinfo(skb)->nr_frags > usable_sge) {
 623		if (skb_linearize(skb) < 0) {
 624			ipoib_warn(priv, "skb could not be linearized\n");
 625			++dev->stats.tx_dropped;
 626			++dev->stats.tx_errors;
 627			dev_kfree_skb_any(skb);
 628			return -1;
 629		}
 630		/* Does skb_linearize return ok without reducing nr_frags? */
 631		if (skb_shinfo(skb)->nr_frags > usable_sge) {
 632			ipoib_warn(priv, "too many frags after skb linearize\n");
 633			++dev->stats.tx_dropped;
 634			++dev->stats.tx_errors;
 635			dev_kfree_skb_any(skb);
 636			return -1;
 637		}
 638	}
 639
 640	ipoib_dbg_data(priv,
 641		       "sending packet, length=%d address=%p dqpn=0x%06x\n",
 642		       skb->len, address, dqpn);
 643
 644	/*
 645	 * We put the skb into the tx_ring _before_ we call post_send()
 646	 * because it's entirely possible that the completion handler will
 647	 * run before we execute anything after the post_send().  That
 648	 * means we have to make sure everything is properly recorded and
 649	 * our state is consistent before we call post_send().
 650	 */
 651	tx_req = &priv->tx_ring[priv->tx_head & (ipoib_sendq_size - 1)];
 652	tx_req->skb = skb;
 653	if (unlikely(ipoib_dma_map_tx(priv->ca, tx_req))) {
 654		++dev->stats.tx_errors;
 655		dev_kfree_skb_any(skb);
 656		return -1;
 657	}
 658
 659	if (skb->ip_summed == CHECKSUM_PARTIAL)
 660		priv->tx_wr.wr.send_flags |= IB_SEND_IP_CSUM;
 661	else
 662		priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
 663	/* increase the tx_head after send success, but use it for queue state */
 664	if ((priv->global_tx_head - priv->global_tx_tail) ==
 665	    ipoib_sendq_size - 1) {
 666		ipoib_dbg(priv, "TX ring full, stopping kernel net queue\n");
 667		netif_stop_queue(dev);
 668	}
 669
 670	skb_orphan(skb);
 671	skb_dst_drop(skb);
 672
 673	if (netif_queue_stopped(dev))
 674		if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP |
 675				     IB_CQ_REPORT_MISSED_EVENTS) < 0)
 676			ipoib_warn(priv, "request notify on send CQ failed\n");
 677
 678	rc = post_send(priv, priv->tx_head & (ipoib_sendq_size - 1),
 679		       address, dqpn, tx_req, phead, hlen);
 680	if (unlikely(rc)) {
 681		ipoib_warn(priv, "post_send failed, error %d\n", rc);
 682		++dev->stats.tx_errors;
 683		ipoib_dma_unmap_tx(priv, tx_req);
 684		dev_kfree_skb_any(skb);
 685		if (netif_queue_stopped(dev))
 686			netif_wake_queue(dev);
 687		rc = 0;
 688	} else {
 689		netif_trans_update(dev);
 690
 691		rc = priv->tx_head;
 692		++priv->tx_head;
 693		++priv->global_tx_head;
 694	}
 695	return rc;
 696}
 697
 698static void ipoib_reap_dead_ahs(struct ipoib_dev_priv *priv)
 699{
 700	struct ipoib_ah *ah, *tah;
 701	unsigned long flags;
 702
 703	netif_tx_lock_bh(priv->dev);
 704	spin_lock_irqsave(&priv->lock, flags);
 705
 706	list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
 707		if ((int) priv->tx_tail - (int) ah->last_send >= 0) {
 708			list_del(&ah->list);
 709			rdma_destroy_ah(ah->ah, 0);
 710			kfree(ah);
 711		}
 712
 713	spin_unlock_irqrestore(&priv->lock, flags);
 714	netif_tx_unlock_bh(priv->dev);
 715}
 716
 717void ipoib_reap_ah(struct work_struct *work)
 718{
 719	struct ipoib_dev_priv *priv =
 720		container_of(work, struct ipoib_dev_priv, ah_reap_task.work);
 721
 722	ipoib_reap_dead_ahs(priv);
 723
 724	if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
 725		queue_delayed_work(priv->wq, &priv->ah_reap_task,
 726				   round_jiffies_relative(HZ));
 727}
 728
 729static void ipoib_start_ah_reaper(struct ipoib_dev_priv *priv)
 730{
 731	clear_bit(IPOIB_STOP_REAPER, &priv->flags);
 732	queue_delayed_work(priv->wq, &priv->ah_reap_task,
 733			   round_jiffies_relative(HZ));
 734}
 735
 736static void ipoib_stop_ah_reaper(struct ipoib_dev_priv *priv)
 737{
 738	set_bit(IPOIB_STOP_REAPER, &priv->flags);
 739	cancel_delayed_work(&priv->ah_reap_task);
 740	/*
 741	 * After ipoib_stop_ah_reaper() we always go through
 742	 * ipoib_reap_dead_ahs() which ensures the work is really stopped and
 743	 * does a final flush out of the dead_ah's list
 744	 */
 745}
 746
 747static int recvs_pending(struct net_device *dev)
 748{
 749	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 750	int pending = 0;
 751	int i;
 752
 753	for (i = 0; i < ipoib_recvq_size; ++i)
 754		if (priv->rx_ring[i].skb)
 755			++pending;
 756
 757	return pending;
 758}
 759
 760static void check_qp_movement_and_print(struct ipoib_dev_priv *priv,
 761					struct ib_qp *qp,
 762					enum ib_qp_state new_state)
 763{
 764	struct ib_qp_attr qp_attr;
 765	struct ib_qp_init_attr query_init_attr;
 766	int ret;
 767
 768	ret = ib_query_qp(qp, &qp_attr, IB_QP_STATE, &query_init_attr);
 769	if (ret) {
 770		ipoib_warn(priv, "%s: Failed to query QP\n", __func__);
 771		return;
 772	}
 773	/* print according to the new-state and the previous state.*/
 774	if (new_state == IB_QPS_ERR && qp_attr.qp_state == IB_QPS_RESET)
 775		ipoib_dbg(priv, "Failed modify QP, IB_QPS_RESET to IB_QPS_ERR, acceptable\n");
 776	else
 777		ipoib_warn(priv, "Failed to modify QP to state: %d from state: %d\n",
 778			   new_state, qp_attr.qp_state);
 779}
 780
 781static void ipoib_napi_enable(struct net_device *dev)
 782{
 783	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 784
 785	netdev_lock_ops_to_full(dev);
 786	napi_enable_locked(&priv->recv_napi);
 787	napi_enable_locked(&priv->send_napi);
 788	netdev_unlock_full_to_ops(dev);
 789}
 790
 791static void ipoib_napi_disable(struct net_device *dev)
 792{
 793	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 794
 795	netdev_lock_ops_to_full(dev);
 796	napi_disable_locked(&priv->recv_napi);
 797	napi_disable_locked(&priv->send_napi);
 798	netdev_unlock_full_to_ops(dev);
 799}
 800
 801int ipoib_ib_dev_stop_default(struct net_device *dev)
 802{
 803	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 804	struct ib_qp_attr qp_attr;
 805	unsigned long begin;
 806	struct ipoib_tx_buf *tx_req;
 807	int i;
 808
 809	if (test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
 810		ipoib_napi_disable(dev);
 811
 812	ipoib_cm_dev_stop(dev);
 813
 814	/*
 815	 * Move our QP to the error state and then reinitialize in
 816	 * when all work requests have completed or have been flushed.
 817	 */
 818	qp_attr.qp_state = IB_QPS_ERR;
 819	if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
 820		check_qp_movement_and_print(priv, priv->qp, IB_QPS_ERR);
 821
 822	/* Wait for all sends and receives to complete */
 823	begin = jiffies;
 824
 825	while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
 826		if (time_after(jiffies, begin + 5 * HZ)) {
 827			ipoib_warn(priv,
 828				   "timing out; %d sends %d receives not completed\n",
 829				   priv->tx_head - priv->tx_tail,
 830				   recvs_pending(dev));
 831
 832			/*
 833			 * assume the HW is wedged and just free up
 834			 * all our pending work requests.
 835			 */
 836			while ((int)priv->tx_tail - (int)priv->tx_head < 0) {
 837				tx_req = &priv->tx_ring[priv->tx_tail &
 838							(ipoib_sendq_size - 1)];
 839				ipoib_dma_unmap_tx(priv, tx_req);
 840				dev_kfree_skb_any(tx_req->skb);
 841				++priv->tx_tail;
 842				++priv->global_tx_tail;
 843			}
 844
 845			for (i = 0; i < ipoib_recvq_size; ++i) {
 846				struct ipoib_rx_buf *rx_req;
 847
 848				rx_req = &priv->rx_ring[i];
 849				if (!rx_req->skb)
 850					continue;
 851				ipoib_ud_dma_unmap_rx(priv,
 852						      priv->rx_ring[i].mapping);
 853				dev_kfree_skb_any(rx_req->skb);
 854				rx_req->skb = NULL;
 855			}
 856
 857			goto timeout;
 858		}
 859
 860		ipoib_drain_cq(dev);
 861
 862		usleep_range(1000, 2000);
 863	}
 864
 865	ipoib_dbg(priv, "All sends and receives done.\n");
 866
 867timeout:
 868	qp_attr.qp_state = IB_QPS_RESET;
 869	if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
 870		ipoib_warn(priv, "Failed to modify QP to RESET state\n");
 871
 872	ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
 873
 874	return 0;
 875}
 876
 877int ipoib_ib_dev_open_default(struct net_device *dev)
 878{
 879	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 880	int ret;
 881
 882	ret = ipoib_init_qp(dev);
 883	if (ret) {
 884		ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
 885		return -1;
 886	}
 887
 888	ret = ipoib_ib_post_receives(dev);
 889	if (ret) {
 890		ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
 891		goto out;
 892	}
 893
 894	ret = ipoib_cm_dev_open(dev);
 895	if (ret) {
 896		ipoib_warn(priv, "ipoib_cm_dev_open returned %d\n", ret);
 897		goto out;
 898	}
 899
 900	if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
 901		ipoib_napi_enable(dev);
 902
 903	return 0;
 904out:
 905	return -1;
 906}
 907
 908int ipoib_ib_dev_open(struct net_device *dev)
 909{
 910	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 911
 912	ipoib_pkey_dev_check_presence(dev);
 913
 914	if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
 915		ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,
 916			   (!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));
 917		return -1;
 918	}
 919
 920	ipoib_start_ah_reaper(priv);
 921	if (priv->rn_ops->ndo_open(dev)) {
 922		pr_warn("%s: Failed to open dev\n", dev->name);
 923		goto dev_stop;
 924	}
 925
 926	set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
 927
 928	return 0;
 929
 930dev_stop:
 931	ipoib_stop_ah_reaper(priv);
 932	return -1;
 933}
 934
 935void ipoib_ib_dev_stop(struct net_device *dev)
 936{
 937	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 938
 939	priv->rn_ops->ndo_stop(dev);
 940
 941	clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
 942	ipoib_stop_ah_reaper(priv);
 943}
 944
 945void ipoib_pkey_dev_check_presence(struct net_device *dev)
 946{
 947	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 948	struct rdma_netdev *rn = netdev_priv(dev);
 949
 950	if (!(priv->pkey & 0x7fff) ||
 951	    ib_find_pkey(priv->ca, priv->port, priv->pkey,
 952			 &priv->pkey_index)) {
 953		clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
 954	} else {
 955		if (rn->set_id)
 956			rn->set_id(dev, priv->pkey_index);
 957		set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
 958	}
 959}
 960
 961void ipoib_ib_dev_up(struct net_device *dev)
 962{
 963	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 964
 965	ipoib_pkey_dev_check_presence(dev);
 966
 967	if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
 968		ipoib_dbg(priv, "PKEY is not assigned.\n");
 969		return;
 970	}
 971
 972	set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
 973
 974	ipoib_mcast_start_thread(dev);
 975}
 976
 977void ipoib_ib_dev_down(struct net_device *dev)
 978{
 979	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 980
 981	ipoib_dbg(priv, "downing ib_dev\n");
 982
 983	clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
 984	netif_carrier_off(dev);
 985
 986	ipoib_mcast_stop_thread(dev);
 987	ipoib_mcast_dev_flush(dev);
 988
 989	ipoib_flush_paths(dev);
 990}
 991
 992void ipoib_drain_cq(struct net_device *dev)
 993{
 994	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 995	int i, n;
 996
 997	/*
 998	 * We call completion handling routines that expect to be
 999	 * called from the BH-disabled NAPI poll context, so disable
1000	 * BHs here too.
1001	 */
1002	local_bh_disable();
1003
1004	do {
1005		n = ib_poll_cq(priv->recv_cq, IPOIB_NUM_WC, priv->ibwc);
1006		for (i = 0; i < n; ++i) {
1007			/*
1008			 * Convert any successful completions to flush
1009			 * errors to avoid passing packets up the
1010			 * stack after bringing the device down.
1011			 */
1012			if (priv->ibwc[i].status == IB_WC_SUCCESS)
1013				priv->ibwc[i].status = IB_WC_WR_FLUSH_ERR;
1014
1015			if (priv->ibwc[i].wr_id & IPOIB_OP_RECV) {
1016				if (priv->ibwc[i].wr_id & IPOIB_OP_CM)
1017					ipoib_cm_handle_rx_wc(dev, priv->ibwc + i);
1018				else
1019					ipoib_ib_handle_rx_wc(dev, priv->ibwc + i);
1020			} else {
1021				pr_warn("%s: Got unexpected wqe id\n", __func__);
1022			}
1023		}
1024	} while (n == IPOIB_NUM_WC);
1025
1026	while (poll_tx(priv))
1027		; /* nothing */
1028
1029	local_bh_enable();
1030}
1031
1032/*
1033 * Takes whatever value which is in pkey index 0 and updates priv->pkey
1034 * returns 0 if the pkey value was changed.
1035 */
1036static inline int update_parent_pkey(struct ipoib_dev_priv *priv)
1037{
1038	int result;
1039	u16 prev_pkey;
1040
1041	prev_pkey = priv->pkey;
1042	result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
1043	if (result) {
1044		ipoib_warn(priv, "ib_query_pkey port %d failed (ret = %d)\n",
1045			   priv->port, result);
1046		return result;
1047	}
1048
1049	priv->pkey |= 0x8000;
1050
1051	if (prev_pkey != priv->pkey) {
1052		ipoib_dbg(priv, "pkey changed from 0x%x to 0x%x\n",
1053			  prev_pkey, priv->pkey);
1054		/*
1055		 * Update the pkey in the broadcast address, while making sure to set
1056		 * the full membership bit, so that we join the right broadcast group.
1057		 */
1058		priv->dev->broadcast[8] = priv->pkey >> 8;
1059		priv->dev->broadcast[9] = priv->pkey & 0xff;
1060		return 0;
1061	}
1062
1063	return 1;
1064}
1065/*
1066 * returns 0 if pkey value was found in a different slot.
1067 */
1068static inline int update_child_pkey(struct ipoib_dev_priv *priv)
1069{
1070	u16 old_index = priv->pkey_index;
1071
1072	priv->pkey_index = 0;
1073	ipoib_pkey_dev_check_presence(priv->dev);
1074
1075	if (test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
1076	    (old_index == priv->pkey_index))
1077		return 1;
1078	return 0;
1079}
1080
1081/*
1082 * returns true if the device address of the ipoib interface has changed and the
1083 * new address is a valid one (i.e in the gid table), return false otherwise.
1084 */
1085static bool ipoib_dev_addr_changed_valid(struct ipoib_dev_priv *priv)
1086{
1087	union ib_gid search_gid;
1088	union ib_gid gid0;
1089	int err;
1090	u16 index;
1091	u32 port;
1092	bool ret = false;
1093
1094	if (rdma_query_gid(priv->ca, priv->port, 0, &gid0))
1095		return false;
1096
1097	netif_addr_lock_bh(priv->dev);
1098
1099	/* The subnet prefix may have changed, update it now so we won't have
1100	 * to do it later
1101	 */
1102	priv->local_gid.global.subnet_prefix = gid0.global.subnet_prefix;
1103	dev_addr_mod(priv->dev, 4, (u8 *)&gid0.global.subnet_prefix,
1104		     sizeof(gid0.global.subnet_prefix));
1105	search_gid.global.subnet_prefix = gid0.global.subnet_prefix;
1106
1107	search_gid.global.interface_id = priv->local_gid.global.interface_id;
1108
1109	netif_addr_unlock_bh(priv->dev);
1110
1111	err = ib_find_gid(priv->ca, &search_gid, &port, &index);
1112
1113	netif_addr_lock_bh(priv->dev);
1114
1115	if (search_gid.global.interface_id !=
1116	    priv->local_gid.global.interface_id)
1117		/* There was a change while we were looking up the gid, bail
1118		 * here and let the next work sort this out
1119		 */
1120		goto out;
1121
1122	/* The next section of code needs some background:
1123	 * Per IB spec the port GUID can't change if the HCA is powered on.
1124	 * port GUID is the basis for GID at index 0 which is the basis for
1125	 * the default device address of a ipoib interface.
1126	 *
1127	 * so it seems the flow should be:
1128	 * if user_changed_dev_addr && gid in gid tbl
1129	 *	set bit dev_addr_set
1130	 *	return true
1131	 * else
1132	 *	return false
1133	 *
1134	 * The issue is that there are devices that don't follow the spec,
1135	 * they change the port GUID when the HCA is powered, so in order
1136	 * not to break userspace applications, We need to check if the
1137	 * user wanted to control the device address and we assume that
1138	 * if he sets the device address back to be based on GID index 0,
1139	 * he no longer wishs to control it.
1140	 *
1141	 * If the user doesn't control the device address,
1142	 * IPOIB_FLAG_DEV_ADDR_SET is set and ib_find_gid failed it means
1143	 * the port GUID has changed and GID at index 0 has changed
1144	 * so we need to change priv->local_gid and priv->dev->dev_addr
1145	 * to reflect the new GID.
1146	 */
1147	if (!test_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags)) {
1148		if (!err && port == priv->port) {
1149			set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
1150			if (index == 0)
1151				clear_bit(IPOIB_FLAG_DEV_ADDR_CTRL,
1152					  &priv->flags);
1153			else
1154				set_bit(IPOIB_FLAG_DEV_ADDR_CTRL, &priv->flags);
1155			ret = true;
1156		} else {
1157			ret = false;
1158		}
1159	} else {
1160		if (!err && port == priv->port) {
1161			ret = true;
1162		} else {
1163			if (!test_bit(IPOIB_FLAG_DEV_ADDR_CTRL, &priv->flags)) {
1164				memcpy(&priv->local_gid, &gid0,
1165				       sizeof(priv->local_gid));
1166				dev_addr_mod(priv->dev, 4, (u8 *)&gid0,
1167					     sizeof(priv->local_gid));
1168				ret = true;
1169			}
1170		}
1171	}
1172
1173out:
1174	netif_addr_unlock_bh(priv->dev);
1175
1176	return ret;
1177}
1178
1179static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
1180				enum ipoib_flush_level level)
1181{
1182	struct net_device *dev = priv->dev;
1183	int result;
1184
1185	if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags) &&
1186	    level != IPOIB_FLUSH_HEAVY) {
1187		/* Make sure the dev_addr is set even if not flushing */
1188		if (level == IPOIB_FLUSH_LIGHT)
1189			ipoib_dev_addr_changed_valid(priv);
1190		ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
1191		return;
1192	}
1193
1194	if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
1195		/* interface is down. update pkey and leave. */
1196		if (level == IPOIB_FLUSH_HEAVY) {
1197			if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
1198				update_parent_pkey(priv);
1199			else
1200				update_child_pkey(priv);
1201		} else if (level == IPOIB_FLUSH_LIGHT)
1202			ipoib_dev_addr_changed_valid(priv);
1203		ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_ADMIN_UP not set.\n");
1204		return;
1205	}
1206
1207	if (level == IPOIB_FLUSH_HEAVY) {
1208		/* child devices chase their origin pkey value, while non-child
1209		 * (parent) devices should always takes what present in pkey index 0
1210		 */
1211		if (test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
1212			result = update_child_pkey(priv);
1213			if (result) {
1214				/* restart QP only if P_Key index is changed */
1215				ipoib_dbg(priv, "Not flushing - P_Key index not changed.\n");
1216				return;
1217			}
1218
1219		} else {
1220			result = update_parent_pkey(priv);
1221			/* restart QP only if P_Key value changed */
1222			if (result) {
1223				ipoib_dbg(priv, "Not flushing - P_Key value not changed.\n");
1224				return;
1225			}
1226		}
1227	}
1228
1229	if (level == IPOIB_FLUSH_LIGHT) {
1230		int oper_up;
1231		ipoib_mark_paths_invalid(dev);
1232		/* Set IPoIB operation as down to prevent races between:
1233		 * the flush flow which leaves MCG and on the fly joins
1234		 * which can happen during that time. mcast restart task
1235		 * should deal with join requests we missed.
1236		 */
1237		oper_up = test_and_clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
1238		ipoib_mcast_dev_flush(dev);
1239		if (oper_up)
1240			set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
1241		ipoib_reap_dead_ahs(priv);
1242	}
1243
1244	if (level >= IPOIB_FLUSH_NORMAL)
1245		ipoib_ib_dev_down(dev);
1246
1247	if (level == IPOIB_FLUSH_HEAVY) {
1248		netdev_lock_ops(dev);
1249		if (test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
1250			ipoib_ib_dev_stop(dev);
1251
1252		result = ipoib_ib_dev_open(dev);
1253		netdev_unlock_ops(dev);
1254
1255		if (result)
1256			return;
1257
1258		if (netif_queue_stopped(dev))
1259			netif_start_queue(dev);
1260	}
1261
1262	/*
1263	 * The device could have been brought down between the start and when
1264	 * we get here, don't bring it back up if it's not configured up
1265	 */
1266	if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
1267		if (level >= IPOIB_FLUSH_NORMAL)
1268			ipoib_ib_dev_up(dev);
1269		if (ipoib_dev_addr_changed_valid(priv))
1270			ipoib_mcast_restart_task(&priv->restart_task);
1271	}
1272}
1273
1274void ipoib_ib_dev_flush_light(struct work_struct *work)
1275{
1276	struct ipoib_dev_priv *priv =
1277		container_of(work, struct ipoib_dev_priv, flush_light);
1278
1279	__ipoib_ib_dev_flush(priv, IPOIB_FLUSH_LIGHT);
1280}
1281
1282void ipoib_ib_dev_flush_normal(struct work_struct *work)
1283{
1284	struct ipoib_dev_priv *priv =
1285		container_of(work, struct ipoib_dev_priv, flush_normal);
1286
1287	__ipoib_ib_dev_flush(priv, IPOIB_FLUSH_NORMAL);
1288}
1289
1290void ipoib_ib_dev_flush_heavy(struct work_struct *work)
1291{
1292	struct ipoib_dev_priv *priv =
1293		container_of(work, struct ipoib_dev_priv, flush_heavy);
1294
1295	rtnl_lock();
1296	__ipoib_ib_dev_flush(priv, IPOIB_FLUSH_HEAVY);
1297	rtnl_unlock();
1298}
1299
1300void ipoib_queue_work(struct ipoib_dev_priv *priv,
1301		      enum ipoib_flush_level level)
1302{
1303	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
1304		struct ipoib_dev_priv *cpriv;
1305
1306		netdev_lock(priv->dev);
1307		list_for_each_entry(cpriv, &priv->child_intfs, list)
1308			ipoib_queue_work(cpriv, level);
1309		netdev_unlock(priv->dev);
1310	}
1311
1312	switch (level) {
1313	case IPOIB_FLUSH_LIGHT:
1314		queue_work(ipoib_workqueue, &priv->flush_light);
1315		break;
1316	case IPOIB_FLUSH_NORMAL:
1317		queue_work(ipoib_workqueue, &priv->flush_normal);
1318		break;
1319	case IPOIB_FLUSH_HEAVY:
1320		queue_work(ipoib_workqueue, &priv->flush_heavy);
1321		break;
1322	}
1323}
1324
1325void ipoib_ib_dev_cleanup(struct net_device *dev)
1326{
1327	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1328
1329	ipoib_dbg(priv, "cleaning up ib_dev\n");
1330	/*
1331	 * We must make sure there are no more (path) completions
1332	 * that may wish to touch priv fields that are no longer valid
1333	 */
1334	ipoib_flush_paths(dev);
1335
1336	ipoib_mcast_stop_thread(dev);
1337	ipoib_mcast_dev_flush(dev);
1338
1339	/*
1340	 * All of our ah references aren't free until after
1341	 * ipoib_mcast_dev_flush(), ipoib_flush_paths, and
1342	 * the neighbor garbage collection is stopped and reaped.
1343	 * That should all be done now, so make a final ah flush.
1344	 */
1345	ipoib_reap_dead_ahs(priv);
1346
1347	clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
1348
1349	priv->rn_ops->ndo_uninit(dev);
1350
1351	if (priv->pd) {
1352		ib_dealloc_pd(priv->pd);
1353		priv->pd = NULL;
1354	}
1355}
1356