mad.c

   1/*
   2 * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
   3 * Copyright (c) 2005 Intel Corporation.  All rights reserved.
   4 * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
   5 * Copyright (c) 2009 HNR Consulting. All rights reserved.
   6 * Copyright (c) 2014,2018 Intel Corporation.  All rights reserved.
   7 *
   8 * This software is available to you under a choice of one of two
   9 * licenses.  You may choose to be licensed under the terms of the GNU
  10 * General Public License (GPL) Version 2, available from the file
  11 * COPYING in the main directory of this source tree, or the
  12 * OpenIB.org BSD license below:
  13 *
  14 *     Redistribution and use in source and binary forms, with or
  15 *     without modification, are permitted provided that the following
  16 *     conditions are met:
  17 *
  18 *      - Redistributions of source code must retain the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer.
  21 *
  22 *      - Redistributions in binary form must reproduce the above
  23 *        copyright notice, this list of conditions and the following
  24 *        disclaimer in the documentation and/or other materials
  25 *        provided with the distribution.
  26 *
  27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34 * SOFTWARE.
  35 *
  36 */
  37
  38#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  39
  40#include <linux/dma-mapping.h>
  41#include <linux/slab.h>
  42#include <linux/module.h>
  43#include <linux/security.h>
  44#include <linux/xarray.h>
  45#include <rdma/ib_cache.h>
  46
  47#include "mad_priv.h"
  48#include "core_priv.h"
  49#include "mad_rmpp.h"
  50#include "smi.h"
  51#include "opa_smi.h"
  52#include "agent.h"
  53
  54#define CREATE_TRACE_POINTS
  55#include <trace/events/ib_mad.h>
  56
  57#ifdef CONFIG_TRACEPOINTS
  58static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,
  59			  struct ib_mad_qp_info *qp_info,
  60			  struct trace_event_raw_ib_mad_send_template *entry)
  61{
  62	struct ib_ud_wr *wr = &mad_send_wr->send_wr;
  63	struct rdma_ah_attr attr = {};
  64
  65	rdma_query_ah(wr->ah, &attr);
  66
  67	/* These are common */
  68	entry->sl = attr.sl;
  69	entry->rqpn = wr->remote_qpn;
  70	entry->rqkey = wr->remote_qkey;
  71	entry->dlid = rdma_ah_get_dlid(&attr);
  72}
  73#endif
  74
  75static int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
  76static int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
  77
  78module_param_named(send_queue_size, mad_sendq_size, int, 0444);
  79MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests");
  80module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
  81MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
  82
  83static DEFINE_XARRAY_ALLOC1(ib_mad_clients);
  84static u32 ib_mad_client_next;
  85static struct list_head ib_mad_port_list;
  86
  87/* Port list lock */
  88static DEFINE_SPINLOCK(ib_mad_port_list_lock);
  89
  90/* Forward declarations */
  91static int method_in_use(struct ib_mad_mgmt_method_table **method,
  92			 struct ib_mad_reg_req *mad_reg_req);
  93static void remove_mad_reg_req(struct ib_mad_agent_private *priv);
  94static struct ib_mad_agent_private *find_mad_agent(
  95					struct ib_mad_port_private *port_priv,
  96					const struct ib_mad_hdr *mad);
  97static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
  98				    struct ib_mad_private *mad);
  99static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
 100static void timeout_sends(struct work_struct *work);
 101static void local_completions(struct work_struct *work);
 102static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 103			      struct ib_mad_agent_private *agent_priv,
 104			      u8 mgmt_class);
 105static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 106			   struct ib_mad_agent_private *agent_priv);
 107static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
 108			      struct ib_wc *wc);
 109static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc);
 110
 111/*
 112 * Returns a ib_mad_port_private structure or NULL for a device/port
 113 * Assumes ib_mad_port_list_lock is being held
 114 */
 115static inline struct ib_mad_port_private *
 116__ib_get_mad_port(struct ib_device *device, u32 port_num)
 117{
 118	struct ib_mad_port_private *entry;
 119
 120	list_for_each_entry(entry, &ib_mad_port_list, port_list) {
 121		if (entry->device == device && entry->port_num == port_num)
 122			return entry;
 123	}
 124	return NULL;
 125}
 126
 127/*
 128 * Wrapper function to return a ib_mad_port_private structure or NULL
 129 * for a device/port
 130 */
 131static inline struct ib_mad_port_private *
 132ib_get_mad_port(struct ib_device *device, u32 port_num)
 133{
 134	struct ib_mad_port_private *entry;
 135	unsigned long flags;
 136
 137	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
 138	entry = __ib_get_mad_port(device, port_num);
 139	spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
 140
 141	return entry;
 142}
 143
 144static inline u8 convert_mgmt_class(u8 mgmt_class)
 145{
 146	/* Alias IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE to 0 */
 147	return mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ?
 148		0 : mgmt_class;
 149}
 150
 151static int get_spl_qp_index(enum ib_qp_type qp_type)
 152{
 153	switch (qp_type) {
 154	case IB_QPT_SMI:
 155		return 0;
 156	case IB_QPT_GSI:
 157		return 1;
 158	default:
 159		return -1;
 160	}
 161}
 162
 163static int vendor_class_index(u8 mgmt_class)
 164{
 165	return mgmt_class - IB_MGMT_CLASS_VENDOR_RANGE2_START;
 166}
 167
 168static int is_vendor_class(u8 mgmt_class)
 169{
 170	if ((mgmt_class < IB_MGMT_CLASS_VENDOR_RANGE2_START) ||
 171	    (mgmt_class > IB_MGMT_CLASS_VENDOR_RANGE2_END))
 172		return 0;
 173	return 1;
 174}
 175
 176static int is_vendor_oui(char *oui)
 177{
 178	if (oui[0] || oui[1] || oui[2])
 179		return 1;
 180	return 0;
 181}
 182
 183static int is_vendor_method_in_use(
 184		struct ib_mad_mgmt_vendor_class *vendor_class,
 185		struct ib_mad_reg_req *mad_reg_req)
 186{
 187	struct ib_mad_mgmt_method_table *method;
 188	int i;
 189
 190	for (i = 0; i < MAX_MGMT_OUI; i++) {
 191		if (!memcmp(vendor_class->oui[i], mad_reg_req->oui, 3)) {
 192			method = vendor_class->method_table[i];
 193			if (method) {
 194				if (method_in_use(&method, mad_reg_req))
 195					return 1;
 196				else
 197					break;
 198			}
 199		}
 200	}
 201	return 0;
 202}
 203
 204int ib_response_mad(const struct ib_mad_hdr *hdr)
 205{
 206	return ((hdr->method & IB_MGMT_METHOD_RESP) ||
 207		(hdr->method == IB_MGMT_METHOD_TRAP_REPRESS) ||
 208		((hdr->mgmt_class == IB_MGMT_CLASS_BM) &&
 209		 (hdr->attr_mod & IB_BM_ATTR_MOD_RESP)));
 210}
 211EXPORT_SYMBOL(ib_response_mad);
 212
 213#define SOL_FC_MAX_DEFAULT_FRAC 4
 214#define SOL_FC_MAX_SA_FRAC 32
 215
 216static int get_sol_fc_max_outstanding(struct ib_mad_reg_req *mad_reg_req)
 217{
 218	if (!mad_reg_req)
 219		/* Send only agent */
 220		return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
 221
 222	switch (mad_reg_req->mgmt_class) {
 223	case IB_MGMT_CLASS_CM:
 224		return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
 225	case IB_MGMT_CLASS_SUBN_ADM:
 226		return mad_recvq_size / SOL_FC_MAX_SA_FRAC;
 227	case IB_MGMT_CLASS_SUBN_LID_ROUTED:
 228	case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
 229		return min(mad_recvq_size, IB_MAD_QP_RECV_SIZE) /
 230		       SOL_FC_MAX_DEFAULT_FRAC;
 231	default:
 232		return 0;
 233	}
 234}
 235
 236/*
 237 * ib_register_mad_agent - Register to send/receive MADs
 238 *
 239 * Context: Process context.
 240 */
 241struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 242					   u32 port_num,
 243					   enum ib_qp_type qp_type,
 244					   struct ib_mad_reg_req *mad_reg_req,
 245					   u8 rmpp_version,
 246					   ib_mad_send_handler send_handler,
 247					   ib_mad_recv_handler recv_handler,
 248					   void *context,
 249					   u32 registration_flags)
 250{
 251	struct ib_mad_port_private *port_priv;
 252	struct ib_mad_agent *ret = ERR_PTR(-EINVAL);
 253	struct ib_mad_agent_private *mad_agent_priv;
 254	struct ib_mad_reg_req *reg_req = NULL;
 255	struct ib_mad_mgmt_class_table *class;
 256	struct ib_mad_mgmt_vendor_class_table *vendor;
 257	struct ib_mad_mgmt_vendor_class *vendor_class;
 258	struct ib_mad_mgmt_method_table *method;
 259	int ret2, qpn;
 260	u8 mgmt_class, vclass;
 261
 262	if ((qp_type == IB_QPT_SMI && !rdma_cap_ib_smi(device, port_num)) ||
 263	    (qp_type == IB_QPT_GSI && !rdma_cap_ib_cm(device, port_num)))
 264		return ERR_PTR(-EPROTONOSUPPORT);
 265
 266	/* Validate parameters */
 267	qpn = get_spl_qp_index(qp_type);
 268	if (qpn == -1) {
 269		dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n",
 270				    __func__, qp_type);
 271		goto error1;
 272	}
 273
 274	if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) {
 275		dev_dbg_ratelimited(&device->dev,
 276				    "%s: invalid RMPP Version %u\n",
 277				    __func__, rmpp_version);
 278		goto error1;
 279	}
 280
 281	/* Validate MAD registration request if supplied */
 282	if (mad_reg_req) {
 283		if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) {
 284			dev_dbg_ratelimited(&device->dev,
 285					    "%s: invalid Class Version %u\n",
 286					    __func__,
 287					    mad_reg_req->mgmt_class_version);
 288			goto error1;
 289		}
 290		if (!recv_handler) {
 291			dev_dbg_ratelimited(&device->dev,
 292					    "%s: no recv_handler\n", __func__);
 293			goto error1;
 294		}
 295		if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) {
 296			/*
 297			 * IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE is the only
 298			 * one in this range currently allowed
 299			 */
 300			if (mad_reg_req->mgmt_class !=
 301			    IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
 302				dev_dbg_ratelimited(&device->dev,
 303					"%s: Invalid Mgmt Class 0x%x\n",
 304					__func__, mad_reg_req->mgmt_class);
 305				goto error1;
 306			}
 307		} else if (mad_reg_req->mgmt_class == 0) {
 308			/*
 309			 * Class 0 is reserved in IBA and is used for
 310			 * aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
 311			 */
 312			dev_dbg_ratelimited(&device->dev,
 313					    "%s: Invalid Mgmt Class 0\n",
 314					    __func__);
 315			goto error1;
 316		} else if (is_vendor_class(mad_reg_req->mgmt_class)) {
 317			/*
 318			 * If class is in "new" vendor range,
 319			 * ensure supplied OUI is not zero
 320			 */
 321			if (!is_vendor_oui(mad_reg_req->oui)) {
 322				dev_dbg_ratelimited(&device->dev,
 323					"%s: No OUI specified for class 0x%x\n",
 324					__func__,
 325					mad_reg_req->mgmt_class);
 326				goto error1;
 327			}
 328		}
 329		/* Make sure class supplied is consistent with RMPP */
 330		if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) {
 331			if (rmpp_version) {
 332				dev_dbg_ratelimited(&device->dev,
 333					"%s: RMPP version for non-RMPP class 0x%x\n",
 334					__func__, mad_reg_req->mgmt_class);
 335				goto error1;
 336			}
 337		}
 338
 339		/* Make sure class supplied is consistent with QP type */
 340		if (qp_type == IB_QPT_SMI) {
 341			if ((mad_reg_req->mgmt_class !=
 342					IB_MGMT_CLASS_SUBN_LID_ROUTED) &&
 343			    (mad_reg_req->mgmt_class !=
 344					IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
 345				dev_dbg_ratelimited(&device->dev,
 346					"%s: Invalid SM QP type: class 0x%x\n",
 347					__func__, mad_reg_req->mgmt_class);
 348				goto error1;
 349			}
 350		} else {
 351			if ((mad_reg_req->mgmt_class ==
 352					IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
 353			    (mad_reg_req->mgmt_class ==
 354					IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
 355				dev_dbg_ratelimited(&device->dev,
 356					"%s: Invalid GS QP type: class 0x%x\n",
 357					__func__, mad_reg_req->mgmt_class);
 358				goto error1;
 359			}
 360		}
 361	} else {
 362		/* No registration request supplied */
 363		if (!send_handler)
 364			goto error1;
 365		if (registration_flags & IB_MAD_USER_RMPP)
 366			goto error1;
 367	}
 368
 369	/* Validate device and port */
 370	port_priv = ib_get_mad_port(device, port_num);
 371	if (!port_priv) {
 372		dev_dbg_ratelimited(&device->dev, "%s: Invalid port %u\n",
 373				    __func__, port_num);
 374		ret = ERR_PTR(-ENODEV);
 375		goto error1;
 376	}
 377
 378	/* Verify the QP requested is supported. For example, Ethernet devices
 379	 * will not have QP0.
 380	 */
 381	if (!port_priv->qp_info[qpn].qp) {
 382		dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n",
 383				    __func__, qpn);
 384		ret = ERR_PTR(-EPROTONOSUPPORT);
 385		goto error1;
 386	}
 387
 388	/* Allocate structures */
 389	mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL);
 390	if (!mad_agent_priv) {
 391		ret = ERR_PTR(-ENOMEM);
 392		goto error1;
 393	}
 394
 395	if (mad_reg_req) {
 396		reg_req = kmemdup(mad_reg_req, sizeof *reg_req, GFP_KERNEL);
 397		if (!reg_req) {
 398			ret = ERR_PTR(-ENOMEM);
 399			goto error3;
 400		}
 401	}
 402
 403	/* Now, fill in the various structures */
 404	mad_agent_priv->qp_info = &port_priv->qp_info[qpn];
 405	mad_agent_priv->reg_req = reg_req;
 406	mad_agent_priv->agent.rmpp_version = rmpp_version;
 407	mad_agent_priv->agent.device = device;
 408	mad_agent_priv->agent.recv_handler = recv_handler;
 409	mad_agent_priv->agent.send_handler = send_handler;
 410	mad_agent_priv->agent.context = context;
 411	mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp;
 412	mad_agent_priv->agent.port_num = port_num;
 413	mad_agent_priv->agent.flags = registration_flags;
 414	spin_lock_init(&mad_agent_priv->lock);
 415	INIT_LIST_HEAD(&mad_agent_priv->send_list);
 416	INIT_LIST_HEAD(&mad_agent_priv->wait_list);
 417	INIT_LIST_HEAD(&mad_agent_priv->rmpp_list);
 418	INIT_LIST_HEAD(&mad_agent_priv->backlog_list);
 419	INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
 420	INIT_LIST_HEAD(&mad_agent_priv->local_list);
 421	INIT_WORK(&mad_agent_priv->local_work, local_completions);
 422	refcount_set(&mad_agent_priv->refcount, 1);
 423	init_completion(&mad_agent_priv->comp);
 424	mad_agent_priv->sol_fc_send_count = 0;
 425	mad_agent_priv->sol_fc_wait_count = 0;
 426	mad_agent_priv->sol_fc_max =
 427		recv_handler ? get_sol_fc_max_outstanding(mad_reg_req) : 0;
 428
 429	ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
 430	if (ret2) {
 431		ret = ERR_PTR(ret2);
 432		goto error4;
 433	}
 434
 435	/*
 436	 * The mlx4 driver uses the top byte to distinguish which virtual
 437	 * function generated the MAD, so we must avoid using it.
 438	 */
 439	ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid,
 440			mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1),
 441			&ib_mad_client_next, GFP_KERNEL);
 442	if (ret2 < 0) {
 443		ret = ERR_PTR(ret2);
 444		goto error5;
 445	}
 446
 447	/*
 448	 * Make sure MAD registration (if supplied)
 449	 * is non overlapping with any existing ones
 450	 */
 451	spin_lock_irq(&port_priv->reg_lock);
 452	if (mad_reg_req) {
 453		mgmt_class = convert_mgmt_class(mad_reg_req->mgmt_class);
 454		if (!is_vendor_class(mgmt_class)) {
 455			class = port_priv->version[mad_reg_req->
 456						   mgmt_class_version].class;
 457			if (class) {
 458				method = class->method_table[mgmt_class];
 459				if (method) {
 460					if (method_in_use(&method,
 461							   mad_reg_req))
 462						goto error6;
 463				}
 464			}
 465			ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv,
 466						  mgmt_class);
 467		} else {
 468			/* "New" vendor class range */
 469			vendor = port_priv->version[mad_reg_req->
 470						    mgmt_class_version].vendor;
 471			if (vendor) {
 472				vclass = vendor_class_index(mgmt_class);
 473				vendor_class = vendor->vendor_class[vclass];
 474				if (vendor_class) {
 475					if (is_vendor_method_in_use(
 476							vendor_class,
 477							mad_reg_req))
 478						goto error6;
 479				}
 480			}
 481			ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv);
 482		}
 483		if (ret2) {
 484			ret = ERR_PTR(ret2);
 485			goto error6;
 486		}
 487	}
 488	spin_unlock_irq(&port_priv->reg_lock);
 489
 490	trace_ib_mad_create_agent(mad_agent_priv);
 491	return &mad_agent_priv->agent;
 492error6:
 493	spin_unlock_irq(&port_priv->reg_lock);
 494	xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
 495error5:
 496	ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
 497error4:
 498	kfree(reg_req);
 499error3:
 500	kfree(mad_agent_priv);
 501error1:
 502	return ret;
 503}
 504EXPORT_SYMBOL(ib_register_mad_agent);
 505
 506static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 507{
 508	if (refcount_dec_and_test(&mad_agent_priv->refcount))
 509		complete(&mad_agent_priv->comp);
 510}
 511
 512static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 513{
 514	struct ib_mad_port_private *port_priv;
 515
 516	/* Note that we could still be handling received MADs */
 517	trace_ib_mad_unregister_agent(mad_agent_priv);
 518
 519	/*
 520	 * Canceling all sends results in dropping received response
 521	 * MADs, preventing us from queuing additional work
 522	 */
 523	cancel_mads(mad_agent_priv);
 524	port_priv = mad_agent_priv->qp_info->port_priv;
 525	cancel_delayed_work(&mad_agent_priv->timed_work);
 526
 527	spin_lock_irq(&port_priv->reg_lock);
 528	remove_mad_reg_req(mad_agent_priv);
 529	spin_unlock_irq(&port_priv->reg_lock);
 530	xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
 531
 532	flush_workqueue(port_priv->wq);
 533
 534	deref_mad_agent(mad_agent_priv);
 535	wait_for_completion(&mad_agent_priv->comp);
 536	ib_cancel_rmpp_recvs(mad_agent_priv);
 537
 538	ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
 539
 540	kfree(mad_agent_priv->reg_req);
 541	kfree_rcu(mad_agent_priv, rcu);
 542}
 543
 544/*
 545 * ib_unregister_mad_agent - Unregisters a client from using MAD services
 546 *
 547 * Context: Process context.
 548 */
 549void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)
 550{
 551	struct ib_mad_agent_private *mad_agent_priv;
 552
 553	mad_agent_priv = container_of(mad_agent,
 554				      struct ib_mad_agent_private,
 555				      agent);
 556	unregister_mad_agent(mad_agent_priv);
 557}
 558EXPORT_SYMBOL(ib_unregister_mad_agent);
 559
 560static void dequeue_mad(struct ib_mad_list_head *mad_list)
 561{
 562	struct ib_mad_queue *mad_queue;
 563	unsigned long flags;
 564
 565	mad_queue = mad_list->mad_queue;
 566	spin_lock_irqsave(&mad_queue->lock, flags);
 567	list_del(&mad_list->list);
 568	mad_queue->count--;
 569	spin_unlock_irqrestore(&mad_queue->lock, flags);
 570}
 571
 572static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,
 573		u16 pkey_index, u32 port_num, struct ib_wc *wc)
 574{
 575	memset(wc, 0, sizeof *wc);
 576	wc->wr_cqe = cqe;
 577	wc->status = IB_WC_SUCCESS;
 578	wc->opcode = IB_WC_RECV;
 579	wc->pkey_index = pkey_index;
 580	wc->byte_len = sizeof(struct ib_mad) + sizeof(struct ib_grh);
 581	wc->src_qp = IB_QP0;
 582	wc->qp = qp;
 583	wc->slid = slid;
 584	wc->sl = 0;
 585	wc->dlid_path_bits = 0;
 586	wc->port_num = port_num;
 587}
 588
 589static size_t mad_priv_size(const struct ib_mad_private *mp)
 590{
 591	return sizeof(struct ib_mad_private) + mp->mad_size;
 592}
 593
 594static struct ib_mad_private *alloc_mad_private(size_t mad_size, gfp_t flags)
 595{
 596	size_t size = sizeof(struct ib_mad_private) + mad_size;
 597	struct ib_mad_private *ret = kzalloc(size, flags);
 598
 599	if (ret)
 600		ret->mad_size = mad_size;
 601
 602	return ret;
 603}
 604
 605static size_t port_mad_size(const struct ib_mad_port_private *port_priv)
 606{
 607	return rdma_max_mad_size(port_priv->device, port_priv->port_num);
 608}
 609
 610static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
 611{
 612	return sizeof(struct ib_grh) + mp->mad_size;
 613}
 614
 615/*
 616 * Return 0 if SMP is to be sent
 617 * Return 1 if SMP was consumed locally (whether or not solicited)
 618 * Return < 0 if error
 619 */
 620static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 621				  struct ib_mad_send_wr_private *mad_send_wr)
 622{
 623	int ret = 0;
 624	struct ib_smp *smp = mad_send_wr->send_buf.mad;
 625	struct opa_smp *opa_smp = (struct opa_smp *)smp;
 626	unsigned long flags;
 627	struct ib_mad_local_private *local;
 628	struct ib_mad_private *mad_priv;
 629	struct ib_mad_port_private *port_priv;
 630	struct ib_mad_agent_private *recv_mad_agent = NULL;
 631	struct ib_device *device = mad_agent_priv->agent.device;
 632	u32 port_num;
 633	struct ib_wc mad_wc;
 634	struct ib_ud_wr *send_wr = &mad_send_wr->send_wr;
 635	size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
 636	u16 out_mad_pkey_index = 0;
 637	u16 drslid;
 638	bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
 639				    mad_agent_priv->qp_info->port_priv->port_num);
 640
 641	if (rdma_cap_ib_switch(device) &&
 642	    smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
 643		port_num = send_wr->port_num;
 644	else
 645		port_num = mad_agent_priv->agent.port_num;
 646
 647	/*
 648	 * Directed route handling starts if the initial LID routed part of
 649	 * a request or the ending LID routed part of a response is empty.
 650	 * If we are at the start of the LID routed part, don't update the
 651	 * hop_ptr or hop_cnt.  See section 14.2.2, Vol 1 IB spec.
 652	 */
 653	if (opa && smp->class_version == OPA_SM_CLASS_VERSION) {
 654		u32 opa_drslid;
 655
 656		trace_ib_mad_handle_out_opa_smi(opa_smp);
 657
 658		if ((opa_get_smp_direction(opa_smp)
 659		     ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) ==
 660		     OPA_LID_PERMISSIVE &&
 661		     opa_smi_handle_dr_smp_send(opa_smp,
 662						rdma_cap_ib_switch(device),
 663						port_num) == IB_SMI_DISCARD) {
 664			ret = -EINVAL;
 665			dev_err(&device->dev, "OPA Invalid directed route\n");
 666			goto out;
 667		}
 668		opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
 669		if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
 670		    opa_drslid & 0xffff0000) {
 671			ret = -EINVAL;
 672			dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
 673			       opa_drslid);
 674			goto out;
 675		}
 676		drslid = (u16)(opa_drslid & 0x0000ffff);
 677
 678		/* Check to post send on QP or process locally */
 679		if (opa_smi_check_local_smp(opa_smp, device) == IB_SMI_DISCARD &&
 680		    opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD)
 681			goto out;
 682	} else {
 683		trace_ib_mad_handle_out_ib_smi(smp);
 684
 685		if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) ==
 686		     IB_LID_PERMISSIVE &&
 687		     smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) ==
 688		     IB_SMI_DISCARD) {
 689			ret = -EINVAL;
 690			dev_err(&device->dev, "Invalid directed route\n");
 691			goto out;
 692		}
 693		drslid = be16_to_cpu(smp->dr_slid);
 694
 695		/* Check to post send on QP or process locally */
 696		if (smi_check_local_smp(smp, device) == IB_SMI_DISCARD &&
 697		    smi_check_local_returning_smp(smp, device) == IB_SMI_DISCARD)
 698			goto out;
 699	}
 700
 701	local = kmalloc(sizeof *local, GFP_ATOMIC);
 702	if (!local) {
 703		ret = -ENOMEM;
 704		goto out;
 705	}
 706	local->mad_priv = NULL;
 707	local->recv_mad_agent = NULL;
 708	mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC);
 709	if (!mad_priv) {
 710		ret = -ENOMEM;
 711		kfree(local);
 712		goto out;
 713	}
 714
 715	build_smp_wc(mad_agent_priv->agent.qp,
 716		     send_wr->wr.wr_cqe, drslid,
 717		     send_wr->pkey_index,
 718		     send_wr->port_num, &mad_wc);
 719
 720	if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
 721		mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
 722					+ mad_send_wr->send_buf.data_len
 723					+ sizeof(struct ib_grh);
 724	}
 725
 726	/* No GRH for DR SMP */
 727	ret = device->ops.process_mad(device, 0, port_num, &mad_wc, NULL,
 728				      (const struct ib_mad *)smp,
 729				      (struct ib_mad *)mad_priv->mad, &mad_size,
 730				      &out_mad_pkey_index);
 731	switch (ret) {
 732	case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY:
 733		if (ib_response_mad((const struct ib_mad_hdr *)mad_priv->mad) &&
 734		    mad_agent_priv->agent.recv_handler) {
 735			local->mad_priv = mad_priv;
 736			local->recv_mad_agent = mad_agent_priv;
 737			/*
 738			 * Reference MAD agent until receive
 739			 * side of local completion handled
 740			 */
 741			refcount_inc(&mad_agent_priv->refcount);
 742		} else
 743			kfree(mad_priv);
 744		break;
 745	case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED:
 746		kfree(mad_priv);
 747		break;
 748	case IB_MAD_RESULT_SUCCESS:
 749		/* Treat like an incoming receive MAD */
 750		port_priv = ib_get_mad_port(mad_agent_priv->agent.device,
 751					    mad_agent_priv->agent.port_num);
 752		if (port_priv) {
 753			memcpy(mad_priv->mad, smp, mad_priv->mad_size);
 754			recv_mad_agent = find_mad_agent(port_priv,
 755						        (const struct ib_mad_hdr *)mad_priv->mad);
 756		}
 757		if (!port_priv || !recv_mad_agent) {
 758			/*
 759			 * No receiving agent so drop packet and
 760			 * generate send completion.
 761			 */
 762			kfree(mad_priv);
 763			break;
 764		}
 765		local->mad_priv = mad_priv;
 766		local->recv_mad_agent = recv_mad_agent;
 767		break;
 768	default:
 769		kfree(mad_priv);
 770		kfree(local);
 771		ret = -EINVAL;
 772		goto out;
 773	}
 774
 775	local->mad_send_wr = mad_send_wr;
 776	if (opa) {
 777		local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
 778		local->return_wc_byte_len = mad_size;
 779	}
 780	/* Reference MAD agent until send side of local completion handled */
 781	refcount_inc(&mad_agent_priv->refcount);
 782	/* Queue local completion to local list */
 783	spin_lock_irqsave(&mad_agent_priv->lock, flags);
 784	list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
 785	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 786	queue_work(mad_agent_priv->qp_info->port_priv->wq,
 787		   &mad_agent_priv->local_work);
 788	ret = 1;
 789out:
 790	return ret;
 791}
 792
 793static int get_pad_size(int hdr_len, int data_len, size_t mad_size)
 794{
 795	int seg_size, pad;
 796
 797	seg_size = mad_size - hdr_len;
 798	if (data_len && seg_size) {
 799		pad = seg_size - data_len % seg_size;
 800		return pad == seg_size ? 0 : pad;
 801	} else
 802		return seg_size;
 803}
 804
 805static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr)
 806{
 807	struct ib_rmpp_segment *s, *t;
 808
 809	list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) {
 810		list_del(&s->list);
 811		kfree(s);
 812	}
 813}
 814
 815static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr,
 816				size_t mad_size, gfp_t gfp_mask)
 817{
 818	struct ib_mad_send_buf *send_buf = &send_wr->send_buf;
 819	struct ib_rmpp_mad *rmpp_mad = send_buf->mad;
 820	struct ib_rmpp_segment *seg = NULL;
 821	int left, seg_size, pad;
 822
 823	send_buf->seg_size = mad_size - send_buf->hdr_len;
 824	send_buf->seg_rmpp_size = mad_size - IB_MGMT_RMPP_HDR;
 825	seg_size = send_buf->seg_size;
 826	pad = send_wr->pad;
 827
 828	/* Allocate data segments. */
 829	for (left = send_buf->data_len + pad; left > 0; left -= seg_size) {
 830		seg = kmalloc(sizeof(*seg) + seg_size, gfp_mask);
 831		if (!seg) {
 832			free_send_rmpp_list(send_wr);
 833			return -ENOMEM;
 834		}
 835		seg->num = ++send_buf->seg_count;
 836		list_add_tail(&seg->list, &send_wr->rmpp_list);
 837	}
 838
 839	/* Zero any padding */
 840	if (pad)
 841		memset(seg->data + seg_size - pad, 0, pad);
 842
 843	rmpp_mad->rmpp_hdr.rmpp_version = send_wr->mad_agent_priv->
 844					  agent.rmpp_version;
 845	rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA;
 846	ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
 847
 848	send_wr->cur_seg = container_of(send_wr->rmpp_list.next,
 849					struct ib_rmpp_segment, list);
 850	send_wr->last_ack_seg = send_wr->cur_seg;
 851	return 0;
 852}
 853
 854int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent)
 855{
 856	return agent->rmpp_version && !(agent->flags & IB_MAD_USER_RMPP);
 857}
 858EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent);
 859
 860struct ib_mad_send_buf *ib_create_send_mad(struct ib_mad_agent *mad_agent,
 861					   u32 remote_qpn, u16 pkey_index,
 862					   int rmpp_active, int hdr_len,
 863					   int data_len, gfp_t gfp_mask,
 864					   u8 base_version)
 865{
 866	struct ib_mad_agent_private *mad_agent_priv;
 867	struct ib_mad_send_wr_private *mad_send_wr;
 868	int pad, message_size, ret, size;
 869	void *buf;
 870	size_t mad_size;
 871	bool opa;
 872
 873	mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
 874				      agent);
 875
 876	opa = rdma_cap_opa_mad(mad_agent->device, mad_agent->port_num);
 877
 878	if (opa && base_version == OPA_MGMT_BASE_VERSION)
 879		mad_size = sizeof(struct opa_mad);
 880	else
 881		mad_size = sizeof(struct ib_mad);
 882
 883	pad = get_pad_size(hdr_len, data_len, mad_size);
 884	message_size = hdr_len + data_len + pad;
 885
 886	if (ib_mad_kernel_rmpp_agent(mad_agent)) {
 887		if (!rmpp_active && message_size > mad_size)
 888			return ERR_PTR(-EINVAL);
 889	} else
 890		if (rmpp_active || message_size > mad_size)
 891			return ERR_PTR(-EINVAL);
 892
 893	size = rmpp_active ? hdr_len : mad_size;
 894	buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask);
 895	if (!buf)
 896		return ERR_PTR(-ENOMEM);
 897
 898	mad_send_wr = buf + size;
 899	INIT_LIST_HEAD(&mad_send_wr->rmpp_list);
 900	mad_send_wr->send_buf.mad = buf;
 901	mad_send_wr->send_buf.hdr_len = hdr_len;
 902	mad_send_wr->send_buf.data_len = data_len;
 903	mad_send_wr->pad = pad;
 904
 905	mad_send_wr->mad_agent_priv = mad_agent_priv;
 906	mad_send_wr->sg_list[0].length = hdr_len;
 907	mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey;
 908
 909	/* OPA MADs don't have to be the full 2048 bytes */
 910	if (opa && base_version == OPA_MGMT_BASE_VERSION &&
 911	    data_len < mad_size - hdr_len)
 912		mad_send_wr->sg_list[1].length = data_len;
 913	else
 914		mad_send_wr->sg_list[1].length = mad_size - hdr_len;
 915
 916	mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey;
 917
 918	mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
 919
 920	mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
 921	mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list;
 922	mad_send_wr->send_wr.wr.num_sge = 2;
 923	mad_send_wr->send_wr.wr.opcode = IB_WR_SEND;
 924	mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED;
 925	mad_send_wr->send_wr.remote_qpn = remote_qpn;
 926	mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY;
 927	mad_send_wr->send_wr.pkey_index = pkey_index;
 928
 929	if (rmpp_active) {
 930		ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask);
 931		if (ret) {
 932			kfree(buf);
 933			return ERR_PTR(ret);
 934		}
 935	}
 936
 937	mad_send_wr->send_buf.mad_agent = mad_agent;
 938	refcount_inc(&mad_agent_priv->refcount);
 939	return &mad_send_wr->send_buf;
 940}
 941EXPORT_SYMBOL(ib_create_send_mad);
 942
 943int ib_get_mad_data_offset(u8 mgmt_class)
 944{
 945	if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM)
 946		return IB_MGMT_SA_HDR;
 947	else if ((mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
 948		 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
 949		 (mgmt_class == IB_MGMT_CLASS_BIS))
 950		return IB_MGMT_DEVICE_HDR;
 951	else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
 952		 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))
 953		return IB_MGMT_VENDOR_HDR;
 954	else
 955		return IB_MGMT_MAD_HDR;
 956}
 957EXPORT_SYMBOL(ib_get_mad_data_offset);
 958
 959int ib_is_mad_class_rmpp(u8 mgmt_class)
 960{
 961	if ((mgmt_class == IB_MGMT_CLASS_SUBN_ADM) ||
 962	    (mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
 963	    (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
 964	    (mgmt_class == IB_MGMT_CLASS_BIS) ||
 965	    ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
 966	     (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END)))
 967		return 1;
 968	return 0;
 969}
 970EXPORT_SYMBOL(ib_is_mad_class_rmpp);
 971
 972void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num)
 973{
 974	struct ib_mad_send_wr_private *mad_send_wr;
 975	struct list_head *list;
 976
 977	mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
 978				   send_buf);
 979	list = &mad_send_wr->cur_seg->list;
 980
 981	if (mad_send_wr->cur_seg->num < seg_num) {
 982		list_for_each_entry(mad_send_wr->cur_seg, list, list)
 983			if (mad_send_wr->cur_seg->num == seg_num)
 984				break;
 985	} else if (mad_send_wr->cur_seg->num > seg_num) {
 986		list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list)
 987			if (mad_send_wr->cur_seg->num == seg_num)
 988				break;
 989	}
 990	return mad_send_wr->cur_seg->data;
 991}
 992EXPORT_SYMBOL(ib_get_rmpp_segment);
 993
 994static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr)
 995{
 996	if (mad_send_wr->send_buf.seg_count)
 997		return ib_get_rmpp_segment(&mad_send_wr->send_buf,
 998					   mad_send_wr->seg_num);
 999	else
1000		return mad_send_wr->send_buf.mad +
1001		       mad_send_wr->send_buf.hdr_len;
1002}
1003
1004void ib_free_send_mad(struct ib_mad_send_buf *send_buf)
1005{
1006	struct ib_mad_agent_private *mad_agent_priv;
1007	struct ib_mad_send_wr_private *mad_send_wr;
1008
1009	mad_agent_priv = container_of(send_buf->mad_agent,
1010				      struct ib_mad_agent_private, agent);
1011	mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
1012				   send_buf);
1013
1014	free_send_rmpp_list(mad_send_wr);
1015	kfree(send_buf->mad);
1016	deref_mad_agent(mad_agent_priv);
1017}
1018EXPORT_SYMBOL(ib_free_send_mad);
1019
1020int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
1021{
1022	struct ib_mad_qp_info *qp_info;
1023	struct list_head *list;
1024	struct ib_mad_agent *mad_agent;
1025	struct ib_sge *sge;
1026	unsigned long flags;
1027	int ret;
1028
1029	/* Set WR ID to find mad_send_wr upon completion */
1030	qp_info = mad_send_wr->mad_agent_priv->qp_info;
1031	mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;
1032	mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
1033	mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
1034
1035	mad_agent = mad_send_wr->send_buf.mad_agent;
1036	sge = mad_send_wr->sg_list;
1037	sge[0].addr = ib_dma_map_single(mad_agent->device,
1038					mad_send_wr->send_buf.mad,
1039					sge[0].length,
1040					DMA_TO_DEVICE);
1041	if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[0].addr)))
1042		return -ENOMEM;
1043
1044	mad_send_wr->header_mapping = sge[0].addr;
1045
1046	sge[1].addr = ib_dma_map_single(mad_agent->device,
1047					ib_get_payload(mad_send_wr),
1048					sge[1].length,
1049					DMA_TO_DEVICE);
1050	if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[1].addr))) {
1051		ib_dma_unmap_single(mad_agent->device,
1052				    mad_send_wr->header_mapping,
1053				    sge[0].length, DMA_TO_DEVICE);
1054		return -ENOMEM;
1055	}
1056	mad_send_wr->payload_mapping = sge[1].addr;
1057
1058	spin_lock_irqsave(&qp_info->send_queue.lock, flags);
1059	if (qp_info->send_queue.count < qp_info->send_queue.max_active) {
1060		trace_ib_mad_ib_send_mad(mad_send_wr, qp_info);
1061		ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr,
1062				   NULL);
1063		list = &qp_info->send_queue.list;
1064	} else {
1065		ret = 0;
1066		list = &qp_info->overflow_list;
1067	}
1068
1069	if (!ret) {
1070		qp_info->send_queue.count++;
1071		list_add_tail(&mad_send_wr->mad_list.list, list);
1072	}
1073	spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
1074	if (ret) {
1075		ib_dma_unmap_single(mad_agent->device,
1076				    mad_send_wr->header_mapping,
1077				    sge[0].length, DMA_TO_DEVICE);
1078		ib_dma_unmap_single(mad_agent->device,
1079				    mad_send_wr->payload_mapping,
1080				    sge[1].length, DMA_TO_DEVICE);
1081	}
1082	return ret;
1083}
1084
1085static void handle_queued_state(struct ib_mad_send_wr_private *mad_send_wr,
1086		       struct ib_mad_agent_private *mad_agent_priv)
1087{
1088	if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) {
1089		mad_agent_priv->sol_fc_wait_count--;
1090		list_move_tail(&mad_send_wr->agent_list,
1091			       &mad_agent_priv->backlog_list);
1092	} else {
1093		expect_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
1094		list_add_tail(&mad_send_wr->agent_list,
1095			      &mad_agent_priv->backlog_list);
1096	}
1097}
1098
1099static void handle_send_state(struct ib_mad_send_wr_private *mad_send_wr,
1100		       struct ib_mad_agent_private *mad_agent_priv)
1101{
1102	if (mad_send_wr->state == IB_MAD_STATE_INIT) {
1103		list_add_tail(&mad_send_wr->agent_list,
1104			      &mad_agent_priv->send_list);
1105	} else {
1106		expect_mad_state2(mad_send_wr, IB_MAD_STATE_WAIT_RESP,
1107				  IB_MAD_STATE_QUEUED);
1108		list_move_tail(&mad_send_wr->agent_list,
1109			       &mad_agent_priv->send_list);
1110	}
1111
1112	if (mad_send_wr->is_solicited_fc) {
1113		if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1114			mad_agent_priv->sol_fc_wait_count--;
1115		mad_agent_priv->sol_fc_send_count++;
1116	}
1117}
1118
1119static void handle_wait_state(struct ib_mad_send_wr_private *mad_send_wr,
1120		       struct ib_mad_agent_private *mad_agent_priv)
1121{
1122	struct ib_mad_send_wr_private *temp_mad_send_wr;
1123	struct list_head *list_item;
1124	unsigned long delay;
1125
1126	expect_mad_state3(mad_send_wr, IB_MAD_STATE_SEND_START,
1127			  IB_MAD_STATE_WAIT_RESP, IB_MAD_STATE_CANCELED);
1128	if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
1129	    mad_send_wr->is_solicited_fc) {
1130		mad_agent_priv->sol_fc_send_count--;
1131		mad_agent_priv->sol_fc_wait_count++;
1132	}
1133
1134	list_del_init(&mad_send_wr->agent_list);
1135	delay = mad_send_wr->timeout;
1136	mad_send_wr->timeout += jiffies;
1137
1138	if (delay) {
1139		list_for_each_prev(list_item,
1140				   &mad_agent_priv->wait_list) {
1141			temp_mad_send_wr = list_entry(
1142				list_item,
1143				struct ib_mad_send_wr_private,
1144				agent_list);
1145			if (time_after(mad_send_wr->timeout,
1146				       temp_mad_send_wr->timeout))
1147				break;
1148		}
1149	} else {
1150		list_item = &mad_agent_priv->wait_list;
1151	}
1152
1153	list_add(&mad_send_wr->agent_list, list_item);
1154}
1155
1156static void handle_early_resp_state(struct ib_mad_send_wr_private *mad_send_wr,
1157			    struct ib_mad_agent_private *mad_agent_priv)
1158{
1159	expect_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
1160	mad_agent_priv->sol_fc_send_count -= mad_send_wr->is_solicited_fc;
1161}
1162
1163static void handle_canceled_state(struct ib_mad_send_wr_private *mad_send_wr,
1164			 struct ib_mad_agent_private *mad_agent_priv)
1165{
1166	not_expect_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
1167	if (mad_send_wr->is_solicited_fc) {
1168		if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
1169			mad_agent_priv->sol_fc_send_count--;
1170		else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1171			mad_agent_priv->sol_fc_wait_count--;
1172	}
1173}
1174
1175static void handle_done_state(struct ib_mad_send_wr_private *mad_send_wr,
1176		       struct ib_mad_agent_private *mad_agent_priv)
1177{
1178	if (mad_send_wr->is_solicited_fc) {
1179		if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
1180			mad_agent_priv->sol_fc_send_count--;
1181		else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1182			mad_agent_priv->sol_fc_wait_count--;
1183	}
1184
1185	list_del_init(&mad_send_wr->agent_list);
1186}
1187
1188void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
1189			     enum ib_mad_state new_state)
1190{
1191	struct ib_mad_agent_private *mad_agent_priv =
1192		mad_send_wr->mad_agent_priv;
1193
1194	switch (new_state) {
1195	case IB_MAD_STATE_INIT:
1196		break;
1197	case IB_MAD_STATE_QUEUED:
1198		handle_queued_state(mad_send_wr, mad_agent_priv);
1199		break;
1200	case IB_MAD_STATE_SEND_START:
1201		handle_send_state(mad_send_wr, mad_agent_priv);
1202		break;
1203	case IB_MAD_STATE_WAIT_RESP:
1204		handle_wait_state(mad_send_wr, mad_agent_priv);
1205		if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
1206			return;
1207		break;
1208	case IB_MAD_STATE_EARLY_RESP:
1209		handle_early_resp_state(mad_send_wr, mad_agent_priv);
1210		break;
1211	case IB_MAD_STATE_CANCELED:
1212		handle_canceled_state(mad_send_wr, mad_agent_priv);
1213		break;
1214	case IB_MAD_STATE_DONE:
1215		handle_done_state(mad_send_wr, mad_agent_priv);
1216		break;
1217	}
1218
1219	mad_send_wr->state = new_state;
1220}
1221
1222static bool is_solicited_fc_mad(struct ib_mad_send_wr_private *mad_send_wr)
1223{
1224	struct ib_rmpp_mad *rmpp_mad;
1225	u8 mgmt_class;
1226
1227	if (!mad_send_wr->timeout)
1228		return 0;
1229
1230	rmpp_mad = mad_send_wr->send_buf.mad;
1231	if (mad_send_wr->mad_agent_priv->agent.rmpp_version &&
1232	    (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE))
1233		return 0;
1234
1235	mgmt_class =
1236		((struct ib_mad_hdr *)mad_send_wr->send_buf.mad)->mgmt_class;
1237	return mgmt_class == IB_MGMT_CLASS_CM ||
1238	       mgmt_class == IB_MGMT_CLASS_SUBN_ADM ||
1239	       mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
1240	       mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE;
1241}
1242
1243static bool mad_is_for_backlog(struct ib_mad_send_wr_private *mad_send_wr)
1244{
1245	struct ib_mad_agent_private *mad_agent_priv =
1246		mad_send_wr->mad_agent_priv;
1247
1248	if (!mad_send_wr->is_solicited_fc || !mad_agent_priv->sol_fc_max)
1249		return false;
1250
1251	if (!list_empty(&mad_agent_priv->backlog_list))
1252		return true;
1253
1254	return mad_agent_priv->sol_fc_send_count +
1255		       mad_agent_priv->sol_fc_wait_count >=
1256	       mad_agent_priv->sol_fc_max;
1257}
1258
1259/*
1260 * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated
1261 *  with the registered client
1262 */
1263int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
1264		     struct ib_mad_send_buf **bad_send_buf)
1265{
1266	struct ib_mad_agent_private *mad_agent_priv;
1267	struct ib_mad_send_buf *next_send_buf;
1268	struct ib_mad_send_wr_private *mad_send_wr;
1269	unsigned long flags;
1270	int ret = -EINVAL;
1271
1272	/* Walk list of send WRs and post each on send list */
1273	for (; send_buf; send_buf = next_send_buf) {
1274		mad_send_wr = container_of(send_buf,
1275					   struct ib_mad_send_wr_private,
1276					   send_buf);
1277		mad_agent_priv = mad_send_wr->mad_agent_priv;
1278
1279		ret = ib_mad_enforce_security(mad_agent_priv,
1280					      mad_send_wr->send_wr.pkey_index);
1281		if (ret)
1282			goto error;
1283
1284		if (!send_buf->mad_agent->send_handler) {
1285			ret = -EINVAL;
1286			goto error;
1287		}
1288
1289		if (!ib_is_mad_class_rmpp(((struct ib_mad_hdr *) send_buf->mad)->mgmt_class)) {
1290			if (mad_agent_priv->agent.rmpp_version) {
1291				ret = -EINVAL;
1292				goto error;
1293			}
1294		}
1295
1296		/*
1297		 * Save pointer to next work request to post in case the
1298		 * current one completes, and the user modifies the work
1299		 * request associated with the completion
1300		 */
1301		next_send_buf = send_buf->next;
1302		mad_send_wr->send_wr.ah = send_buf->ah;
1303
1304		if (((struct ib_mad_hdr *) send_buf->mad)->mgmt_class ==
1305		    IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
1306			ret = handle_outgoing_dr_smp(mad_agent_priv,
1307						     mad_send_wr);
1308			if (ret < 0)		/* error */
1309				goto error;
1310			else if (ret == 1)	/* locally consumed */
1311				continue;
1312		}
1313
1314		mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid;
1315		/* Timeout will be updated after send completes */
1316		mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms);
1317		mad_send_wr->max_retries = send_buf->retries;
1318		mad_send_wr->retries_left = send_buf->retries;
1319		send_buf->retries = 0;
1320		change_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
1321
1322		/* Reference MAD agent until send completes */
1323		refcount_inc(&mad_agent_priv->refcount);
1324		spin_lock_irqsave(&mad_agent_priv->lock, flags);
1325		mad_send_wr->is_solicited_fc = is_solicited_fc_mad(mad_send_wr);
1326		if (mad_is_for_backlog(mad_send_wr)) {
1327			change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
1328			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1329			return 0;
1330		}
1331
1332		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
1333		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1334
1335		if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
1336			ret = ib_send_rmpp_mad(mad_send_wr);
1337			if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED)
1338				ret = ib_send_mad(mad_send_wr);
1339		} else
1340			ret = ib_send_mad(mad_send_wr);
1341		if (ret < 0) {
1342			/* Fail send request */
1343			spin_lock_irqsave(&mad_agent_priv->lock, flags);
1344			change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
1345			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1346			deref_mad_agent(mad_agent_priv);
1347			goto error;
1348		}
1349	}
1350	return 0;
1351error:
1352	if (bad_send_buf)
1353		*bad_send_buf = send_buf;
1354	return ret;
1355}
1356EXPORT_SYMBOL(ib_post_send_mad);
1357
1358/*
1359 * ib_free_recv_mad - Returns data buffers used to receive
1360 *  a MAD to the access layer
1361 */
1362void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc)
1363{
1364	struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf;
1365	struct ib_mad_private_header *mad_priv_hdr;
1366	struct ib_mad_private *priv;
1367	struct list_head free_list;
1368
1369	INIT_LIST_HEAD(&free_list);
1370	list_splice_init(&mad_recv_wc->rmpp_list, &free_list);
1371
1372	list_for_each_entry_safe(mad_recv_buf, temp_recv_buf,
1373					&free_list, list) {
1374		mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc,
1375					   recv_buf);
1376		mad_priv_hdr = container_of(mad_recv_wc,
1377					    struct ib_mad_private_header,
1378					    recv_wc);
1379		priv = container_of(mad_priv_hdr, struct ib_mad_private,
1380				    header);
1381		kfree(priv);
1382	}
1383}
1384EXPORT_SYMBOL(ib_free_recv_mad);
1385
1386static int method_in_use(struct ib_mad_mgmt_method_table **method,
1387			 struct ib_mad_reg_req *mad_reg_req)
1388{
1389	int i;
1390
1391	for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) {
1392		if ((*method)->agent[i]) {
1393			pr_err("Method %d already in use\n", i);
1394			return -EINVAL;
1395		}
1396	}
1397	return 0;
1398}
1399
1400static int allocate_method_table(struct ib_mad_mgmt_method_table **method)
1401{
1402	/* Allocate management method table */
1403	*method = kzalloc(sizeof **method, GFP_ATOMIC);
1404	return (*method) ? 0 : (-ENOMEM);
1405}
1406
1407/*
1408 * Check to see if there are any methods still in use
1409 */
1410static int check_method_table(struct ib_mad_mgmt_method_table *method)
1411{
1412	int i;
1413
1414	for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
1415		if (method->agent[i])
1416			return 1;
1417	return 0;
1418}
1419
1420/*
1421 * Check to see if there are any method tables for this class still in use
1422 */
1423static int check_class_table(struct ib_mad_mgmt_class_table *class)
1424{
1425	int i;
1426
1427	for (i = 0; i < MAX_MGMT_CLASS; i++)
1428		if (class->method_table[i])
1429			return 1;
1430	return 0;
1431}
1432
1433static int check_vendor_class(struct ib_mad_mgmt_vendor_class *vendor_class)
1434{
1435	int i;
1436
1437	for (i = 0; i < MAX_MGMT_OUI; i++)
1438		if (vendor_class->method_table[i])
1439			return 1;
1440	return 0;
1441}
1442
1443static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class,
1444			   const char *oui)
1445{
1446	int i;
1447
1448	for (i = 0; i < MAX_MGMT_OUI; i++)
1449		/* Is there matching OUI for this vendor class ? */
1450		if (!memcmp(vendor_class->oui[i], oui, 3))
1451			return i;
1452
1453	return -1;
1454}
1455
1456static int check_vendor_table(struct ib_mad_mgmt_vendor_class_table *vendor)
1457{
1458	int i;
1459
1460	for (i = 0; i < MAX_MGMT_VENDOR_RANGE2; i++)
1461		if (vendor->vendor_class[i])
1462			return 1;
1463
1464	return 0;
1465}
1466
1467static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method,
1468				     struct ib_mad_agent_private *agent)
1469{
1470	int i;
1471
1472	/* Remove any methods for this mad agent */
1473	for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
1474		if (method->agent[i] == agent)
1475			method->agent[i] = NULL;
1476}
1477
1478static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
1479			      struct ib_mad_agent_private *agent_priv,
1480			      u8 mgmt_class)
1481{
1482	struct ib_mad_port_private *port_priv;
1483	struct ib_mad_mgmt_class_table **class;
1484	struct ib_mad_mgmt_method_table **method;
1485	int i, ret;
1486
1487	port_priv = agent_priv->qp_info->port_priv;
1488	class = &port_priv->version[mad_reg_req->mgmt_class_version].class;
1489	if (!*class) {
1490		/* Allocate management class table for "new" class version */
1491		*class = kzalloc(sizeof **class, GFP_ATOMIC);
1492		if (!*class) {
1493			ret = -ENOMEM;
1494			goto error1;
1495		}
1496
1497		/* Allocate method table for this management class */
1498		method = &(*class)->method_table[mgmt_class];
1499		if ((ret = allocate_method_table(method)))
1500			goto error2;
1501	} else {
1502		method = &(*class)->method_table[mgmt_class];
1503		if (!*method) {
1504			/* Allocate method table for this management class */
1505			if ((ret = allocate_method_table(method)))
1506				goto error1;
1507		}
1508	}
1509
1510	/* Now, make sure methods are not already in use */
1511	if (method_in_use(method, mad_reg_req))
1512		goto error3;
1513
1514	/* Finally, add in methods being registered */
1515	for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
1516		(*method)->agent[i] = agent_priv;
1517
1518	return 0;
1519
1520error3:
1521	/* Remove any methods for this mad agent */
1522	remove_methods_mad_agent(*method, agent_priv);
1523	/* Now, check to see if there are any methods in use */
1524	if (!check_method_table(*method)) {
1525		/* If not, release management method table */
1526		kfree(*method);
1527		*method = NULL;
1528	}
1529	ret = -EINVAL;
1530	goto error1;
1531error2:
1532	kfree(*class);
1533	*class = NULL;
1534error1:
1535	return ret;
1536}
1537
1538static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
1539			   struct ib_mad_agent_private *agent_priv)
1540{
1541	struct ib_mad_port_private *port_priv;
1542	struct ib_mad_mgmt_vendor_class_table **vendor_table;
1543	struct ib_mad_mgmt_vendor_class_table *vendor = NULL;
1544	struct ib_mad_mgmt_vendor_class *vendor_class = NULL;
1545	struct ib_mad_mgmt_method_table **method;
1546	int i, ret = -ENOMEM;
1547	u8 vclass;
1548
1549	/* "New" vendor (with OUI) class */
1550	vclass = vendor_class_index(mad_reg_req->mgmt_class);
1551	port_priv = agent_priv->qp_info->port_priv;
1552	vendor_table = &port_priv->version[
1553				mad_reg_req->mgmt_class_version].vendor;
1554	if (!*vendor_table) {
1555		/* Allocate mgmt vendor class table for "new" class version */
1556		vendor = kzalloc(sizeof *vendor, GFP_ATOMIC);
1557		if (!vendor)
1558			goto error1;
1559
1560		*vendor_table = vendor;
1561	}
1562	if (!(*vendor_table)->vendor_class[vclass]) {
1563		/* Allocate table for this management vendor class */
1564		vendor_class = kzalloc(sizeof *vendor_class, GFP_ATOMIC);
1565		if (!vendor_class)
1566			goto error2;
1567
1568		(*vendor_table)->vendor_class[vclass] = vendor_class;
1569	}
1570	for (i = 0; i < MAX_MGMT_OUI; i++) {
1571		/* Is there matching OUI for this vendor class ? */
1572		if (!memcmp((*vendor_table)->vendor_class[vclass]->oui[i],
1573			    mad_reg_req->oui, 3)) {
1574			method = &(*vendor_table)->vendor_class[
1575						vclass]->method_table[i];
1576			if (!*method)
1577				goto error3;
1578			goto check_in_use;
1579		}
1580	}
1581	for (i = 0; i < MAX_MGMT_OUI; i++) {
1582		/* OUI slot available ? */
1583		if (!is_vendor_oui((*vendor_table)->vendor_class[
1584				vclass]->oui[i])) {
1585			method = &(*vendor_table)->vendor_class[
1586				vclass]->method_table[i];
1587			/* Allocate method table for this OUI */
1588			if (!*method) {
1589				ret = allocate_method_table(method);
1590				if (ret)
1591					goto error3;
1592			}
1593			memcpy((*vendor_table)->vendor_class[vclass]->oui[i],
1594			       mad_reg_req->oui, 3);
1595			goto check_in_use;
1596		}
1597	}
1598	dev_err(&agent_priv->agent.device->dev, "All OUI slots in use\n");
1599	goto error3;
1600
1601check_in_use:
1602	/* Now, make sure methods are not already in use */
1603	if (method_in_use(method, mad_reg_req))
1604		goto error4;
1605
1606	/* Finally, add in methods being registered */
1607	for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
1608		(*method)->agent[i] = agent_priv;
1609
1610	return 0;
1611
1612error4:
1613	/* Remove any methods for this mad agent */
1614	remove_methods_mad_agent(*method, agent_priv);
1615	/* Now, check to see if there are any methods in use */
1616	if (!check_method_table(*method)) {
1617		/* If not, release management method table */
1618		kfree(*method);
1619		*method = NULL;
1620	}
1621	ret = -EINVAL;
1622error3:
1623	if (vendor_class) {
1624		(*vendor_table)->vendor_class[vclass] = NULL;
1625		kfree(vendor_class);
1626	}
1627error2:
1628	if (vendor) {
1629		*vendor_table = NULL;
1630		kfree(vendor);
1631	}
1632error1:
1633	return ret;
1634}
1635
1636static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv)
1637{
1638	struct ib_mad_port_private *port_priv;
1639	struct ib_mad_mgmt_class_table *class;
1640	struct ib_mad_mgmt_method_table *method;
1641	struct ib_mad_mgmt_vendor_class_table *vendor;
1642	struct ib_mad_mgmt_vendor_class *vendor_class;
1643	int index;
1644	u8 mgmt_class;
1645
1646	/*
1647	 * Was MAD registration request supplied
1648	 * with original registration ?
1649	 */
1650	if (!agent_priv->reg_req)
1651		goto out;
1652
1653	port_priv = agent_priv->qp_info->port_priv;
1654	mgmt_class = convert_mgmt_class(agent_priv->reg_req->mgmt_class);
1655	class = port_priv->version[
1656			agent_priv->reg_req->mgmt_class_version].class;
1657	if (!class)
1658		goto vendor_check;
1659
1660	method = class->method_table[mgmt_class];
1661	if (method) {
1662		/* Remove any methods for this mad agent */
1663		remove_methods_mad_agent(method, agent_priv);
1664		/* Now, check to see if there are any methods still in use */
1665		if (!check_method_table(method)) {
1666			/* If not, release management method table */
1667			kfree(method);
1668			class->method_table[mgmt_class] = NULL;
1669			/* Any management classes left ? */
1670			if (!check_class_table(class)) {
1671				/* If not, release management class table */
1672				kfree(class);
1673				port_priv->version[
1674					agent_priv->reg_req->
1675					mgmt_class_version].class = NULL;
1676			}
1677		}
1678	}
1679
1680vendor_check:
1681	if (!is_vendor_class(mgmt_class))
1682		goto out;
1683
1684	/* normalize mgmt_class to vendor range 2 */
1685	mgmt_class = vendor_class_index(agent_priv->reg_req->mgmt_class);
1686	vendor = port_priv->version[
1687			agent_priv->reg_req->mgmt_class_version].vendor;
1688
1689	if (!vendor)
1690		goto out;
1691
1692	vendor_class = vendor->vendor_class[mgmt_class];
1693	if (vendor_class) {
1694		index = find_vendor_oui(vendor_class, agent_priv->reg_req->oui);
1695		if (index < 0)
1696			goto out;
1697		method = vendor_class->method_table[index];
1698		if (method) {
1699			/* Remove any methods for this mad agent */
1700			remove_methods_mad_agent(method, agent_priv);
1701			/*
1702			 * Now, check to see if there are
1703			 * any methods still in use
1704			 */
1705			if (!check_method_table(method)) {
1706				/* If not, release management method table */
1707				kfree(method);
1708				vendor_class->method_table[index] = NULL;
1709				memset(vendor_class->oui[index], 0, 3);
1710				/* Any OUIs left ? */
1711				if (!check_vendor_class(vendor_class)) {
1712					/* If not, release vendor class table */
1713					kfree(vendor_class);
1714					vendor->vendor_class[mgmt_class] = NULL;
1715					/* Any other vendor classes left ? */
1716					if (!check_vendor_table(vendor)) {
1717						kfree(vendor);
1718						port_priv->version[
1719							agent_priv->reg_req->
1720							mgmt_class_version].
1721							vendor = NULL;
1722					}
1723				}
1724			}
1725		}
1726	}
1727
1728out:
1729	return;
1730}
1731
1732static struct ib_mad_agent_private *
1733find_mad_agent(struct ib_mad_port_private *port_priv,
1734	       const struct ib_mad_hdr *mad_hdr)
1735{
1736	struct ib_mad_agent_private *mad_agent = NULL;
1737	unsigned long flags;
1738
1739	if (ib_response_mad(mad_hdr)) {
1740		u32 hi_tid;
1741
1742		/*
1743		 * Routing is based on high 32 bits of transaction ID
1744		 * of MAD.
1745		 */
1746		hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
1747		rcu_read_lock();
1748		mad_agent = xa_load(&ib_mad_clients, hi_tid);
1749		if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount))
1750			mad_agent = NULL;
1751		rcu_read_unlock();
1752	} else {
1753		struct ib_mad_mgmt_class_table *class;
1754		struct ib_mad_mgmt_method_table *method;
1755		struct ib_mad_mgmt_vendor_class_table *vendor;
1756		struct ib_mad_mgmt_vendor_class *vendor_class;
1757		const struct ib_vendor_mad *vendor_mad;
1758		int index;
1759
1760		spin_lock_irqsave(&port_priv->reg_lock, flags);
1761		/*
1762		 * Routing is based on version, class, and method
1763		 * For "newer" vendor MADs, also based on OUI
1764		 */
1765		if (mad_hdr->class_version >= MAX_MGMT_VERSION)
1766			goto out;
1767		if (!is_vendor_class(mad_hdr->mgmt_class)) {
1768			class = port_priv->version[
1769					mad_hdr->class_version].class;
1770			if (!class)
1771				goto out;
1772			if (convert_mgmt_class(mad_hdr->mgmt_class) >=
1773			    ARRAY_SIZE(class->method_table))
1774				goto out;
1775			method = class->method_table[convert_mgmt_class(
1776							mad_hdr->mgmt_class)];
1777			if (method)
1778				mad_agent = method->agent[mad_hdr->method &
1779							  ~IB_MGMT_METHOD_RESP];
1780		} else {
1781			vendor = port_priv->version[
1782					mad_hdr->class_version].vendor;
1783			if (!vendor)
1784				goto out;
1785			vendor_class = vendor->vendor_class[vendor_class_index(
1786						mad_hdr->mgmt_class)];
1787			if (!vendor_class)
1788				goto out;
1789			/* Find matching OUI */
1790			vendor_mad = (const struct ib_vendor_mad *)mad_hdr;
1791			index = find_vendor_oui(vendor_class, vendor_mad->oui);
1792			if (index == -1)
1793				goto out;
1794			method = vendor_class->method_table[index];
1795			if (method) {
1796				mad_agent = method->agent[mad_hdr->method &
1797							  ~IB_MGMT_METHOD_RESP];
1798			}
1799		}
1800		if (mad_agent)
1801			refcount_inc(&mad_agent->refcount);
1802out:
1803		spin_unlock_irqrestore(&port_priv->reg_lock, flags);
1804	}
1805
1806	if (mad_agent && !mad_agent->agent.recv_handler) {
1807		dev_notice(&port_priv->device->dev,
1808			   "No receive handler for client %p on port %u\n",
1809			   &mad_agent->agent, port_priv->port_num);
1810		deref_mad_agent(mad_agent);
1811		mad_agent = NULL;
1812	}
1813
1814	return mad_agent;
1815}
1816
1817static int validate_mad(const struct ib_mad_hdr *mad_hdr,
1818			const struct ib_mad_qp_info *qp_info,
1819			bool opa)
1820{
1821	int valid = 0;
1822	u32 qp_num = qp_info->qp->qp_num;
1823
1824	/* Make sure MAD base version is understood */
1825	if (mad_hdr->base_version != IB_MGMT_BASE_VERSION &&
1826	    (!opa || mad_hdr->base_version != OPA_MGMT_BASE_VERSION)) {
1827		pr_err("MAD received with unsupported base version %u %s\n",
1828		       mad_hdr->base_version, opa ? "(opa)" : "");
1829		goto out;
1830	}
1831
1832	/* Filter SMI packets sent to other than QP0 */
1833	if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
1834	    (mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
1835		if (qp_num == 0)
1836			valid = 1;
1837	} else {
1838		/* CM attributes other than ClassPortInfo only use Send method */
1839		if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_CM) &&
1840		    (mad_hdr->attr_id != IB_MGMT_CLASSPORTINFO_ATTR_ID) &&
1841		    (mad_hdr->method != IB_MGMT_METHOD_SEND))
1842			goto out;
1843		/* Filter GSI packets sent to QP0 */
1844		if (qp_num != 0)
1845			valid = 1;
1846	}
1847
1848out:
1849	return valid;
1850}
1851
1852static int is_rmpp_data_mad(const struct ib_mad_agent_private *mad_agent_priv,
1853			    const struct ib_mad_hdr *mad_hdr)
1854{
1855	struct ib_rmpp_mad *rmpp_mad;
1856
1857	rmpp_mad = (struct ib_rmpp_mad *)mad_hdr;
1858	return !mad_agent_priv->agent.rmpp_version ||
1859		!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) ||
1860		!(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
1861				    IB_MGMT_RMPP_FLAG_ACTIVE) ||
1862		(rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA);
1863}
1864
1865static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr,
1866				     const struct ib_mad_recv_wc *rwc)
1867{
1868	return ((struct ib_mad_hdr *)(wr->send_buf.mad))->mgmt_class ==
1869		rwc->recv_buf.mad->mad_hdr.mgmt_class;
1870}
1871
1872static inline int
1873rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv,
1874		 const struct ib_mad_send_wr_private *wr,
1875		 const struct ib_mad_recv_wc *rwc)
1876{
1877	struct rdma_ah_attr attr;
1878	u8 send_resp, rcv_resp;
1879	union ib_gid sgid;
1880	struct ib_device *device = mad_agent_priv->agent.device;
1881	u32 port_num = mad_agent_priv->agent.port_num;
1882	u8 lmc;
1883	bool has_grh;
1884
1885	send_resp = ib_response_mad((struct ib_mad_hdr *)wr->send_buf.mad);
1886	rcv_resp = ib_response_mad(&rwc->recv_buf.mad->mad_hdr);
1887
1888	if (send_resp == rcv_resp)
1889		/* both requests, or both responses. GIDs different */
1890		return 0;
1891
1892	if (rdma_query_ah(wr->send_buf.ah, &attr))
1893		/* Assume not equal, to avoid false positives. */
1894		return 0;
1895
1896	has_grh = !!(rdma_ah_get_ah_flags(&attr) & IB_AH_GRH);
1897	if (has_grh != !!(rwc->wc->wc_flags & IB_WC_GRH))
1898		/* one has GID, other does not.  Assume different */
1899		return 0;
1900
1901	if (!send_resp && rcv_resp) {
1902		/* is request/response. */
1903		if (!has_grh) {
1904			if (ib_get_cached_lmc(device, port_num, &lmc))
1905				return 0;
1906			return (!lmc || !((rdma_ah_get_path_bits(&attr) ^
1907					   rwc->wc->dlid_path_bits) &
1908					  ((1 << lmc) - 1)));
1909		} else {
1910			const struct ib_global_route *grh =
1911					rdma_ah_read_grh(&attr);
1912
1913			if (rdma_query_gid(device, port_num,
1914					   grh->sgid_index, &sgid))
1915				return 0;
1916			return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw,
1917				       16);
1918		}
1919	}
1920
1921	if (!has_grh)
1922		return rdma_ah_get_dlid(&attr) == rwc->wc->slid;
1923	else
1924		return !memcmp(rdma_ah_read_grh(&attr)->dgid.raw,
1925			       rwc->recv_buf.grh->sgid.raw,
1926			       16);
1927}
1928
1929static inline int is_direct(u8 class)
1930{
1931	return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE);
1932}
1933
1934struct ib_mad_send_wr_private*
1935ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
1936		 const struct ib_mad_recv_wc *wc)
1937{
1938	struct ib_mad_send_wr_private *wr;
1939	const struct ib_mad_hdr *mad_hdr;
1940
1941	mad_hdr = &wc->recv_buf.mad->mad_hdr;
1942
1943	list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) {
1944		if ((wr->tid == mad_hdr->tid) &&
1945		    rcv_has_same_class(wr, wc) &&
1946		    /*
1947		     * Don't check GID for direct routed MADs.
1948		     * These might have permissive LIDs.
1949		     */
1950		    (is_direct(mad_hdr->mgmt_class) ||
1951		     rcv_has_same_gid(mad_agent_priv, wr, wc)))
1952			return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1953	}
1954
1955	list_for_each_entry(wr, &mad_agent_priv->backlog_list, agent_list) {
1956		if ((wr->tid == mad_hdr->tid) &&
1957		    rcv_has_same_class(wr, wc) &&
1958		    /*
1959		     * Don't check GID for direct routed MADs.
1960		     * These might have permissive LIDs.
1961		     */
1962		    (is_direct(mad_hdr->mgmt_class) ||
1963		     rcv_has_same_gid(mad_agent_priv, wr, wc)))
1964			return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1965	}
1966
1967	/*
1968	 * It's possible to receive the response before we've
1969	 * been notified that the send has completed
1970	 */
1971	list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) {
1972		if (is_rmpp_data_mad(mad_agent_priv, wr->send_buf.mad) &&
1973		    wr->tid == mad_hdr->tid &&
1974		    wr->timeout &&
1975		    rcv_has_same_class(wr, wc) &&
1976		    /*
1977		     * Don't check GID for direct routed MADs.
1978		     * These might have permissive LIDs.
1979		     */
1980		    (is_direct(mad_hdr->mgmt_class) ||
1981		     rcv_has_same_gid(mad_agent_priv, wr, wc)))
1982			/* Verify request has not been canceled */
1983			return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1984	}
1985	return NULL;
1986}
1987
1988static void
1989process_backlog_mads(struct ib_mad_agent_private *mad_agent_priv)
1990{
1991	struct ib_mad_send_wr_private *mad_send_wr;
1992	struct ib_mad_send_wc mad_send_wc = {};
1993	unsigned long flags;
1994	int ret;
1995
1996	spin_lock_irqsave(&mad_agent_priv->lock, flags);
1997	while (!list_empty(&mad_agent_priv->backlog_list) &&
1998	       (mad_agent_priv->sol_fc_send_count +
1999			mad_agent_priv->sol_fc_wait_count <
2000		mad_agent_priv->sol_fc_max)) {
2001		mad_send_wr = list_entry(mad_agent_priv->backlog_list.next,
2002					 struct ib_mad_send_wr_private,
2003					 agent_list);
2004		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
2005		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2006		ret = ib_send_mad(mad_send_wr);
2007		if (ret) {
2008			spin_lock_irqsave(&mad_agent_priv->lock, flags);
2009			deref_mad_agent(mad_agent_priv);
2010			change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2011			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2012			mad_send_wc.send_buf = &mad_send_wr->send_buf;
2013			mad_send_wc.status = IB_WC_LOC_QP_OP_ERR;
2014			mad_agent_priv->agent.send_handler(
2015				&mad_agent_priv->agent, &mad_send_wc);
2016		}
2017
2018		spin_lock_irqsave(&mad_agent_priv->lock, flags);
2019	}
2020
2021	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2022}
2023
2024void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
2025{
2026	mad_send_wr->timeout = 0;
2027	if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP ||
2028	    mad_send_wr->state == IB_MAD_STATE_QUEUED)
2029		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2030	else
2031		change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP);
2032}
2033
2034static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
2035				 struct ib_mad_recv_wc *mad_recv_wc)
2036{
2037	struct ib_mad_send_wr_private *mad_send_wr;
2038	struct ib_mad_send_wc mad_send_wc;
2039	unsigned long flags;
2040	bool is_mad_done;
2041	int ret;
2042
2043	INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
2044	ret = ib_mad_enforce_security(mad_agent_priv,
2045				      mad_recv_wc->wc->pkey_index);
2046	if (ret) {
2047		ib_free_recv_mad(mad_recv_wc);
2048		deref_mad_agent(mad_agent_priv);
2049		return;
2050	}
2051
2052	list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
2053	if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
2054		mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
2055						      mad_recv_wc);
2056		if (!mad_recv_wc) {
2057			deref_mad_agent(mad_agent_priv);
2058			return;
2059		}
2060	}
2061
2062	/* Complete corresponding request */
2063	if (ib_response_mad(&mad_recv_wc->recv_buf.mad->mad_hdr)) {
2064		spin_lock_irqsave(&mad_agent_priv->lock, flags);
2065		mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
2066		if (!mad_send_wr) {
2067			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2068			if (!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)
2069			   && ib_is_mad_class_rmpp(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class)
2070			   && (ib_get_rmpp_flags(&((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr)
2071					& IB_MGMT_RMPP_FLAG_ACTIVE)) {
2072				/* user rmpp is in effect
2073				 * and this is an active RMPP MAD
2074				 */
2075				mad_agent_priv->agent.recv_handler(
2076						&mad_agent_priv->agent, NULL,
2077						mad_recv_wc);
2078				deref_mad_agent(mad_agent_priv);
2079			} else {
2080				/* not user rmpp, revert to normal behavior and
2081				 * drop the mad
2082				 */
2083				ib_free_recv_mad(mad_recv_wc);
2084				deref_mad_agent(mad_agent_priv);
2085				return;
2086			}
2087		} else {
2088			ib_mark_mad_done(mad_send_wr);
2089			is_mad_done = (mad_send_wr->state == IB_MAD_STATE_DONE);
2090			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2091
2092			/* Defined behavior is to complete response before request */
2093			mad_agent_priv->agent.recv_handler(
2094					&mad_agent_priv->agent,
2095					&mad_send_wr->send_buf,
2096					mad_recv_wc);
2097			deref_mad_agent(mad_agent_priv);
2098
2099			if (is_mad_done) {
2100				mad_send_wc.status = IB_WC_SUCCESS;
2101				mad_send_wc.vendor_err = 0;
2102				mad_send_wc.send_buf = &mad_send_wr->send_buf;
2103				ib_mad_complete_send_wr(mad_send_wr,
2104							&mad_send_wc);
2105			}
2106		}
2107	} else {
2108		mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL,
2109						   mad_recv_wc);
2110		deref_mad_agent(mad_agent_priv);
2111	}
2112}
2113
2114static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv,
2115				     const struct ib_mad_qp_info *qp_info,
2116				     const struct ib_wc *wc,
2117				     u32 port_num,
2118				     struct ib_mad_private *recv,
2119				     struct ib_mad_private *response)
2120{
2121	enum smi_forward_action retsmi;
2122	struct ib_smp *smp = (struct ib_smp *)recv->mad;
2123
2124	trace_ib_mad_handle_ib_smi(smp);
2125
2126	if (smi_handle_dr_smp_recv(smp,
2127				   rdma_cap_ib_switch(port_priv->device),
2128				   port_num,
2129				   port_priv->device->phys_port_cnt) ==
2130				   IB_SMI_DISCARD)
2131		return IB_SMI_DISCARD;
2132
2133	retsmi = smi_check_forward_dr_smp(smp);
2134	if (retsmi == IB_SMI_LOCAL)
2135		return IB_SMI_HANDLE;
2136
2137	if (retsmi == IB_SMI_SEND) { /* don't forward */
2138		if (smi_handle_dr_smp_send(smp,
2139					   rdma_cap_ib_switch(port_priv->device),
2140					   port_num) == IB_SMI_DISCARD)
2141			return IB_SMI_DISCARD;
2142
2143		if (smi_check_local_smp(smp, port_priv->device) == IB_SMI_DISCARD)
2144			return IB_SMI_DISCARD;
2145	} else if (rdma_cap_ib_switch(port_priv->device)) {
2146		/* forward case for switches */
2147		memcpy(response, recv, mad_priv_size(response));
2148		response->header.recv_wc.wc = &response->header.wc;
2149		response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
2150		response->header.recv_wc.recv_buf.grh = &response->grh;
2151
2152		agent_send_response((const struct ib_mad_hdr *)response->mad,
2153				    &response->grh, wc,
2154				    port_priv->device,
2155				    smi_get_fwd_port(smp),
2156				    qp_info->qp->qp_num,
2157				    response->mad_size,
2158				    false);
2159
2160		return IB_SMI_DISCARD;
2161	}
2162	return IB_SMI_HANDLE;
2163}
2164
2165static bool generate_unmatched_resp(const struct ib_mad_private *recv,
2166				    struct ib_mad_private *response,
2167				    size_t *resp_len, bool opa)
2168{
2169	const struct ib_mad_hdr *recv_hdr = (const struct ib_mad_hdr *)recv->mad;
2170	struct ib_mad_hdr *resp_hdr = (struct ib_mad_hdr *)response->mad;
2171
2172	if (recv_hdr->method == IB_MGMT_METHOD_GET ||
2173	    recv_hdr->method == IB_MGMT_METHOD_SET) {
2174		memcpy(response, recv, mad_priv_size(response));
2175		response->header.recv_wc.wc = &response->header.wc;
2176		response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
2177		response->header.recv_wc.recv_buf.grh = &response->grh;
2178		resp_hdr->method = IB_MGMT_METHOD_GET_RESP;
2179		resp_hdr->status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
2180		if (recv_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
2181			resp_hdr->status |= IB_SMP_DIRECTION;
2182
2183		if (opa && recv_hdr->base_version == OPA_MGMT_BASE_VERSION) {
2184			if (recv_hdr->mgmt_class ==
2185			    IB_MGMT_CLASS_SUBN_LID_ROUTED ||
2186			    recv_hdr->mgmt_class ==
2187			    IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
2188				*resp_len = opa_get_smp_header_size(
2189							(struct opa_smp *)recv->mad);
2190			else
2191				*resp_len = sizeof(struct ib_mad_hdr);
2192		}
2193
2194		return true;
2195	} else {
2196		return false;
2197	}
2198}
2199
2200static enum smi_action
2201handle_opa_smi(struct ib_mad_port_private *port_priv,
2202	       struct ib_mad_qp_info *qp_info,
2203	       struct ib_wc *wc,
2204	       u32 port_num,
2205	       struct ib_mad_private *recv,
2206	       struct ib_mad_private *response)
2207{
2208	enum smi_forward_action retsmi;
2209	struct opa_smp *smp = (struct opa_smp *)recv->mad;
2210
2211	trace_ib_mad_handle_opa_smi(smp);
2212
2213	if (opa_smi_handle_dr_smp_recv(smp,
2214				   rdma_cap_ib_switch(port_priv->device),
2215				   port_num,
2216				   port_priv->device->phys_port_cnt) ==
2217				   IB_SMI_DISCARD)
2218		return IB_SMI_DISCARD;
2219
2220	retsmi = opa_smi_check_forward_dr_smp(smp);
2221	if (retsmi == IB_SMI_LOCAL)
2222		return IB_SMI_HANDLE;
2223
2224	if (retsmi == IB_SMI_SEND) { /* don't forward */
2225		if (opa_smi_handle_dr_smp_send(smp,
2226					   rdma_cap_ib_switch(port_priv->device),
2227					   port_num) == IB_SMI_DISCARD)
2228			return IB_SMI_DISCARD;
2229
2230		if (opa_smi_check_local_smp(smp, port_priv->device) ==
2231		    IB_SMI_DISCARD)
2232			return IB_SMI_DISCARD;
2233
2234	} else if (rdma_cap_ib_switch(port_priv->device)) {
2235		/* forward case for switches */
2236		memcpy(response, recv, mad_priv_size(response));
2237		response->header.recv_wc.wc = &response->header.wc;
2238		response->header.recv_wc.recv_buf.opa_mad =
2239				(struct opa_mad *)response->mad;
2240		response->header.recv_wc.recv_buf.grh = &response->grh;
2241
2242		agent_send_response((const struct ib_mad_hdr *)response->mad,
2243				    &response->grh, wc,
2244				    port_priv->device,
2245				    opa_smi_get_fwd_port(smp),
2246				    qp_info->qp->qp_num,
2247				    recv->header.wc.byte_len,
2248				    true);
2249
2250		return IB_SMI_DISCARD;
2251	}
2252
2253	return IB_SMI_HANDLE;
2254}
2255
2256static enum smi_action
2257handle_smi(struct ib_mad_port_private *port_priv,
2258	   struct ib_mad_qp_info *qp_info,
2259	   struct ib_wc *wc,
2260	   u32 port_num,
2261	   struct ib_mad_private *recv,
2262	   struct ib_mad_private *response,
2263	   bool opa)
2264{
2265	struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad;
2266
2267	if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION &&
2268	    mad_hdr->class_version == OPA_SM_CLASS_VERSION)
2269		return handle_opa_smi(port_priv, qp_info, wc, port_num, recv,
2270				      response);
2271
2272	return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response);
2273}
2274
2275static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2276{
2277	struct ib_mad_port_private *port_priv = cq->cq_context;
2278	struct ib_mad_list_head *mad_list =
2279		container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2280	struct ib_mad_qp_info *qp_info;
2281	struct ib_mad_private_header *mad_priv_hdr;
2282	struct ib_mad_private *recv, *response = NULL;
2283	struct ib_mad_agent_private *mad_agent;
2284	u32 port_num;
2285	int ret = IB_MAD_RESULT_SUCCESS;
2286	size_t mad_size;
2287	u16 resp_mad_pkey_index = 0;
2288	bool opa;
2289
2290	if (list_empty_careful(&port_priv->port_list))
2291		return;
2292
2293	if (wc->status != IB_WC_SUCCESS) {
2294		/*
2295		 * Receive errors indicate that the QP has entered the error
2296		 * state - error handling/shutdown code will cleanup
2297		 */
2298		return;
2299	}
2300
2301	qp_info = mad_list->mad_queue->qp_info;
2302	dequeue_mad(mad_list);
2303
2304	opa = rdma_cap_opa_mad(qp_info->port_priv->device,
2305			       qp_info->port_priv->port_num);
2306
2307	mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header,
2308				    mad_list);
2309	recv = container_of(mad_priv_hdr, struct ib_mad_private, header);
2310	ib_dma_unmap_single(port_priv->device,
2311			    recv->header.mapping,
2312			    mad_priv_dma_size(recv),
2313			    DMA_FROM_DEVICE);
2314
2315	/* Setup MAD receive work completion from "normal" work completion */
2316	recv->header.wc = *wc;
2317	recv->header.recv_wc.wc = &recv->header.wc;
2318
2319	if (opa && ((struct ib_mad_hdr *)(recv->mad))->base_version == OPA_MGMT_BASE_VERSION) {
2320		recv->header.recv_wc.mad_len = wc->byte_len - sizeof(struct ib_grh);
2321		recv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
2322	} else {
2323		recv->header.recv_wc.mad_len = sizeof(struct ib_mad);
2324		recv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
2325	}
2326
2327	recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad;
2328	recv->header.recv_wc.recv_buf.grh = &recv->grh;
2329
2330	/* Validate MAD */
2331	if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa))
2332		goto out;
2333
2334	trace_ib_mad_recv_done_handler(qp_info, wc,
2335				       (struct ib_mad_hdr *)recv->mad);
2336
2337	mad_size = recv->mad_size;
2338	response = alloc_mad_private(mad_size, GFP_KERNEL);
2339	if (!response)
2340		goto out;
2341
2342	if (rdma_cap_ib_switch(port_priv->device))
2343		port_num = wc->port_num;
2344	else
2345		port_num = port_priv->port_num;
2346
2347	if (((struct ib_mad_hdr *)recv->mad)->mgmt_class ==
2348	    IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
2349		if (handle_smi(port_priv, qp_info, wc, port_num, recv,
2350			       response, opa)
2351		    == IB_SMI_DISCARD)
2352			goto out;
2353	}
2354
2355	/* Give driver "right of first refusal" on incoming MAD */
2356	if (port_priv->device->ops.process_mad) {
2357		ret = port_priv->device->ops.process_mad(
2358			port_priv->device, 0, port_priv->port_num, wc,
2359			&recv->grh, (const struct ib_mad *)recv->mad,
2360			(struct ib_mad *)response->mad, &mad_size,
2361			&resp_mad_pkey_index);
2362
2363		if (opa)
2364			wc->pkey_index = resp_mad_pkey_index;
2365
2366		if (ret & IB_MAD_RESULT_SUCCESS) {
2367			if (ret & IB_MAD_RESULT_CONSUMED)
2368				goto out;
2369			if (ret & IB_MAD_RESULT_REPLY) {
2370				agent_send_response((const struct ib_mad_hdr *)response->mad,
2371						    &recv->grh, wc,
2372						    port_priv->device,
2373						    port_num,
2374						    qp_info->qp->qp_num,
2375						    mad_size, opa);
2376				goto out;
2377			}
2378		}
2379	}
2380
2381	mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad);
2382	if (mad_agent) {
2383		trace_ib_mad_recv_done_agent(mad_agent);
2384		ib_mad_complete_recv(mad_agent, &recv->header.recv_wc);
2385		/*
2386		 * recv is freed up in error cases in ib_mad_complete_recv
2387		 * or via recv_handler in ib_mad_complete_recv()
2388		 */
2389		recv = NULL;
2390	} else if ((ret & IB_MAD_RESULT_SUCCESS) &&
2391		   generate_unmatched_resp(recv, response, &mad_size, opa)) {
2392		agent_send_response((const struct ib_mad_hdr *)response->mad, &recv->grh, wc,
2393				    port_priv->device, port_num,
2394				    qp_info->qp->qp_num, mad_size, opa);
2395	}
2396
2397out:
2398	/* Post another receive request for this QP */
2399	if (response) {
2400		ib_mad_post_receive_mads(qp_info, response);
2401		kfree(recv);
2402	} else
2403		ib_mad_post_receive_mads(qp_info, recv);
2404}
2405
2406static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
2407{
2408	struct ib_mad_send_wr_private *mad_send_wr;
2409	unsigned long delay;
2410
2411	if (list_empty(&mad_agent_priv->wait_list)) {
2412		cancel_delayed_work(&mad_agent_priv->timed_work);
2413	} else {
2414		mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
2415					 struct ib_mad_send_wr_private,
2416					 agent_list);
2417
2418		if (time_after(mad_agent_priv->timeout,
2419			       mad_send_wr->timeout)) {
2420			mad_agent_priv->timeout = mad_send_wr->timeout;
2421			delay = mad_send_wr->timeout - jiffies;
2422			if ((long)delay <= 0)
2423				delay = 1;
2424			mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
2425					 &mad_agent_priv->timed_work, delay);
2426		}
2427	}
2428}
2429
2430static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)
2431{
2432	struct ib_mad_agent_private *mad_agent_priv;
2433	unsigned long delay;
2434
2435	mad_agent_priv = mad_send_wr->mad_agent_priv;
2436	delay = mad_send_wr->timeout;
2437	change_mad_state(mad_send_wr, IB_MAD_STATE_WAIT_RESP);
2438
2439	/* Reschedule a work item if we have a shorter timeout */
2440	if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list)
2441		mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
2442				 &mad_agent_priv->timed_work, delay);
2443}
2444
2445void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
2446			  unsigned long timeout_ms)
2447{
2448	mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
2449	wait_for_response(mad_send_wr);
2450}
2451
2452/*
2453 * Process a send work completion
2454 */
2455void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
2456			     struct ib_mad_send_wc *mad_send_wc)
2457{
2458	struct ib_mad_agent_private	*mad_agent_priv;
2459	unsigned long			flags;
2460	int				ret;
2461
2462	mad_agent_priv = mad_send_wr->mad_agent_priv;
2463	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2464	if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
2465		ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc);
2466		if (ret == IB_RMPP_RESULT_CONSUMED)
2467			goto done;
2468	} else
2469		ret = IB_RMPP_RESULT_UNHANDLED;
2470
2471	if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
2472		mad_send_wc->status = IB_WC_WR_FLUSH_ERR;
2473	else if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
2474		 mad_send_wr->timeout) {
2475		wait_for_response(mad_send_wr);
2476		goto done;
2477	}
2478
2479	/* Remove send from MAD agent and notify client of completion */
2480	if (mad_send_wr->state != IB_MAD_STATE_DONE)
2481		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2482	adjust_timeout(mad_agent_priv);
2483	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2484
2485	if (ret == IB_RMPP_RESULT_INTERNAL) {
2486		ib_rmpp_send_handler(mad_send_wc);
2487	} else {
2488		if (mad_send_wr->is_solicited_fc)
2489			process_backlog_mads(mad_agent_priv);
2490		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2491						   mad_send_wc);
2492	}
2493
2494	/* Release reference on agent taken when sending */
2495	deref_mad_agent(mad_agent_priv);
2496	return;
2497done:
2498	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2499}
2500
2501static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc)
2502{
2503	struct ib_mad_port_private *port_priv = cq->cq_context;
2504	struct ib_mad_list_head *mad_list =
2505		container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2506	struct ib_mad_send_wr_private	*mad_send_wr, *queued_send_wr;
2507	struct ib_mad_qp_info		*qp_info;
2508	struct ib_mad_queue		*send_queue;
2509	struct ib_mad_send_wc		mad_send_wc;
2510	unsigned long flags;
2511	int ret;
2512
2513	if (list_empty_careful(&port_priv->port_list))
2514		return;
2515
2516	if (wc->status != IB_WC_SUCCESS) {
2517		if (!ib_mad_send_error(port_priv, wc))
2518			return;
2519	}
2520
2521	mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
2522				   mad_list);
2523	send_queue = mad_list->mad_queue;
2524	qp_info = send_queue->qp_info;
2525
2526	trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv);
2527	trace_ib_mad_send_done_handler(mad_send_wr, wc);
2528
2529retry:
2530	ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
2531			    mad_send_wr->header_mapping,
2532			    mad_send_wr->sg_list[0].length, DMA_TO_DEVICE);
2533	ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
2534			    mad_send_wr->payload_mapping,
2535			    mad_send_wr->sg_list[1].length, DMA_TO_DEVICE);
2536	queued_send_wr = NULL;
2537	spin_lock_irqsave(&send_queue->lock, flags);
2538	list_del(&mad_list->list);
2539
2540	/* Move queued send to the send queue */
2541	if (send_queue->count-- > send_queue->max_active) {
2542		mad_list = container_of(qp_info->overflow_list.next,
2543					struct ib_mad_list_head, list);
2544		queued_send_wr = container_of(mad_list,
2545					struct ib_mad_send_wr_private,
2546					mad_list);
2547		list_move_tail(&mad_list->list, &send_queue->list);
2548	}
2549	spin_unlock_irqrestore(&send_queue->lock, flags);
2550
2551	mad_send_wc.send_buf = &mad_send_wr->send_buf;
2552	mad_send_wc.status = wc->status;
2553	mad_send_wc.vendor_err = wc->vendor_err;
2554	ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
2555
2556	if (queued_send_wr) {
2557		trace_ib_mad_send_done_resend(queued_send_wr, qp_info);
2558		ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr,
2559				   NULL);
2560		if (ret) {
2561			dev_err(&port_priv->device->dev,
2562				"ib_post_send failed: %d\n", ret);
2563			mad_send_wr = queued_send_wr;
2564			wc->status = IB_WC_LOC_QP_OP_ERR;
2565			goto retry;
2566		}
2567	}
2568}
2569
2570static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info)
2571{
2572	struct ib_mad_send_wr_private *mad_send_wr;
2573	struct ib_mad_list_head *mad_list;
2574	unsigned long flags;
2575
2576	spin_lock_irqsave(&qp_info->send_queue.lock, flags);
2577	list_for_each_entry(mad_list, &qp_info->send_queue.list, list) {
2578		mad_send_wr = container_of(mad_list,
2579					   struct ib_mad_send_wr_private,
2580					   mad_list);
2581		mad_send_wr->retry = 1;
2582	}
2583	spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
2584}
2585
2586static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
2587		struct ib_wc *wc)
2588{
2589	struct ib_mad_list_head *mad_list =
2590		container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2591	struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info;
2592	struct ib_mad_send_wr_private *mad_send_wr;
2593	int ret;
2594
2595	/*
2596	 * Send errors will transition the QP to SQE - move
2597	 * QP to RTS and repost flushed work requests
2598	 */
2599	mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
2600				   mad_list);
2601	if (wc->status == IB_WC_WR_FLUSH_ERR) {
2602		if (mad_send_wr->retry) {
2603			/* Repost send */
2604			mad_send_wr->retry = 0;
2605			trace_ib_mad_error_handler(mad_send_wr, qp_info);
2606			ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,
2607					   NULL);
2608			if (!ret)
2609				return false;
2610		}
2611	} else {
2612		struct ib_qp_attr *attr;
2613
2614		/* Transition QP to RTS and fail offending send */
2615		attr = kmalloc(sizeof *attr, GFP_KERNEL);
2616		if (attr) {
2617			attr->qp_state = IB_QPS_RTS;
2618			attr->cur_qp_state = IB_QPS_SQE;
2619			ret = ib_modify_qp(qp_info->qp, attr,
2620					   IB_QP_STATE | IB_QP_CUR_STATE);
2621			kfree(attr);
2622			if (ret)
2623				dev_err(&port_priv->device->dev,
2624					"%s - ib_modify_qp to RTS: %d\n",
2625					__func__, ret);
2626			else
2627				mark_sends_for_retry(qp_info);
2628		}
2629	}
2630
2631	return true;
2632}
2633
2634static void clear_mad_error_list(struct list_head *list,
2635				 enum ib_wc_status wc_status,
2636				 struct ib_mad_agent_private *mad_agent_priv)
2637{
2638	struct ib_mad_send_wr_private *mad_send_wr, *n;
2639	struct ib_mad_send_wc mad_send_wc;
2640
2641	mad_send_wc.status = wc_status;
2642	mad_send_wc.vendor_err = 0;
2643
2644	list_for_each_entry_safe(mad_send_wr, n, list, agent_list) {
2645		mad_send_wc.send_buf = &mad_send_wr->send_buf;
2646		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2647						   &mad_send_wc);
2648		deref_mad_agent(mad_agent_priv);
2649	}
2650}
2651
2652static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
2653{
2654	unsigned long flags;
2655	struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;
2656	struct list_head cancel_list;
2657
2658	INIT_LIST_HEAD(&cancel_list);
2659
2660	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2661	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2662				 &mad_agent_priv->send_list, agent_list)
2663		change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
2664
2665	/* Empty wait & backlog list to prevent receives from finding request */
2666	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2667				 &mad_agent_priv->wait_list, agent_list) {
2668		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2669		list_add_tail(&mad_send_wr->agent_list, &cancel_list);
2670	}
2671
2672	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2673				 &mad_agent_priv->backlog_list, agent_list) {
2674		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2675		list_add_tail(&mad_send_wr->agent_list, &cancel_list);
2676	}
2677
2678	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2679	/* Report all cancelled requests */
2680	clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
2681}
2682
2683static struct ib_mad_send_wr_private*
2684find_send_wr(struct ib_mad_agent_private *mad_agent_priv,
2685	     struct ib_mad_send_buf *send_buf)
2686{
2687	struct ib_mad_send_wr_private *mad_send_wr;
2688
2689	list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
2690			    agent_list) {
2691		if (&mad_send_wr->send_buf == send_buf)
2692			return mad_send_wr;
2693	}
2694
2695	list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
2696			    agent_list) {
2697		if (is_rmpp_data_mad(mad_agent_priv,
2698				     mad_send_wr->send_buf.mad) &&
2699		    &mad_send_wr->send_buf == send_buf)
2700			return mad_send_wr;
2701	}
2702
2703	list_for_each_entry(mad_send_wr, &mad_agent_priv->backlog_list,
2704			    agent_list) {
2705		if (&mad_send_wr->send_buf == send_buf)
2706			return mad_send_wr;
2707	}
2708
2709	return NULL;
2710}
2711
2712int ib_modify_mad(struct ib_mad_send_buf *send_buf, u32 timeout_ms)
2713{
2714	struct ib_mad_agent_private *mad_agent_priv;
2715	struct ib_mad_send_wr_private *mad_send_wr;
2716	unsigned long flags;
2717	int active;
2718
2719	if (!send_buf)
2720		return -EINVAL;
2721
2722	mad_agent_priv = container_of(send_buf->mad_agent,
2723				      struct ib_mad_agent_private, agent);
2724	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2725	mad_send_wr = find_send_wr(mad_agent_priv, send_buf);
2726	if (!mad_send_wr || mad_send_wr->state == IB_MAD_STATE_CANCELED) {
2727		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2728		return -EINVAL;
2729	}
2730
2731	active = ((mad_send_wr->state == IB_MAD_STATE_SEND_START) ||
2732		  (mad_send_wr->state == IB_MAD_STATE_EARLY_RESP) ||
2733		  (mad_send_wr->state == IB_MAD_STATE_QUEUED && timeout_ms));
2734	if (!timeout_ms)
2735		change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
2736
2737	mad_send_wr->send_buf.timeout_ms = timeout_ms;
2738	if (active)
2739		mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
2740	else
2741		ib_reset_mad_timeout(mad_send_wr, timeout_ms);
2742
2743	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2744	return 0;
2745}
2746EXPORT_SYMBOL(ib_modify_mad);
2747
2748static void local_completions(struct work_struct *work)
2749{
2750	struct ib_mad_agent_private *mad_agent_priv;
2751	struct ib_mad_local_private *local;
2752	struct ib_mad_agent_private *recv_mad_agent;
2753	unsigned long flags;
2754	int free_mad;
2755	struct ib_wc wc;
2756	struct ib_mad_send_wc mad_send_wc;
2757	bool opa;
2758
2759	mad_agent_priv =
2760		container_of(work, struct ib_mad_agent_private, local_work);
2761
2762	opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
2763			       mad_agent_priv->qp_info->port_priv->port_num);
2764
2765	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2766	while (!list_empty(&mad_agent_priv->local_list)) {
2767		local = list_entry(mad_agent_priv->local_list.next,
2768				   struct ib_mad_local_private,
2769				   completion_list);
2770		list_del(&local->completion_list);
2771		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2772		free_mad = 0;
2773		if (local->mad_priv) {
2774			u8 base_version;
2775			recv_mad_agent = local->recv_mad_agent;
2776			if (!recv_mad_agent) {
2777				dev_err(&mad_agent_priv->agent.device->dev,
2778					"No receive MAD agent for local completion\n");
2779				free_mad = 1;
2780				goto local_send_completion;
2781			}
2782
2783			/*
2784			 * Defined behavior is to complete response
2785			 * before request
2786			 */
2787			build_smp_wc(recv_mad_agent->agent.qp,
2788				     local->mad_send_wr->send_wr.wr.wr_cqe,
2789				     be16_to_cpu(IB_LID_PERMISSIVE),
2790				     local->mad_send_wr->send_wr.pkey_index,
2791				     recv_mad_agent->agent.port_num, &wc);
2792
2793			local->mad_priv->header.recv_wc.wc = &wc;
2794
2795			base_version = ((struct ib_mad_hdr *)(local->mad_priv->mad))->base_version;
2796			if (opa && base_version == OPA_MGMT_BASE_VERSION) {
2797				local->mad_priv->header.recv_wc.mad_len = local->return_wc_byte_len;
2798				local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
2799			} else {
2800				local->mad_priv->header.recv_wc.mad_len = sizeof(struct ib_mad);
2801				local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
2802			}
2803
2804			INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list);
2805			list_add(&local->mad_priv->header.recv_wc.recv_buf.list,
2806				 &local->mad_priv->header.recv_wc.rmpp_list);
2807			local->mad_priv->header.recv_wc.recv_buf.grh = NULL;
2808			local->mad_priv->header.recv_wc.recv_buf.mad =
2809						(struct ib_mad *)local->mad_priv->mad;
2810			recv_mad_agent->agent.recv_handler(
2811						&recv_mad_agent->agent,
2812						&local->mad_send_wr->send_buf,
2813						&local->mad_priv->header.recv_wc);
2814			spin_lock_irqsave(&recv_mad_agent->lock, flags);
2815			deref_mad_agent(recv_mad_agent);
2816			spin_unlock_irqrestore(&recv_mad_agent->lock, flags);
2817		}
2818
2819local_send_completion:
2820		/* Complete send */
2821		mad_send_wc.status = IB_WC_SUCCESS;
2822		mad_send_wc.vendor_err = 0;
2823		mad_send_wc.send_buf = &local->mad_send_wr->send_buf;
2824		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2825						   &mad_send_wc);
2826
2827		spin_lock_irqsave(&mad_agent_priv->lock, flags);
2828		deref_mad_agent(mad_agent_priv);
2829		if (free_mad)
2830			kfree(local->mad_priv);
2831		kfree(local);
2832	}
2833	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2834}
2835
2836static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
2837{
2838	int ret;
2839
2840	if (!mad_send_wr->retries_left)
2841		return -ETIMEDOUT;
2842
2843	mad_send_wr->retries_left--;
2844	mad_send_wr->send_buf.retries++;
2845
2846	mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);
2847	if (mad_send_wr->is_solicited_fc &&
2848	    !list_empty(&mad_send_wr->mad_agent_priv->backlog_list)) {
2849		change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
2850		return 0;
2851	}
2852
2853	if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) {
2854		ret = ib_retry_rmpp(mad_send_wr);
2855		switch (ret) {
2856		case IB_RMPP_RESULT_UNHANDLED:
2857			ret = ib_send_mad(mad_send_wr);
2858			break;
2859		case IB_RMPP_RESULT_CONSUMED:
2860			ret = 0;
2861			break;
2862		default:
2863			ret = -ECOMM;
2864			break;
2865		}
2866	} else
2867		ret = ib_send_mad(mad_send_wr);
2868
2869	if (!ret)
2870		change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
2871
2872	return ret;
2873}
2874
2875static void timeout_sends(struct work_struct *work)
2876{
2877	struct ib_mad_send_wr_private *mad_send_wr;
2878	struct ib_mad_agent_private *mad_agent_priv;
2879	struct list_head timeout_list;
2880	struct list_head cancel_list;
2881	struct list_head *list_item;
2882	unsigned long flags, delay;
2883
2884	mad_agent_priv = container_of(work, struct ib_mad_agent_private,
2885				      timed_work.work);
2886	INIT_LIST_HEAD(&timeout_list);
2887	INIT_LIST_HEAD(&cancel_list);
2888
2889	spin_lock_irqsave(&mad_agent_priv->lock, flags);
2890	while (!list_empty(&mad_agent_priv->wait_list)) {
2891		mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
2892					 struct ib_mad_send_wr_private,
2893					 agent_list);
2894
2895		if (time_after(mad_send_wr->timeout, jiffies)) {
2896			delay = mad_send_wr->timeout - jiffies;
2897			if ((long)delay <= 0)
2898				delay = 1;
2899			queue_delayed_work(mad_agent_priv->qp_info->
2900					   port_priv->wq,
2901					   &mad_agent_priv->timed_work, delay);
2902			break;
2903		}
2904
2905		if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
2906			list_item = &cancel_list;
2907		else if (retry_send(mad_send_wr))
2908			list_item = &timeout_list;
2909		else
2910			continue;
2911
2912		change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2913		list_add_tail(&mad_send_wr->agent_list, list_item);
2914	}
2915
2916	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2917	process_backlog_mads(mad_agent_priv);
2918	clear_mad_error_list(&timeout_list, IB_WC_RESP_TIMEOUT_ERR,
2919			     mad_agent_priv);
2920	clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
2921}
2922
2923/*
2924 * Allocate receive MADs and post receive WRs for them
2925 */
2926static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
2927				    struct ib_mad_private *mad)
2928{
2929	unsigned long flags;
2930	struct ib_mad_private *mad_priv;
2931	struct ib_sge sg_list;
2932	struct ib_recv_wr recv_wr;
2933	struct ib_mad_queue *recv_queue = &qp_info->recv_queue;
2934	int ret = 0;
2935
2936	/* Initialize common scatter list fields */
2937	sg_list.lkey = qp_info->port_priv->pd->local_dma_lkey;
2938
2939	/* Initialize common receive WR fields */
2940	recv_wr.next = NULL;
2941	recv_wr.sg_list = &sg_list;
2942	recv_wr.num_sge = 1;
2943
2944	while (true) {
2945		/* Allocate and map receive buffer */
2946		if (mad) {
2947			mad_priv = mad;
2948			mad = NULL;
2949		} else {
2950			mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv),
2951						     GFP_ATOMIC);
2952			if (!mad_priv)
2953				return -ENOMEM;
2954		}
2955		sg_list.length = mad_priv_dma_size(mad_priv);
2956		sg_list.addr = ib_dma_map_single(qp_info->port_priv->device,
2957						 &mad_priv->grh,
2958						 mad_priv_dma_size(mad_priv),
2959						 DMA_FROM_DEVICE);
2960		if (unlikely(ib_dma_mapping_error(qp_info->port_priv->device,
2961						  sg_list.addr))) {
2962			ret = -ENOMEM;
2963			goto free_mad_priv;
2964		}
2965		mad_priv->header.mapping = sg_list.addr;
2966		mad_priv->header.mad_list.mad_queue = recv_queue;
2967		mad_priv->header.mad_list.cqe.done = ib_mad_recv_done;
2968		recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe;
2969		spin_lock_irqsave(&recv_queue->lock, flags);
2970		if (recv_queue->count >= recv_queue->max_active) {
2971			/* Fully populated the receive queue */
2972			spin_unlock_irqrestore(&recv_queue->lock, flags);
2973			break;
2974		}
2975		recv_queue->count++;
2976		list_add_tail(&mad_priv->header.mad_list.list,
2977			      &recv_queue->list);
2978		spin_unlock_irqrestore(&recv_queue->lock, flags);
2979
2980		ret = ib_post_recv(qp_info->qp, &recv_wr, NULL);
2981		if (ret) {
2982			spin_lock_irqsave(&recv_queue->lock, flags);
2983			list_del(&mad_priv->header.mad_list.list);
2984			recv_queue->count--;
2985			spin_unlock_irqrestore(&recv_queue->lock, flags);
2986			dev_err(&qp_info->port_priv->device->dev,
2987				"ib_post_recv failed: %d\n", ret);
2988			break;
2989		}
2990	}
2991
2992	ib_dma_unmap_single(qp_info->port_priv->device,
2993			    mad_priv->header.mapping,
2994			    mad_priv_dma_size(mad_priv), DMA_FROM_DEVICE);
2995free_mad_priv:
2996	kfree(mad_priv);
2997	return ret;
2998}
2999
3000/*
3001 * Return all the posted receive MADs
3002 */
3003static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info)
3004{
3005	struct ib_mad_private_header *mad_priv_hdr;
3006	struct ib_mad_private *recv;
3007	struct ib_mad_list_head *mad_list;
3008
3009	if (!qp_info->qp)
3010		return;
3011
3012	while (!list_empty(&qp_info->recv_queue.list)) {
3013
3014		mad_list = list_entry(qp_info->recv_queue.list.next,
3015				      struct ib_mad_list_head, list);
3016		mad_priv_hdr = container_of(mad_list,
3017					    struct ib_mad_private_header,
3018					    mad_list);
3019		recv = container_of(mad_priv_hdr, struct ib_mad_private,
3020				    header);
3021
3022		/* Remove from posted receive MAD list */
3023		list_del(&mad_list->list);
3024
3025		ib_dma_unmap_single(qp_info->port_priv->device,
3026				    recv->header.mapping,
3027				    mad_priv_dma_size(recv),
3028				    DMA_FROM_DEVICE);
3029		kfree(recv);
3030	}
3031
3032	qp_info->recv_queue.count = 0;
3033}
3034
3035/*
3036 * Start the port
3037 */
3038static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
3039{
3040	int ret, i;
3041	struct ib_qp_attr *attr;
3042	struct ib_qp *qp;
3043	u16 pkey_index;
3044
3045	attr = kmalloc(sizeof *attr, GFP_KERNEL);
3046	if (!attr)
3047		return -ENOMEM;
3048
3049	ret = ib_find_pkey(port_priv->device, port_priv->port_num,
3050			   IB_DEFAULT_PKEY_FULL, &pkey_index);
3051	if (ret)
3052		pkey_index = 0;
3053
3054	for (i = 0; i < IB_MAD_QPS_CORE; i++) {
3055		qp = port_priv->qp_info[i].qp;
3056		if (!qp)
3057			continue;
3058
3059		/*
3060		 * PKey index for QP1 is irrelevant but
3061		 * one is needed for the Reset to Init transition
3062		 */
3063		attr->qp_state = IB_QPS_INIT;
3064		attr->pkey_index = pkey_index;
3065		attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY;
3066		ret = ib_modify_qp(qp, attr, IB_QP_STATE |
3067					     IB_QP_PKEY_INDEX | IB_QP_QKEY);
3068		if (ret) {
3069			dev_err(&port_priv->device->dev,
3070				"Couldn't change QP%d state to INIT: %d\n",
3071				i, ret);
3072			goto out;
3073		}
3074
3075		attr->qp_state = IB_QPS_RTR;
3076		ret = ib_modify_qp(qp, attr, IB_QP_STATE);
3077		if (ret) {
3078			dev_err(&port_priv->device->dev,
3079				"Couldn't change QP%d state to RTR: %d\n",
3080				i, ret);
3081			goto out;
3082		}
3083
3084		attr->qp_state = IB_QPS_RTS;
3085		attr->sq_psn = IB_MAD_SEND_Q_PSN;
3086		ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN);
3087		if (ret) {
3088			dev_err(&port_priv->device->dev,
3089				"Couldn't change QP%d state to RTS: %d\n",
3090				i, ret);
3091			goto out;
3092		}
3093	}
3094
3095	ret = ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP);
3096	if (ret) {
3097		dev_err(&port_priv->device->dev,
3098			"Failed to request completion notification: %d\n",
3099			ret);
3100		goto out;
3101	}
3102
3103	for (i = 0; i < IB_MAD_QPS_CORE; i++) {
3104		if (!port_priv->qp_info[i].qp)
3105			continue;
3106
3107		ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL);
3108		if (ret) {
3109			dev_err(&port_priv->device->dev,
3110				"Couldn't post receive WRs\n");
3111			goto out;
3112		}
3113	}
3114out:
3115	kfree(attr);
3116	return ret;
3117}
3118
3119static void qp_event_handler(struct ib_event *event, void *qp_context)
3120{
3121	struct ib_mad_qp_info	*qp_info = qp_context;
3122
3123	/* It's worse than that! He's dead, Jim! */
3124	dev_err(&qp_info->port_priv->device->dev,
3125		"Fatal error (%d) on MAD QP (%u)\n",
3126		event->event, qp_info->qp->qp_num);
3127}
3128
3129static void init_mad_queue(struct ib_mad_qp_info *qp_info,
3130			   struct ib_mad_queue *mad_queue)
3131{
3132	mad_queue->qp_info = qp_info;
3133	mad_queue->count = 0;
3134	spin_lock_init(&mad_queue->lock);
3135	INIT_LIST_HEAD(&mad_queue->list);
3136}
3137
3138static void init_mad_qp(struct ib_mad_port_private *port_priv,
3139			struct ib_mad_qp_info *qp_info)
3140{
3141	qp_info->port_priv = port_priv;
3142	init_mad_queue(qp_info, &qp_info->send_queue);
3143	init_mad_queue(qp_info, &qp_info->recv_queue);
3144	INIT_LIST_HEAD(&qp_info->overflow_list);
3145}
3146
3147static int create_mad_qp(struct ib_mad_qp_info *qp_info,
3148			 enum ib_qp_type qp_type)
3149{
3150	struct ib_qp_init_attr	qp_init_attr;
3151	int ret;
3152
3153	memset(&qp_init_attr, 0, sizeof qp_init_attr);
3154	qp_init_attr.send_cq = qp_info->port_priv->cq;
3155	qp_init_attr.recv_cq = qp_info->port_priv->cq;
3156	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
3157	qp_init_attr.cap.max_send_wr = mad_sendq_size;
3158	qp_init_attr.cap.max_recv_wr = mad_recvq_size;
3159	qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
3160	qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
3161	qp_init_attr.qp_type = qp_type;
3162	qp_init_attr.port_num = qp_info->port_priv->port_num;
3163	qp_init_attr.qp_context = qp_info;
3164	qp_init_attr.event_handler = qp_event_handler;
3165	qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr);
3166	if (IS_ERR(qp_info->qp)) {
3167		dev_err(&qp_info->port_priv->device->dev,
3168			"Couldn't create ib_mad QP%d\n",
3169			get_spl_qp_index(qp_type));
3170		ret = PTR_ERR(qp_info->qp);
3171		goto error;
3172	}
3173	/* Use minimum queue sizes unless the CQ is resized */
3174	qp_info->send_queue.max_active = mad_sendq_size;
3175	qp_info->recv_queue.max_active = mad_recvq_size;
3176	return 0;
3177
3178error:
3179	return ret;
3180}
3181
3182static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)
3183{
3184	if (!qp_info->qp)
3185		return;
3186
3187	ib_destroy_qp(qp_info->qp);
3188}
3189
3190/*
3191 * Open the port
3192 * Create the QP, PD, MR, and CQ if needed
3193 */
3194static int ib_mad_port_open(struct ib_device *device,
3195			    u32 port_num)
3196{
3197	int ret, cq_size;
3198	struct ib_mad_port_private *port_priv;
3199	unsigned long flags;
3200	int has_smi;
3201
3202	if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE))
3203		return -EFAULT;
3204
3205	if (WARN_ON(rdma_cap_opa_mad(device, port_num) &&
3206		    rdma_max_mad_size(device, port_num) < OPA_MGMT_MAD_SIZE))
3207		return -EFAULT;
3208
3209	/* Create new device info */
3210	port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
3211	if (!port_priv)
3212		return -ENOMEM;
3213
3214	port_priv->device = device;
3215	port_priv->port_num = port_num;
3216	spin_lock_init(&port_priv->reg_lock);
3217	init_mad_qp(port_priv, &port_priv->qp_info[0]);
3218	init_mad_qp(port_priv, &port_priv->qp_info[1]);
3219
3220	cq_size = mad_sendq_size + mad_recvq_size;
3221	has_smi = rdma_cap_ib_smi(device, port_num);
3222	if (has_smi)
3223		cq_size *= 2;
3224
3225	port_priv->pd = ib_alloc_pd(device, 0);
3226	if (IS_ERR(port_priv->pd)) {
3227		dev_err(&device->dev, "Couldn't create ib_mad PD\n");
3228		ret = PTR_ERR(port_priv->pd);
3229		goto error3;
3230	}
3231
3232	port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0,
3233			IB_POLL_UNBOUND_WORKQUEUE);
3234	if (IS_ERR(port_priv->cq)) {
3235		dev_err(&device->dev, "Couldn't create ib_mad CQ\n");
3236		ret = PTR_ERR(port_priv->cq);
3237		goto error4;
3238	}
3239
3240	if (has_smi) {
3241		ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI);
3242		if (ret)
3243			goto error6;
3244	}
3245
3246	if (rdma_cap_ib_cm(device, port_num)) {
3247		ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI);
3248		if (ret)
3249			goto error7;
3250	}
3251
3252	port_priv->wq = alloc_ordered_workqueue("ib_mad%u", WQ_MEM_RECLAIM,
3253						port_num);
3254	if (!port_priv->wq) {
3255		ret = -ENOMEM;
3256		goto error8;
3257	}
3258
3259	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3260	list_add_tail(&port_priv->port_list, &ib_mad_port_list);
3261	spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3262
3263	ret = ib_mad_port_start(port_priv);
3264	if (ret) {
3265		dev_err(&device->dev, "Couldn't start port\n");
3266		goto error9;
3267	}
3268
3269	return 0;
3270
3271error9:
3272	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3273	list_del_init(&port_priv->port_list);
3274	spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3275
3276	destroy_workqueue(port_priv->wq);
3277error8:
3278	destroy_mad_qp(&port_priv->qp_info[1]);
3279error7:
3280	destroy_mad_qp(&port_priv->qp_info[0]);
3281error6:
3282	ib_free_cq(port_priv->cq);
3283	cleanup_recv_queue(&port_priv->qp_info[1]);
3284	cleanup_recv_queue(&port_priv->qp_info[0]);
3285error4:
3286	ib_dealloc_pd(port_priv->pd);
3287error3:
3288	kfree(port_priv);
3289
3290	return ret;
3291}
3292
3293/*
3294 * Close the port
3295 * If there are no classes using the port, free the port
3296 * resources (CQ, MR, PD, QP) and remove the port's info structure
3297 */
3298static int ib_mad_port_close(struct ib_device *device, u32 port_num)
3299{
3300	struct ib_mad_port_private *port_priv;
3301	unsigned long flags;
3302
3303	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3304	port_priv = __ib_get_mad_port(device, port_num);
3305	if (port_priv == NULL) {
3306		spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3307		dev_err(&device->dev, "Port %u not found\n", port_num);
3308		return -ENODEV;
3309	}
3310	list_del_init(&port_priv->port_list);
3311	spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3312
3313	destroy_workqueue(port_priv->wq);
3314	destroy_mad_qp(&port_priv->qp_info[1]);
3315	destroy_mad_qp(&port_priv->qp_info[0]);
3316	ib_free_cq(port_priv->cq);
3317	ib_dealloc_pd(port_priv->pd);
3318	cleanup_recv_queue(&port_priv->qp_info[1]);
3319	cleanup_recv_queue(&port_priv->qp_info[0]);
3320	/* XXX: Handle deallocation of MAD registration tables */
3321
3322	kfree(port_priv);
3323
3324	return 0;
3325}
3326
3327static int ib_mad_init_device(struct ib_device *device)
3328{
3329	int start, i;
3330	unsigned int count = 0;
3331	int ret;
3332
3333	start = rdma_start_port(device);
3334
3335	for (i = start; i <= rdma_end_port(device); i++) {
3336		if (!rdma_cap_ib_mad(device, i))
3337			continue;
3338
3339		ret = ib_mad_port_open(device, i);
3340		if (ret) {
3341			dev_err(&device->dev, "Couldn't open port %d\n", i);
3342			goto error;
3343		}
3344		ret = ib_agent_port_open(device, i);
3345		if (ret) {
3346			dev_err(&device->dev,
3347				"Couldn't open port %d for agents\n", i);
3348			goto error_agent;
3349		}
3350		count++;
3351	}
3352	if (!count)
3353		return -EOPNOTSUPP;
3354
3355	return 0;
3356
3357error_agent:
3358	if (ib_mad_port_close(device, i))
3359		dev_err(&device->dev, "Couldn't close port %d\n", i);
3360
3361error:
3362	while (--i >= start) {
3363		if (!rdma_cap_ib_mad(device, i))
3364			continue;
3365
3366		if (ib_agent_port_close(device, i))
3367			dev_err(&device->dev,
3368				"Couldn't close port %d for agents\n", i);
3369		if (ib_mad_port_close(device, i))
3370			dev_err(&device->dev, "Couldn't close port %d\n", i);
3371	}
3372	return ret;
3373}
3374
3375static void ib_mad_remove_device(struct ib_device *device, void *client_data)
3376{
3377	unsigned int i;
3378
3379	rdma_for_each_port (device, i) {
3380		if (!rdma_cap_ib_mad(device, i))
3381			continue;
3382
3383		if (ib_agent_port_close(device, i))
3384			dev_err(&device->dev,
3385				"Couldn't close port %u for agents\n", i);
3386		if (ib_mad_port_close(device, i))
3387			dev_err(&device->dev, "Couldn't close port %u\n", i);
3388	}
3389}
3390
3391static struct ib_client mad_client = {
3392	.name   = "mad",
3393	.add = ib_mad_init_device,
3394	.remove = ib_mad_remove_device
3395};
3396
3397int ib_mad_init(void)
3398{
3399	mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE);
3400	mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE);
3401
3402	mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE);
3403	mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE);
3404
3405	INIT_LIST_HEAD(&ib_mad_port_list);
3406
3407	if (ib_register_client(&mad_client)) {
3408		pr_err("Couldn't register ib_mad client\n");
3409		return -EINVAL;
3410	}
3411
3412	return 0;
3413}
3414
3415void ib_mad_cleanup(void)
3416{
3417	ib_unregister_client(&mad_client);
3418}