rxe_qp.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/*
  3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
  4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  5 */
  6
  7#include <linux/skbuff.h>
  8#include <linux/delay.h>
  9#include <linux/sched.h>
 10#include <linux/vmalloc.h>
 11#include <rdma/uverbs_ioctl.h>
 12
 13#include "rxe.h"
 14#include "rxe_loc.h"
 15#include "rxe_queue.h"
 16#include "rxe_task.h"
 17
 18#ifdef CONFIG_DEBUG_LOCK_ALLOC
 19/*
 20 * lockdep can detect false positive circular dependencies
 21 * when there are user-space socket API users or in kernel
 22 * users switching between a tcp and rdma transport.
 23 * Maybe also switching between siw and rxe may cause
 24 * problems as per default sockets are only classified
 25 * by family and not by ip protocol. And there might
 26 * be different locks used between the application
 27 * and the low level sockets.
 28 *
 29 * Problems were seen with ksmbd.ko and cifs.ko,
 30 * switching transports, use git blame to find
 31 * more details.
 32 */
 33static struct lock_class_key rxe_send_sk_key[2];
 34static struct lock_class_key rxe_send_slock_key[2];
 35#endif /* CONFIG_DEBUG_LOCK_ALLOC */
 36
 37static inline void rxe_reclassify_send_socket(struct socket *sock)
 38{
 39#ifdef CONFIG_DEBUG_LOCK_ALLOC
 40	struct sock *sk = sock->sk;
 41
 42	if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
 43		return;
 44
 45	switch (sk->sk_family) {
 46	case AF_INET:
 47		sock_lock_init_class_and_name(sk,
 48					      "slock-AF_INET-RDMA-RXE-SEND",
 49					      &rxe_send_slock_key[0],
 50					      "sk_lock-AF_INET-RDMA-RXE-SEND",
 51					      &rxe_send_sk_key[0]);
 52		break;
 53	case AF_INET6:
 54		sock_lock_init_class_and_name(sk,
 55					      "slock-AF_INET6-RDMA-RXE-SEND",
 56					      &rxe_send_slock_key[1],
 57					      "sk_lock-AF_INET6-RDMA-RXE-SEND",
 58					      &rxe_send_sk_key[1]);
 59		break;
 60	default:
 61		WARN_ON_ONCE(1);
 62	}
 63#endif /* CONFIG_DEBUG_LOCK_ALLOC */
 64}
 65
 66static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap,
 67			  int has_srq)
 68{
 69	if (cap->max_send_wr > rxe->attr.max_qp_wr) {
 70		rxe_dbg_dev(rxe, "invalid send wr = %u > %d\n",
 71			 cap->max_send_wr, rxe->attr.max_qp_wr);
 72		goto err1;
 73	}
 74
 75	if (cap->max_send_sge > rxe->attr.max_send_sge) {
 76		rxe_dbg_dev(rxe, "invalid send sge = %u > %d\n",
 77			 cap->max_send_sge, rxe->attr.max_send_sge);
 78		goto err1;
 79	}
 80
 81	if (!has_srq) {
 82		if (cap->max_recv_wr > rxe->attr.max_qp_wr) {
 83			rxe_dbg_dev(rxe, "invalid recv wr = %u > %d\n",
 84				 cap->max_recv_wr, rxe->attr.max_qp_wr);
 85			goto err1;
 86		}
 87
 88		if (cap->max_recv_sge > rxe->attr.max_recv_sge) {
 89			rxe_dbg_dev(rxe, "invalid recv sge = %u > %d\n",
 90				 cap->max_recv_sge, rxe->attr.max_recv_sge);
 91			goto err1;
 92		}
 93	}
 94
 95	if (cap->max_inline_data > rxe->max_inline_data) {
 96		rxe_dbg_dev(rxe, "invalid max inline data = %u > %d\n",
 97			 cap->max_inline_data, rxe->max_inline_data);
 98		goto err1;
 99	}
100
101	return 0;
102
103err1:
104	return -EINVAL;
105}
106
107int rxe_qp_chk_init(struct rxe_dev *rxe, struct ib_qp_init_attr *init)
108{
109	struct ib_qp_cap *cap = &init->cap;
110	struct rxe_port *port;
111	int port_num = init->port_num;
112
113	switch (init->qp_type) {
114	case IB_QPT_GSI:
115	case IB_QPT_RC:
116	case IB_QPT_UC:
117	case IB_QPT_UD:
118		break;
119	default:
120		return -EOPNOTSUPP;
121	}
122
123	if (!init->recv_cq || !init->send_cq) {
124		rxe_dbg_dev(rxe, "missing cq\n");
125		goto err1;
126	}
127
128	if (rxe_qp_chk_cap(rxe, cap, !!init->srq))
129		goto err1;
130
131	if (init->qp_type == IB_QPT_GSI) {
132		if (!rdma_is_port_valid(&rxe->ib_dev, port_num)) {
133			rxe_dbg_dev(rxe, "invalid port = %d\n", port_num);
134			goto err1;
135		}
136
137		port = &rxe->port;
138
139		if (init->qp_type == IB_QPT_GSI && port->qp_gsi_index) {
140			rxe_dbg_dev(rxe, "GSI QP exists for port %d\n", port_num);
141			goto err1;
142		}
143	}
144
145	return 0;
146
147err1:
148	return -EINVAL;
149}
150
151static int alloc_rd_atomic_resources(struct rxe_qp *qp, unsigned int n)
152{
153	qp->resp.res_head = 0;
154	qp->resp.res_tail = 0;
155	qp->resp.resources = kcalloc(n, sizeof(struct resp_res), GFP_KERNEL);
156
157	if (!qp->resp.resources)
158		return -ENOMEM;
159
160	return 0;
161}
162
163static void free_rd_atomic_resources(struct rxe_qp *qp)
164{
165	if (qp->resp.resources) {
166		int i;
167
168		for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
169			struct resp_res *res = &qp->resp.resources[i];
170
171			free_rd_atomic_resource(res);
172		}
173		kfree(qp->resp.resources);
174		qp->resp.resources = NULL;
175	}
176}
177
178void free_rd_atomic_resource(struct resp_res *res)
179{
180	res->type = 0;
181}
182
183static void cleanup_rd_atomic_resources(struct rxe_qp *qp)
184{
185	int i;
186	struct resp_res *res;
187
188	if (qp->resp.resources) {
189		for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
190			res = &qp->resp.resources[i];
191			free_rd_atomic_resource(res);
192		}
193	}
194}
195
196static void rxe_qp_init_misc(struct rxe_dev *rxe, struct rxe_qp *qp,
197			     struct ib_qp_init_attr *init)
198{
199	struct rxe_port *port;
200	u32 qpn;
201
202	qp->sq_sig_type		= init->sq_sig_type;
203	qp->attr.path_mtu	= 1;
204	qp->mtu			= ib_mtu_enum_to_int(qp->attr.path_mtu);
205
206	qpn			= qp->elem.index;
207	port			= &rxe->port;
208
209	switch (init->qp_type) {
210	case IB_QPT_GSI:
211		qp->ibqp.qp_num		= 1;
212		port->qp_gsi_index	= qpn;
213		qp->attr.port_num	= init->port_num;
214		break;
215
216	default:
217		qp->ibqp.qp_num		= qpn;
218		break;
219	}
220
221	spin_lock_init(&qp->state_lock);
222
223	spin_lock_init(&qp->sq.sq_lock);
224	spin_lock_init(&qp->rq.producer_lock);
225	spin_lock_init(&qp->rq.consumer_lock);
226
227	skb_queue_head_init(&qp->req_pkts);
228	skb_queue_head_init(&qp->resp_pkts);
229
230	atomic_set(&qp->ssn, 0);
231	atomic_set(&qp->skb_out, 0);
232}
233
234static int rxe_init_sq(struct rxe_qp *qp, struct ib_qp_init_attr *init,
235		       struct ib_udata *udata,
236		       struct rxe_create_qp_resp __user *uresp)
237{
238	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
239	int wqe_size;
240	int err;
241
242	qp->sq.max_wr = init->cap.max_send_wr;
243	wqe_size = max_t(int, init->cap.max_send_sge * sizeof(struct ib_sge),
244			 init->cap.max_inline_data);
245	qp->sq.max_sge = wqe_size / sizeof(struct ib_sge);
246	qp->sq.max_inline = wqe_size;
247	wqe_size += sizeof(struct rxe_send_wqe);
248
249	qp->sq.queue = rxe_queue_init(rxe, &qp->sq.max_wr, wqe_size,
250				      QUEUE_TYPE_FROM_CLIENT);
251	if (!qp->sq.queue) {
252		rxe_err_qp(qp, "Unable to allocate send queue\n");
253		err = -ENOMEM;
254		goto err_out;
255	}
256
257	/* prepare info for caller to mmap send queue if user space qp */
258	err = do_mmap_info(rxe, uresp ? &uresp->sq_mi : NULL, udata,
259			   qp->sq.queue->buf, qp->sq.queue->buf_size,
260			   &qp->sq.queue->ip);
261	if (err) {
262		rxe_err_qp(qp, "do_mmap_info failed, err = %d\n", err);
263		goto err_free;
264	}
265
266	/* return actual capabilities to caller which may be larger
267	 * than requested
268	 */
269	init->cap.max_send_wr = qp->sq.max_wr;
270	init->cap.max_send_sge = qp->sq.max_sge;
271	init->cap.max_inline_data = qp->sq.max_inline;
272
273	return 0;
274
275err_free:
276	vfree(qp->sq.queue->buf);
277	kfree(qp->sq.queue);
278	qp->sq.queue = NULL;
279err_out:
280	return err;
281}
282
283static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
284			   struct ib_qp_init_attr *init, struct ib_udata *udata,
285			   struct rxe_create_qp_resp __user *uresp)
286{
287	int err;
288
289	/* if we don't finish qp create make sure queue is valid */
290	skb_queue_head_init(&qp->req_pkts);
291
292	err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk);
293	if (err < 0)
294		return err;
295	rxe_reclassify_send_socket(qp->sk);
296	qp->sk->sk->sk_user_data = qp;
297
298	/* pick a source UDP port number for this QP based on
299	 * the source QPN. this spreads traffic for different QPs
300	 * across different NIC RX queues (while using a single
301	 * flow for a given QP to maintain packet order).
302	 * the port number must be in the Dynamic Ports range
303	 * (0xc000 - 0xffff).
304	 */
305	qp->src_port = RXE_ROCE_V2_SPORT + (hash_32(qp_num(qp), 14) & 0x3fff);
306
307	err = rxe_init_sq(qp, init, udata, uresp);
308	if (err)
309		return err;
310
311	qp->req.wqe_index = queue_get_producer(qp->sq.queue,
312					       QUEUE_TYPE_FROM_CLIENT);
313
314	qp->req.opcode		= -1;
315	qp->comp.opcode		= -1;
316
317	rxe_init_task(&qp->send_task, qp, rxe_sender);
318
319	qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */
320	if (init->qp_type == IB_QPT_RC) {
321		timer_setup(&qp->rnr_nak_timer, rnr_nak_timer, 0);
322		timer_setup(&qp->retrans_timer, retransmit_timer, 0);
323	}
324	return 0;
325}
326
327static int rxe_init_rq(struct rxe_qp *qp, struct ib_qp_init_attr *init,
328		       struct ib_udata *udata,
329		       struct rxe_create_qp_resp __user *uresp)
330{
331	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
332	int wqe_size;
333	int err;
334
335	qp->rq.max_wr = init->cap.max_recv_wr;
336	qp->rq.max_sge = init->cap.max_recv_sge;
337	wqe_size = sizeof(struct rxe_recv_wqe) +
338				qp->rq.max_sge*sizeof(struct ib_sge);
339
340	qp->rq.queue = rxe_queue_init(rxe, &qp->rq.max_wr, wqe_size,
341				      QUEUE_TYPE_FROM_CLIENT);
342	if (!qp->rq.queue) {
343		rxe_err_qp(qp, "Unable to allocate recv queue\n");
344		err = -ENOMEM;
345		goto err_out;
346	}
347
348	/* prepare info for caller to mmap recv queue if user space qp */
349	err = do_mmap_info(rxe, uresp ? &uresp->rq_mi : NULL, udata,
350			   qp->rq.queue->buf, qp->rq.queue->buf_size,
351			   &qp->rq.queue->ip);
352	if (err) {
353		rxe_err_qp(qp, "do_mmap_info failed, err = %d\n", err);
354		goto err_free;
355	}
356
357	/* return actual capabilities to caller which may be larger
358	 * than requested
359	 */
360	init->cap.max_recv_wr = qp->rq.max_wr;
361
362	return 0;
363
364err_free:
365	vfree(qp->rq.queue->buf);
366	kfree(qp->rq.queue);
367	qp->rq.queue = NULL;
368err_out:
369	return err;
370}
371
372static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
373			    struct ib_qp_init_attr *init,
374			    struct ib_udata *udata,
375			    struct rxe_create_qp_resp __user *uresp)
376{
377	int err;
378
379	/* if we don't finish qp create make sure queue is valid */
380	skb_queue_head_init(&qp->resp_pkts);
381
382	if (!qp->srq) {
383		err = rxe_init_rq(qp, init, udata, uresp);
384		if (err)
385			return err;
386	}
387
388	rxe_init_task(&qp->recv_task, qp, rxe_receiver);
389
390	qp->resp.opcode		= OPCODE_NONE;
391	qp->resp.msn		= 0;
392
393	return 0;
394}
395
396/* called by the create qp verb */
397int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd,
398		     struct ib_qp_init_attr *init,
399		     struct rxe_create_qp_resp __user *uresp,
400		     struct ib_pd *ibpd,
401		     struct ib_udata *udata)
402{
403	int err;
404	struct rxe_cq *rcq = to_rcq(init->recv_cq);
405	struct rxe_cq *scq = to_rcq(init->send_cq);
406	struct rxe_srq *srq = init->srq ? to_rsrq(init->srq) : NULL;
407	unsigned long flags;
408
409	rxe_get(pd);
410	rxe_get(rcq);
411	rxe_get(scq);
412	if (srq)
413		rxe_get(srq);
414
415	qp->pd = pd;
416	qp->rcq = rcq;
417	qp->scq = scq;
418	qp->srq = srq;
419
420	atomic_inc(&rcq->num_wq);
421	atomic_inc(&scq->num_wq);
422
423	rxe_qp_init_misc(rxe, qp, init);
424
425	err = rxe_qp_init_req(rxe, qp, init, udata, uresp);
426	if (err)
427		goto err1;
428
429	err = rxe_qp_init_resp(rxe, qp, init, udata, uresp);
430	if (err)
431		goto err2;
432
433	spin_lock_irqsave(&qp->state_lock, flags);
434	qp->attr.qp_state = IB_QPS_RESET;
435	qp->valid = 1;
436	spin_unlock_irqrestore(&qp->state_lock, flags);
437
438	return 0;
439
440err2:
441	rxe_queue_cleanup(qp->sq.queue);
442	qp->sq.queue = NULL;
443err1:
444	atomic_dec(&rcq->num_wq);
445	atomic_dec(&scq->num_wq);
446
447	qp->pd = NULL;
448	qp->rcq = NULL;
449	qp->scq = NULL;
450	qp->srq = NULL;
451
452	if (srq)
453		rxe_put(srq);
454	rxe_put(scq);
455	rxe_put(rcq);
456	rxe_put(pd);
457
458	return err;
459}
460
461/* called by the query qp verb */
462int rxe_qp_to_init(struct rxe_qp *qp, struct ib_qp_init_attr *init)
463{
464	init->event_handler		= qp->ibqp.event_handler;
465	init->qp_context		= qp->ibqp.qp_context;
466	init->send_cq			= qp->ibqp.send_cq;
467	init->recv_cq			= qp->ibqp.recv_cq;
468	init->srq			= qp->ibqp.srq;
469
470	init->cap.max_send_wr		= qp->sq.max_wr;
471	init->cap.max_send_sge		= qp->sq.max_sge;
472	init->cap.max_inline_data	= qp->sq.max_inline;
473
474	if (!qp->srq) {
475		init->cap.max_recv_wr		= qp->rq.max_wr;
476		init->cap.max_recv_sge		= qp->rq.max_sge;
477	}
478
479	init->sq_sig_type		= qp->sq_sig_type;
480
481	init->qp_type			= qp->ibqp.qp_type;
482	init->port_num			= 1;
483
484	return 0;
485}
486
487int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp,
488		    struct ib_qp_attr *attr, int mask)
489{
490	if (mask & IB_QP_PORT) {
491		if (!rdma_is_port_valid(&rxe->ib_dev, attr->port_num)) {
492			rxe_dbg_qp(qp, "invalid port %d\n", attr->port_num);
493			goto err1;
494		}
495	}
496
497	if (mask & IB_QP_CAP && rxe_qp_chk_cap(rxe, &attr->cap, !!qp->srq))
498		goto err1;
499
500	if (mask & IB_QP_ACCESS_FLAGS) {
501		if (!(qp_type(qp) == IB_QPT_RC || qp_type(qp) == IB_QPT_UC))
502			goto err1;
503		if (attr->qp_access_flags & ~RXE_ACCESS_SUPPORTED_QP)
504			goto err1;
505	}
506
507	if (mask & IB_QP_AV && rxe_av_chk_attr(qp, &attr->ah_attr))
508		goto err1;
509
510	if (mask & IB_QP_ALT_PATH) {
511		if (rxe_av_chk_attr(qp, &attr->alt_ah_attr))
512			goto err1;
513		if (!rdma_is_port_valid(&rxe->ib_dev, attr->alt_port_num))  {
514			rxe_dbg_qp(qp, "invalid alt port %d\n", attr->alt_port_num);
515			goto err1;
516		}
517		if (attr->alt_timeout > 31) {
518			rxe_dbg_qp(qp, "invalid alt timeout %d > 31\n",
519				 attr->alt_timeout);
520			goto err1;
521		}
522	}
523
524	if (mask & IB_QP_PATH_MTU) {
525		struct rxe_port *port = &rxe->port;
526
527		enum ib_mtu max_mtu = port->attr.max_mtu;
528		enum ib_mtu mtu = attr->path_mtu;
529
530		if (mtu > max_mtu) {
531			rxe_dbg_qp(qp, "invalid mtu (%d) > (%d)\n",
532				 ib_mtu_enum_to_int(mtu),
533				 ib_mtu_enum_to_int(max_mtu));
534			goto err1;
535		}
536	}
537
538	if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
539		if (attr->max_rd_atomic > rxe->attr.max_qp_rd_atom) {
540			rxe_dbg_qp(qp, "invalid max_rd_atomic %d > %d\n",
541				 attr->max_rd_atomic,
542				 rxe->attr.max_qp_rd_atom);
543			goto err1;
544		}
545	}
546
547	if (mask & IB_QP_TIMEOUT) {
548		if (attr->timeout > 31) {
549			rxe_dbg_qp(qp, "invalid timeout %d > 31\n",
550					attr->timeout);
551			goto err1;
552		}
553	}
554
555	return 0;
556
557err1:
558	return -EINVAL;
559}
560
561/* move the qp to the reset state */
562static void rxe_qp_reset(struct rxe_qp *qp)
563{
564	/* stop tasks from running */
565	rxe_disable_task(&qp->recv_task);
566	rxe_disable_task(&qp->send_task);
567
568	/* drain work and packet queuesc */
569	rxe_sender(qp);
570	rxe_receiver(qp);
571
572	if (qp->rq.queue)
573		rxe_queue_reset(qp->rq.queue);
574	if (qp->sq.queue)
575		rxe_queue_reset(qp->sq.queue);
576
577	/* cleanup attributes */
578	atomic_set(&qp->ssn, 0);
579	qp->req.opcode = -1;
580	qp->req.need_retry = 0;
581	qp->req.wait_for_rnr_timer = 0;
582	qp->req.noack_pkts = 0;
583	qp->resp.msn = 0;
584	qp->resp.opcode = -1;
585	qp->resp.drop_msg = 0;
586	qp->resp.goto_error = 0;
587	qp->resp.sent_psn_nak = 0;
588
589	if (qp->resp.mr) {
590		rxe_put(qp->resp.mr);
591		qp->resp.mr = NULL;
592	}
593
594	cleanup_rd_atomic_resources(qp);
595
596	/* reenable tasks */
597	rxe_enable_task(&qp->recv_task);
598	rxe_enable_task(&qp->send_task);
599}
600
601/* move the qp to the error state */
602void rxe_qp_error(struct rxe_qp *qp)
603{
604	unsigned long flags;
605
606	spin_lock_irqsave(&qp->state_lock, flags);
607	qp->attr.qp_state = IB_QPS_ERR;
608
609	/* drain work and packet queues */
610	rxe_sched_task(&qp->recv_task);
611	rxe_sched_task(&qp->send_task);
612	spin_unlock_irqrestore(&qp->state_lock, flags);
613}
614
615static void rxe_qp_sqd(struct rxe_qp *qp, struct ib_qp_attr *attr,
616		       int mask)
617{
618	unsigned long flags;
619
620	spin_lock_irqsave(&qp->state_lock, flags);
621	qp->attr.sq_draining = 1;
622	rxe_sched_task(&qp->send_task);
623	spin_unlock_irqrestore(&qp->state_lock, flags);
624}
625
626/* caller should hold qp->state_lock */
627static int __qp_chk_state(struct rxe_qp *qp, struct ib_qp_attr *attr,
628			    int mask)
629{
630	enum ib_qp_state cur_state;
631	enum ib_qp_state new_state;
632
633	cur_state = (mask & IB_QP_CUR_STATE) ?
634				attr->cur_qp_state : qp->attr.qp_state;
635	new_state = (mask & IB_QP_STATE) ?
636				attr->qp_state : cur_state;
637
638	if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask))
639		return -EINVAL;
640
641	if (mask & IB_QP_STATE && cur_state == IB_QPS_SQD) {
642		if (qp->attr.sq_draining && new_state != IB_QPS_ERR)
643			return -EINVAL;
644	}
645
646	return 0;
647}
648
649static const char *const qps2str[] = {
650	[IB_QPS_RESET]	= "RESET",
651	[IB_QPS_INIT]	= "INIT",
652	[IB_QPS_RTR]	= "RTR",
653	[IB_QPS_RTS]	= "RTS",
654	[IB_QPS_SQD]	= "SQD",
655	[IB_QPS_SQE]	= "SQE",
656	[IB_QPS_ERR]	= "ERR",
657};
658
659/* called by the modify qp verb */
660int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
661		     struct ib_udata *udata)
662{
663	int err;
664
665	if (mask & IB_QP_CUR_STATE)
666		qp->attr.cur_qp_state = attr->qp_state;
667
668	if (mask & IB_QP_STATE) {
669		unsigned long flags;
670
671		spin_lock_irqsave(&qp->state_lock, flags);
672		err = __qp_chk_state(qp, attr, mask);
673		if (!err) {
674			qp->attr.qp_state = attr->qp_state;
675			rxe_dbg_qp(qp, "state -> %s\n",
676					qps2str[attr->qp_state]);
677		}
678		spin_unlock_irqrestore(&qp->state_lock, flags);
679
680		if (err)
681			return err;
682
683		switch (attr->qp_state) {
684		case IB_QPS_RESET:
685			rxe_qp_reset(qp);
686			break;
687		case IB_QPS_SQD:
688			rxe_qp_sqd(qp, attr, mask);
689			break;
690		case IB_QPS_ERR:
691			rxe_qp_error(qp);
692			break;
693		default:
694			break;
695		}
696	}
697
698	if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
699		int max_rd_atomic = attr->max_rd_atomic ?
700			roundup_pow_of_two(attr->max_rd_atomic) : 0;
701
702		qp->attr.max_rd_atomic = max_rd_atomic;
703		atomic_set(&qp->req.rd_atomic, max_rd_atomic);
704	}
705
706	if (mask & IB_QP_MAX_DEST_RD_ATOMIC) {
707		int max_dest_rd_atomic = attr->max_dest_rd_atomic ?
708			roundup_pow_of_two(attr->max_dest_rd_atomic) : 0;
709
710		qp->attr.max_dest_rd_atomic = max_dest_rd_atomic;
711
712		free_rd_atomic_resources(qp);
713
714		err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic);
715		if (err)
716			return err;
717	}
718
719	if (mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
720		qp->attr.en_sqd_async_notify = attr->en_sqd_async_notify;
721
722	if (mask & IB_QP_ACCESS_FLAGS)
723		qp->attr.qp_access_flags = attr->qp_access_flags;
724
725	if (mask & IB_QP_PKEY_INDEX)
726		qp->attr.pkey_index = attr->pkey_index;
727
728	if (mask & IB_QP_PORT)
729		qp->attr.port_num = attr->port_num;
730
731	if (mask & IB_QP_QKEY)
732		qp->attr.qkey = attr->qkey;
733
734	if (mask & IB_QP_AV)
735		rxe_init_av(&attr->ah_attr, &qp->pri_av);
736
737	if (mask & IB_QP_ALT_PATH) {
738		rxe_init_av(&attr->alt_ah_attr, &qp->alt_av);
739		qp->attr.alt_port_num = attr->alt_port_num;
740		qp->attr.alt_pkey_index = attr->alt_pkey_index;
741		qp->attr.alt_timeout = attr->alt_timeout;
742	}
743
744	if (mask & IB_QP_PATH_MTU) {
745		qp->attr.path_mtu = attr->path_mtu;
746		qp->mtu = ib_mtu_enum_to_int(attr->path_mtu);
747	}
748
749	if (mask & IB_QP_TIMEOUT) {
750		qp->attr.timeout = attr->timeout;
751		if (attr->timeout == 0) {
752			qp->qp_timeout_jiffies = 0;
753		} else {
754			/* According to the spec, timeout = 4.096 * 2 ^ attr->timeout [us] */
755			int j = nsecs_to_jiffies(4096ULL << attr->timeout);
756
757			qp->qp_timeout_jiffies = j ? j : 1;
758		}
759	}
760
761	if (mask & IB_QP_RETRY_CNT) {
762		qp->attr.retry_cnt = attr->retry_cnt;
763		qp->comp.retry_cnt = attr->retry_cnt;
764		rxe_dbg_qp(qp, "set retry count = %d\n", attr->retry_cnt);
765	}
766
767	if (mask & IB_QP_RNR_RETRY) {
768		qp->attr.rnr_retry = attr->rnr_retry;
769		qp->comp.rnr_retry = attr->rnr_retry;
770		rxe_dbg_qp(qp, "set rnr retry count = %d\n", attr->rnr_retry);
771	}
772
773	if (mask & IB_QP_RQ_PSN) {
774		qp->attr.rq_psn = (attr->rq_psn & BTH_PSN_MASK);
775		qp->resp.psn = qp->attr.rq_psn;
776		rxe_dbg_qp(qp, "set resp psn = 0x%x\n", qp->resp.psn);
777	}
778
779	if (mask & IB_QP_MIN_RNR_TIMER) {
780		qp->attr.min_rnr_timer = attr->min_rnr_timer;
781		rxe_dbg_qp(qp, "set min rnr timer = 0x%x\n",
782			 attr->min_rnr_timer);
783	}
784
785	if (mask & IB_QP_SQ_PSN) {
786		qp->attr.sq_psn = (attr->sq_psn & BTH_PSN_MASK);
787		qp->req.psn = qp->attr.sq_psn;
788		qp->comp.psn = qp->attr.sq_psn;
789		rxe_dbg_qp(qp, "set req psn = 0x%x\n", qp->req.psn);
790	}
791
792	if (mask & IB_QP_PATH_MIG_STATE)
793		qp->attr.path_mig_state = attr->path_mig_state;
794
795	if (mask & IB_QP_DEST_QPN)
796		qp->attr.dest_qp_num = attr->dest_qp_num;
797
798	return 0;
799}
800
801/* called by the query qp verb */
802int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask)
803{
804	unsigned long flags;
805
806	*attr = qp->attr;
807
808	attr->rq_psn				= qp->resp.psn;
809	attr->sq_psn				= qp->req.psn;
810
811	attr->cap.max_send_wr			= qp->sq.max_wr;
812	attr->cap.max_send_sge			= qp->sq.max_sge;
813	attr->cap.max_inline_data		= qp->sq.max_inline;
814
815	if (!qp->srq) {
816		attr->cap.max_recv_wr		= qp->rq.max_wr;
817		attr->cap.max_recv_sge		= qp->rq.max_sge;
818	}
819
820	rxe_av_to_attr(&qp->pri_av, &attr->ah_attr);
821	rxe_av_to_attr(&qp->alt_av, &attr->alt_ah_attr);
822
823	/* Applications that get this state typically spin on it.
824	 * Yield the processor
825	 */
826	spin_lock_irqsave(&qp->state_lock, flags);
827	attr->cur_qp_state = qp_state(qp);
828	if (qp->attr.sq_draining) {
829		spin_unlock_irqrestore(&qp->state_lock, flags);
830		cond_resched();
831	} else {
832		spin_unlock_irqrestore(&qp->state_lock, flags);
833	}
834
835	return 0;
836}
837
838int rxe_qp_chk_destroy(struct rxe_qp *qp)
839{
840	/* See IBA o10-2.2.3
841	 * An attempt to destroy a QP while attached to a mcast group
842	 * will fail immediately.
843	 */
844	if (atomic_read(&qp->mcg_num)) {
845		rxe_dbg_qp(qp, "Attempt to destroy while attached to multicast group\n");
846		return -EBUSY;
847	}
848
849	return 0;
850}
851
852/* called when the last reference to the qp is dropped */
853static void rxe_qp_do_cleanup(struct work_struct *work)
854{
855	struct rxe_qp *qp = container_of(work, typeof(*qp), cleanup_work.work);
856	unsigned long flags;
857
858	spin_lock_irqsave(&qp->state_lock, flags);
859	qp->valid = 0;
860	spin_unlock_irqrestore(&qp->state_lock, flags);
861	qp->qp_timeout_jiffies = 0;
862
863	/* In the function timer_setup, .function is initialized. If .function
864	 * is NULL, it indicates the function timer_setup is not called, the
865	 * timer is not initialized. Or else, the timer is initialized.
866	 */
867	if (qp_type(qp) == IB_QPT_RC && qp->retrans_timer.function &&
868		qp->rnr_nak_timer.function) {
869		timer_delete_sync(&qp->retrans_timer);
870		timer_delete_sync(&qp->rnr_nak_timer);
871	}
872
873	if (qp->recv_task.func)
874		rxe_cleanup_task(&qp->recv_task);
875
876	if (qp->send_task.func)
877		rxe_cleanup_task(&qp->send_task);
878
879	/* flush out any receive wr's or pending requests */
880	rxe_sender(qp);
881	rxe_receiver(qp);
882
883	if (qp->sq.queue)
884		rxe_queue_cleanup(qp->sq.queue);
885
886	if (qp->srq)
887		rxe_put(qp->srq);
888
889	if (qp->rq.queue)
890		rxe_queue_cleanup(qp->rq.queue);
891
892	if (qp->scq) {
893		atomic_dec(&qp->scq->num_wq);
894		rxe_put(qp->scq);
895	}
896
897	if (qp->rcq) {
898		atomic_dec(&qp->rcq->num_wq);
899		rxe_put(qp->rcq);
900	}
901
902	if (qp->pd)
903		rxe_put(qp->pd);
904
905	if (qp->resp.mr)
906		rxe_put(qp->resp.mr);
907
908	free_rd_atomic_resources(qp);
909
910	if (qp->sk) {
911		if (qp_type(qp) == IB_QPT_RC)
912			sk_dst_reset(qp->sk->sk);
913
914		kernel_sock_shutdown(qp->sk, SHUT_RDWR);
915		sock_release(qp->sk);
916	}
917}
918
919/* called when the last reference to the qp is dropped */
920void rxe_qp_cleanup(struct rxe_pool_elem *elem)
921{
922	struct rxe_qp *qp = container_of(elem, typeof(*qp), elem);
923
924	execute_in_process_context(rxe_qp_do_cleanup, &qp->cleanup_work);
925}