siw_qp.c

   1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4/* Copyright (c) 2008-2019, IBM Corporation */
   5
   6#include <linux/errno.h>
   7#include <linux/types.h>
   8#include <linux/net.h>
   9#include <linux/scatterlist.h>
  10#include <linux/llist.h>
  11#include <asm/barrier.h>
  12#include <net/tcp.h>
  13#include <trace/events/sock.h>
  14
  15#include "siw.h"
  16#include "siw_verbs.h"
  17#include "siw_mem.h"
  18
  19static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = {
  20	[SIW_QP_STATE_IDLE] = "IDLE",
  21	[SIW_QP_STATE_RTR] = "RTR",
  22	[SIW_QP_STATE_RTS] = "RTS",
  23	[SIW_QP_STATE_CLOSING] = "CLOSING",
  24	[SIW_QP_STATE_TERMINATE] = "TERMINATE",
  25	[SIW_QP_STATE_ERROR] = "ERROR"
  26};
  27
  28/*
  29 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a
  30 * per-RDMAP message basis. Please keep order of initializer. All MPA len
  31 * is initialized to minimum packet size.
  32 */
  33struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = {
  34	{ /* RDMAP_RDMA_WRITE */
  35	  .hdr_len = sizeof(struct iwarp_rdma_write),
  36	  .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
  37	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
  38				 cpu_to_be16(DDP_VERSION << 8) |
  39				 cpu_to_be16(RDMAP_VERSION << 6) |
  40				 cpu_to_be16(RDMAP_RDMA_WRITE),
  41	  .rx_data = siw_proc_write },
  42	{ /* RDMAP_RDMA_READ_REQ */
  43	  .hdr_len = sizeof(struct iwarp_rdma_rreq),
  44	  .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
  45	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  46				 cpu_to_be16(RDMAP_VERSION << 6) |
  47				 cpu_to_be16(RDMAP_RDMA_READ_REQ),
  48	  .rx_data = siw_proc_rreq },
  49	{ /* RDMAP_RDMA_READ_RESP */
  50	  .hdr_len = sizeof(struct iwarp_rdma_rresp),
  51	  .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
  52	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
  53				 cpu_to_be16(DDP_VERSION << 8) |
  54				 cpu_to_be16(RDMAP_VERSION << 6) |
  55				 cpu_to_be16(RDMAP_RDMA_READ_RESP),
  56	  .rx_data = siw_proc_rresp },
  57	{ /* RDMAP_SEND */
  58	  .hdr_len = sizeof(struct iwarp_send),
  59	  .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
  60	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  61				 cpu_to_be16(RDMAP_VERSION << 6) |
  62				 cpu_to_be16(RDMAP_SEND),
  63	  .rx_data = siw_proc_send },
  64	{ /* RDMAP_SEND_INVAL */
  65	  .hdr_len = sizeof(struct iwarp_send_inv),
  66	  .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
  67	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  68				 cpu_to_be16(RDMAP_VERSION << 6) |
  69				 cpu_to_be16(RDMAP_SEND_INVAL),
  70	  .rx_data = siw_proc_send },
  71	{ /* RDMAP_SEND_SE */
  72	  .hdr_len = sizeof(struct iwarp_send),
  73	  .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
  74	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  75				 cpu_to_be16(RDMAP_VERSION << 6) |
  76				 cpu_to_be16(RDMAP_SEND_SE),
  77	  .rx_data = siw_proc_send },
  78	{ /* RDMAP_SEND_SE_INVAL */
  79	  .hdr_len = sizeof(struct iwarp_send_inv),
  80	  .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
  81	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  82				 cpu_to_be16(RDMAP_VERSION << 6) |
  83				 cpu_to_be16(RDMAP_SEND_SE_INVAL),
  84	  .rx_data = siw_proc_send },
  85	{ /* RDMAP_TERMINATE */
  86	  .hdr_len = sizeof(struct iwarp_terminate),
  87	  .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
  88	  .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  89				 cpu_to_be16(RDMAP_VERSION << 6) |
  90				 cpu_to_be16(RDMAP_TERMINATE),
  91	  .rx_data = siw_proc_terminate }
  92};
  93
  94void siw_qp_llp_data_ready(struct sock *sk)
  95{
  96	struct siw_qp *qp;
  97
  98	trace_sk_data_ready(sk);
  99
 100	read_lock(&sk->sk_callback_lock);
 101
 102	if (unlikely(!sk->sk_user_data || !sk_to_qp(sk)))
 103		goto done;
 104
 105	qp = sk_to_qp(sk);
 106
 107	if (likely(!qp->rx_stream.rx_suspend &&
 108		   down_read_trylock(&qp->state_lock))) {
 109		read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };
 110
 111		if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
 112			/*
 113			 * Implements data receive operation during
 114			 * socket callback. TCP gracefully catches
 115			 * the case where there is nothing to receive
 116			 * (not calling siw_tcp_rx_data() then).
 117			 */
 118			tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
 119
 120		up_read(&qp->state_lock);
 121	} else {
 122		siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",
 123			   qp->rx_stream.rx_suspend);
 124	}
 125done:
 126	read_unlock(&sk->sk_callback_lock);
 127}
 128
 129void siw_qp_llp_close(struct siw_qp *qp)
 130{
 131	siw_dbg_qp(qp, "enter llp close, state = %s\n",
 132		   siw_qp_state_to_string[qp->attrs.state]);
 133
 134	down_write(&qp->state_lock);
 135
 136	qp->rx_stream.rx_suspend = 1;
 137	qp->tx_ctx.tx_suspend = 1;
 138	qp->attrs.sk = NULL;
 139
 140	switch (qp->attrs.state) {
 141	case SIW_QP_STATE_RTS:
 142	case SIW_QP_STATE_RTR:
 143	case SIW_QP_STATE_IDLE:
 144	case SIW_QP_STATE_TERMINATE:
 145		qp->attrs.state = SIW_QP_STATE_ERROR;
 146		break;
 147	/*
 148	 * SIW_QP_STATE_CLOSING:
 149	 *
 150	 * This is a forced close. shall the QP be moved to
 151	 * ERROR or IDLE ?
 152	 */
 153	case SIW_QP_STATE_CLOSING:
 154		if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
 155			qp->attrs.state = SIW_QP_STATE_ERROR;
 156		else
 157			qp->attrs.state = SIW_QP_STATE_IDLE;
 158		break;
 159
 160	default:
 161		siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",
 162			   siw_qp_state_to_string[qp->attrs.state]);
 163		break;
 164	}
 165	siw_sq_flush(qp);
 166	siw_rq_flush(qp);
 167
 168	/*
 169	 * Dereference closing CEP
 170	 */
 171	if (qp->cep) {
 172		siw_cep_put(qp->cep);
 173		qp->cep = NULL;
 174	}
 175
 176	up_write(&qp->state_lock);
 177
 178	siw_dbg_qp(qp, "llp close exit: state %s\n",
 179		   siw_qp_state_to_string[qp->attrs.state]);
 180}
 181
 182/*
 183 * socket callback routine informing about newly available send space.
 184 * Function schedules SQ work for processing SQ items.
 185 */
 186void siw_qp_llp_write_space(struct sock *sk)
 187{
 188	struct siw_cep *cep;
 189
 190	read_lock(&sk->sk_callback_lock);
 191
 192	cep  = sk_to_cep(sk);
 193	if (cep) {
 194		cep->sk_write_space(sk);
 195
 196		if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
 197			(void)siw_sq_start(cep->qp);
 198	}
 199
 200	read_unlock(&sk->sk_callback_lock);
 201}
 202
 203static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)
 204{
 205	if (irq_size) {
 206		irq_size = roundup_pow_of_two(irq_size);
 207		qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe));
 208		if (!qp->irq) {
 209			qp->attrs.irq_size = 0;
 210			return -ENOMEM;
 211		}
 212	}
 213	if (orq_size) {
 214		orq_size = roundup_pow_of_two(orq_size);
 215		qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe));
 216		if (!qp->orq) {
 217			qp->attrs.orq_size = 0;
 218			qp->attrs.irq_size = 0;
 219			vfree(qp->irq);
 220			return -ENOMEM;
 221		}
 222	}
 223	qp->attrs.irq_size = irq_size;
 224	qp->attrs.orq_size = orq_size;
 225	siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);
 226	return 0;
 227}
 228
 229/*
 230 * Send a non signalled READ or WRITE to peer side as negotiated
 231 * with MPAv2 P2P setup protocol. The work request is only created
 232 * as a current active WR and does not consume Send Queue space.
 233 *
 234 * Caller must hold QP state lock.
 235 */
 236int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)
 237{
 238	struct siw_wqe *wqe = tx_wqe(qp);
 239	unsigned long flags;
 240	int rv = 0;
 241
 242	spin_lock_irqsave(&qp->sq_lock, flags);
 243
 244	if (unlikely(wqe->wr_status != SIW_WR_IDLE)) {
 245		spin_unlock_irqrestore(&qp->sq_lock, flags);
 246		return -EIO;
 247	}
 248	memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
 249
 250	wqe->wr_status = SIW_WR_QUEUED;
 251	wqe->sqe.flags = 0;
 252	wqe->sqe.num_sge = 1;
 253	wqe->sqe.sge[0].length = 0;
 254	wqe->sqe.sge[0].laddr = 0;
 255	wqe->sqe.sge[0].lkey = 0;
 256	/*
 257	 * While it must not be checked for inbound zero length
 258	 * READ/WRITE, some HW may treat STag 0 special.
 259	 */
 260	wqe->sqe.rkey = 1;
 261	wqe->sqe.raddr = 0;
 262	wqe->processed = 0;
 263
 264	if (ctrl & MPA_V2_RDMA_WRITE_RTR)
 265		wqe->sqe.opcode = SIW_OP_WRITE;
 266	else if (ctrl & MPA_V2_RDMA_READ_RTR) {
 267		struct siw_sqe *rreq = NULL;
 268
 269		wqe->sqe.opcode = SIW_OP_READ;
 270
 271		spin_lock(&qp->orq_lock);
 272
 273		if (qp->attrs.orq_size)
 274			rreq = orq_get_free(qp);
 275		if (rreq) {
 276			siw_read_to_orq(rreq, &wqe->sqe);
 277			qp->orq_put++;
 278		} else
 279			rv = -EIO;
 280
 281		spin_unlock(&qp->orq_lock);
 282	} else
 283		rv = -EINVAL;
 284
 285	if (rv)
 286		wqe->wr_status = SIW_WR_IDLE;
 287
 288	spin_unlock_irqrestore(&qp->sq_lock, flags);
 289
 290	if (!rv)
 291		rv = siw_sq_start(qp);
 292
 293	return rv;
 294}
 295
 296/*
 297 * Map memory access error to DDP tagged error
 298 */
 299enum ddp_ecode siw_tagged_error(enum siw_access_state state)
 300{
 301	switch (state) {
 302	case E_STAG_INVALID:
 303		return DDP_ECODE_T_INVALID_STAG;
 304	case E_BASE_BOUNDS:
 305		return DDP_ECODE_T_BASE_BOUNDS;
 306	case E_PD_MISMATCH:
 307		return DDP_ECODE_T_STAG_NOT_ASSOC;
 308	case E_ACCESS_PERM:
 309		/*
 310		 * RFC 5041 (DDP) lacks an ecode for insufficient access
 311		 * permissions. 'Invalid STag' seem to be the closest
 312		 * match though.
 313		 */
 314		return DDP_ECODE_T_INVALID_STAG;
 315	default:
 316		WARN_ON(1);
 317		return DDP_ECODE_T_INVALID_STAG;
 318	}
 319}
 320
 321/*
 322 * Map memory access error to RDMAP protection error
 323 */
 324enum rdmap_ecode siw_rdmap_error(enum siw_access_state state)
 325{
 326	switch (state) {
 327	case E_STAG_INVALID:
 328		return RDMAP_ECODE_INVALID_STAG;
 329	case E_BASE_BOUNDS:
 330		return RDMAP_ECODE_BASE_BOUNDS;
 331	case E_PD_MISMATCH:
 332		return RDMAP_ECODE_STAG_NOT_ASSOC;
 333	case E_ACCESS_PERM:
 334		return RDMAP_ECODE_ACCESS_RIGHTS;
 335	default:
 336		return RDMAP_ECODE_UNSPECIFIED;
 337	}
 338}
 339
 340void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,
 341			u8 ecode, int in_tx)
 342{
 343	if (!qp->term_info.valid) {
 344		memset(&qp->term_info, 0, sizeof(qp->term_info));
 345		qp->term_info.layer = layer;
 346		qp->term_info.etype = etype;
 347		qp->term_info.ecode = ecode;
 348		qp->term_info.in_tx = in_tx;
 349		qp->term_info.valid = 1;
 350	}
 351	siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
 352		   layer, etype, ecode, in_tx ? "yes" : "no");
 353}
 354
 355/*
 356 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581.
 357 * Sending TERMINATE messages is best effort - such messages
 358 * can only be send if the QP is still connected and it does
 359 * not have another outbound message in-progress, i.e. the
 360 * TERMINATE message must not interfer with an incomplete current
 361 * transmit operation.
 362 */
 363void siw_send_terminate(struct siw_qp *qp)
 364{
 365	struct kvec iov[3];
 366	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
 367	struct iwarp_terminate *term = NULL;
 368	union iwarp_hdr *err_hdr = NULL;
 369	struct socket *s = qp->attrs.sk;
 370	struct siw_rx_stream *srx = &qp->rx_stream;
 371	union iwarp_hdr *rx_hdr = &srx->hdr;
 372	u32 crc = 0;
 373	int num_frags, len_terminate, rv;
 374
 375	if (!qp->term_info.valid)
 376		return;
 377
 378	qp->term_info.valid = 0;
 379
 380	if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
 381		siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",
 382			   tx_type(tx_wqe(qp)));
 383		return;
 384	}
 385	if (!s && qp->cep)
 386		/* QP not yet in RTS. Take socket from connection end point */
 387		s = qp->cep->sock;
 388
 389	if (!s) {
 390		siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");
 391		return;
 392	}
 393
 394	term = kzalloc(sizeof(*term), GFP_KERNEL);
 395	if (!term)
 396		return;
 397
 398	term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE);
 399	term->ddp_mo = 0;
 400	term->ddp_msn = cpu_to_be32(1);
 401
 402	iov[0].iov_base = term;
 403	iov[0].iov_len = sizeof(*term);
 404
 405	if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
 406	    ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
 407	     (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
 408		err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL);
 409		if (!err_hdr) {
 410			kfree(term);
 411			return;
 412		}
 413	}
 414	memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl,
 415	       sizeof(struct iwarp_ctrl));
 416
 417	__rdmap_term_set_layer(term, qp->term_info.layer);
 418	__rdmap_term_set_etype(term, qp->term_info.etype);
 419	__rdmap_term_set_ecode(term, qp->term_info.ecode);
 420
 421	switch (qp->term_info.layer) {
 422	case TERM_ERROR_LAYER_RDMAP:
 423		if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
 424			/* No additional DDP/RDMAP header to be included */
 425			break;
 426
 427		if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
 428			/*
 429			 * Complete RDMAP frame will get attached, and
 430			 * DDP segment length is valid
 431			 */
 432			term->flag_m = 1;
 433			term->flag_d = 1;
 434			term->flag_r = 1;
 435
 436			if (qp->term_info.in_tx) {
 437				struct iwarp_rdma_rreq *rreq;
 438				struct siw_wqe *wqe = tx_wqe(qp);
 439
 440				/* Inbound RREQ error, detected during
 441				 * RRESP creation. Take state from
 442				 * current TX work queue element to
 443				 * reconstruct peers RREQ.
 444				 */
 445				rreq = (struct iwarp_rdma_rreq *)err_hdr;
 446
 447				memcpy(&rreq->ctrl,
 448				       &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
 449				       sizeof(struct iwarp_ctrl));
 450
 451				rreq->rsvd = 0;
 452				rreq->ddp_qn =
 453					htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
 454
 455				/* Provide RREQ's MSN as kept aside */
 456				rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);
 457
 458				rreq->ddp_mo = htonl(wqe->processed);
 459				rreq->sink_stag = htonl(wqe->sqe.rkey);
 460				rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);
 461				rreq->read_size = htonl(wqe->sqe.sge[0].length);
 462				rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);
 463				rreq->source_to =
 464					cpu_to_be64(wqe->sqe.sge[0].laddr);
 465
 466				iov[1].iov_base = rreq;
 467				iov[1].iov_len = sizeof(*rreq);
 468
 469				rx_hdr = (union iwarp_hdr *)rreq;
 470			} else {
 471				/* Take RDMAP/DDP information from
 472				 * current (failed) inbound frame.
 473				 */
 474				iov[1].iov_base = rx_hdr;
 475
 476				if (__rdmap_get_opcode(&rx_hdr->ctrl) ==
 477				    RDMAP_RDMA_READ_REQ)
 478					iov[1].iov_len =
 479						sizeof(struct iwarp_rdma_rreq);
 480				else /* SEND type */
 481					iov[1].iov_len =
 482						sizeof(struct iwarp_send);
 483			}
 484		} else {
 485			/* Do not report DDP hdr information if packet
 486			 * layout is unknown
 487			 */
 488			if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
 489			    (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
 490				break;
 491
 492			iov[1].iov_base = rx_hdr;
 493
 494			/* Only DDP frame will get attached */
 495			if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
 496				iov[1].iov_len =
 497					sizeof(struct iwarp_rdma_write);
 498			else
 499				iov[1].iov_len = sizeof(struct iwarp_send);
 500
 501			term->flag_m = 1;
 502			term->flag_d = 1;
 503		}
 504		term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len);
 505		break;
 506
 507	case TERM_ERROR_LAYER_DDP:
 508		/* Report error encountered while DDP processing.
 509		 * This can only happen as a result of inbound
 510		 * DDP processing
 511		 */
 512
 513		/* Do not report DDP hdr information if packet
 514		 * layout is unknown
 515		 */
 516		if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
 517		     (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
 518		    ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
 519		     (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
 520			break;
 521
 522		iov[1].iov_base = rx_hdr;
 523
 524		if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
 525			iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged);
 526		else
 527			iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged);
 528
 529		term->flag_m = 1;
 530		term->flag_d = 1;
 531		break;
 532
 533	default:
 534		break;
 535	}
 536	if (term->flag_m || term->flag_d || term->flag_r) {
 537		iov[2].iov_base = &crc;
 538		iov[2].iov_len = sizeof(crc);
 539		len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE;
 540		num_frags = 3;
 541	} else {
 542		iov[1].iov_base = &crc;
 543		iov[1].iov_len = sizeof(crc);
 544		len_terminate = sizeof(*term) + MPA_CRC_SIZE;
 545		num_frags = 2;
 546	}
 547
 548	/* Adjust DDP Segment Length parameter, if valid */
 549	if (term->flag_m) {
 550		u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len);
 551		enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl);
 552
 553		real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE;
 554		rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len);
 555	}
 556
 557	term->ctrl.mpa_len =
 558		cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE));
 559	if (qp->tx_ctx.mpa_crc_enabled) {
 560		siw_crc_init(&qp->tx_ctx.mpa_crc);
 561		siw_crc_update(&qp->tx_ctx.mpa_crc,
 562			       iov[0].iov_base, iov[0].iov_len);
 563		if (num_frags == 3) {
 564			siw_crc_update(&qp->tx_ctx.mpa_crc,
 565				       iov[1].iov_base, iov[1].iov_len);
 566		}
 567		siw_crc_final(&qp->tx_ctx.mpa_crc, (u8 *)&crc);
 568	}
 569
 570	rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate);
 571	siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
 572		   rv == len_terminate ? "success" : "failure",
 573		   __rdmap_term_layer(term), __rdmap_term_etype(term),
 574		   __rdmap_term_ecode(term), rv);
 575	kfree(term);
 576	kfree(err_hdr);
 577}
 578
 579/*
 580 * Handle all attrs other than state
 581 */
 582static void siw_qp_modify_nonstate(struct siw_qp *qp,
 583				   struct siw_qp_attrs *attrs,
 584				   enum siw_qp_attr_mask mask)
 585{
 586	if (mask & SIW_QP_ATTR_ACCESS_FLAGS) {
 587		if (attrs->flags & SIW_RDMA_BIND_ENABLED)
 588			qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
 589		else
 590			qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
 591
 592		if (attrs->flags & SIW_RDMA_WRITE_ENABLED)
 593			qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 594		else
 595			qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
 596
 597		if (attrs->flags & SIW_RDMA_READ_ENABLED)
 598			qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
 599		else
 600			qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
 601	}
 602}
 603
 604static int siw_qp_nextstate_from_idle(struct siw_qp *qp,
 605				      struct siw_qp_attrs *attrs,
 606				      enum siw_qp_attr_mask mask)
 607{
 608	int rv = 0;
 609
 610	switch (attrs->state) {
 611	case SIW_QP_STATE_RTS:
 612		if (attrs->flags & SIW_MPA_CRC) {
 613			siw_crc_init(&qp->tx_ctx.mpa_crc);
 614			qp->tx_ctx.mpa_crc_enabled = true;
 615			siw_crc_init(&qp->rx_stream.mpa_crc);
 616			qp->rx_stream.mpa_crc_enabled = true;
 617		}
 618		if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) {
 619			siw_dbg_qp(qp, "no socket\n");
 620			rv = -EINVAL;
 621			break;
 622		}
 623		if (!(mask & SIW_QP_ATTR_MPA)) {
 624			siw_dbg_qp(qp, "no MPA\n");
 625			rv = -EINVAL;
 626			break;
 627		}
 628		/*
 629		 * Initialize iWARP TX state
 630		 */
 631		qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
 632		qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
 633		qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
 634
 635		/*
 636		 * Initialize iWARP RX state
 637		 */
 638		qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
 639		qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
 640		qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
 641
 642		/*
 643		 * init IRD free queue, caller has already checked
 644		 * limits.
 645		 */
 646		rv = siw_qp_readq_init(qp, attrs->irq_size,
 647				       attrs->orq_size);
 648		if (rv)
 649			break;
 650
 651		qp->attrs.sk = attrs->sk;
 652		qp->attrs.state = SIW_QP_STATE_RTS;
 653
 654		siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",
 655			   attrs->flags & SIW_MPA_CRC ? "y" : "n",
 656			   qp->attrs.orq_size, qp->attrs.irq_size);
 657		break;
 658
 659	case SIW_QP_STATE_ERROR:
 660		siw_rq_flush(qp);
 661		qp->attrs.state = SIW_QP_STATE_ERROR;
 662		if (qp->cep) {
 663			siw_cep_put(qp->cep);
 664			qp->cep = NULL;
 665		}
 666		break;
 667
 668	default:
 669		break;
 670	}
 671	return rv;
 672}
 673
 674static int siw_qp_nextstate_from_rts(struct siw_qp *qp,
 675				     struct siw_qp_attrs *attrs)
 676{
 677	int drop_conn = 0;
 678
 679	switch (attrs->state) {
 680	case SIW_QP_STATE_CLOSING:
 681		/*
 682		 * Verbs: move to IDLE if SQ and ORQ are empty.
 683		 * Move to ERROR otherwise. But first of all we must
 684		 * close the connection. So we keep CLOSING or ERROR
 685		 * as a transient state, schedule connection drop work
 686		 * and wait for the socket state change upcall to
 687		 * come back closed.
 688		 */
 689		if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
 690			qp->attrs.state = SIW_QP_STATE_CLOSING;
 691		} else {
 692			qp->attrs.state = SIW_QP_STATE_ERROR;
 693			siw_sq_flush(qp);
 694		}
 695		siw_rq_flush(qp);
 696
 697		drop_conn = 1;
 698		break;
 699
 700	case SIW_QP_STATE_TERMINATE:
 701		qp->attrs.state = SIW_QP_STATE_TERMINATE;
 702
 703		siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
 704				   RDMAP_ETYPE_CATASTROPHIC,
 705				   RDMAP_ECODE_UNSPECIFIED, 1);
 706		drop_conn = 1;
 707		break;
 708
 709	case SIW_QP_STATE_ERROR:
 710		/*
 711		 * This is an emergency close.
 712		 *
 713		 * Any in progress transmit operation will get
 714		 * cancelled.
 715		 * This will likely result in a protocol failure,
 716		 * if a TX operation is in transit. The caller
 717		 * could unconditional wait to give the current
 718		 * operation a chance to complete.
 719		 * Esp., how to handle the non-empty IRQ case?
 720		 * The peer was asking for data transfer at a valid
 721		 * point in time.
 722		 */
 723		siw_sq_flush(qp);
 724		siw_rq_flush(qp);
 725		qp->attrs.state = SIW_QP_STATE_ERROR;
 726		drop_conn = 1;
 727		break;
 728
 729	default:
 730		break;
 731	}
 732	return drop_conn;
 733}
 734
 735static void siw_qp_nextstate_from_term(struct siw_qp *qp,
 736				       struct siw_qp_attrs *attrs)
 737{
 738	switch (attrs->state) {
 739	case SIW_QP_STATE_ERROR:
 740		siw_rq_flush(qp);
 741		qp->attrs.state = SIW_QP_STATE_ERROR;
 742
 743		if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
 744			siw_sq_flush(qp);
 745		break;
 746
 747	default:
 748		break;
 749	}
 750}
 751
 752static int siw_qp_nextstate_from_close(struct siw_qp *qp,
 753				       struct siw_qp_attrs *attrs)
 754{
 755	int rv = 0;
 756
 757	switch (attrs->state) {
 758	case SIW_QP_STATE_IDLE:
 759		WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
 760		qp->attrs.state = SIW_QP_STATE_IDLE;
 761		break;
 762
 763	case SIW_QP_STATE_CLOSING:
 764		/*
 765		 * The LLP may already moved the QP to closing
 766		 * due to graceful peer close init
 767		 */
 768		break;
 769
 770	case SIW_QP_STATE_ERROR:
 771		/*
 772		 * QP was moved to CLOSING by LLP event
 773		 * not yet seen by user.
 774		 */
 775		qp->attrs.state = SIW_QP_STATE_ERROR;
 776
 777		if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
 778			siw_sq_flush(qp);
 779
 780		siw_rq_flush(qp);
 781		break;
 782
 783	default:
 784		siw_dbg_qp(qp, "state transition undefined: %s => %s\n",
 785			   siw_qp_state_to_string[qp->attrs.state],
 786			   siw_qp_state_to_string[attrs->state]);
 787
 788		rv = -ECONNABORTED;
 789	}
 790	return rv;
 791}
 792
 793/*
 794 * Caller must hold qp->state_lock
 795 */
 796int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,
 797		  enum siw_qp_attr_mask mask)
 798{
 799	int drop_conn = 0, rv = 0;
 800
 801	if (!mask)
 802		return 0;
 803
 804	siw_dbg_qp(qp, "state: %s => %s\n",
 805		   siw_qp_state_to_string[qp->attrs.state],
 806		   siw_qp_state_to_string[attrs->state]);
 807
 808	if (mask != SIW_QP_ATTR_STATE)
 809		siw_qp_modify_nonstate(qp, attrs, mask);
 810
 811	if (!(mask & SIW_QP_ATTR_STATE))
 812		return 0;
 813
 814	switch (qp->attrs.state) {
 815	case SIW_QP_STATE_IDLE:
 816	case SIW_QP_STATE_RTR:
 817		rv = siw_qp_nextstate_from_idle(qp, attrs, mask);
 818		break;
 819
 820	case SIW_QP_STATE_RTS:
 821		drop_conn = siw_qp_nextstate_from_rts(qp, attrs);
 822		break;
 823
 824	case SIW_QP_STATE_TERMINATE:
 825		siw_qp_nextstate_from_term(qp, attrs);
 826		break;
 827
 828	case SIW_QP_STATE_CLOSING:
 829		siw_qp_nextstate_from_close(qp, attrs);
 830		break;
 831	default:
 832		break;
 833	}
 834	if (drop_conn)
 835		siw_qp_cm_drop(qp, 0);
 836
 837	return rv;
 838}
 839
 840void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)
 841{
 842	rreq->id = sqe->id;
 843	rreq->opcode = sqe->opcode;
 844	rreq->sge[0].laddr = sqe->sge[0].laddr;
 845	rreq->sge[0].length = sqe->sge[0].length;
 846	rreq->sge[0].lkey = sqe->sge[0].lkey;
 847	rreq->sge[1].lkey = sqe->sge[1].lkey;
 848	rreq->flags = sqe->flags | SIW_WQE_VALID;
 849	rreq->num_sge = 1;
 850}
 851
 852static int siw_activate_tx_from_sq(struct siw_qp *qp)
 853{
 854	struct siw_sqe *sqe;
 855	struct siw_wqe *wqe = tx_wqe(qp);
 856	int rv = 1;
 857
 858	sqe = sq_get_next(qp);
 859	if (!sqe)
 860		return 0;
 861
 862	memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
 863	wqe->wr_status = SIW_WR_QUEUED;
 864
 865	/* First copy SQE to kernel private memory */
 866	memcpy(&wqe->sqe, sqe, sizeof(*sqe));
 867
 868	if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {
 869		rv = -EINVAL;
 870		goto out;
 871	}
 872	if (wqe->sqe.flags & SIW_WQE_INLINE) {
 873		if (wqe->sqe.opcode != SIW_OP_SEND &&
 874		    wqe->sqe.opcode != SIW_OP_WRITE) {
 875			rv = -EINVAL;
 876			goto out;
 877		}
 878		if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {
 879			rv = -EINVAL;
 880			goto out;
 881		}
 882		wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];
 883		wqe->sqe.sge[0].lkey = 0;
 884		wqe->sqe.num_sge = 1;
 885	}
 886	if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {
 887		/* A READ cannot be fenced */
 888		if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||
 889			     wqe->sqe.opcode ==
 890				     SIW_OP_READ_LOCAL_INV)) {
 891			siw_dbg_qp(qp, "cannot fence read\n");
 892			rv = -EINVAL;
 893			goto out;
 894		}
 895		spin_lock(&qp->orq_lock);
 896
 897		if (qp->attrs.orq_size && !siw_orq_empty(qp)) {
 898			qp->tx_ctx.orq_fence = 1;
 899			rv = 0;
 900		}
 901		spin_unlock(&qp->orq_lock);
 902
 903	} else if (wqe->sqe.opcode == SIW_OP_READ ||
 904		   wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
 905		struct siw_sqe *rreq;
 906
 907		if (unlikely(!qp->attrs.orq_size)) {
 908			/* We negotiated not to send READ req's */
 909			rv = -EINVAL;
 910			goto out;
 911		}
 912		wqe->sqe.num_sge = 1;
 913
 914		spin_lock(&qp->orq_lock);
 915
 916		rreq = orq_get_free(qp);
 917		if (rreq) {
 918			/*
 919			 * Make an immediate copy in ORQ to be ready
 920			 * to process loopback READ reply
 921			 */
 922			siw_read_to_orq(rreq, &wqe->sqe);
 923			qp->orq_put++;
 924		} else {
 925			qp->tx_ctx.orq_fence = 1;
 926			rv = 0;
 927		}
 928		spin_unlock(&qp->orq_lock);
 929	}
 930
 931	/* Clear SQE, can be re-used by application */
 932	smp_store_mb(sqe->flags, 0);
 933	qp->sq_get++;
 934out:
 935	if (unlikely(rv < 0)) {
 936		siw_dbg_qp(qp, "error %d\n", rv);
 937		wqe->wr_status = SIW_WR_IDLE;
 938	}
 939	return rv;
 940}
 941
 942/*
 943 * Must be called with SQ locked.
 944 * To avoid complete SQ starvation by constant inbound READ requests,
 945 * the active IRQ will not be served after qp->irq_burst, if the
 946 * SQ has pending work.
 947 */
 948int siw_activate_tx(struct siw_qp *qp)
 949{
 950	struct siw_sqe *irqe;
 951	struct siw_wqe *wqe = tx_wqe(qp);
 952
 953	if (!qp->attrs.irq_size)
 954		return siw_activate_tx_from_sq(qp);
 955
 956	irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
 957
 958	if (!(irqe->flags & SIW_WQE_VALID))
 959		return siw_activate_tx_from_sq(qp);
 960
 961	/*
 962	 * Avoid local WQE processing starvation in case
 963	 * of constant inbound READ request stream
 964	 */
 965	if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
 966		qp->irq_burst = 0;
 967		return siw_activate_tx_from_sq(qp);
 968	}
 969	memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
 970	wqe->wr_status = SIW_WR_QUEUED;
 971
 972	/* start READ RESPONSE */
 973	wqe->sqe.opcode = SIW_OP_READ_RESPONSE;
 974	wqe->sqe.flags = 0;
 975	if (irqe->num_sge) {
 976		wqe->sqe.num_sge = 1;
 977		wqe->sqe.sge[0].length = irqe->sge[0].length;
 978		wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;
 979		wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;
 980	} else {
 981		wqe->sqe.num_sge = 0;
 982	}
 983
 984	/* Retain original RREQ's message sequence number for
 985	 * potential error reporting cases.
 986	 */
 987	wqe->sqe.sge[1].length = irqe->sge[1].length;
 988
 989	wqe->sqe.rkey = irqe->rkey;
 990	wqe->sqe.raddr = irqe->raddr;
 991
 992	wqe->processed = 0;
 993	qp->irq_get++;
 994
 995	/* mark current IRQ entry free */
 996	smp_store_mb(irqe->flags, 0);
 997
 998	return 1;
 999}
1000
1001/*
1002 * Check if current CQ state qualifies for calling CQ completion
1003 * handler. Must be called with CQ lock held.
1004 */
1005static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags)
1006{
1007	u32 cq_notify;
1008
1009	if (!cq->base_cq.comp_handler)
1010		return false;
1011
1012	/* Read application shared notification state */
1013	cq_notify = READ_ONCE(cq->notify->flags);
1014
1015	if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) ||
1016	    ((cq_notify & SIW_NOTIFY_SOLICITED) &&
1017	     (flags & SIW_WQE_SOLICITED))) {
1018		/*
1019		 * CQ notification is one-shot: Since the
1020		 * current CQE causes user notification,
1021		 * the CQ gets dis-aremd and must be re-aremd
1022		 * by the user for a new notification.
1023		 */
1024		WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT);
1025
1026		return true;
1027	}
1028	return false;
1029}
1030
1031int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
1032		     enum siw_wc_status status)
1033{
1034	struct siw_cq *cq = qp->scq;
1035	int rv = 0;
1036
1037	if (cq) {
1038		u32 sqe_flags = sqe->flags;
1039		struct siw_cqe *cqe;
1040		u32 idx;
1041		unsigned long flags;
1042
1043		spin_lock_irqsave(&cq->lock, flags);
1044
1045		idx = cq->cq_put % cq->num_cqe;
1046		cqe = &cq->queue[idx];
1047
1048		if (!READ_ONCE(cqe->flags)) {
1049			bool notify;
1050
1051			cqe->id = sqe->id;
1052			cqe->opcode = sqe->opcode;
1053			cqe->status = status;
1054			cqe->imm_data = 0;
1055			cqe->bytes = bytes;
1056
1057			if (rdma_is_kernel_res(&cq->base_cq.res))
1058				cqe->base_qp = &qp->base_qp;
1059			else
1060				cqe->qp_id = qp_id(qp);
1061
1062			/* mark CQE valid for application */
1063			WRITE_ONCE(cqe->flags, SIW_WQE_VALID);
1064			/* recycle SQE */
1065			smp_store_mb(sqe->flags, 0);
1066
1067			cq->cq_put++;
1068			notify = siw_cq_notify_now(cq, sqe_flags);
1069
1070			spin_unlock_irqrestore(&cq->lock, flags);
1071
1072			if (notify) {
1073				siw_dbg_cq(cq, "Call completion handler\n");
1074				cq->base_cq.comp_handler(&cq->base_cq,
1075						cq->base_cq.cq_context);
1076			}
1077		} else {
1078			spin_unlock_irqrestore(&cq->lock, flags);
1079			rv = -ENOMEM;
1080			siw_cq_event(cq, IB_EVENT_CQ_ERR);
1081		}
1082	} else {
1083		/* recycle SQE */
1084		smp_store_mb(sqe->flags, 0);
1085	}
1086	return rv;
1087}
1088
1089int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
1090		     u32 inval_stag, enum siw_wc_status status)
1091{
1092	struct siw_cq *cq = qp->rcq;
1093	int rv = 0;
1094
1095	if (cq) {
1096		struct siw_cqe *cqe;
1097		u32 idx;
1098		unsigned long flags;
1099
1100		spin_lock_irqsave(&cq->lock, flags);
1101
1102		idx = cq->cq_put % cq->num_cqe;
1103		cqe = &cq->queue[idx];
1104
1105		if (!READ_ONCE(cqe->flags)) {
1106			bool notify;
1107			u8 cqe_flags = SIW_WQE_VALID;
1108
1109			cqe->id = rqe->id;
1110			cqe->opcode = SIW_OP_RECEIVE;
1111			cqe->status = status;
1112			cqe->imm_data = 0;
1113			cqe->bytes = bytes;
1114
1115			if (rdma_is_kernel_res(&cq->base_cq.res)) {
1116				cqe->base_qp = &qp->base_qp;
1117				if (inval_stag) {
1118					cqe_flags |= SIW_WQE_REM_INVAL;
1119					cqe->inval_stag = inval_stag;
1120				}
1121			} else {
1122				cqe->qp_id = qp_id(qp);
1123			}
1124			/* mark CQE valid for application */
1125			WRITE_ONCE(cqe->flags, cqe_flags);
1126			/* recycle RQE */
1127			smp_store_mb(rqe->flags, 0);
1128
1129			cq->cq_put++;
1130			notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED);
1131
1132			spin_unlock_irqrestore(&cq->lock, flags);
1133
1134			if (notify) {
1135				siw_dbg_cq(cq, "Call completion handler\n");
1136				cq->base_cq.comp_handler(&cq->base_cq,
1137						cq->base_cq.cq_context);
1138			}
1139		} else {
1140			spin_unlock_irqrestore(&cq->lock, flags);
1141			rv = -ENOMEM;
1142			siw_cq_event(cq, IB_EVENT_CQ_ERR);
1143		}
1144	} else {
1145		/* recycle RQE */
1146		smp_store_mb(rqe->flags, 0);
1147	}
1148	return rv;
1149}
1150
1151/*
1152 * siw_sq_flush()
1153 *
1154 * Flush SQ and ORQ entries to CQ.
1155 *
1156 * Must be called with QP state write lock held.
1157 * Therefore, SQ and ORQ lock must not be taken.
1158 */
1159void siw_sq_flush(struct siw_qp *qp)
1160{
1161	struct siw_sqe *sqe;
1162	struct siw_wqe *wqe = tx_wqe(qp);
1163	int async_event = 0;
1164
1165	/*
1166	 * Start with completing any work currently on the ORQ
1167	 */
1168	while (qp->attrs.orq_size) {
1169		sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1170		if (!READ_ONCE(sqe->flags))
1171			break;
1172
1173		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1174			break;
1175
1176		WRITE_ONCE(sqe->flags, 0);
1177		qp->orq_get++;
1178	}
1179	/*
1180	 * Flush an in-progress WQE if present
1181	 */
1182	if (wqe->wr_status != SIW_WR_IDLE) {
1183		siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",
1184			   tx_type(wqe), wqe->wr_status);
1185
1186		siw_wqe_put_mem(wqe, tx_type(wqe));
1187
1188		if (tx_type(wqe) != SIW_OP_READ_RESPONSE &&
1189		    ((tx_type(wqe) != SIW_OP_READ &&
1190		      tx_type(wqe) != SIW_OP_READ_LOCAL_INV) ||
1191		     wqe->wr_status == SIW_WR_QUEUED))
1192			/*
1193			 * An in-progress Read Request is already in
1194			 * the ORQ
1195			 */
1196			siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1197					 SIW_WC_WR_FLUSH_ERR);
1198
1199		wqe->wr_status = SIW_WR_IDLE;
1200	}
1201	/*
1202	 * Flush the Send Queue
1203	 */
1204	while (qp->attrs.sq_size) {
1205		sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1206		if (!READ_ONCE(sqe->flags))
1207			break;
1208
1209		async_event = 1;
1210		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1211			/*
1212			 * Shall IB_EVENT_SQ_DRAINED be supressed if work
1213			 * completion fails?
1214			 */
1215			break;
1216
1217		WRITE_ONCE(sqe->flags, 0);
1218		qp->sq_get++;
1219	}
1220	if (async_event)
1221		siw_qp_event(qp, IB_EVENT_SQ_DRAINED);
1222}
1223
1224/*
1225 * siw_rq_flush()
1226 *
1227 * Flush recv queue entries to CQ. Also
1228 * takes care of pending active tagged and untagged
1229 * inbound transfers, which have target memory
1230 * referenced.
1231 *
1232 * Must be called with QP state write lock held.
1233 * Therefore, RQ lock must not be taken.
1234 */
1235void siw_rq_flush(struct siw_qp *qp)
1236{
1237	struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1238
1239	/*
1240	 * Flush an in-progress untagged operation if present
1241	 */
1242	if (wqe->wr_status != SIW_WR_IDLE) {
1243		siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",
1244			   rx_type(wqe), wqe->wr_status);
1245
1246		siw_wqe_put_mem(wqe, rx_type(wqe));
1247
1248		if (rx_type(wqe) == SIW_OP_RECEIVE) {
1249			siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1250					 0, SIW_WC_WR_FLUSH_ERR);
1251		} else if (rx_type(wqe) != SIW_OP_READ &&
1252			   rx_type(wqe) != SIW_OP_READ_RESPONSE &&
1253			   rx_type(wqe) != SIW_OP_WRITE) {
1254			siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1255		}
1256		wqe->wr_status = SIW_WR_IDLE;
1257	}
1258	wqe = &qp->rx_tagged.wqe_active;
1259
1260	if (wqe->wr_status != SIW_WR_IDLE) {
1261		siw_wqe_put_mem(wqe, rx_type(wqe));
1262		wqe->wr_status = SIW_WR_IDLE;
1263	}
1264	/*
1265	 * Flush the Receive Queue
1266	 */
1267	while (qp->attrs.rq_size) {
1268		struct siw_rqe *rqe =
1269			&qp->recvq[qp->rq_get % qp->attrs.rq_size];
1270
1271		if (!READ_ONCE(rqe->flags))
1272			break;
1273
1274		if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1275			break;
1276
1277		WRITE_ONCE(rqe->flags, 0);
1278		qp->rq_get++;
1279	}
1280}
1281
1282int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)
1283{
1284	int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,
1285			  GFP_KERNEL);
1286
1287	if (!rv) {
1288		kref_init(&qp->ref);
1289		qp->sdev = sdev;
1290		siw_dbg_qp(qp, "new QP\n");
1291	}
1292	return rv;
1293}
1294
1295void siw_free_qp(struct kref *ref)
1296{
1297	struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);
1298	struct siw_device *sdev = qp->sdev;
1299	unsigned long flags;
1300
1301	if (qp->cep)
1302		siw_cep_put(qp->cep);
1303
1304	found = xa_erase(&sdev->qp_xa, qp_id(qp));
1305	WARN_ON(found != qp);
1306	spin_lock_irqsave(&sdev->lock, flags);
1307	list_del(&qp->devq);
1308	spin_unlock_irqrestore(&sdev->lock, flags);
1309
1310	vfree(qp->sendq);
1311	vfree(qp->recvq);
1312	vfree(qp->irq);
1313	vfree(qp->orq);
1314
1315	siw_put_tx_cpu(qp->tx_cpu);
1316	complete(&qp->qp_free);
1317	atomic_dec(&sdev->num_qp);
1318}