siw_qp_tx.c

   1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4/* Copyright (c) 2008-2019, IBM Corporation */
   5
   6#include <linux/errno.h>
   7#include <linux/types.h>
   8#include <linux/net.h>
   9#include <linux/scatterlist.h>
  10#include <linux/highmem.h>
  11#include <net/tcp.h>
  12
  13#include <rdma/iw_cm.h>
  14#include <rdma/ib_verbs.h>
  15#include <rdma/ib_user_verbs.h>
  16
  17#include "siw.h"
  18#include "siw_verbs.h"
  19#include "siw_mem.h"
  20
  21#define MAX_HDR_INLINE					\
  22	(((uint32_t)(sizeof(struct siw_rreq_pkt) -	\
  23		     sizeof(struct iwarp_send))) & 0xF8)
  24
  25static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx)
  26{
  27	struct siw_pbl *pbl = mem->pbl;
  28	u64 offset = addr - mem->va;
  29	dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx);
  30
  31	if (paddr)
  32		return ib_virt_dma_to_page(paddr);
  33
  34	return NULL;
  35}
  36
  37static struct page *siw_get_page(struct siw_mem *mem, struct siw_sge *sge,
  38				 unsigned long offset, int *pbl_idx)
  39{
  40	if (!mem->is_pbl)
  41		return siw_get_upage(mem->umem, sge->laddr + offset);
  42	else
  43		return siw_get_pblpage(mem, sge->laddr + offset, pbl_idx);
  44}
  45
  46/*
  47 * Copy short payload at provided destination payload address
  48 */
  49static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr)
  50{
  51	struct siw_wqe *wqe = &c_tx->wqe_active;
  52	struct siw_sge *sge = &wqe->sqe.sge[0];
  53	u32 bytes = sge->length;
  54
  55	if (bytes > MAX_HDR_INLINE || wqe->sqe.num_sge != 1)
  56		return MAX_HDR_INLINE + 1;
  57
  58	if (!bytes)
  59		return 0;
  60
  61	if (tx_flags(wqe) & SIW_WQE_INLINE) {
  62		memcpy(paddr, &wqe->sqe.sge[1], bytes);
  63	} else {
  64		struct siw_mem *mem = wqe->mem[0];
  65
  66		if (!mem->mem_obj) {
  67			/* Kernel client using kva */
  68			memcpy(paddr, ib_virt_dma_to_ptr(sge->laddr), bytes);
  69		} else if (c_tx->in_syscall) {
  70			if (copy_from_user(paddr, u64_to_user_ptr(sge->laddr),
  71					   bytes))
  72				return -EFAULT;
  73		} else {
  74			unsigned int off = sge->laddr & ~PAGE_MASK;
  75			struct page *p;
  76			char *buffer;
  77			int pbl_idx = 0;
  78
  79			p = siw_get_page(mem, sge, 0, &pbl_idx);
  80			if (unlikely(!p))
  81				return -EFAULT;
  82
  83			buffer = kmap_local_page(p);
  84
  85			if (likely(PAGE_SIZE - off >= bytes)) {
  86				memcpy(paddr, buffer + off, bytes);
  87			} else {
  88				unsigned long part = bytes - (PAGE_SIZE - off);
  89
  90				memcpy(paddr, buffer + off, part);
  91				kunmap_local(buffer);
  92
  93				p = siw_get_page(mem, sge, part, &pbl_idx);
  94				if (unlikely(!p))
  95					return -EFAULT;
  96
  97				buffer = kmap_local_page(p);
  98				memcpy(paddr + part, buffer, bytes - part);
  99			}
 100			kunmap_local(buffer);
 101		}
 102	}
 103	return (int)bytes;
 104}
 105
 106#define PKT_FRAGMENTED 1
 107#define PKT_COMPLETE 0
 108
 109/*
 110 * siw_qp_prepare_tx()
 111 *
 112 * Prepare tx state for sending out one fpdu. Builds complete pkt
 113 * if no user data or only immediate data are present.
 114 *
 115 * returns PKT_COMPLETE if complete pkt built, PKT_FRAGMENTED otherwise.
 116 */
 117static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx)
 118{
 119	struct siw_wqe *wqe = &c_tx->wqe_active;
 120	char *crc = NULL;
 121	int data = 0;
 122
 123	switch (tx_type(wqe)) {
 124	case SIW_OP_READ:
 125	case SIW_OP_READ_LOCAL_INV:
 126		memcpy(&c_tx->pkt.ctrl,
 127		       &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
 128		       sizeof(struct iwarp_ctrl));
 129
 130		c_tx->pkt.rreq.rsvd = 0;
 131		c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
 132		c_tx->pkt.rreq.ddp_msn =
 133			htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]);
 134		c_tx->pkt.rreq.ddp_mo = 0;
 135		c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey);
 136		c_tx->pkt.rreq.sink_to =
 137			cpu_to_be64(wqe->sqe.sge[0].laddr);
 138		c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey);
 139		c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr);
 140		c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length);
 141
 142		c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq);
 143		crc = (char *)&c_tx->pkt.rreq_pkt.crc;
 144		break;
 145
 146	case SIW_OP_SEND:
 147		if (tx_flags(wqe) & SIW_WQE_SOLICITED)
 148			memcpy(&c_tx->pkt.ctrl,
 149			       &iwarp_pktinfo[RDMAP_SEND_SE].ctrl,
 150			       sizeof(struct iwarp_ctrl));
 151		else
 152			memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl,
 153			       sizeof(struct iwarp_ctrl));
 154
 155		c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND;
 156		c_tx->pkt.send.ddp_msn =
 157			htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]);
 158		c_tx->pkt.send.ddp_mo = 0;
 159
 160		c_tx->pkt.send_inv.inval_stag = 0;
 161
 162		c_tx->ctrl_len = sizeof(struct iwarp_send);
 163
 164		crc = (char *)&c_tx->pkt.send_pkt.crc;
 165		data = siw_try_1seg(c_tx, crc);
 166		break;
 167
 168	case SIW_OP_SEND_REMOTE_INV:
 169		if (tx_flags(wqe) & SIW_WQE_SOLICITED)
 170			memcpy(&c_tx->pkt.ctrl,
 171			       &iwarp_pktinfo[RDMAP_SEND_SE_INVAL].ctrl,
 172			       sizeof(struct iwarp_ctrl));
 173		else
 174			memcpy(&c_tx->pkt.ctrl,
 175			       &iwarp_pktinfo[RDMAP_SEND_INVAL].ctrl,
 176			       sizeof(struct iwarp_ctrl));
 177
 178		c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND;
 179		c_tx->pkt.send.ddp_msn =
 180			htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]);
 181		c_tx->pkt.send.ddp_mo = 0;
 182
 183		c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey);
 184
 185		c_tx->ctrl_len = sizeof(struct iwarp_send_inv);
 186
 187		crc = (char *)&c_tx->pkt.send_pkt.crc;
 188		data = siw_try_1seg(c_tx, crc);
 189		break;
 190
 191	case SIW_OP_WRITE:
 192		memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl,
 193		       sizeof(struct iwarp_ctrl));
 194
 195		c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey);
 196		c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr);
 197		c_tx->ctrl_len = sizeof(struct iwarp_rdma_write);
 198
 199		crc = (char *)&c_tx->pkt.write_pkt.crc;
 200		data = siw_try_1seg(c_tx, crc);
 201		break;
 202
 203	case SIW_OP_READ_RESPONSE:
 204		memcpy(&c_tx->pkt.ctrl,
 205		       &iwarp_pktinfo[RDMAP_RDMA_READ_RESP].ctrl,
 206		       sizeof(struct iwarp_ctrl));
 207
 208		/* NBO */
 209		c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey);
 210		c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr);
 211
 212		c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp);
 213
 214		crc = (char *)&c_tx->pkt.write_pkt.crc;
 215		data = siw_try_1seg(c_tx, crc);
 216		break;
 217
 218	default:
 219		siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe));
 220		return -EOPNOTSUPP;
 221	}
 222	if (unlikely(data < 0))
 223		return data;
 224
 225	c_tx->ctrl_sent = 0;
 226
 227	if (data <= MAX_HDR_INLINE) {
 228		if (data) {
 229			wqe->processed = data;
 230
 231			c_tx->pkt.ctrl.mpa_len =
 232				htons(c_tx->ctrl_len + data - MPA_HDR_SIZE);
 233
 234			/* Add pad, if needed */
 235			data += -(int)data & 0x3;
 236			/* advance CRC location after payload */
 237			crc += data;
 238			c_tx->ctrl_len += data;
 239
 240			if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED))
 241				c_tx->pkt.c_untagged.ddp_mo = 0;
 242			else
 243				c_tx->pkt.c_tagged.ddp_to =
 244					cpu_to_be64(wqe->sqe.raddr);
 245		}
 246
 247		*(u32 *)crc = 0;
 248		/*
 249		 * Do complete CRC if enabled and short packet
 250		 */
 251		if (c_tx->mpa_crc_enabled)
 252			siw_crc_oneshot(&c_tx->pkt, c_tx->ctrl_len, (u8 *)crc);
 253		c_tx->ctrl_len += MPA_CRC_SIZE;
 254
 255		return PKT_COMPLETE;
 256	}
 257	c_tx->ctrl_len += MPA_CRC_SIZE;
 258	c_tx->sge_idx = 0;
 259	c_tx->sge_off = 0;
 260	c_tx->pbl_idx = 0;
 261
 262	/*
 263	 * Allow direct sending out of user buffer if WR is non signalled
 264	 * and payload is over threshold.
 265	 * Per RDMA verbs, the application should not change the send buffer
 266	 * until the work completed. In iWarp, work completion is only
 267	 * local delivery to TCP. TCP may reuse the buffer for
 268	 * retransmission. Changing unsent data also breaks the CRC,
 269	 * if applied.
 270	 */
 271	if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH &&
 272	    !(tx_flags(wqe) & SIW_WQE_SIGNALLED))
 273		c_tx->use_sendpage = 1;
 274	else
 275		c_tx->use_sendpage = 0;
 276
 277	return PKT_FRAGMENTED;
 278}
 279
 280static noinline_for_stack int
 281siw_sendmsg(struct socket *sock, unsigned int msg_flags,
 282	    struct kvec *vec, size_t num, size_t len)
 283{
 284	struct msghdr msg = { .msg_flags = msg_flags };
 285
 286	return kernel_sendmsg(sock, &msg, vec, num, len);
 287}
 288
 289/*
 290 * Send out one complete control type FPDU, or header of FPDU carrying
 291 * data. Used for fixed sized packets like Read.Requests or zero length
 292 * SENDs, WRITEs, READ.Responses, or header only.
 293 */
 294static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s,
 295			      int flags)
 296{
 297	struct kvec iov = { .iov_base =
 298				    (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent,
 299			    .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent };
 300
 301	int rv = siw_sendmsg(s, flags, &iov, 1, iov.iov_len);
 302
 303	if (rv >= 0) {
 304		c_tx->ctrl_sent += rv;
 305
 306		if (c_tx->ctrl_sent == c_tx->ctrl_len)
 307			rv = 0;
 308		else
 309			rv = -EAGAIN;
 310	}
 311	return rv;
 312}
 313
 314/*
 315 * 0copy TCP transmit interface: Use MSG_SPLICE_PAGES.
 316 *
 317 * Using sendpage to push page by page appears to be less efficient
 318 * than using sendmsg, even if data are copied.
 319 *
 320 * A general performance limitation might be the extra four bytes
 321 * trailer checksum segment to be pushed after user data.
 322 */
 323static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset,
 324			     size_t size)
 325{
 326	struct bio_vec bvec;
 327	struct msghdr msg = {
 328		.msg_flags = (MSG_MORE | MSG_DONTWAIT | MSG_SPLICE_PAGES),
 329	};
 330	struct sock *sk = s->sk;
 331	int i = 0, rv = 0, sent = 0;
 332
 333	while (size) {
 334		size_t bytes = min_t(size_t, PAGE_SIZE - offset, size);
 335
 336		if (size + offset <= PAGE_SIZE)
 337			msg.msg_flags &= ~MSG_MORE;
 338
 339		tcp_rate_check_app_limited(sk);
 340		if (!sendpage_ok(page[i]))
 341			msg.msg_flags &= ~MSG_SPLICE_PAGES;
 342		bvec_set_page(&bvec, page[i], bytes, offset);
 343		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, bytes);
 344
 345try_page_again:
 346		lock_sock(sk);
 347		rv = tcp_sendmsg_locked(sk, &msg, bytes);
 348		release_sock(sk);
 349
 350		if (rv > 0) {
 351			size -= rv;
 352			sent += rv;
 353			if (rv != bytes) {
 354				bytes -= rv;
 355				goto try_page_again;
 356			}
 357			offset = 0;
 358		} else {
 359			if (rv == -EAGAIN || rv == 0)
 360				break;
 361			return rv;
 362		}
 363		i++;
 364	}
 365	return sent;
 366}
 367
 368/*
 369 * siw_0copy_tx()
 370 *
 371 * Pushes list of pages to TCP socket. If pages from multiple
 372 * SGE's, all referenced pages of each SGE are pushed in one
 373 * shot.
 374 */
 375static int siw_0copy_tx(struct socket *s, struct page **page,
 376			struct siw_sge *sge, unsigned int offset,
 377			unsigned int size)
 378{
 379	int i = 0, sent = 0, rv;
 380	int sge_bytes = min(sge->length - offset, size);
 381
 382	offset = (sge->laddr + offset) & ~PAGE_MASK;
 383
 384	while (sent != size) {
 385		rv = siw_tcp_sendpages(s, &page[i], offset, sge_bytes);
 386		if (rv >= 0) {
 387			sent += rv;
 388			if (size == sent || sge_bytes > rv)
 389				break;
 390
 391			i += PAGE_ALIGN(sge_bytes + offset) >> PAGE_SHIFT;
 392			sge++;
 393			sge_bytes = min(sge->length, size - sent);
 394			offset = sge->laddr & ~PAGE_MASK;
 395		} else {
 396			sent = rv;
 397			break;
 398		}
 399	}
 400	return sent;
 401}
 402
 403#define MAX_TRAILER (MPA_CRC_SIZE + 4)
 404
 405static void siw_unmap_pages(struct kvec *iov, unsigned long kmap_mask, int len)
 406{
 407	int i;
 408
 409	/*
 410	 * Work backwards through the array to honor the kmap_local_page()
 411	 * ordering requirements.
 412	 */
 413	for (i = (len-1); i >= 0; i--) {
 414		if (kmap_mask & BIT(i)) {
 415			unsigned long addr = (unsigned long)iov[i].iov_base;
 416
 417			kunmap_local((void *)(addr & PAGE_MASK));
 418		}
 419	}
 420}
 421
 422/*
 423 * siw_tx_hdt() tries to push a complete packet to TCP where all
 424 * packet fragments are referenced by the elements of one iovec.
 425 * For the data portion, each involved page must be referenced by
 426 * one extra element. All sge's data can be non-aligned to page
 427 * boundaries. Two more elements are referencing iWARP header
 428 * and trailer:
 429 * MAX_ARRAY = 64KB/PAGE_SIZE + 1 + (2 * (SIW_MAX_SGE - 1) + HDR + TRL
 430 */
 431#define MAX_ARRAY ((0xffff / PAGE_SIZE) + 1 + (2 * (SIW_MAX_SGE - 1) + 2))
 432
 433/*
 434 * Write out iov referencing hdr, data and trailer of current FPDU.
 435 * Update transmit state dependent on write return status
 436 */
 437static noinline_for_stack int siw_tx_hdt(struct siw_iwarp_tx *c_tx,
 438					 struct socket *s)
 439{
 440	struct siw_wqe *wqe = &c_tx->wqe_active;
 441	struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx];
 442	struct kvec iov[MAX_ARRAY];
 443	struct page *page_array[MAX_ARRAY];
 444
 445	int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv;
 446	unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0,
 447		     sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx,
 448		     pbl_idx = c_tx->pbl_idx;
 449	unsigned long kmap_mask = 0L;
 450
 451	if (c_tx->state == SIW_SEND_HDR) {
 452		if (c_tx->use_sendpage) {
 453			rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE);
 454			if (rv)
 455				goto done;
 456
 457			c_tx->state = SIW_SEND_DATA;
 458		} else {
 459			iov[0].iov_base =
 460				(char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent;
 461			iov[0].iov_len = hdr_len =
 462				c_tx->ctrl_len - c_tx->ctrl_sent;
 463			seg = 1;
 464		}
 465	}
 466
 467	wqe->processed += data_len;
 468
 469	while (data_len) { /* walk the list of SGE's */
 470		unsigned int sge_len = min(sge->length - sge_off, data_len);
 471		unsigned int fp_off = (sge->laddr + sge_off) & ~PAGE_MASK;
 472		struct siw_mem *mem;
 473
 474		if (!(tx_flags(wqe) & SIW_WQE_INLINE)) {
 475			mem = wqe->mem[sge_idx];
 476			is_kva = mem->mem_obj == NULL ? 1 : 0;
 477		} else {
 478			is_kva = 1;
 479		}
 480		if (is_kva && !c_tx->use_sendpage) {
 481			/*
 482			 * tx from kernel virtual address: either inline data
 483			 * or memory region with assigned kernel buffer
 484			 */
 485			iov[seg].iov_base =
 486				ib_virt_dma_to_ptr(sge->laddr + sge_off);
 487			iov[seg].iov_len = sge_len;
 488
 489			if (do_crc)
 490				siw_crc_update(&c_tx->mpa_crc,
 491					       iov[seg].iov_base, sge_len);
 492			sge_off += sge_len;
 493			data_len -= sge_len;
 494			seg++;
 495			goto sge_done;
 496		}
 497
 498		while (sge_len) {
 499			size_t plen = min((int)PAGE_SIZE - fp_off, sge_len);
 500			void *kaddr;
 501
 502			if (!is_kva) {
 503				struct page *p;
 504
 505				p = siw_get_page(mem, sge, sge_off, &pbl_idx);
 506				if (unlikely(!p)) {
 507					siw_unmap_pages(iov, kmap_mask, seg);
 508					wqe->processed -= c_tx->bytes_unsent;
 509					rv = -EFAULT;
 510					goto done_crc;
 511				}
 512				page_array[seg] = p;
 513
 514				if (!c_tx->use_sendpage) {
 515					void *kaddr = kmap_local_page(p);
 516
 517					/* Remember for later kunmap() */
 518					kmap_mask |= BIT(seg);
 519					iov[seg].iov_base = kaddr + fp_off;
 520					iov[seg].iov_len = plen;
 521
 522					if (do_crc)
 523						siw_crc_update(
 524							&c_tx->mpa_crc,
 525							iov[seg].iov_base,
 526							plen);
 527				} else if (do_crc) {
 528					kaddr = kmap_local_page(p);
 529					siw_crc_update(&c_tx->mpa_crc,
 530						       kaddr + fp_off, plen);
 531					kunmap_local(kaddr);
 532				}
 533			} else {
 534				/*
 535				 * Cast to an uintptr_t to preserve all 64 bits
 536				 * in sge->laddr.
 537				 */
 538				u64 va = sge->laddr + sge_off;
 539
 540				page_array[seg] = ib_virt_dma_to_page(va);
 541				if (do_crc)
 542					siw_crc_update(&c_tx->mpa_crc,
 543						       ib_virt_dma_to_ptr(va),
 544						       plen);
 545			}
 546
 547			sge_len -= plen;
 548			sge_off += plen;
 549			data_len -= plen;
 550			fp_off = 0;
 551
 552			if (++seg >= (int)MAX_ARRAY) {
 553				siw_dbg_qp(tx_qp(c_tx), "to many fragments\n");
 554				siw_unmap_pages(iov, kmap_mask, seg-1);
 555				wqe->processed -= c_tx->bytes_unsent;
 556				rv = -EMSGSIZE;
 557				goto done_crc;
 558			}
 559		}
 560sge_done:
 561		/* Update SGE variables at end of SGE */
 562		if (sge_off == sge->length &&
 563		    (data_len != 0 || wqe->processed < wqe->bytes)) {
 564			sge_idx++;
 565			sge++;
 566			sge_off = 0;
 567		}
 568	}
 569	/* trailer */
 570	if (likely(c_tx->state != SIW_SEND_TRAILER)) {
 571		iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad];
 572		iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad);
 573	} else {
 574		iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent];
 575		iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent;
 576	}
 577
 578	if (c_tx->pad) {
 579		*(u32 *)c_tx->trailer.pad = 0;
 580		if (do_crc)
 581			siw_crc_update(&c_tx->mpa_crc,
 582				       (u8 *)&c_tx->trailer.crc - c_tx->pad,
 583				       c_tx->pad);
 584	}
 585	if (!c_tx->mpa_crc_enabled)
 586		c_tx->trailer.crc = 0;
 587	else if (do_crc)
 588		siw_crc_final(&c_tx->mpa_crc, (u8 *)&c_tx->trailer.crc);
 589
 590	data_len = c_tx->bytes_unsent;
 591
 592	if (c_tx->use_sendpage) {
 593		rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx],
 594				  c_tx->sge_off, data_len);
 595		if (rv == data_len) {
 596
 597			rv = siw_sendmsg(s, MSG_DONTWAIT | MSG_EOR, &iov[seg],
 598					 1, trl_len);
 599			if (rv > 0)
 600				rv += data_len;
 601			else
 602				rv = data_len;
 603		}
 604	} else {
 605		rv = siw_sendmsg(s, MSG_DONTWAIT | MSG_EOR, iov, seg + 1,
 606				    hdr_len + data_len + trl_len);
 607		siw_unmap_pages(iov, kmap_mask, seg);
 608	}
 609	if (rv < (int)hdr_len) {
 610		/* Not even complete hdr pushed or negative rv */
 611		wqe->processed -= data_len;
 612		if (rv >= 0) {
 613			c_tx->ctrl_sent += rv;
 614			rv = -EAGAIN;
 615		}
 616		goto done_crc;
 617	}
 618	rv -= hdr_len;
 619
 620	if (rv >= (int)data_len) {
 621		/* all user data pushed to TCP or no data to push */
 622		if (data_len > 0 && wqe->processed < wqe->bytes) {
 623			/* Save the current state for next tx */
 624			c_tx->sge_idx = sge_idx;
 625			c_tx->sge_off = sge_off;
 626			c_tx->pbl_idx = pbl_idx;
 627		}
 628		rv -= data_len;
 629
 630		if (rv == trl_len) /* all pushed */
 631			rv = 0;
 632		else {
 633			c_tx->state = SIW_SEND_TRAILER;
 634			c_tx->ctrl_len = MAX_TRAILER;
 635			c_tx->ctrl_sent = rv + 4 - c_tx->pad;
 636			c_tx->bytes_unsent = 0;
 637			rv = -EAGAIN;
 638		}
 639
 640	} else if (data_len > 0) {
 641		/* Maybe some user data pushed to TCP */
 642		c_tx->state = SIW_SEND_DATA;
 643		wqe->processed -= data_len - rv;
 644
 645		if (rv) {
 646			/*
 647			 * Some bytes out. Recompute tx state based
 648			 * on old state and bytes pushed
 649			 */
 650			unsigned int sge_unsent;
 651
 652			c_tx->bytes_unsent -= rv;
 653			sge = &wqe->sqe.sge[c_tx->sge_idx];
 654			sge_unsent = sge->length - c_tx->sge_off;
 655
 656			while (sge_unsent <= rv) {
 657				rv -= sge_unsent;
 658				c_tx->sge_idx++;
 659				c_tx->sge_off = 0;
 660				sge++;
 661				sge_unsent = sge->length;
 662			}
 663			c_tx->sge_off += rv;
 664		}
 665		rv = -EAGAIN;
 666	}
 667done_crc:
 668	c_tx->do_crc = 0;
 669done:
 670	return rv;
 671}
 672
 673static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx,
 674				     struct socket *s)
 675{
 676	struct tcp_sock *tp = tcp_sk(s->sk);
 677
 678	if (tp->gso_segs) {
 679		if (c_tx->gso_seg_limit == 0)
 680			c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs;
 681		else
 682			c_tx->tcp_seglen =
 683				tp->mss_cache *
 684				min_t(u16, c_tx->gso_seg_limit, tp->gso_segs);
 685	} else {
 686		c_tx->tcp_seglen = tp->mss_cache;
 687	}
 688	/* Loopback may give odd numbers */
 689	c_tx->tcp_seglen &= 0xfffffff8;
 690}
 691
 692/*
 693 * siw_prepare_fpdu()
 694 *
 695 * Prepares transmit context to send out one FPDU if FPDU will contain
 696 * user data and user data are not immediate data.
 697 * Computes maximum FPDU length to fill up TCP MSS if possible.
 698 *
 699 * @qp:		QP from which to transmit
 700 * @wqe:	Current WQE causing transmission
 701 *
 702 * TODO: Take into account real available sendspace on socket
 703 *       to avoid header misalignment due to send pausing within
 704 *       fpdu transmission
 705 */
 706static void siw_prepare_fpdu(struct siw_qp *qp, struct siw_wqe *wqe)
 707{
 708	struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
 709	int data_len;
 710
 711	c_tx->ctrl_len =
 712		iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len;
 713	c_tx->ctrl_sent = 0;
 714
 715	/*
 716	 * Update target buffer offset if any
 717	 */
 718	if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED))
 719		/* Untagged message */
 720		c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed);
 721	else /* Tagged message */
 722		c_tx->pkt.c_tagged.ddp_to =
 723			cpu_to_be64(wqe->sqe.raddr + wqe->processed);
 724
 725	data_len = wqe->bytes - wqe->processed;
 726	if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) {
 727		/* Trim DDP payload to fit into current TCP segment */
 728		data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE);
 729		c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST;
 730		c_tx->pad = 0;
 731	} else {
 732		c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST;
 733		c_tx->pad = -data_len & 0x3;
 734	}
 735	c_tx->bytes_unsent = data_len;
 736
 737	c_tx->pkt.ctrl.mpa_len =
 738		htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE);
 739
 740	/*
 741	 * Init MPA CRC computation
 742	 */
 743	if (c_tx->mpa_crc_enabled) {
 744		siw_crc_init(&c_tx->mpa_crc);
 745		siw_crc_update(&c_tx->mpa_crc, &c_tx->pkt, c_tx->ctrl_len);
 746		c_tx->do_crc = 1;
 747	}
 748}
 749
 750/*
 751 * siw_check_sgl_tx()
 752 *
 753 * Check permissions for a list of SGE's (SGL).
 754 * A successful check will have all memory referenced
 755 * for transmission resolved and assigned to the WQE.
 756 *
 757 * @pd:		Protection Domain SGL should belong to
 758 * @wqe:	WQE to be checked
 759 * @perms:	requested access permissions
 760 *
 761 */
 762
 763static int siw_check_sgl_tx(struct ib_pd *pd, struct siw_wqe *wqe,
 764			    enum ib_access_flags perms)
 765{
 766	struct siw_sge *sge = &wqe->sqe.sge[0];
 767	int i, len, num_sge = wqe->sqe.num_sge;
 768
 769	if (unlikely(num_sge > SIW_MAX_SGE))
 770		return -EINVAL;
 771
 772	for (i = 0, len = 0; num_sge; num_sge--, i++, sge++) {
 773		/*
 774		 * rdma verbs: do not check stag for a zero length sge
 775		 */
 776		if (sge->length) {
 777			int rv = siw_check_sge(pd, sge, &wqe->mem[i], perms, 0,
 778					       sge->length);
 779
 780			if (unlikely(rv != E_ACCESS_OK))
 781				return rv;
 782		}
 783		len += sge->length;
 784	}
 785	return len;
 786}
 787
 788/*
 789 * siw_qp_sq_proc_tx()
 790 *
 791 * Process one WQE which needs transmission on the wire.
 792 */
 793static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe)
 794{
 795	struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
 796	struct socket *s = qp->attrs.sk;
 797	int rv = 0, burst_len = qp->tx_ctx.burst;
 798	enum rdmap_ecode ecode = RDMAP_ECODE_CATASTROPHIC_STREAM;
 799
 800	if (unlikely(wqe->wr_status == SIW_WR_IDLE))
 801		return 0;
 802
 803	if (!burst_len)
 804		burst_len = SQ_USER_MAXBURST;
 805
 806	if (wqe->wr_status == SIW_WR_QUEUED) {
 807		if (!(wqe->sqe.flags & SIW_WQE_INLINE)) {
 808			if (tx_type(wqe) == SIW_OP_READ_RESPONSE)
 809				wqe->sqe.num_sge = 1;
 810
 811			if (tx_type(wqe) != SIW_OP_READ &&
 812			    tx_type(wqe) != SIW_OP_READ_LOCAL_INV) {
 813				/*
 814				 * Reference memory to be tx'd w/o checking
 815				 * access for LOCAL_READ permission, since
 816				 * not defined in RDMA core.
 817				 */
 818				rv = siw_check_sgl_tx(qp->pd, wqe, 0);
 819				if (rv < 0) {
 820					if (tx_type(wqe) ==
 821					    SIW_OP_READ_RESPONSE)
 822						ecode = siw_rdmap_error(-rv);
 823					rv = -EINVAL;
 824					goto tx_error;
 825				}
 826				wqe->bytes = rv;
 827			} else {
 828				wqe->bytes = 0;
 829			}
 830		} else {
 831			wqe->bytes = wqe->sqe.sge[0].length;
 832			if (!rdma_is_kernel_res(&qp->base_qp.res)) {
 833				if (wqe->bytes > SIW_MAX_INLINE) {
 834					rv = -EINVAL;
 835					goto tx_error;
 836				}
 837				wqe->sqe.sge[0].laddr =
 838					(u64)(uintptr_t)&wqe->sqe.sge[1];
 839			}
 840		}
 841		wqe->wr_status = SIW_WR_INPROGRESS;
 842		wqe->processed = 0;
 843
 844		siw_update_tcpseg(c_tx, s);
 845
 846		rv = siw_qp_prepare_tx(c_tx);
 847		if (rv == PKT_FRAGMENTED) {
 848			c_tx->state = SIW_SEND_HDR;
 849			siw_prepare_fpdu(qp, wqe);
 850		} else if (rv == PKT_COMPLETE) {
 851			c_tx->state = SIW_SEND_SHORT_FPDU;
 852		} else {
 853			goto tx_error;
 854		}
 855	}
 856
 857next_segment:
 858	siw_dbg_qp(qp, "wr type %d, state %d, data %u, sent %u, id %llx\n",
 859		   tx_type(wqe), wqe->wr_status, wqe->bytes, wqe->processed,
 860		   wqe->sqe.id);
 861
 862	if (--burst_len == 0) {
 863		rv = -EINPROGRESS;
 864		goto tx_done;
 865	}
 866	if (c_tx->state == SIW_SEND_SHORT_FPDU) {
 867		enum siw_opcode tx_type = tx_type(wqe);
 868		unsigned int msg_flags;
 869
 870		if (siw_sq_empty(qp) || !siw_tcp_nagle || burst_len == 1)
 871			/*
 872			 * End current TCP segment, if SQ runs empty,
 873			 * or siw_tcp_nagle is not set, or we bail out
 874			 * soon due to no burst credit left.
 875			 */
 876			msg_flags = MSG_DONTWAIT;
 877		else
 878			msg_flags = MSG_DONTWAIT | MSG_MORE;
 879
 880		rv = siw_tx_ctrl(c_tx, s, msg_flags);
 881
 882		if (!rv && tx_type != SIW_OP_READ &&
 883		    tx_type != SIW_OP_READ_LOCAL_INV)
 884			wqe->processed = wqe->bytes;
 885
 886		goto tx_done;
 887
 888	} else {
 889		rv = siw_tx_hdt(c_tx, s);
 890	}
 891	if (!rv) {
 892		/*
 893		 * One segment sent. Processing completed if last
 894		 * segment, Do next segment otherwise.
 895		 */
 896		if (unlikely(c_tx->tx_suspend)) {
 897			/*
 898			 * Verbs, 6.4.: Try stopping sending after a full
 899			 * DDP segment if the connection goes down
 900			 * (== peer halfclose)
 901			 */
 902			rv = -ECONNABORTED;
 903			goto tx_done;
 904		}
 905		if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) {
 906			siw_dbg_qp(qp, "WQE completed\n");
 907			goto tx_done;
 908		}
 909		c_tx->state = SIW_SEND_HDR;
 910
 911		siw_update_tcpseg(c_tx, s);
 912
 913		siw_prepare_fpdu(qp, wqe);
 914		goto next_segment;
 915	}
 916tx_done:
 917	qp->tx_ctx.burst = burst_len;
 918	return rv;
 919
 920tx_error:
 921	if (ecode != RDMAP_ECODE_CATASTROPHIC_STREAM)
 922		siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
 923				   RDMAP_ETYPE_REMOTE_PROTECTION, ecode, 1);
 924	else
 925		siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
 926				   RDMAP_ETYPE_CATASTROPHIC,
 927				   RDMAP_ECODE_UNSPECIFIED, 1);
 928	return rv;
 929}
 930
 931static int siw_fastreg_mr(struct ib_pd *pd, struct siw_sqe *sqe)
 932{
 933	struct ib_mr *base_mr = (struct ib_mr *)(uintptr_t)sqe->base_mr;
 934	struct siw_device *sdev = to_siw_dev(pd->device);
 935	struct siw_mem *mem;
 936	int rv = 0;
 937
 938	siw_dbg_pd(pd, "STag 0x%08x\n", sqe->rkey);
 939
 940	if (unlikely(!base_mr)) {
 941		pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey);
 942		return -EINVAL;
 943	}
 944
 945	if (unlikely(base_mr->rkey >> 8 != sqe->rkey  >> 8)) {
 946		pr_warn("siw: fastreg: STag 0x%08x: bad MR\n", sqe->rkey);
 947		return -EINVAL;
 948	}
 949
 950	mem = siw_mem_id2obj(sdev, sqe->rkey  >> 8);
 951	if (unlikely(!mem)) {
 952		pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey);
 953		return -EINVAL;
 954	}
 955
 956	if (unlikely(mem->pd != pd)) {
 957		pr_warn("siw: fastreg: PD mismatch\n");
 958		rv = -EINVAL;
 959		goto out;
 960	}
 961	if (unlikely(mem->stag_valid)) {
 962		pr_warn("siw: fastreg: STag 0x%08x already valid\n", sqe->rkey);
 963		rv = -EINVAL;
 964		goto out;
 965	}
 966	/* Refresh STag since user may have changed key part */
 967	mem->stag = sqe->rkey;
 968	mem->perms = sqe->access;
 969
 970	siw_dbg_mem(mem, "STag 0x%08x now valid\n", sqe->rkey);
 971	mem->va = base_mr->iova;
 972	mem->stag_valid = 1;
 973out:
 974	siw_mem_put(mem);
 975	return rv;
 976}
 977
 978static int siw_qp_sq_proc_local(struct siw_qp *qp, struct siw_wqe *wqe)
 979{
 980	int rv;
 981
 982	switch (tx_type(wqe)) {
 983	case SIW_OP_REG_MR:
 984		rv = siw_fastreg_mr(qp->pd, &wqe->sqe);
 985		break;
 986
 987	case SIW_OP_INVAL_STAG:
 988		rv = siw_invalidate_stag(qp->pd, wqe->sqe.rkey);
 989		break;
 990
 991	default:
 992		rv = -EINVAL;
 993	}
 994	return rv;
 995}
 996
 997/*
 998 * siw_qp_sq_process()
 999 *
1000 * Core TX path routine for RDMAP/DDP/MPA using a TCP kernel socket.
1001 * Sends RDMAP payload for the current SQ WR @wqe of @qp in one or more
1002 * MPA FPDUs, each containing a DDP segment.
1003 *
1004 * SQ processing may occur in user context as a result of posting
1005 * new WQE's or from siw_tx_thread context. Processing in
1006 * user context is limited to non-kernel verbs users.
1007 *
1008 * SQ processing may get paused anytime, possibly in the middle of a WR
1009 * or FPDU, if insufficient send space is available. SQ processing
1010 * gets resumed from siw_tx_thread, if send space becomes available again.
1011 *
1012 * Must be called with the QP state read-locked.
1013 *
1014 * Note:
1015 * An outbound RREQ can be satisfied by the corresponding RRESP
1016 * _before_ it gets assigned to the ORQ. This happens regularly
1017 * in RDMA READ via loopback case. Since both outbound RREQ and
1018 * inbound RRESP can be handled by the same CPU, locking the ORQ
1019 * is dead-lock prone and thus not an option. With that, the
1020 * RREQ gets assigned to the ORQ _before_ being sent - see
1021 * siw_activate_tx() - and pulled back in case of send failure.
1022 */
1023int siw_qp_sq_process(struct siw_qp *qp)
1024{
1025	struct siw_wqe *wqe = tx_wqe(qp);
1026	enum siw_opcode tx_type;
1027	unsigned long flags;
1028	int rv = 0;
1029
1030	siw_dbg_qp(qp, "enter for type %d\n", tx_type(wqe));
1031
1032next_wqe:
1033	/*
1034	 * Stop QP processing if SQ state changed
1035	 */
1036	if (unlikely(qp->tx_ctx.tx_suspend)) {
1037		siw_dbg_qp(qp, "tx suspended\n");
1038		goto done;
1039	}
1040	tx_type = tx_type(wqe);
1041
1042	if (tx_type <= SIW_OP_READ_RESPONSE)
1043		rv = siw_qp_sq_proc_tx(qp, wqe);
1044	else
1045		rv = siw_qp_sq_proc_local(qp, wqe);
1046
1047	if (!rv) {
1048		/*
1049		 * WQE processing done
1050		 */
1051		switch (tx_type) {
1052		case SIW_OP_SEND:
1053		case SIW_OP_SEND_REMOTE_INV:
1054		case SIW_OP_WRITE:
1055			siw_wqe_put_mem(wqe, tx_type);
1056			fallthrough;
1057
1058		case SIW_OP_INVAL_STAG:
1059		case SIW_OP_REG_MR:
1060			if (tx_flags(wqe) & SIW_WQE_SIGNALLED)
1061				siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1062						 SIW_WC_SUCCESS);
1063			break;
1064
1065		case SIW_OP_READ:
1066		case SIW_OP_READ_LOCAL_INV:
1067			/*
1068			 * already enqueued to ORQ queue
1069			 */
1070			break;
1071
1072		case SIW_OP_READ_RESPONSE:
1073			siw_wqe_put_mem(wqe, tx_type);
1074			break;
1075
1076		default:
1077			WARN(1, "undefined WQE type %d\n", tx_type);
1078			rv = -EINVAL;
1079			goto done;
1080		}
1081
1082		spin_lock_irqsave(&qp->sq_lock, flags);
1083		wqe->wr_status = SIW_WR_IDLE;
1084		rv = siw_activate_tx(qp);
1085		spin_unlock_irqrestore(&qp->sq_lock, flags);
1086
1087		if (rv <= 0)
1088			goto done;
1089
1090		goto next_wqe;
1091
1092	} else if (rv == -EAGAIN) {
1093		siw_dbg_qp(qp, "sq paused: hd/tr %d of %d, data %d\n",
1094			   qp->tx_ctx.ctrl_sent, qp->tx_ctx.ctrl_len,
1095			   qp->tx_ctx.bytes_unsent);
1096		rv = 0;
1097		goto done;
1098	} else if (rv == -EINPROGRESS) {
1099		rv = siw_sq_start(qp);
1100		goto done;
1101	} else {
1102		/*
1103		 * WQE processing failed.
1104		 * Verbs 8.3.2:
1105		 * o It turns any WQE into a signalled WQE.
1106		 * o Local catastrophic error must be surfaced
1107		 * o QP must be moved into Terminate state: done by code
1108		 *   doing socket state change processing
1109		 *
1110		 * o TODO: Termination message must be sent.
1111		 * o TODO: Implement more precise work completion errors,
1112		 *         see enum ib_wc_status in ib_verbs.h
1113		 */
1114		siw_dbg_qp(qp, "wqe type %d processing failed: %d\n",
1115			   tx_type(wqe), rv);
1116
1117		spin_lock_irqsave(&qp->sq_lock, flags);
1118		/*
1119		 * RREQ may have already been completed by inbound RRESP!
1120		 */
1121		if ((tx_type == SIW_OP_READ ||
1122		     tx_type == SIW_OP_READ_LOCAL_INV) && qp->attrs.orq_size) {
1123			/* Cleanup pending entry in ORQ */
1124			qp->orq_put--;
1125			qp->orq[qp->orq_put % qp->attrs.orq_size].flags = 0;
1126		}
1127		spin_unlock_irqrestore(&qp->sq_lock, flags);
1128		/*
1129		 * immediately suspends further TX processing
1130		 */
1131		if (!qp->tx_ctx.tx_suspend)
1132			siw_qp_cm_drop(qp, 0);
1133
1134		switch (tx_type) {
1135		case SIW_OP_SEND:
1136		case SIW_OP_SEND_REMOTE_INV:
1137		case SIW_OP_SEND_WITH_IMM:
1138		case SIW_OP_WRITE:
1139		case SIW_OP_READ:
1140		case SIW_OP_READ_LOCAL_INV:
1141			siw_wqe_put_mem(wqe, tx_type);
1142			fallthrough;
1143
1144		case SIW_OP_INVAL_STAG:
1145		case SIW_OP_REG_MR:
1146			siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1147					 SIW_WC_LOC_QP_OP_ERR);
1148
1149			siw_qp_event(qp, IB_EVENT_QP_FATAL);
1150
1151			break;
1152
1153		case SIW_OP_READ_RESPONSE:
1154			siw_dbg_qp(qp, "proc. read.response failed: %d\n", rv);
1155
1156			siw_qp_event(qp, IB_EVENT_QP_REQ_ERR);
1157
1158			siw_wqe_put_mem(wqe, SIW_OP_READ_RESPONSE);
1159
1160			break;
1161
1162		default:
1163			WARN(1, "undefined WQE type %d\n", tx_type);
1164			rv = -EINVAL;
1165		}
1166		wqe->wr_status = SIW_WR_IDLE;
1167	}
1168done:
1169	return rv;
1170}
1171
1172static void siw_sq_resume(struct siw_qp *qp)
1173{
1174	if (down_read_trylock(&qp->state_lock)) {
1175		if (likely(qp->attrs.state == SIW_QP_STATE_RTS &&
1176			   !qp->tx_ctx.tx_suspend)) {
1177			int rv = siw_qp_sq_process(qp);
1178
1179			up_read(&qp->state_lock);
1180
1181			if (unlikely(rv < 0)) {
1182				siw_dbg_qp(qp, "SQ task failed: err %d\n", rv);
1183
1184				if (!qp->tx_ctx.tx_suspend)
1185					siw_qp_cm_drop(qp, 0);
1186			}
1187		} else {
1188			up_read(&qp->state_lock);
1189		}
1190	} else {
1191		siw_dbg_qp(qp, "Resume SQ while QP locked\n");
1192	}
1193	siw_qp_put(qp);
1194}
1195
1196struct tx_task_t {
1197	struct llist_head active;
1198	wait_queue_head_t waiting;
1199};
1200
1201static DEFINE_PER_CPU(struct tx_task_t, siw_tx_task_g);
1202
1203int siw_create_tx_threads(void)
1204{
1205	int cpu, assigned = 0;
1206
1207	for_each_online_cpu(cpu) {
1208		struct tx_task_t *tx_task;
1209
1210		/* Skip HT cores */
1211		if (cpu % cpumask_weight(topology_sibling_cpumask(cpu)))
1212			continue;
1213
1214		tx_task = &per_cpu(siw_tx_task_g, cpu);
1215		init_llist_head(&tx_task->active);
1216		init_waitqueue_head(&tx_task->waiting);
1217
1218		siw_tx_thread[cpu] =
1219			kthread_run_on_cpu(siw_run_sq,
1220					   (unsigned long *)(long)cpu,
1221					   cpu, "siw_tx/%u");
1222		if (IS_ERR(siw_tx_thread[cpu])) {
1223			siw_tx_thread[cpu] = NULL;
1224			continue;
1225		}
1226		assigned++;
1227	}
1228	return assigned;
1229}
1230
1231void siw_stop_tx_threads(void)
1232{
1233	int cpu;
1234
1235	for_each_possible_cpu(cpu) {
1236		if (siw_tx_thread[cpu]) {
1237			kthread_stop(siw_tx_thread[cpu]);
1238			wake_up(&per_cpu(siw_tx_task_g, cpu).waiting);
1239			siw_tx_thread[cpu] = NULL;
1240		}
1241	}
1242}
1243
1244int siw_run_sq(void *data)
1245{
1246	const int nr_cpu = (unsigned int)(long)data;
1247	struct llist_node *active;
1248	struct siw_qp *qp;
1249	struct tx_task_t *tx_task = &per_cpu(siw_tx_task_g, nr_cpu);
1250
1251	while (1) {
1252		struct llist_node *fifo_list = NULL;
1253
1254		wait_event_interruptible(tx_task->waiting,
1255					 !llist_empty(&tx_task->active) ||
1256						 kthread_should_stop());
1257
1258		if (kthread_should_stop())
1259			break;
1260
1261		active = llist_del_all(&tx_task->active);
1262		/*
1263		 * llist_del_all returns a list with newest entry first.
1264		 * Re-order list for fairness among QP's.
1265		 */
1266		fifo_list = llist_reverse_order(active);
1267		while (fifo_list) {
1268			qp = container_of(fifo_list, struct siw_qp, tx_list);
1269			fifo_list = llist_next(fifo_list);
1270			qp->tx_list.next = NULL;
1271
1272			siw_sq_resume(qp);
1273		}
1274	}
1275	active = llist_del_all(&tx_task->active);
1276	if (active) {
1277		llist_for_each_entry(qp, active, tx_list) {
1278			qp->tx_list.next = NULL;
1279			siw_sq_resume(qp);
1280		}
1281	}
1282	return 0;
1283}
1284
1285int siw_sq_start(struct siw_qp *qp)
1286{
1287	if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
1288		return 0;
1289
1290	if (unlikely(!cpu_online(qp->tx_cpu))) {
1291		siw_put_tx_cpu(qp->tx_cpu);
1292		qp->tx_cpu = siw_get_tx_cpu(qp->sdev);
1293		if (qp->tx_cpu < 0) {
1294			pr_warn("siw: no tx cpu available\n");
1295
1296			return -EIO;
1297		}
1298	}
1299	siw_qp_get(qp);
1300
1301	llist_add(&qp->tx_list, &per_cpu(siw_tx_task_g, qp->tx_cpu).active);
1302
1303	wake_up(&per_cpu(siw_tx_task_g, qp->tx_cpu).waiting);
1304
1305	return 0;
1306}