efct_scsi.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2021 Broadcom. All Rights Reserved. The term
   4 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
   5 */
   6
   7#include "efct_driver.h"
   8#include "efct_hw.h"
   9
  10#define enable_tsend_auto_resp(efct)	1
  11#define enable_treceive_auto_resp(efct)	0
  12
  13#define SCSI_IOFMT "[%04x][i:%04x t:%04x h:%04x]"
  14
  15#define scsi_io_printf(io, fmt, ...) \
  16	efc_log_debug(io->efct, "[%s]" SCSI_IOFMT fmt, \
  17		io->node->display_name, io->instance_index,\
  18		io->init_task_tag, io->tgt_task_tag, io->hw_tag, ##__VA_ARGS__)
  19
  20#define EFCT_LOG_ENABLE_SCSI_TRACE(efct)                \
  21		(((efct) != NULL) ? (((efct)->logmask & (1U << 2)) != 0) : 0)
  22
  23#define scsi_io_trace(io, fmt, ...) \
  24	do { \
  25		if (EFCT_LOG_ENABLE_SCSI_TRACE(io->efct)) \
  26			scsi_io_printf(io, fmt, ##__VA_ARGS__); \
  27	} while (0)
  28
  29struct efct_io *
  30efct_scsi_io_alloc(struct efct_node *node)
  31{
  32	struct efct *efct;
  33	struct efct_xport *xport;
  34	struct efct_io *io;
  35	unsigned long flags;
  36
  37	efct = node->efct;
  38
  39	xport = efct->xport;
  40
  41	io = efct_io_pool_io_alloc(efct->xport->io_pool);
  42	if (!io) {
  43		efc_log_err(efct, "IO alloc Failed\n");
  44		atomic_add_return(1, &xport->io_alloc_failed_count);
  45		return NULL;
  46	}
  47
  48	/* initialize refcount */
  49	kref_init(&io->ref);
  50	io->release = _efct_scsi_io_free;
  51
  52	/* set generic fields */
  53	io->efct = efct;
  54	io->node = node;
  55	kref_get(&node->ref);
  56
  57	/* set type and name */
  58	io->io_type = EFCT_IO_TYPE_IO;
  59	io->display_name = "scsi_io";
  60
  61	io->cmd_ini = false;
  62	io->cmd_tgt = true;
  63
  64	/* Add to node's active_ios list */
  65	INIT_LIST_HEAD(&io->list_entry);
  66	spin_lock_irqsave(&node->active_ios_lock, flags);
  67	list_add(&io->list_entry, &node->active_ios);
  68
  69	spin_unlock_irqrestore(&node->active_ios_lock, flags);
  70
  71	return io;
  72}
  73
  74void
  75_efct_scsi_io_free(struct kref *arg)
  76{
  77	struct efct_io *io = container_of(arg, struct efct_io, ref);
  78	struct efct *efct = io->efct;
  79	struct efct_node *node = io->node;
  80	unsigned long flags = 0;
  81
  82	scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
  83
  84	if (io->io_free) {
  85		efc_log_err(efct, "IO already freed.\n");
  86		return;
  87	}
  88
  89	spin_lock_irqsave(&node->active_ios_lock, flags);
  90	list_del_init(&io->list_entry);
  91	spin_unlock_irqrestore(&node->active_ios_lock, flags);
  92
  93	kref_put(&node->ref, node->release);
  94	io->node = NULL;
  95	efct_io_pool_io_free(efct->xport->io_pool, io);
  96}
  97
  98void
  99efct_scsi_io_free(struct efct_io *io)
 100{
 101	scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
 102	WARN_ON(!refcount_read(&io->ref.refcount));
 103	kref_put(&io->ref, io->release);
 104}
 105
 106static void
 107efct_target_io_cb(struct efct_hw_io *hio, u32 length, int status,
 108		  u32 ext_status, void *app)
 109{
 110	u32 flags = 0;
 111	struct efct_io *io = app;
 112	struct efct *efct;
 113	enum efct_scsi_io_status scsi_stat = EFCT_SCSI_STATUS_GOOD;
 114	efct_scsi_io_cb_t cb;
 115
 116	if (!io || !io->efct) {
 117		pr_err("%s: IO can not be NULL\n", __func__);
 118		return;
 119	}
 120
 121	scsi_io_trace(io, "status x%x ext_status x%x\n", status, ext_status);
 122
 123	efct = io->efct;
 124
 125	io->transferred += length;
 126
 127	if (!io->scsi_tgt_cb) {
 128		efct_scsi_check_pending(efct);
 129		return;
 130	}
 131
 132	/* Call target server completion */
 133	cb = io->scsi_tgt_cb;
 134
 135	/* Clear the callback before invoking the callback */
 136	io->scsi_tgt_cb = NULL;
 137
 138	/* if status was good, and auto-good-response was set,
 139	 * then callback target-server with IO_CMPL_RSP_SENT,
 140	 * otherwise send IO_CMPL
 141	 */
 142	if (status == 0 && io->auto_resp)
 143		flags |= EFCT_SCSI_IO_CMPL_RSP_SENT;
 144	else
 145		flags |= EFCT_SCSI_IO_CMPL;
 146
 147	switch (status) {
 148	case SLI4_FC_WCQE_STATUS_SUCCESS:
 149		scsi_stat = EFCT_SCSI_STATUS_GOOD;
 150		break;
 151	case SLI4_FC_WCQE_STATUS_DI_ERROR:
 152		if (ext_status & SLI4_FC_DI_ERROR_GE)
 153			scsi_stat = EFCT_SCSI_STATUS_DIF_GUARD_ERR;
 154		else if (ext_status & SLI4_FC_DI_ERROR_AE)
 155			scsi_stat = EFCT_SCSI_STATUS_DIF_APP_TAG_ERROR;
 156		else if (ext_status & SLI4_FC_DI_ERROR_RE)
 157			scsi_stat = EFCT_SCSI_STATUS_DIF_REF_TAG_ERROR;
 158		else
 159			scsi_stat = EFCT_SCSI_STATUS_DIF_UNKNOWN_ERROR;
 160		break;
 161	case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
 162		switch (ext_status) {
 163		case SLI4_FC_LOCAL_REJECT_INVALID_RELOFFSET:
 164		case SLI4_FC_LOCAL_REJECT_ABORT_REQUESTED:
 165			scsi_stat = EFCT_SCSI_STATUS_ABORTED;
 166			break;
 167		case SLI4_FC_LOCAL_REJECT_INVALID_RPI:
 168			scsi_stat = EFCT_SCSI_STATUS_NEXUS_LOST;
 169			break;
 170		case SLI4_FC_LOCAL_REJECT_NO_XRI:
 171			scsi_stat = EFCT_SCSI_STATUS_NO_IO;
 172			break;
 173		default:
 174			/*we have seen 0x0d(TX_DMA_FAILED err)*/
 175			scsi_stat = EFCT_SCSI_STATUS_ERROR;
 176			break;
 177		}
 178		break;
 179
 180	case SLI4_FC_WCQE_STATUS_TARGET_WQE_TIMEOUT:
 181		/* target IO timed out */
 182		scsi_stat = EFCT_SCSI_STATUS_TIMEDOUT_AND_ABORTED;
 183		break;
 184
 185	case SLI4_FC_WCQE_STATUS_SHUTDOWN:
 186		/* Target IO cancelled by HW */
 187		scsi_stat = EFCT_SCSI_STATUS_SHUTDOWN;
 188		break;
 189
 190	default:
 191		scsi_stat = EFCT_SCSI_STATUS_ERROR;
 192		break;
 193	}
 194
 195	cb(io, scsi_stat, flags, io->scsi_tgt_cb_arg);
 196
 197	efct_scsi_check_pending(efct);
 198}
 199
 200static int
 201efct_scsi_build_sgls(struct efct_hw *hw, struct efct_hw_io *hio,
 202		     struct efct_scsi_sgl *sgl, u32 sgl_count,
 203		     enum efct_hw_io_type type)
 204{
 205	int rc;
 206	u32 i;
 207	struct efct *efct = hw->os;
 208
 209	/* Initialize HW SGL */
 210	rc = efct_hw_io_init_sges(hw, hio, type);
 211	if (rc) {
 212		efc_log_err(efct, "efct_hw_io_init_sges failed: %d\n", rc);
 213		return -EIO;
 214	}
 215
 216	for (i = 0; i < sgl_count; i++) {
 217		/* Add data SGE */
 218		rc = efct_hw_io_add_sge(hw, hio, sgl[i].addr, sgl[i].len);
 219		if (rc) {
 220			efc_log_err(efct, "add sge failed cnt=%d rc=%d\n",
 221				    sgl_count, rc);
 222			return rc;
 223		}
 224	}
 225
 226	return 0;
 227}
 228
 229static void efc_log_sgl(struct efct_io *io)
 230{
 231	struct efct_hw_io *hio = io->hio;
 232	struct sli4_sge *data = NULL;
 233	u32 *dword = NULL;
 234	u32 i;
 235	u32 n_sge;
 236
 237	scsi_io_trace(io, "def_sgl at 0x%x 0x%08x\n",
 238		      upper_32_bits(hio->def_sgl.phys),
 239		      lower_32_bits(hio->def_sgl.phys));
 240	n_sge = (hio->sgl == &hio->def_sgl) ? hio->n_sge : hio->def_sgl_count;
 241	for (i = 0, data = hio->def_sgl.virt; i < n_sge; i++, data++) {
 242		dword = (u32 *)data;
 243
 244		scsi_io_trace(io, "SGL %2d 0x%08x 0x%08x 0x%08x 0x%08x\n",
 245			      i, dword[0], dword[1], dword[2], dword[3]);
 246
 247		if (dword[2] & (1U << 31))
 248			break;
 249	}
 250}
 251
 252static void
 253efct_scsi_check_pending_async_cb(struct efct_hw *hw, int status,
 254				 u8 *mqe, void *arg)
 255{
 256	struct efct_io *io = arg;
 257
 258	if (io) {
 259		efct_hw_done_t cb = io->hw_cb;
 260
 261		if (!io->hw_cb)
 262			return;
 263
 264		io->hw_cb = NULL;
 265		(cb)(io->hio, 0, SLI4_FC_WCQE_STATUS_DISPATCH_ERROR, 0, io);
 266	}
 267}
 268
 269static int
 270efct_scsi_io_dispatch_hw_io(struct efct_io *io, struct efct_hw_io *hio)
 271{
 272	int rc = 0;
 273	struct efct *efct = io->efct;
 274
 275	/* Got a HW IO;
 276	 * update ini/tgt_task_tag with HW IO info and dispatch
 277	 */
 278	io->hio = hio;
 279	if (io->cmd_tgt)
 280		io->tgt_task_tag = hio->indicator;
 281	else if (io->cmd_ini)
 282		io->init_task_tag = hio->indicator;
 283	io->hw_tag = hio->reqtag;
 284
 285	hio->eq = io->hw_priv;
 286
 287	/* Copy WQ steering */
 288	switch (io->wq_steering) {
 289	case EFCT_SCSI_WQ_STEERING_CLASS >> EFCT_SCSI_WQ_STEERING_SHIFT:
 290		hio->wq_steering = EFCT_HW_WQ_STEERING_CLASS;
 291		break;
 292	case EFCT_SCSI_WQ_STEERING_REQUEST >> EFCT_SCSI_WQ_STEERING_SHIFT:
 293		hio->wq_steering = EFCT_HW_WQ_STEERING_REQUEST;
 294		break;
 295	case EFCT_SCSI_WQ_STEERING_CPU >> EFCT_SCSI_WQ_STEERING_SHIFT:
 296		hio->wq_steering = EFCT_HW_WQ_STEERING_CPU;
 297		break;
 298	}
 299
 300	switch (io->io_type) {
 301	case EFCT_IO_TYPE_IO:
 302		rc = efct_scsi_build_sgls(&efct->hw, io->hio,
 303					  io->sgl, io->sgl_count, io->hio_type);
 304		if (rc)
 305			break;
 306
 307		if (EFCT_LOG_ENABLE_SCSI_TRACE(efct))
 308			efc_log_sgl(io);
 309
 310		if (io->app_id)
 311			io->iparam.fcp_tgt.app_id = io->app_id;
 312
 313		io->iparam.fcp_tgt.vpi = io->node->vpi;
 314		io->iparam.fcp_tgt.rpi = io->node->rpi;
 315		io->iparam.fcp_tgt.s_id = io->node->port_fc_id;
 316		io->iparam.fcp_tgt.d_id = io->node->node_fc_id;
 317		io->iparam.fcp_tgt.xmit_len = io->wire_len;
 318
 319		rc = efct_hw_io_send(&io->efct->hw, io->hio_type, io->hio,
 320				     &io->iparam, io->hw_cb, io);
 321		break;
 322	default:
 323		scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
 324		rc = -EIO;
 325		break;
 326	}
 327	return rc;
 328}
 329
 330static int
 331efct_scsi_io_dispatch_no_hw_io(struct efct_io *io)
 332{
 333	int rc;
 334
 335	switch (io->io_type) {
 336	case EFCT_IO_TYPE_ABORT: {
 337		struct efct_hw_io *hio_to_abort = NULL;
 338
 339		hio_to_abort = io->io_to_abort->hio;
 340
 341		if (!hio_to_abort) {
 342			/*
 343			 * If "IO to abort" does not have an
 344			 * associated HW IO, immediately make callback with
 345			 * success. The command must have been sent to
 346			 * the backend, but the data phase has not yet
 347			 * started, so we don't have a HW IO.
 348			 *
 349			 * Note: since the backend shims should be
 350			 * taking a reference on io_to_abort, it should not
 351			 * be possible to have been completed and freed by
 352			 * the backend before the abort got here.
 353			 */
 354			scsi_io_printf(io, "IO: not active\n");
 355			((efct_hw_done_t)io->hw_cb)(io->hio, 0,
 356					SLI4_FC_WCQE_STATUS_SUCCESS, 0, io);
 357			rc = 0;
 358			break;
 359		}
 360
 361		/* HW IO is valid, abort it */
 362		scsi_io_printf(io, "aborting\n");
 363		rc = efct_hw_io_abort(&io->efct->hw, hio_to_abort,
 364				      io->send_abts, io->hw_cb, io);
 365		if (rc) {
 366			int status = SLI4_FC_WCQE_STATUS_SUCCESS;
 367			efct_hw_done_t cb = io->hw_cb;
 368
 369			if (rc != -ENOENT && rc != -EINPROGRESS) {
 370				status = -1;
 371				scsi_io_printf(io, "Failed to abort IO rc=%d\n",
 372					       rc);
 373			}
 374			cb(io->hio, 0, status, 0, io);
 375			rc = 0;
 376		}
 377
 378		break;
 379	}
 380	default:
 381		scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);
 382		rc = -EIO;
 383		break;
 384	}
 385	return rc;
 386}
 387
 388static struct efct_io *
 389efct_scsi_dispatch_pending(struct efct *efct)
 390{
 391	struct efct_xport *xport = efct->xport;
 392	struct efct_io *io = NULL;
 393	struct efct_hw_io *hio;
 394	unsigned long flags = 0;
 395	int status;
 396
 397	spin_lock_irqsave(&xport->io_pending_lock, flags);
 398
 399	if (!list_empty(&xport->io_pending_list)) {
 400		io = list_first_entry(&xport->io_pending_list, struct efct_io,
 401				      io_pending_link);
 402		list_del_init(&io->io_pending_link);
 403	}
 404
 405	if (!io) {
 406		spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 407		return NULL;
 408	}
 409
 410	if (io->io_type == EFCT_IO_TYPE_ABORT) {
 411		hio = NULL;
 412	} else {
 413		hio = efct_hw_io_alloc(&efct->hw);
 414		if (!hio) {
 415			/*
 416			 * No HW IO available.Put IO back on
 417			 * the front of pending list
 418			 */
 419			list_add(&xport->io_pending_list, &io->io_pending_link);
 420			io = NULL;
 421		} else {
 422			hio->eq = io->hw_priv;
 423		}
 424	}
 425
 426	/* Must drop the lock before dispatching the IO */
 427	spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 428
 429	if (!io)
 430		return NULL;
 431
 432	/*
 433	 * We pulled an IO off the pending list,
 434	 * and either got an HW IO or don't need one
 435	 */
 436	atomic_sub_return(1, &xport->io_pending_count);
 437	if (!hio)
 438		status = efct_scsi_io_dispatch_no_hw_io(io);
 439	else
 440		status = efct_scsi_io_dispatch_hw_io(io, hio);
 441	if (status) {
 442		/*
 443		 * Invoke the HW callback, but do so in the
 444		 * separate execution context,provided by the
 445		 * NOP mailbox completion processing context
 446		 * by using efct_hw_async_call()
 447		 */
 448		if (efct_hw_async_call(&efct->hw,
 449				       efct_scsi_check_pending_async_cb, io)) {
 450			efc_log_debug(efct, "call hw async failed\n");
 451		}
 452	}
 453
 454	return io;
 455}
 456
 457void
 458efct_scsi_check_pending(struct efct *efct)
 459{
 460	struct efct_xport *xport = efct->xport;
 461	struct efct_io *io = NULL;
 462	int count = 0;
 463	unsigned long flags = 0;
 464	int dispatch = 0;
 465
 466	/* Guard against recursion */
 467	if (atomic_add_return(1, &xport->io_pending_recursing)) {
 468		/* This function is already running.  Decrement and return. */
 469		atomic_sub_return(1, &xport->io_pending_recursing);
 470		return;
 471	}
 472
 473	while (efct_scsi_dispatch_pending(efct))
 474		count++;
 475
 476	if (count) {
 477		atomic_sub_return(1, &xport->io_pending_recursing);
 478		return;
 479	}
 480
 481	/*
 482	 * If nothing was removed from the list,
 483	 * we might be in a case where we need to abort an
 484	 * active IO and the abort is on the pending list.
 485	 * Look for an abort we can dispatch.
 486	 */
 487
 488	spin_lock_irqsave(&xport->io_pending_lock, flags);
 489
 490	list_for_each_entry(io, &xport->io_pending_list, io_pending_link) {
 491		if (io->io_type == EFCT_IO_TYPE_ABORT && io->io_to_abort->hio) {
 492			/* This IO has a HW IO, so it is
 493			 * active.  Dispatch the abort.
 494			 */
 495			dispatch = 1;
 496			list_del_init(&io->io_pending_link);
 497			atomic_sub_return(1, &xport->io_pending_count);
 498			break;
 499		}
 500	}
 501
 502	spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 503
 504	if (dispatch) {
 505		if (efct_scsi_io_dispatch_no_hw_io(io)) {
 506			if (efct_hw_async_call(&efct->hw,
 507				efct_scsi_check_pending_async_cb, io)) {
 508				efc_log_debug(efct, "hw async failed\n");
 509			}
 510		}
 511	}
 512
 513	atomic_sub_return(1, &xport->io_pending_recursing);
 514}
 515
 516int
 517efct_scsi_io_dispatch(struct efct_io *io, void *cb)
 518{
 519	struct efct_hw_io *hio;
 520	struct efct *efct = io->efct;
 521	struct efct_xport *xport = efct->xport;
 522	unsigned long flags = 0;
 523
 524	io->hw_cb = cb;
 525
 526	/*
 527	 * if this IO already has a HW IO, then this is either
 528	 * not the first phase of the IO. Send it to the HW.
 529	 */
 530	if (io->hio)
 531		return efct_scsi_io_dispatch_hw_io(io, io->hio);
 532
 533	/*
 534	 * We don't already have a HW IO associated with the IO. First check
 535	 * the pending list. If not empty, add IO to the tail and process the
 536	 * pending list.
 537	 */
 538	spin_lock_irqsave(&xport->io_pending_lock, flags);
 539	if (!list_empty(&xport->io_pending_list)) {
 540		/*
 541		 * If this is a low latency request,
 542		 * the put at the front of the IO pending
 543		 * queue, otherwise put it at the end of the queue.
 544		 */
 545		if (io->low_latency) {
 546			INIT_LIST_HEAD(&io->io_pending_link);
 547			list_add(&xport->io_pending_list, &io->io_pending_link);
 548		} else {
 549			INIT_LIST_HEAD(&io->io_pending_link);
 550			list_add_tail(&io->io_pending_link,
 551				      &xport->io_pending_list);
 552		}
 553		spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 554		atomic_add_return(1, &xport->io_pending_count);
 555		atomic_add_return(1, &xport->io_total_pending);
 556
 557		/* process pending list */
 558		efct_scsi_check_pending(efct);
 559		return 0;
 560	}
 561	spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 562
 563	/*
 564	 * We don't have a HW IO associated with the IO and there's nothing
 565	 * on the pending list. Attempt to allocate a HW IO and dispatch it.
 566	 */
 567	hio = efct_hw_io_alloc(&io->efct->hw);
 568	if (!hio) {
 569		/* Couldn't get a HW IO. Save this IO on the pending list */
 570		spin_lock_irqsave(&xport->io_pending_lock, flags);
 571		INIT_LIST_HEAD(&io->io_pending_link);
 572		list_add_tail(&io->io_pending_link, &xport->io_pending_list);
 573		spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 574
 575		atomic_add_return(1, &xport->io_total_pending);
 576		atomic_add_return(1, &xport->io_pending_count);
 577		return 0;
 578	}
 579
 580	/* We successfully allocated a HW IO; dispatch to HW */
 581	return efct_scsi_io_dispatch_hw_io(io, hio);
 582}
 583
 584int
 585efct_scsi_io_dispatch_abort(struct efct_io *io, void *cb)
 586{
 587	struct efct *efct = io->efct;
 588	struct efct_xport *xport = efct->xport;
 589	unsigned long flags = 0;
 590
 591	io->hw_cb = cb;
 592
 593	/*
 594	 * For aborts, we don't need a HW IO, but we still want
 595	 * to pass through the pending list to preserve ordering.
 596	 * Thus, if the pending list is not empty, add this abort
 597	 * to the pending list and process the pending list.
 598	 */
 599	spin_lock_irqsave(&xport->io_pending_lock, flags);
 600	if (!list_empty(&xport->io_pending_list)) {
 601		INIT_LIST_HEAD(&io->io_pending_link);
 602		list_add_tail(&io->io_pending_link, &xport->io_pending_list);
 603		spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 604		atomic_add_return(1, &xport->io_pending_count);
 605		atomic_add_return(1, &xport->io_total_pending);
 606
 607		/* process pending list */
 608		efct_scsi_check_pending(efct);
 609		return 0;
 610	}
 611	spin_unlock_irqrestore(&xport->io_pending_lock, flags);
 612
 613	/* nothing on pending list, dispatch abort */
 614	return efct_scsi_io_dispatch_no_hw_io(io);
 615}
 616
 617static inline int
 618efct_scsi_xfer_data(struct efct_io *io, u32 flags,
 619		    struct efct_scsi_sgl *sgl, u32 sgl_count, u64 xwire_len,
 620		    enum efct_hw_io_type type, int enable_ar,
 621		    efct_scsi_io_cb_t cb, void *arg)
 622{
 623	struct efct *efct;
 624	size_t residual = 0;
 625
 626	io->sgl_count = sgl_count;
 627
 628	efct = io->efct;
 629
 630	scsi_io_trace(io, "%s wire_len %llu\n",
 631		      (type == EFCT_HW_IO_TARGET_READ) ? "send" : "recv",
 632		      xwire_len);
 633
 634	io->hio_type = type;
 635
 636	io->scsi_tgt_cb = cb;
 637	io->scsi_tgt_cb_arg = arg;
 638
 639	residual = io->exp_xfer_len - io->transferred;
 640	io->wire_len = (xwire_len < residual) ? xwire_len : residual;
 641	residual = (xwire_len - io->wire_len);
 642
 643	memset(&io->iparam, 0, sizeof(io->iparam));
 644	io->iparam.fcp_tgt.ox_id = io->init_task_tag;
 645	io->iparam.fcp_tgt.offset = io->transferred;
 646	io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
 647	io->iparam.fcp_tgt.timeout = io->timeout;
 648
 649	/* if this is the last data phase and there is no residual, enable
 650	 * auto-good-response
 651	 */
 652	if (enable_ar && (flags & EFCT_SCSI_LAST_DATAPHASE) && residual == 0 &&
 653	    ((io->transferred + io->wire_len) == io->exp_xfer_len) &&
 654	    (!(flags & EFCT_SCSI_NO_AUTO_RESPONSE))) {
 655		io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
 656		io->auto_resp = true;
 657	} else {
 658		io->auto_resp = false;
 659	}
 660
 661	/* save this transfer length */
 662	io->xfer_req = io->wire_len;
 663
 664	/* Adjust the transferred count to account for overrun
 665	 * when the residual is calculated in efct_scsi_send_resp
 666	 */
 667	io->transferred += residual;
 668
 669	/* Adjust the SGL size if there is overrun */
 670
 671	if (residual) {
 672		struct efct_scsi_sgl  *sgl_ptr = &io->sgl[sgl_count - 1];
 673
 674		while (residual) {
 675			size_t len = sgl_ptr->len;
 676
 677			if (len > residual) {
 678				sgl_ptr->len = len - residual;
 679				residual = 0;
 680			} else {
 681				sgl_ptr->len = 0;
 682				residual -= len;
 683				io->sgl_count--;
 684			}
 685			sgl_ptr--;
 686		}
 687	}
 688
 689	/* Set latency and WQ steering */
 690	io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
 691	io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
 692				EFCT_SCSI_WQ_STEERING_SHIFT;
 693	io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
 694				EFCT_SCSI_WQ_CLASS_SHIFT;
 695
 696	if (efct->xport) {
 697		struct efct_xport *xport = efct->xport;
 698
 699		if (type == EFCT_HW_IO_TARGET_READ) {
 700			xport->fcp_stats.input_requests++;
 701			xport->fcp_stats.input_bytes += xwire_len;
 702		} else if (type == EFCT_HW_IO_TARGET_WRITE) {
 703			xport->fcp_stats.output_requests++;
 704			xport->fcp_stats.output_bytes += xwire_len;
 705		}
 706	}
 707	return efct_scsi_io_dispatch(io, efct_target_io_cb);
 708}
 709
 710int
 711efct_scsi_send_rd_data(struct efct_io *io, u32 flags,
 712		       struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
 713		       efct_scsi_io_cb_t cb, void *arg)
 714{
 715	return efct_scsi_xfer_data(io, flags, sgl, sgl_count,
 716				   len, EFCT_HW_IO_TARGET_READ,
 717				   enable_tsend_auto_resp(io->efct), cb, arg);
 718}
 719
 720int
 721efct_scsi_recv_wr_data(struct efct_io *io, u32 flags,
 722		       struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,
 723		       efct_scsi_io_cb_t cb, void *arg)
 724{
 725	return efct_scsi_xfer_data(io, flags, sgl, sgl_count, len,
 726				   EFCT_HW_IO_TARGET_WRITE,
 727				   enable_treceive_auto_resp(io->efct), cb, arg);
 728}
 729
 730int
 731efct_scsi_send_resp(struct efct_io *io, u32 flags,
 732		    struct efct_scsi_cmd_resp *rsp,
 733		    efct_scsi_io_cb_t cb, void *arg)
 734{
 735	struct efct *efct;
 736	int residual;
 737	/* Always try auto resp */
 738	bool auto_resp = true;
 739	u8 scsi_status = 0;
 740	u16 scsi_status_qualifier = 0;
 741	u8 *sense_data = NULL;
 742	u32 sense_data_length = 0;
 743
 744	efct = io->efct;
 745
 746	if (rsp) {
 747		scsi_status = rsp->scsi_status;
 748		scsi_status_qualifier = rsp->scsi_status_qualifier;
 749		sense_data = rsp->sense_data;
 750		sense_data_length = rsp->sense_data_length;
 751		residual = rsp->residual;
 752	} else {
 753		residual = io->exp_xfer_len - io->transferred;
 754	}
 755
 756	io->wire_len = 0;
 757	io->hio_type = EFCT_HW_IO_TARGET_RSP;
 758
 759	io->scsi_tgt_cb = cb;
 760	io->scsi_tgt_cb_arg = arg;
 761
 762	memset(&io->iparam, 0, sizeof(io->iparam));
 763	io->iparam.fcp_tgt.ox_id = io->init_task_tag;
 764	io->iparam.fcp_tgt.offset = 0;
 765	io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
 766	io->iparam.fcp_tgt.timeout = io->timeout;
 767
 768	/* Set low latency queueing request */
 769	io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;
 770	io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>
 771				EFCT_SCSI_WQ_STEERING_SHIFT;
 772	io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>
 773				EFCT_SCSI_WQ_CLASS_SHIFT;
 774
 775	if (scsi_status != 0 || residual || sense_data_length) {
 776		struct fcp_resp_with_ext *fcprsp = io->rspbuf.virt;
 777		u8 *sns_data;
 778
 779		if (!fcprsp) {
 780			efc_log_err(efct, "NULL response buffer\n");
 781			return -EIO;
 782		}
 783
 784		sns_data = (u8 *)io->rspbuf.virt + sizeof(*fcprsp);
 785
 786		auto_resp = false;
 787
 788		memset(fcprsp, 0, sizeof(*fcprsp));
 789
 790		io->wire_len += sizeof(*fcprsp);
 791
 792		fcprsp->resp.fr_status = scsi_status;
 793		fcprsp->resp.fr_retry_delay =
 794			cpu_to_be16(scsi_status_qualifier);
 795
 796		/* set residual status if necessary */
 797		if (residual != 0) {
 798			/* FCP: if data transferred is less than the
 799			 * amount expected, then this is an underflow.
 800			 * If data transferred would have been greater
 801			 * than the amount expected this is an overflow
 802			 */
 803			if (residual > 0) {
 804				fcprsp->resp.fr_flags |= FCP_RESID_UNDER;
 805				fcprsp->ext.fr_resid =	cpu_to_be32(residual);
 806			} else {
 807				fcprsp->resp.fr_flags |= FCP_RESID_OVER;
 808				fcprsp->ext.fr_resid = cpu_to_be32(-residual);
 809			}
 810		}
 811
 812		if (EFCT_SCSI_SNS_BUF_VALID(sense_data) && sense_data_length) {
 813			if (sense_data_length > SCSI_SENSE_BUFFERSIZE) {
 814				efc_log_err(efct, "Sense exceeds max size.\n");
 815				return -EIO;
 816			}
 817
 818			fcprsp->resp.fr_flags |= FCP_SNS_LEN_VAL;
 819			memcpy(sns_data, sense_data, sense_data_length);
 820			fcprsp->ext.fr_sns_len = cpu_to_be32(sense_data_length);
 821			io->wire_len += sense_data_length;
 822		}
 823
 824		io->sgl[0].addr = io->rspbuf.phys;
 825		io->sgl[0].dif_addr = 0;
 826		io->sgl[0].len = io->wire_len;
 827		io->sgl_count = 1;
 828	}
 829
 830	if (auto_resp)
 831		io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;
 832
 833	return efct_scsi_io_dispatch(io, efct_target_io_cb);
 834}
 835
 836static int
 837efct_target_bls_resp_cb(struct efct_hw_io *hio,	u32 length, int status,
 838			u32 ext_status, void *app)
 839{
 840	struct efct_io *io = app;
 841	struct efct *efct;
 842	enum efct_scsi_io_status bls_status;
 843
 844	efct = io->efct;
 845
 846	/* BLS isn't really a "SCSI" concept, but use SCSI status */
 847	if (status) {
 848		io_error_log(io, "s=%#x x=%#x\n", status, ext_status);
 849		bls_status = EFCT_SCSI_STATUS_ERROR;
 850	} else {
 851		bls_status = EFCT_SCSI_STATUS_GOOD;
 852	}
 853
 854	if (io->bls_cb) {
 855		efct_scsi_io_cb_t bls_cb = io->bls_cb;
 856		void *bls_cb_arg = io->bls_cb_arg;
 857
 858		io->bls_cb = NULL;
 859		io->bls_cb_arg = NULL;
 860
 861		/* invoke callback */
 862		bls_cb(io, bls_status, 0, bls_cb_arg);
 863	}
 864
 865	efct_scsi_check_pending(efct);
 866	return 0;
 867}
 868
 869static int
 870efct_target_send_bls_resp(struct efct_io *io,
 871			  efct_scsi_io_cb_t cb, void *arg)
 872{
 873	struct efct_node *node = io->node;
 874	struct sli_bls_params *bls = &io->iparam.bls;
 875	struct efct *efct = node->efct;
 876	struct fc_ba_acc *acc;
 877	int rc;
 878
 879	/* fill out IO structure with everything needed to send BA_ACC */
 880	memset(&io->iparam, 0, sizeof(io->iparam));
 881	bls->ox_id = io->init_task_tag;
 882	bls->rx_id = io->abort_rx_id;
 883	bls->vpi = io->node->vpi;
 884	bls->rpi = io->node->rpi;
 885	bls->s_id = U32_MAX;
 886	bls->d_id = io->node->node_fc_id;
 887	bls->rpi_registered = true;
 888
 889	acc = (void *)bls->payload;
 890	acc->ba_ox_id = cpu_to_be16(bls->ox_id);
 891	acc->ba_rx_id = cpu_to_be16(bls->rx_id);
 892	acc->ba_high_seq_cnt = cpu_to_be16(U16_MAX);
 893
 894	/* generic io fields have already been populated */
 895
 896	/* set type and BLS-specific fields */
 897	io->io_type = EFCT_IO_TYPE_BLS_RESP;
 898	io->display_name = "bls_rsp";
 899	io->hio_type = EFCT_HW_BLS_ACC;
 900	io->bls_cb = cb;
 901	io->bls_cb_arg = arg;
 902
 903	/* dispatch IO */
 904	rc = efct_hw_bls_send(efct, FC_RCTL_BA_ACC, bls,
 905			      efct_target_bls_resp_cb, io);
 906	return rc;
 907}
 908
 909static int efct_bls_send_rjt_cb(struct efct_hw_io *hio, u32 length, int status,
 910				u32 ext_status, void *app)
 911{
 912	struct efct_io *io = app;
 913
 914	efct_scsi_io_free(io);
 915	return 0;
 916}
 917
 918struct efct_io *
 919efct_bls_send_rjt(struct efct_io *io, struct fc_frame_header *hdr)
 920{
 921	struct efct_node *node = io->node;
 922	struct sli_bls_params *bls = &io->iparam.bls;
 923	struct efct *efct = node->efct;
 924	struct fc_ba_rjt *acc;
 925	int rc;
 926
 927	/* fill out BLS Response-specific fields */
 928	io->io_type = EFCT_IO_TYPE_BLS_RESP;
 929	io->display_name = "ba_rjt";
 930	io->hio_type = EFCT_HW_BLS_RJT;
 931	io->init_task_tag = be16_to_cpu(hdr->fh_ox_id);
 932
 933	/* fill out iparam fields */
 934	memset(&io->iparam, 0, sizeof(io->iparam));
 935	bls->ox_id = be16_to_cpu(hdr->fh_ox_id);
 936	bls->rx_id = be16_to_cpu(hdr->fh_rx_id);
 937	bls->vpi = io->node->vpi;
 938	bls->rpi = io->node->rpi;
 939	bls->s_id = U32_MAX;
 940	bls->d_id = io->node->node_fc_id;
 941	bls->rpi_registered = true;
 942
 943	acc = (void *)bls->payload;
 944	acc->br_reason = ELS_RJT_UNAB;
 945	acc->br_explan = ELS_EXPL_NONE;
 946
 947	rc = efct_hw_bls_send(efct, FC_RCTL_BA_RJT, bls, efct_bls_send_rjt_cb,
 948			      io);
 949	if (rc) {
 950		efc_log_err(efct, "efct_scsi_io_dispatch() failed: %d\n", rc);
 951		efct_scsi_io_free(io);
 952		io = NULL;
 953	}
 954	return io;
 955}
 956
 957int
 958efct_scsi_send_tmf_resp(struct efct_io *io,
 959			enum efct_scsi_tmf_resp rspcode,
 960			u8 addl_rsp_info[3],
 961			efct_scsi_io_cb_t cb, void *arg)
 962{
 963	int rc;
 964	struct {
 965		struct fcp_resp_with_ext rsp_ext;
 966		struct fcp_resp_rsp_info info;
 967	} *fcprsp;
 968	u8 fcp_rspcode;
 969
 970	io->wire_len = 0;
 971
 972	switch (rspcode) {
 973	case EFCT_SCSI_TMF_FUNCTION_COMPLETE:
 974		fcp_rspcode = FCP_TMF_CMPL;
 975		break;
 976	case EFCT_SCSI_TMF_FUNCTION_SUCCEEDED:
 977	case EFCT_SCSI_TMF_FUNCTION_IO_NOT_FOUND:
 978		fcp_rspcode = FCP_TMF_CMPL;
 979		break;
 980	case EFCT_SCSI_TMF_FUNCTION_REJECTED:
 981		fcp_rspcode = FCP_TMF_REJECTED;
 982		break;
 983	case EFCT_SCSI_TMF_INCORRECT_LOGICAL_UNIT_NUMBER:
 984		fcp_rspcode = FCP_TMF_INVALID_LUN;
 985		break;
 986	case EFCT_SCSI_TMF_SERVICE_DELIVERY:
 987		fcp_rspcode = FCP_TMF_FAILED;
 988		break;
 989	default:
 990		fcp_rspcode = FCP_TMF_REJECTED;
 991		break;
 992	}
 993
 994	io->hio_type = EFCT_HW_IO_TARGET_RSP;
 995
 996	io->scsi_tgt_cb = cb;
 997	io->scsi_tgt_cb_arg = arg;
 998
 999	if (io->tmf_cmd == EFCT_SCSI_TMF_ABORT_TASK) {
1000		rc = efct_target_send_bls_resp(io, cb, arg);
1001		return rc;
1002	}
1003
1004	/* populate the FCP TMF response */
1005	fcprsp = io->rspbuf.virt;
1006	memset(fcprsp, 0, sizeof(*fcprsp));
1007
1008	fcprsp->rsp_ext.resp.fr_flags |= FCP_SNS_LEN_VAL;
1009
1010	if (addl_rsp_info) {
1011		memcpy(fcprsp->info._fr_resvd, addl_rsp_info,
1012		       sizeof(fcprsp->info._fr_resvd));
1013	}
1014	fcprsp->info.rsp_code = fcp_rspcode;
1015
1016	io->wire_len = sizeof(*fcprsp);
1017
1018	fcprsp->rsp_ext.ext.fr_rsp_len =
1019			cpu_to_be32(sizeof(struct fcp_resp_rsp_info));
1020
1021	io->sgl[0].addr = io->rspbuf.phys;
1022	io->sgl[0].dif_addr = 0;
1023	io->sgl[0].len = io->wire_len;
1024	io->sgl_count = 1;
1025
1026	memset(&io->iparam, 0, sizeof(io->iparam));
1027	io->iparam.fcp_tgt.ox_id = io->init_task_tag;
1028	io->iparam.fcp_tgt.offset = 0;
1029	io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;
1030	io->iparam.fcp_tgt.timeout = io->timeout;
1031
1032	rc = efct_scsi_io_dispatch(io, efct_target_io_cb);
1033
1034	return rc;
1035}
1036
1037static int
1038efct_target_abort_cb(struct efct_hw_io *hio, u32 length, int status,
1039		     u32 ext_status, void *app)
1040{
1041	struct efct_io *io = app;
1042	struct efct *efct;
1043	enum efct_scsi_io_status scsi_status;
1044	efct_scsi_io_cb_t abort_cb;
1045	void *abort_cb_arg;
1046
1047	efct = io->efct;
1048
1049	if (!io->abort_cb)
1050		goto done;
1051
1052	abort_cb = io->abort_cb;
1053	abort_cb_arg = io->abort_cb_arg;
1054
1055	io->abort_cb = NULL;
1056	io->abort_cb_arg = NULL;
1057
1058	switch (status) {
1059	case SLI4_FC_WCQE_STATUS_SUCCESS:
1060		scsi_status = EFCT_SCSI_STATUS_GOOD;
1061		break;
1062	case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:
1063		switch (ext_status) {
1064		case SLI4_FC_LOCAL_REJECT_NO_XRI:
1065			scsi_status = EFCT_SCSI_STATUS_NO_IO;
1066			break;
1067		case SLI4_FC_LOCAL_REJECT_ABORT_IN_PROGRESS:
1068			scsi_status = EFCT_SCSI_STATUS_ABORT_IN_PROGRESS;
1069			break;
1070		default:
1071			/*we have seen 0x15 (abort in progress)*/
1072			scsi_status = EFCT_SCSI_STATUS_ERROR;
1073			break;
1074		}
1075		break;
1076	case SLI4_FC_WCQE_STATUS_FCP_RSP_FAILURE:
1077		scsi_status = EFCT_SCSI_STATUS_CHECK_RESPONSE;
1078		break;
1079	default:
1080		scsi_status = EFCT_SCSI_STATUS_ERROR;
1081		break;
1082	}
1083	/* invoke callback */
1084	abort_cb(io->io_to_abort, scsi_status, 0, abort_cb_arg);
1085
1086done:
1087	/* done with IO to abort,efct_ref_get(): efct_scsi_tgt_abort_io() */
1088	kref_put(&io->io_to_abort->ref, io->io_to_abort->release);
1089
1090	efct_io_pool_io_free(efct->xport->io_pool, io);
1091
1092	efct_scsi_check_pending(efct);
1093	return 0;
1094}
1095
1096int
1097efct_scsi_tgt_abort_io(struct efct_io *io, efct_scsi_io_cb_t cb, void *arg)
1098{
1099	struct efct *efct;
1100	struct efct_xport *xport;
1101	int rc;
1102	struct efct_io *abort_io = NULL;
1103
1104	efct = io->efct;
1105	xport = efct->xport;
1106
1107	/* take a reference on IO being aborted */
1108	if (kref_get_unless_zero(&io->ref) == 0) {
1109		/* command no longer active */
1110		scsi_io_printf(io, "command no longer active\n");
1111		return -EIO;
1112	}
1113
1114	/*
1115	 * allocate a new IO to send the abort request. Use efct_io_alloc()
1116	 * directly, as we need an IO object that will not fail allocation
1117	 * due to allocations being disabled (in efct_scsi_io_alloc())
1118	 */
1119	abort_io = efct_io_pool_io_alloc(efct->xport->io_pool);
1120	if (!abort_io) {
1121		atomic_add_return(1, &xport->io_alloc_failed_count);
1122		kref_put(&io->ref, io->release);
1123		return -EIO;
1124	}
1125
1126	/* Save the target server callback and argument */
1127	/* set generic fields */
1128	abort_io->cmd_tgt = true;
1129	abort_io->node = io->node;
1130
1131	/* set type and abort-specific fields */
1132	abort_io->io_type = EFCT_IO_TYPE_ABORT;
1133	abort_io->display_name = "tgt_abort";
1134	abort_io->io_to_abort = io;
1135	abort_io->send_abts = false;
1136	abort_io->abort_cb = cb;
1137	abort_io->abort_cb_arg = arg;
1138
1139	/* now dispatch IO */
1140	rc = efct_scsi_io_dispatch_abort(abort_io, efct_target_abort_cb);
1141	if (rc)
1142		kref_put(&io->ref, io->release);
1143	return rc;
1144}
1145
1146void
1147efct_scsi_io_complete(struct efct_io *io)
1148{
1149	if (io->io_free) {
1150		efc_log_debug(io->efct, "completion for non-busy io tag 0x%x\n",
1151			      io->tag);
1152		return;
1153	}
1154
1155	scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);
1156	kref_put(&io->ref, io->release);
1157}