fc.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2016 Avago Technologies.  All rights reserved.
   4 */
   5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   6#include <linux/module.h>
   7#include <linux/parser.h>
   8#include <uapi/scsi/fc/fc_fs.h>
   9#include <uapi/scsi/fc/fc_els.h>
  10#include <linux/delay.h>
  11#include <linux/overflow.h>
  12#include <linux/blk-cgroup.h>
  13#include "nvme.h"
  14#include "fabrics.h"
  15#include <linux/nvme-fc-driver.h>
  16#include <linux/nvme-fc.h>
  17#include "fc.h"
  18#include <scsi/scsi_transport_fc.h>
  19
  20/* *************************** Data Structures/Defines ****************** */
  21
  22
  23enum nvme_fc_queue_flags {
  24	NVME_FC_Q_CONNECTED = 0,
  25	NVME_FC_Q_LIVE,
  26};
  27
  28#define NVME_FC_DEFAULT_DEV_LOSS_TMO	60	/* seconds */
  29#define NVME_FC_DEFAULT_RECONNECT_TMO	2	/* delay between reconnects
  30						 * when connected and a
  31						 * connection failure.
  32						 */
  33
  34struct nvme_fc_queue {
  35	struct nvme_fc_ctrl	*ctrl;
  36	struct device		*dev;
  37	struct blk_mq_hw_ctx	*hctx;
  38	void			*lldd_handle;
  39	size_t			cmnd_capsule_len;
  40	u32			qnum;
  41	u32			rqcnt;
  42	u32			seqno;
  43
  44	u64			connection_id;
  45	atomic_t		csn;
  46
  47	unsigned long		flags;
  48} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
  49
  50enum nvme_fcop_flags {
  51	FCOP_FLAGS_TERMIO	= (1 << 0),
  52	FCOP_FLAGS_AEN		= (1 << 1),
  53};
  54
  55struct nvmefc_ls_req_op {
  56	struct nvmefc_ls_req	ls_req;
  57
  58	struct nvme_fc_rport	*rport;
  59	struct nvme_fc_queue	*queue;
  60	struct request		*rq;
  61	u32			flags;
  62
  63	int			ls_error;
  64	struct completion	ls_done;
  65	struct list_head	lsreq_list;	/* rport->ls_req_list */
  66	bool			req_queued;
  67};
  68
  69struct nvmefc_ls_rcv_op {
  70	struct nvme_fc_rport		*rport;
  71	struct nvmefc_ls_rsp		*lsrsp;
  72	union nvmefc_ls_requests	*rqstbuf;
  73	union nvmefc_ls_responses	*rspbuf;
  74	u16				rqstdatalen;
  75	bool				handled;
  76	dma_addr_t			rspdma;
  77	struct list_head		lsrcv_list;	/* rport->ls_rcv_list */
  78} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
  79
  80enum nvme_fcpop_state {
  81	FCPOP_STATE_UNINIT	= 0,
  82	FCPOP_STATE_IDLE	= 1,
  83	FCPOP_STATE_ACTIVE	= 2,
  84	FCPOP_STATE_ABORTED	= 3,
  85	FCPOP_STATE_COMPLETE	= 4,
  86};
  87
  88struct nvme_fc_fcp_op {
  89	struct nvme_request	nreq;		/*
  90						 * nvme/host/core.c
  91						 * requires this to be
  92						 * the 1st element in the
  93						 * private structure
  94						 * associated with the
  95						 * request.
  96						 */
  97	struct nvmefc_fcp_req	fcp_req;
  98
  99	struct nvme_fc_ctrl	*ctrl;
 100	struct nvme_fc_queue	*queue;
 101	struct request		*rq;
 102
 103	atomic_t		state;
 104	u32			flags;
 105	u32			rqno;
 106	u32			nents;
 107
 108	struct nvme_fc_cmd_iu	cmd_iu;
 109	struct nvme_fc_ersp_iu	rsp_iu;
 110};
 111
 112struct nvme_fcp_op_w_sgl {
 113	struct nvme_fc_fcp_op	op;
 114	struct scatterlist	sgl[NVME_INLINE_SG_CNT];
 115	uint8_t			priv[];
 116};
 117
 118struct nvme_fc_lport {
 119	struct nvme_fc_local_port	localport;
 120
 121	struct ida			endp_cnt;
 122	struct list_head		port_list;	/* nvme_fc_port_list */
 123	struct list_head		endp_list;
 124	struct device			*dev;	/* physical device for dma */
 125	struct nvme_fc_port_template	*ops;
 126	struct kref			ref;
 127	atomic_t                        act_rport_cnt;
 128} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
 129
 130struct nvme_fc_rport {
 131	struct nvme_fc_remote_port	remoteport;
 132
 133	struct list_head		endp_list; /* for lport->endp_list */
 134	struct list_head		ctrl_list;
 135	struct list_head		ls_req_list;
 136	struct list_head		ls_rcv_list;
 137	struct list_head		disc_list;
 138	struct device			*dev;	/* physical device for dma */
 139	struct nvme_fc_lport		*lport;
 140	spinlock_t			lock;
 141	struct kref			ref;
 142	atomic_t                        act_ctrl_cnt;
 143	unsigned long			dev_loss_end;
 144	struct work_struct		lsrcv_work;
 145} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
 146
 147/* fc_ctrl flags values - specified as bit positions */
 148#define ASSOC_ACTIVE		0
 149#define ASSOC_FAILED		1
 150#define FCCTRL_TERMIO		2
 151
 152struct nvme_fc_ctrl {
 153	spinlock_t		lock;
 154	struct nvme_fc_queue	*queues;
 155	struct device		*dev;
 156	struct nvme_fc_lport	*lport;
 157	struct nvme_fc_rport	*rport;
 158	u32			cnum;
 159
 160	bool			ioq_live;
 161	u64			association_id;
 162	struct nvmefc_ls_rcv_op	*rcv_disconn;
 163
 164	struct list_head	ctrl_list;	/* rport->ctrl_list */
 165
 166	struct blk_mq_tag_set	admin_tag_set;
 167	struct blk_mq_tag_set	tag_set;
 168
 169	struct work_struct	ioerr_work;
 170	struct delayed_work	connect_work;
 171
 172	struct kref		ref;
 173	unsigned long		flags;
 174	u32			iocnt;
 175	wait_queue_head_t	ioabort_wait;
 176
 177	struct nvme_fc_fcp_op	aen_ops[NVME_NR_AEN_COMMANDS];
 178
 179	struct nvme_ctrl	ctrl;
 180};
 181
 182static inline struct nvme_fc_ctrl *
 183to_fc_ctrl(struct nvme_ctrl *ctrl)
 184{
 185	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
 186}
 187
 188static inline struct nvme_fc_lport *
 189localport_to_lport(struct nvme_fc_local_port *portptr)
 190{
 191	return container_of(portptr, struct nvme_fc_lport, localport);
 192}
 193
 194static inline struct nvme_fc_rport *
 195remoteport_to_rport(struct nvme_fc_remote_port *portptr)
 196{
 197	return container_of(portptr, struct nvme_fc_rport, remoteport);
 198}
 199
 200static inline struct nvmefc_ls_req_op *
 201ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
 202{
 203	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
 204}
 205
 206static inline struct nvme_fc_fcp_op *
 207fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
 208{
 209	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
 210}
 211
 212
 213
 214/* *************************** Globals **************************** */
 215
 216
 217static DEFINE_SPINLOCK(nvme_fc_lock);
 218
 219static LIST_HEAD(nvme_fc_lport_list);
 220static DEFINE_IDA(nvme_fc_local_port_cnt);
 221static DEFINE_IDA(nvme_fc_ctrl_cnt);
 222
 223/*
 224 * These items are short-term. They will eventually be moved into
 225 * a generic FC class. See comments in module init.
 226 */
 227static struct device *fc_udev_device;
 228
 229static void nvme_fc_complete_rq(struct request *rq);
 230
 231/* *********************** FC-NVME Port Management ************************ */
 232
 233static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
 234			struct nvme_fc_queue *, unsigned int);
 235
 236static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
 237
 238
 239static void
 240nvme_fc_free_lport(struct kref *ref)
 241{
 242	struct nvme_fc_lport *lport =
 243		container_of(ref, struct nvme_fc_lport, ref);
 244	unsigned long flags;
 245
 246	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
 247	WARN_ON(!list_empty(&lport->endp_list));
 248
 249	/* remove from transport list */
 250	spin_lock_irqsave(&nvme_fc_lock, flags);
 251	list_del(&lport->port_list);
 252	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 253
 254	ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
 255	ida_destroy(&lport->endp_cnt);
 256
 257	put_device(lport->dev);
 258
 259	kfree(lport);
 260}
 261
 262static void
 263nvme_fc_lport_put(struct nvme_fc_lport *lport)
 264{
 265	kref_put(&lport->ref, nvme_fc_free_lport);
 266}
 267
 268static int
 269nvme_fc_lport_get(struct nvme_fc_lport *lport)
 270{
 271	return kref_get_unless_zero(&lport->ref);
 272}
 273
 274
 275static struct nvme_fc_lport *
 276nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
 277			struct nvme_fc_port_template *ops,
 278			struct device *dev)
 279{
 280	struct nvme_fc_lport *lport;
 281	unsigned long flags;
 282
 283	spin_lock_irqsave(&nvme_fc_lock, flags);
 284
 285	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
 286		if (lport->localport.node_name != pinfo->node_name ||
 287		    lport->localport.port_name != pinfo->port_name)
 288			continue;
 289
 290		if (lport->dev != dev) {
 291			lport = ERR_PTR(-EXDEV);
 292			goto out_done;
 293		}
 294
 295		if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
 296			lport = ERR_PTR(-EEXIST);
 297			goto out_done;
 298		}
 299
 300		if (!nvme_fc_lport_get(lport)) {
 301			/*
 302			 * fails if ref cnt already 0. If so,
 303			 * act as if lport already deleted
 304			 */
 305			lport = NULL;
 306			goto out_done;
 307		}
 308
 309		/* resume the lport */
 310
 311		lport->ops = ops;
 312		lport->localport.port_role = pinfo->port_role;
 313		lport->localport.port_id = pinfo->port_id;
 314		lport->localport.port_state = FC_OBJSTATE_ONLINE;
 315
 316		spin_unlock_irqrestore(&nvme_fc_lock, flags);
 317
 318		return lport;
 319	}
 320
 321	lport = NULL;
 322
 323out_done:
 324	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 325
 326	return lport;
 327}
 328
 329/**
 330 * nvme_fc_register_localport - transport entry point called by an
 331 *                              LLDD to register the existence of a NVME
 332 *                              host FC port.
 333 * @pinfo:     pointer to information about the port to be registered
 334 * @template:  LLDD entrypoints and operational parameters for the port
 335 * @dev:       physical hardware device node port corresponds to. Will be
 336 *             used for DMA mappings
 337 * @portptr:   pointer to a local port pointer. Upon success, the routine
 338 *             will allocate a nvme_fc_local_port structure and place its
 339 *             address in the local port pointer. Upon failure, local port
 340 *             pointer will be set to 0.
 341 *
 342 * Returns:
 343 * a completion status. Must be 0 upon success; a negative errno
 344 * (ex: -ENXIO) upon failure.
 345 */
 346int
 347nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
 348			struct nvme_fc_port_template *template,
 349			struct device *dev,
 350			struct nvme_fc_local_port **portptr)
 351{
 352	struct nvme_fc_lport *newrec;
 353	unsigned long flags;
 354	int ret, idx;
 355
 356	if (!template->localport_delete || !template->remoteport_delete ||
 357	    !template->ls_req || !template->fcp_io ||
 358	    !template->ls_abort || !template->fcp_abort ||
 359	    !template->max_hw_queues || !template->max_sgl_segments ||
 360	    !template->max_dif_sgl_segments || !template->dma_boundary) {
 361		ret = -EINVAL;
 362		goto out_reghost_failed;
 363	}
 364
 365	/*
 366	 * look to see if there is already a localport that had been
 367	 * deregistered and in the process of waiting for all the
 368	 * references to fully be removed.  If the references haven't
 369	 * expired, we can simply re-enable the localport. Remoteports
 370	 * and controller reconnections should resume naturally.
 371	 */
 372	newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
 373
 374	/* found an lport, but something about its state is bad */
 375	if (IS_ERR(newrec)) {
 376		ret = PTR_ERR(newrec);
 377		goto out_reghost_failed;
 378
 379	/* found existing lport, which was resumed */
 380	} else if (newrec) {
 381		*portptr = &newrec->localport;
 382		return 0;
 383	}
 384
 385	/* nothing found - allocate a new localport struct */
 386
 387	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
 388			 GFP_KERNEL);
 389	if (!newrec) {
 390		ret = -ENOMEM;
 391		goto out_reghost_failed;
 392	}
 393
 394	idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
 395	if (idx < 0) {
 396		ret = -ENOSPC;
 397		goto out_fail_kfree;
 398	}
 399
 400	if (!get_device(dev) && dev) {
 401		ret = -ENODEV;
 402		goto out_ida_put;
 403	}
 404
 405	INIT_LIST_HEAD(&newrec->port_list);
 406	INIT_LIST_HEAD(&newrec->endp_list);
 407	kref_init(&newrec->ref);
 408	atomic_set(&newrec->act_rport_cnt, 0);
 409	newrec->ops = template;
 410	newrec->dev = dev;
 411	ida_init(&newrec->endp_cnt);
 412	if (template->local_priv_sz)
 413		newrec->localport.private = &newrec[1];
 414	else
 415		newrec->localport.private = NULL;
 416	newrec->localport.node_name = pinfo->node_name;
 417	newrec->localport.port_name = pinfo->port_name;
 418	newrec->localport.port_role = pinfo->port_role;
 419	newrec->localport.port_id = pinfo->port_id;
 420	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
 421	newrec->localport.port_num = idx;
 422
 423	spin_lock_irqsave(&nvme_fc_lock, flags);
 424	list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
 425	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 426
 427	if (dev)
 428		dma_set_seg_boundary(dev, template->dma_boundary);
 429
 430	*portptr = &newrec->localport;
 431	return 0;
 432
 433out_ida_put:
 434	ida_free(&nvme_fc_local_port_cnt, idx);
 435out_fail_kfree:
 436	kfree(newrec);
 437out_reghost_failed:
 438	*portptr = NULL;
 439
 440	return ret;
 441}
 442EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
 443
 444/**
 445 * nvme_fc_unregister_localport - transport entry point called by an
 446 *                              LLDD to deregister/remove a previously
 447 *                              registered a NVME host FC port.
 448 * @portptr: pointer to the (registered) local port that is to be deregistered.
 449 *
 450 * Returns:
 451 * a completion status. Must be 0 upon success; a negative errno
 452 * (ex: -ENXIO) upon failure.
 453 */
 454int
 455nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
 456{
 457	struct nvme_fc_lport *lport = localport_to_lport(portptr);
 458	unsigned long flags;
 459
 460	if (!portptr)
 461		return -EINVAL;
 462
 463	spin_lock_irqsave(&nvme_fc_lock, flags);
 464
 465	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 466		spin_unlock_irqrestore(&nvme_fc_lock, flags);
 467		return -EINVAL;
 468	}
 469	portptr->port_state = FC_OBJSTATE_DELETED;
 470
 471	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 472
 473	if (atomic_read(&lport->act_rport_cnt) == 0)
 474		lport->ops->localport_delete(&lport->localport);
 475
 476	nvme_fc_lport_put(lport);
 477
 478	return 0;
 479}
 480EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
 481
 482/*
 483 * TRADDR strings, per FC-NVME are fixed format:
 484 *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
 485 * udev event will only differ by prefix of what field is
 486 * being specified:
 487 *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
 488 *  19 + 43 + null_fudge = 64 characters
 489 */
 490#define FCNVME_TRADDR_LENGTH		64
 491
 492static void
 493nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
 494		struct nvme_fc_rport *rport)
 495{
 496	char hostaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_HOST_TRADDR=...*/
 497	char tgtaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_TRADDR=...*/
 498	char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
 499
 500	if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
 501		return;
 502
 503	snprintf(hostaddr, sizeof(hostaddr),
 504		"NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
 505		lport->localport.node_name, lport->localport.port_name);
 506	snprintf(tgtaddr, sizeof(tgtaddr),
 507		"NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
 508		rport->remoteport.node_name, rport->remoteport.port_name);
 509	kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
 510}
 511
 512static void
 513nvme_fc_free_rport(struct kref *ref)
 514{
 515	struct nvme_fc_rport *rport =
 516		container_of(ref, struct nvme_fc_rport, ref);
 517	struct nvme_fc_lport *lport =
 518			localport_to_lport(rport->remoteport.localport);
 519	unsigned long flags;
 520
 521	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
 522	WARN_ON(!list_empty(&rport->ctrl_list));
 523	WARN_ON(!list_empty(&rport->ls_req_list));
 524	WARN_ON(!list_empty(&rport->ls_rcv_list));
 525
 526	/* remove from lport list */
 527	spin_lock_irqsave(&nvme_fc_lock, flags);
 528	list_del(&rport->endp_list);
 529	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 530
 531	WARN_ON(!list_empty(&rport->disc_list));
 532	ida_free(&lport->endp_cnt, rport->remoteport.port_num);
 533
 534	kfree(rport);
 535
 536	nvme_fc_lport_put(lport);
 537}
 538
 539static void
 540nvme_fc_rport_put(struct nvme_fc_rport *rport)
 541{
 542	kref_put(&rport->ref, nvme_fc_free_rport);
 543}
 544
 545static int
 546nvme_fc_rport_get(struct nvme_fc_rport *rport)
 547{
 548	return kref_get_unless_zero(&rport->ref);
 549}
 550
 551static void
 552nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
 553{
 554	switch (nvme_ctrl_state(&ctrl->ctrl)) {
 555	case NVME_CTRL_NEW:
 556	case NVME_CTRL_CONNECTING:
 557		/*
 558		 * As all reconnects were suppressed, schedule a
 559		 * connect.
 560		 */
 561		dev_info(ctrl->ctrl.device,
 562			"NVME-FC{%d}: connectivity re-established. "
 563			"Attempting reconnect\n", ctrl->cnum);
 564
 565		queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
 566		break;
 567
 568	case NVME_CTRL_RESETTING:
 569		/*
 570		 * Controller is already in the process of terminating the
 571		 * association. No need to do anything further. The reconnect
 572		 * step will naturally occur after the reset completes.
 573		 */
 574		break;
 575
 576	default:
 577		/* no action to take - let it delete */
 578		break;
 579	}
 580}
 581
 582static struct nvme_fc_rport *
 583nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
 584				struct nvme_fc_port_info *pinfo)
 585{
 586	struct nvme_fc_rport *rport;
 587	struct nvme_fc_ctrl *ctrl;
 588	unsigned long flags;
 589
 590	spin_lock_irqsave(&nvme_fc_lock, flags);
 591
 592	list_for_each_entry(rport, &lport->endp_list, endp_list) {
 593		if (rport->remoteport.node_name != pinfo->node_name ||
 594		    rport->remoteport.port_name != pinfo->port_name)
 595			continue;
 596
 597		if (!nvme_fc_rport_get(rport)) {
 598			rport = ERR_PTR(-ENOLCK);
 599			goto out_done;
 600		}
 601
 602		spin_unlock_irqrestore(&nvme_fc_lock, flags);
 603
 604		spin_lock_irqsave(&rport->lock, flags);
 605
 606		/* has it been unregistered */
 607		if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
 608			/* means lldd called us twice */
 609			spin_unlock_irqrestore(&rport->lock, flags);
 610			nvme_fc_rport_put(rport);
 611			return ERR_PTR(-ESTALE);
 612		}
 613
 614		rport->remoteport.port_role = pinfo->port_role;
 615		rport->remoteport.port_id = pinfo->port_id;
 616		rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
 617		rport->dev_loss_end = 0;
 618
 619		/*
 620		 * kick off a reconnect attempt on all associations to the
 621		 * remote port. A successful reconnects will resume i/o.
 622		 */
 623		list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
 624			nvme_fc_resume_controller(ctrl);
 625
 626		spin_unlock_irqrestore(&rport->lock, flags);
 627
 628		return rport;
 629	}
 630
 631	rport = NULL;
 632
 633out_done:
 634	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 635
 636	return rport;
 637}
 638
 639static inline void
 640__nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
 641			struct nvme_fc_port_info *pinfo)
 642{
 643	if (pinfo->dev_loss_tmo)
 644		rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
 645	else
 646		rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
 647}
 648
 649/**
 650 * nvme_fc_register_remoteport - transport entry point called by an
 651 *                              LLDD to register the existence of a NVME
 652 *                              subsystem FC port on its fabric.
 653 * @localport: pointer to the (registered) local port that the remote
 654 *             subsystem port is connected to.
 655 * @pinfo:     pointer to information about the port to be registered
 656 * @portptr:   pointer to a remote port pointer. Upon success, the routine
 657 *             will allocate a nvme_fc_remote_port structure and place its
 658 *             address in the remote port pointer. Upon failure, remote port
 659 *             pointer will be set to 0.
 660 *
 661 * Returns:
 662 * a completion status. Must be 0 upon success; a negative errno
 663 * (ex: -ENXIO) upon failure.
 664 */
 665int
 666nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
 667				struct nvme_fc_port_info *pinfo,
 668				struct nvme_fc_remote_port **portptr)
 669{
 670	struct nvme_fc_lport *lport = localport_to_lport(localport);
 671	struct nvme_fc_rport *newrec;
 672	unsigned long flags;
 673	int ret, idx;
 674
 675	if (!nvme_fc_lport_get(lport)) {
 676		ret = -ESHUTDOWN;
 677		goto out_reghost_failed;
 678	}
 679
 680	/*
 681	 * look to see if there is already a remoteport that is waiting
 682	 * for a reconnect (within dev_loss_tmo) with the same WWN's.
 683	 * If so, transition to it and reconnect.
 684	 */
 685	newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
 686
 687	/* found an rport, but something about its state is bad */
 688	if (IS_ERR(newrec)) {
 689		ret = PTR_ERR(newrec);
 690		goto out_lport_put;
 691
 692	/* found existing rport, which was resumed */
 693	} else if (newrec) {
 694		nvme_fc_lport_put(lport);
 695		__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 696		nvme_fc_signal_discovery_scan(lport, newrec);
 697		*portptr = &newrec->remoteport;
 698		return 0;
 699	}
 700
 701	/* nothing found - allocate a new remoteport struct */
 702
 703	newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
 704			 GFP_KERNEL);
 705	if (!newrec) {
 706		ret = -ENOMEM;
 707		goto out_lport_put;
 708	}
 709
 710	idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
 711	if (idx < 0) {
 712		ret = -ENOSPC;
 713		goto out_kfree_rport;
 714	}
 715
 716	INIT_LIST_HEAD(&newrec->endp_list);
 717	INIT_LIST_HEAD(&newrec->ctrl_list);
 718	INIT_LIST_HEAD(&newrec->ls_req_list);
 719	INIT_LIST_HEAD(&newrec->disc_list);
 720	kref_init(&newrec->ref);
 721	atomic_set(&newrec->act_ctrl_cnt, 0);
 722	spin_lock_init(&newrec->lock);
 723	newrec->remoteport.localport = &lport->localport;
 724	INIT_LIST_HEAD(&newrec->ls_rcv_list);
 725	newrec->dev = lport->dev;
 726	newrec->lport = lport;
 727	if (lport->ops->remote_priv_sz)
 728		newrec->remoteport.private = &newrec[1];
 729	else
 730		newrec->remoteport.private = NULL;
 731	newrec->remoteport.port_role = pinfo->port_role;
 732	newrec->remoteport.node_name = pinfo->node_name;
 733	newrec->remoteport.port_name = pinfo->port_name;
 734	newrec->remoteport.port_id = pinfo->port_id;
 735	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
 736	newrec->remoteport.port_num = idx;
 737	__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 738	INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
 739
 740	spin_lock_irqsave(&nvme_fc_lock, flags);
 741	list_add_tail(&newrec->endp_list, &lport->endp_list);
 742	spin_unlock_irqrestore(&nvme_fc_lock, flags);
 743
 744	nvme_fc_signal_discovery_scan(lport, newrec);
 745
 746	*portptr = &newrec->remoteport;
 747	return 0;
 748
 749out_kfree_rport:
 750	kfree(newrec);
 751out_lport_put:
 752	nvme_fc_lport_put(lport);
 753out_reghost_failed:
 754	*portptr = NULL;
 755	return ret;
 756}
 757EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
 758
 759static int
 760nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
 761{
 762	struct nvmefc_ls_req_op *lsop;
 763	unsigned long flags;
 764
 765restart:
 766	spin_lock_irqsave(&rport->lock, flags);
 767
 768	list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
 769		if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
 770			lsop->flags |= FCOP_FLAGS_TERMIO;
 771			spin_unlock_irqrestore(&rport->lock, flags);
 772			rport->lport->ops->ls_abort(&rport->lport->localport,
 773						&rport->remoteport,
 774						&lsop->ls_req);
 775			goto restart;
 776		}
 777	}
 778	spin_unlock_irqrestore(&rport->lock, flags);
 779
 780	return 0;
 781}
 782
 783static void
 784nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
 785{
 786	dev_info(ctrl->ctrl.device,
 787		"NVME-FC{%d}: controller connectivity lost. Awaiting "
 788		"Reconnect", ctrl->cnum);
 789
 790	set_bit(ASSOC_FAILED, &ctrl->flags);
 791	nvme_reset_ctrl(&ctrl->ctrl);
 792}
 793
 794/**
 795 * nvme_fc_unregister_remoteport - transport entry point called by an
 796 *                              LLDD to deregister/remove a previously
 797 *                              registered a NVME subsystem FC port.
 798 * @portptr: pointer to the (registered) remote port that is to be
 799 *           deregistered.
 800 *
 801 * Returns:
 802 * a completion status. Must be 0 upon success; a negative errno
 803 * (ex: -ENXIO) upon failure.
 804 */
 805int
 806nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
 807{
 808	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 809	struct nvme_fc_ctrl *ctrl;
 810	unsigned long flags;
 811
 812	if (!portptr)
 813		return -EINVAL;
 814
 815	spin_lock_irqsave(&rport->lock, flags);
 816
 817	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 818		spin_unlock_irqrestore(&rport->lock, flags);
 819		return -EINVAL;
 820	}
 821	portptr->port_state = FC_OBJSTATE_DELETED;
 822
 823	rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
 824
 825	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
 826		/* if dev_loss_tmo==0, dev loss is immediate */
 827		if (!portptr->dev_loss_tmo) {
 828			dev_warn(ctrl->ctrl.device,
 829				"NVME-FC{%d}: controller connectivity lost.\n",
 830				ctrl->cnum);
 831			nvme_delete_ctrl(&ctrl->ctrl);
 832		} else
 833			nvme_fc_ctrl_connectivity_loss(ctrl);
 834	}
 835
 836	spin_unlock_irqrestore(&rport->lock, flags);
 837
 838	nvme_fc_abort_lsops(rport);
 839
 840	if (atomic_read(&rport->act_ctrl_cnt) == 0)
 841		rport->lport->ops->remoteport_delete(portptr);
 842
 843	/*
 844	 * release the reference, which will allow, if all controllers
 845	 * go away, which should only occur after dev_loss_tmo occurs,
 846	 * for the rport to be torn down.
 847	 */
 848	nvme_fc_rport_put(rport);
 849
 850	return 0;
 851}
 852EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
 853
 854/**
 855 * nvme_fc_rescan_remoteport - transport entry point called by an
 856 *                              LLDD to request a nvme device rescan.
 857 * @remoteport: pointer to the (registered) remote port that is to be
 858 *              rescanned.
 859 *
 860 * Returns: N/A
 861 */
 862void
 863nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
 864{
 865	struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
 866
 867	nvme_fc_signal_discovery_scan(rport->lport, rport);
 868}
 869EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
 870
 871int
 872nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
 873			u32 dev_loss_tmo)
 874{
 875	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 876	unsigned long flags;
 877
 878	spin_lock_irqsave(&rport->lock, flags);
 879
 880	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 881		spin_unlock_irqrestore(&rport->lock, flags);
 882		return -EINVAL;
 883	}
 884
 885	/* a dev_loss_tmo of 0 (immediate) is allowed to be set */
 886	rport->remoteport.dev_loss_tmo = dev_loss_tmo;
 887
 888	spin_unlock_irqrestore(&rport->lock, flags);
 889
 890	return 0;
 891}
 892EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
 893
 894
 895/* *********************** FC-NVME DMA Handling **************************** */
 896
 897/*
 898 * The fcloop device passes in a NULL device pointer. Real LLD's will
 899 * pass in a valid device pointer. If NULL is passed to the dma mapping
 900 * routines, depending on the platform, it may or may not succeed, and
 901 * may crash.
 902 *
 903 * As such:
 904 * Wrap all the dma routines and check the dev pointer.
 905 *
 906 * If simple mappings (return just a dma address, we'll noop them,
 907 * returning a dma address of 0.
 908 *
 909 * On more complex mappings (dma_map_sg), a pseudo routine fills
 910 * in the scatter list, setting all dma addresses to 0.
 911 */
 912
 913static inline dma_addr_t
 914fc_dma_map_single(struct device *dev, void *ptr, size_t size,
 915		enum dma_data_direction dir)
 916{
 917	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
 918}
 919
 920static inline int
 921fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 922{
 923	return dev ? dma_mapping_error(dev, dma_addr) : 0;
 924}
 925
 926static inline void
 927fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
 928	enum dma_data_direction dir)
 929{
 930	if (dev)
 931		dma_unmap_single(dev, addr, size, dir);
 932}
 933
 934static inline void
 935fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 936		enum dma_data_direction dir)
 937{
 938	if (dev)
 939		dma_sync_single_for_cpu(dev, addr, size, dir);
 940}
 941
 942static inline void
 943fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
 944		enum dma_data_direction dir)
 945{
 946	if (dev)
 947		dma_sync_single_for_device(dev, addr, size, dir);
 948}
 949
 950/* pseudo dma_map_sg call */
 951static int
 952fc_map_sg(struct scatterlist *sg, int nents)
 953{
 954	struct scatterlist *s;
 955	int i;
 956
 957	WARN_ON(nents == 0 || sg[0].length == 0);
 958
 959	for_each_sg(sg, s, nents, i) {
 960		s->dma_address = 0L;
 961#ifdef CONFIG_NEED_SG_DMA_LENGTH
 962		s->dma_length = s->length;
 963#endif
 964	}
 965	return nents;
 966}
 967
 968static inline int
 969fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 970		enum dma_data_direction dir)
 971{
 972	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
 973}
 974
 975static inline void
 976fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 977		enum dma_data_direction dir)
 978{
 979	if (dev)
 980		dma_unmap_sg(dev, sg, nents, dir);
 981}
 982
 983/* *********************** FC-NVME LS Handling **************************** */
 984
 985static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
 986static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
 987
 988static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
 989
 990static void
 991__nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
 992{
 993	struct nvme_fc_rport *rport = lsop->rport;
 994	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
 995	unsigned long flags;
 996
 997	spin_lock_irqsave(&rport->lock, flags);
 998
 999	if (!lsop->req_queued) {
1000		spin_unlock_irqrestore(&rport->lock, flags);
1001		return;
1002	}
1003
1004	list_del(&lsop->lsreq_list);
1005
1006	lsop->req_queued = false;
1007
1008	spin_unlock_irqrestore(&rport->lock, flags);
1009
1010	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1011				  (lsreq->rqstlen + lsreq->rsplen),
1012				  DMA_BIDIRECTIONAL);
1013
1014	nvme_fc_rport_put(rport);
1015}
1016
1017static int
1018__nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1019		struct nvmefc_ls_req_op *lsop,
1020		void (*done)(struct nvmefc_ls_req *req, int status))
1021{
1022	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1023	unsigned long flags;
1024	int ret = 0;
1025
1026	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1027		return -ECONNREFUSED;
1028
1029	if (!nvme_fc_rport_get(rport))
1030		return -ESHUTDOWN;
1031
1032	lsreq->done = done;
1033	lsop->rport = rport;
1034	lsop->req_queued = false;
1035	INIT_LIST_HEAD(&lsop->lsreq_list);
1036	init_completion(&lsop->ls_done);
1037
1038	lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1039				  lsreq->rqstlen + lsreq->rsplen,
1040				  DMA_BIDIRECTIONAL);
1041	if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1042		ret = -EFAULT;
1043		goto out_putrport;
1044	}
1045	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1046
1047	spin_lock_irqsave(&rport->lock, flags);
1048
1049	list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1050
1051	lsop->req_queued = true;
1052
1053	spin_unlock_irqrestore(&rport->lock, flags);
1054
1055	ret = rport->lport->ops->ls_req(&rport->lport->localport,
1056					&rport->remoteport, lsreq);
1057	if (ret)
1058		goto out_unlink;
1059
1060	return 0;
1061
1062out_unlink:
1063	lsop->ls_error = ret;
1064	spin_lock_irqsave(&rport->lock, flags);
1065	lsop->req_queued = false;
1066	list_del(&lsop->lsreq_list);
1067	spin_unlock_irqrestore(&rport->lock, flags);
1068	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1069				  (lsreq->rqstlen + lsreq->rsplen),
1070				  DMA_BIDIRECTIONAL);
1071out_putrport:
1072	nvme_fc_rport_put(rport);
1073
1074	return ret;
1075}
1076
1077static void
1078nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1079{
1080	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1081
1082	lsop->ls_error = status;
1083	complete(&lsop->ls_done);
1084}
1085
1086static int
1087nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1088{
1089	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1090	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1091	int ret;
1092
1093	ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1094
1095	if (!ret) {
1096		/*
1097		 * No timeout/not interruptible as we need the struct
1098		 * to exist until the lldd calls us back. Thus mandate
1099		 * wait until driver calls back. lldd responsible for
1100		 * the timeout action
1101		 */
1102		wait_for_completion(&lsop->ls_done);
1103
1104		__nvme_fc_finish_ls_req(lsop);
1105
1106		ret = lsop->ls_error;
1107	}
1108
1109	if (ret)
1110		return ret;
1111
1112	/* ACC or RJT payload ? */
1113	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1114		return -ENXIO;
1115
1116	return 0;
1117}
1118
1119static int
1120nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1121		struct nvmefc_ls_req_op *lsop,
1122		void (*done)(struct nvmefc_ls_req *req, int status))
1123{
1124	/* don't wait for completion */
1125
1126	return __nvme_fc_send_ls_req(rport, lsop, done);
1127}
1128
1129static int
1130nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1131	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1132{
1133	struct nvmefc_ls_req_op *lsop;
1134	struct nvmefc_ls_req *lsreq;
1135	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1136	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1137	unsigned long flags;
1138	int ret, fcret = 0;
1139
1140	lsop = kzalloc((sizeof(*lsop) +
1141			 sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1142			 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1143	if (!lsop) {
1144		dev_info(ctrl->ctrl.device,
1145			"NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1146			ctrl->cnum);
1147		ret = -ENOMEM;
1148		goto out_no_memory;
1149	}
1150
1151	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1152	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1153	lsreq = &lsop->ls_req;
1154	if (ctrl->lport->ops->lsrqst_priv_sz)
1155		lsreq->private = &assoc_acc[1];
1156	else
1157		lsreq->private = NULL;
1158
1159	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1160	assoc_rqst->desc_list_len =
1161			cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1162
1163	assoc_rqst->assoc_cmd.desc_tag =
1164			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1165	assoc_rqst->assoc_cmd.desc_len =
1166			fcnvme_lsdesc_len(
1167				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1168
1169	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1170	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1171	/* Linux supports only Dynamic controllers */
1172	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1173	uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1174	strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1175		sizeof(assoc_rqst->assoc_cmd.hostnqn));
1176	strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1177		sizeof(assoc_rqst->assoc_cmd.subnqn));
1178
1179	lsop->queue = queue;
1180	lsreq->rqstaddr = assoc_rqst;
1181	lsreq->rqstlen = sizeof(*assoc_rqst);
1182	lsreq->rspaddr = assoc_acc;
1183	lsreq->rsplen = sizeof(*assoc_acc);
1184	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1185
1186	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1187	if (ret)
1188		goto out_free_buffer;
1189
1190	/* process connect LS completion */
1191
1192	/* validate the ACC response */
1193	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1194		fcret = VERR_LSACC;
1195	else if (assoc_acc->hdr.desc_list_len !=
1196			fcnvme_lsdesc_len(
1197				sizeof(struct fcnvme_ls_cr_assoc_acc)))
1198		fcret = VERR_CR_ASSOC_ACC_LEN;
1199	else if (assoc_acc->hdr.rqst.desc_tag !=
1200			cpu_to_be32(FCNVME_LSDESC_RQST))
1201		fcret = VERR_LSDESC_RQST;
1202	else if (assoc_acc->hdr.rqst.desc_len !=
1203			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1204		fcret = VERR_LSDESC_RQST_LEN;
1205	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1206		fcret = VERR_CR_ASSOC;
1207	else if (assoc_acc->associd.desc_tag !=
1208			cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1209		fcret = VERR_ASSOC_ID;
1210	else if (assoc_acc->associd.desc_len !=
1211			fcnvme_lsdesc_len(
1212				sizeof(struct fcnvme_lsdesc_assoc_id)))
1213		fcret = VERR_ASSOC_ID_LEN;
1214	else if (assoc_acc->connectid.desc_tag !=
1215			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1216		fcret = VERR_CONN_ID;
1217	else if (assoc_acc->connectid.desc_len !=
1218			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1219		fcret = VERR_CONN_ID_LEN;
1220
1221	if (fcret) {
1222		ret = -EBADF;
1223		dev_err(ctrl->dev,
1224			"q %d Create Association LS failed: %s\n",
1225			queue->qnum, validation_errors[fcret]);
1226	} else {
1227		spin_lock_irqsave(&ctrl->lock, flags);
1228		ctrl->association_id =
1229			be64_to_cpu(assoc_acc->associd.association_id);
1230		queue->connection_id =
1231			be64_to_cpu(assoc_acc->connectid.connection_id);
1232		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1233		spin_unlock_irqrestore(&ctrl->lock, flags);
1234	}
1235
1236out_free_buffer:
1237	kfree(lsop);
1238out_no_memory:
1239	if (ret)
1240		dev_err(ctrl->dev,
1241			"queue %d connect admin queue failed (%d).\n",
1242			queue->qnum, ret);
1243	return ret;
1244}
1245
1246static int
1247nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1248			u16 qsize, u16 ersp_ratio)
1249{
1250	struct nvmefc_ls_req_op *lsop;
1251	struct nvmefc_ls_req *lsreq;
1252	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1253	struct fcnvme_ls_cr_conn_acc *conn_acc;
1254	int ret, fcret = 0;
1255
1256	lsop = kzalloc((sizeof(*lsop) +
1257			 sizeof(*conn_rqst) + sizeof(*conn_acc) +
1258			 ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1259	if (!lsop) {
1260		dev_info(ctrl->ctrl.device,
1261			"NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1262			ctrl->cnum);
1263		ret = -ENOMEM;
1264		goto out_no_memory;
1265	}
1266
1267	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1268	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1269	lsreq = &lsop->ls_req;
1270	if (ctrl->lport->ops->lsrqst_priv_sz)
1271		lsreq->private = (void *)&conn_acc[1];
1272	else
1273		lsreq->private = NULL;
1274
1275	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1276	conn_rqst->desc_list_len = cpu_to_be32(
1277				sizeof(struct fcnvme_lsdesc_assoc_id) +
1278				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1279
1280	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1281	conn_rqst->associd.desc_len =
1282			fcnvme_lsdesc_len(
1283				sizeof(struct fcnvme_lsdesc_assoc_id));
1284	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1285	conn_rqst->connect_cmd.desc_tag =
1286			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1287	conn_rqst->connect_cmd.desc_len =
1288			fcnvme_lsdesc_len(
1289				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1290	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1291	conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1292	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1293
1294	lsop->queue = queue;
1295	lsreq->rqstaddr = conn_rqst;
1296	lsreq->rqstlen = sizeof(*conn_rqst);
1297	lsreq->rspaddr = conn_acc;
1298	lsreq->rsplen = sizeof(*conn_acc);
1299	lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1300
1301	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1302	if (ret)
1303		goto out_free_buffer;
1304
1305	/* process connect LS completion */
1306
1307	/* validate the ACC response */
1308	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1309		fcret = VERR_LSACC;
1310	else if (conn_acc->hdr.desc_list_len !=
1311			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1312		fcret = VERR_CR_CONN_ACC_LEN;
1313	else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1314		fcret = VERR_LSDESC_RQST;
1315	else if (conn_acc->hdr.rqst.desc_len !=
1316			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1317		fcret = VERR_LSDESC_RQST_LEN;
1318	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1319		fcret = VERR_CR_CONN;
1320	else if (conn_acc->connectid.desc_tag !=
1321			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1322		fcret = VERR_CONN_ID;
1323	else if (conn_acc->connectid.desc_len !=
1324			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1325		fcret = VERR_CONN_ID_LEN;
1326
1327	if (fcret) {
1328		ret = -EBADF;
1329		dev_err(ctrl->dev,
1330			"q %d Create I/O Connection LS failed: %s\n",
1331			queue->qnum, validation_errors[fcret]);
1332	} else {
1333		queue->connection_id =
1334			be64_to_cpu(conn_acc->connectid.connection_id);
1335		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1336	}
1337
1338out_free_buffer:
1339	kfree(lsop);
1340out_no_memory:
1341	if (ret)
1342		dev_err(ctrl->dev,
1343			"queue %d connect I/O queue failed (%d).\n",
1344			queue->qnum, ret);
1345	return ret;
1346}
1347
1348static void
1349nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1350{
1351	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1352
1353	__nvme_fc_finish_ls_req(lsop);
1354
1355	/* fc-nvme initiator doesn't care about success or failure of cmd */
1356
1357	kfree(lsop);
1358}
1359
1360/*
1361 * This routine sends a FC-NVME LS to disconnect (aka terminate)
1362 * the FC-NVME Association.  Terminating the association also
1363 * terminates the FC-NVME connections (per queue, both admin and io
1364 * queues) that are part of the association. E.g. things are torn
1365 * down, and the related FC-NVME Association ID and Connection IDs
1366 * become invalid.
1367 *
1368 * The behavior of the fc-nvme initiator is such that its
1369 * understanding of the association and connections will implicitly
1370 * be torn down. The action is implicit as it may be due to a loss of
1371 * connectivity with the fc-nvme target, so you may never get a
1372 * response even if you tried.  As such, the action of this routine
1373 * is to asynchronously send the LS, ignore any results of the LS, and
1374 * continue on with terminating the association. If the fc-nvme target
1375 * is present and receives the LS, it too can tear down.
1376 */
1377static void
1378nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1379{
1380	struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1381	struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1382	struct nvmefc_ls_req_op *lsop;
1383	struct nvmefc_ls_req *lsreq;
1384	int ret;
1385
1386	lsop = kzalloc((sizeof(*lsop) +
1387			sizeof(*discon_rqst) + sizeof(*discon_acc) +
1388			ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1389	if (!lsop) {
1390		dev_info(ctrl->ctrl.device,
1391			"NVME-FC{%d}: send Disconnect Association "
1392			"failed: ENOMEM\n",
1393			ctrl->cnum);
1394		return;
1395	}
1396
1397	discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1398	discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1399	lsreq = &lsop->ls_req;
1400	if (ctrl->lport->ops->lsrqst_priv_sz)
1401		lsreq->private = (void *)&discon_acc[1];
1402	else
1403		lsreq->private = NULL;
1404
1405	nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1406				ctrl->association_id);
1407
1408	ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1409				nvme_fc_disconnect_assoc_done);
1410	if (ret)
1411		kfree(lsop);
1412}
1413
1414static void
1415nvme_fc_xmt_ls_rsp_free(struct nvmefc_ls_rcv_op *lsop)
1416{
1417	struct nvme_fc_rport *rport = lsop->rport;
1418	struct nvme_fc_lport *lport = rport->lport;
1419	unsigned long flags;
1420
1421	spin_lock_irqsave(&rport->lock, flags);
1422	list_del(&lsop->lsrcv_list);
1423	spin_unlock_irqrestore(&rport->lock, flags);
1424
1425	fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1426				sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1427	fc_dma_unmap_single(lport->dev, lsop->rspdma,
1428			sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1429
1430	kfree(lsop->rspbuf);
1431	kfree(lsop->rqstbuf);
1432	kfree(lsop);
1433
1434	nvme_fc_rport_put(rport);
1435}
1436
1437static void
1438nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1439{
1440	struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1441
1442	nvme_fc_xmt_ls_rsp_free(lsop);
1443}
1444
1445static void
1446nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1447{
1448	struct nvme_fc_rport *rport = lsop->rport;
1449	struct nvme_fc_lport *lport = rport->lport;
1450	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1451	int ret;
1452
1453	fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1454				  sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1455
1456	ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1457				     lsop->lsrsp);
1458	if (ret) {
1459		dev_warn(lport->dev,
1460			"LLDD rejected LS RSP xmt: LS %d status %d\n",
1461			w0->ls_cmd, ret);
1462		nvme_fc_xmt_ls_rsp_free(lsop);
1463		return;
1464	}
1465}
1466
1467static struct nvme_fc_ctrl *
1468nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1469		      struct nvmefc_ls_rcv_op *lsop)
1470{
1471	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1472					&lsop->rqstbuf->rq_dis_assoc;
1473	struct nvme_fc_ctrl *ctrl, *tmp, *ret = NULL;
1474	struct nvmefc_ls_rcv_op *oldls = NULL;
1475	u64 association_id = be64_to_cpu(rqst->associd.association_id);
1476	unsigned long flags;
1477
1478	spin_lock_irqsave(&rport->lock, flags);
1479
1480	list_for_each_entry_safe(ctrl, tmp, &rport->ctrl_list, ctrl_list) {
1481		if (!nvme_fc_ctrl_get(ctrl))
1482			continue;
1483		spin_lock(&ctrl->lock);
1484		if (association_id == ctrl->association_id) {
1485			oldls = ctrl->rcv_disconn;
1486			ctrl->rcv_disconn = lsop;
1487			ret = ctrl;
1488		}
1489		spin_unlock(&ctrl->lock);
1490		if (ret)
1491			/* leave the ctrl get reference */
1492			break;
1493		spin_unlock_irqrestore(&rport->lock, flags);
1494		nvme_fc_ctrl_put(ctrl);
1495		spin_lock_irqsave(&rport->lock, flags);
1496	}
1497
1498	spin_unlock_irqrestore(&rport->lock, flags);
1499
1500	/* transmit a response for anything that was pending */
1501	if (oldls) {
1502		dev_info(rport->lport->dev,
1503			"NVME-FC{%d}: Multiple Disconnect Association "
1504			"LS's received\n", ctrl->cnum);
1505		/* overwrite good response with bogus failure */
1506		oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1507						sizeof(*oldls->rspbuf),
1508						rqst->w0.ls_cmd,
1509						FCNVME_RJT_RC_UNAB,
1510						FCNVME_RJT_EXP_NONE, 0);
1511		nvme_fc_xmt_ls_rsp(oldls);
1512	}
1513
1514	return ret;
1515}
1516
1517/*
1518 * returns true to mean LS handled and ls_rsp can be sent
1519 * returns false to defer ls_rsp xmt (will be done as part of
1520 *     association termination)
1521 */
1522static bool
1523nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1524{
1525	struct nvme_fc_rport *rport = lsop->rport;
1526	struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1527					&lsop->rqstbuf->rq_dis_assoc;
1528	struct fcnvme_ls_disconnect_assoc_acc *acc =
1529					&lsop->rspbuf->rsp_dis_assoc;
1530	struct nvme_fc_ctrl *ctrl = NULL;
1531	int ret = 0;
1532
1533	memset(acc, 0, sizeof(*acc));
1534
1535	ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1536	if (!ret) {
1537		/* match an active association */
1538		ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1539		if (!ctrl)
1540			ret = VERR_NO_ASSOC;
1541	}
1542
1543	if (ret) {
1544		dev_info(rport->lport->dev,
1545			"Disconnect LS failed: %s\n",
1546			validation_errors[ret]);
1547		lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1548					sizeof(*acc), rqst->w0.ls_cmd,
1549					(ret == VERR_NO_ASSOC) ?
1550						FCNVME_RJT_RC_INV_ASSOC :
1551						FCNVME_RJT_RC_LOGIC,
1552					FCNVME_RJT_EXP_NONE, 0);
1553		return true;
1554	}
1555
1556	/* format an ACCept response */
1557
1558	lsop->lsrsp->rsplen = sizeof(*acc);
1559
1560	nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1561			fcnvme_lsdesc_len(
1562				sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1563			FCNVME_LS_DISCONNECT_ASSOC);
1564
1565	/*
1566	 * the transmit of the response will occur after the exchanges
1567	 * for the association have been ABTS'd by
1568	 * nvme_fc_delete_association().
1569	 */
1570
1571	/* fail the association */
1572	nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1573
1574	/* release the reference taken by nvme_fc_match_disconn_ls() */
1575	nvme_fc_ctrl_put(ctrl);
1576
1577	return false;
1578}
1579
1580/*
1581 * Actual Processing routine for received FC-NVME LS Requests from the LLD
1582 * returns true if a response should be sent afterward, false if rsp will
1583 * be sent asynchronously.
1584 */
1585static bool
1586nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1587{
1588	struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1589	bool ret = true;
1590
1591	lsop->lsrsp->nvme_fc_private = lsop;
1592	lsop->lsrsp->rspbuf = lsop->rspbuf;
1593	lsop->lsrsp->rspdma = lsop->rspdma;
1594	lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1595	/* Be preventative. handlers will later set to valid length */
1596	lsop->lsrsp->rsplen = 0;
1597
1598	/*
1599	 * handlers:
1600	 *   parse request input, execute the request, and format the
1601	 *   LS response
1602	 */
1603	switch (w0->ls_cmd) {
1604	case FCNVME_LS_DISCONNECT_ASSOC:
1605		ret = nvme_fc_ls_disconnect_assoc(lsop);
1606		break;
1607	case FCNVME_LS_DISCONNECT_CONN:
1608		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1609				sizeof(*lsop->rspbuf), w0->ls_cmd,
1610				FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1611		break;
1612	case FCNVME_LS_CREATE_ASSOCIATION:
1613	case FCNVME_LS_CREATE_CONNECTION:
1614		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1615				sizeof(*lsop->rspbuf), w0->ls_cmd,
1616				FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1617		break;
1618	default:
1619		lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1620				sizeof(*lsop->rspbuf), w0->ls_cmd,
1621				FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1622		break;
1623	}
1624
1625	return(ret);
1626}
1627
1628static void
1629nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1630{
1631	struct nvme_fc_rport *rport =
1632		container_of(work, struct nvme_fc_rport, lsrcv_work);
1633	struct fcnvme_ls_rqst_w0 *w0;
1634	struct nvmefc_ls_rcv_op *lsop;
1635	unsigned long flags;
1636	bool sendrsp;
1637
1638restart:
1639	sendrsp = true;
1640	spin_lock_irqsave(&rport->lock, flags);
1641	list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1642		if (lsop->handled)
1643			continue;
1644
1645		lsop->handled = true;
1646		if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1647			spin_unlock_irqrestore(&rport->lock, flags);
1648			sendrsp = nvme_fc_handle_ls_rqst(lsop);
1649		} else {
1650			spin_unlock_irqrestore(&rport->lock, flags);
1651			w0 = &lsop->rqstbuf->w0;
1652			lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1653						lsop->rspbuf,
1654						sizeof(*lsop->rspbuf),
1655						w0->ls_cmd,
1656						FCNVME_RJT_RC_UNAB,
1657						FCNVME_RJT_EXP_NONE, 0);
1658		}
1659		if (sendrsp)
1660			nvme_fc_xmt_ls_rsp(lsop);
1661		goto restart;
1662	}
1663	spin_unlock_irqrestore(&rport->lock, flags);
1664}
1665
1666static
1667void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1668				struct fcnvme_ls_rqst_w0 *w0)
1669{
1670	dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1671		(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1672			nvmefc_ls_names[w0->ls_cmd] : "");
1673}
1674
1675/**
1676 * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1677 *                       upon the reception of a NVME LS request.
1678 *
1679 * The nvme-fc layer will copy payload to an internal structure for
1680 * processing.  As such, upon completion of the routine, the LLDD may
1681 * immediately free/reuse the LS request buffer passed in the call.
1682 *
1683 * If this routine returns error, the LLDD should abort the exchange.
1684 *
1685 * @portptr:    pointer to the (registered) remote port that the LS
1686 *              was received from. The remoteport is associated with
1687 *              a specific localport.
1688 * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1689 *              used to reference the exchange corresponding to the LS
1690 *              when issuing an ls response.
1691 * @lsreqbuf:   pointer to the buffer containing the LS Request
1692 * @lsreqbuf_len: length, in bytes, of the received LS request
1693 */
1694int
1695nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1696			struct nvmefc_ls_rsp *lsrsp,
1697			void *lsreqbuf, u32 lsreqbuf_len)
1698{
1699	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1700	struct nvme_fc_lport *lport = rport->lport;
1701	struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1702	struct nvmefc_ls_rcv_op *lsop;
1703	unsigned long flags;
1704	int ret;
1705
1706	nvme_fc_rport_get(rport);
1707
1708	/* validate there's a routine to transmit a response */
1709	if (!lport->ops->xmt_ls_rsp) {
1710		dev_info(lport->dev,
1711			"RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1712			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1713				nvmefc_ls_names[w0->ls_cmd] : "");
1714		ret = -EINVAL;
1715		goto out_put;
1716	}
1717
1718	if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1719		dev_info(lport->dev,
1720			"RCV %s LS failed: payload too large\n",
1721			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1722				nvmefc_ls_names[w0->ls_cmd] : "");
1723		ret = -E2BIG;
1724		goto out_put;
1725	}
1726
1727	lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1728	if (!lsop) {
1729		nvme_fc_rcv_ls_req_err_msg(lport, w0);
1730		ret = -ENOMEM;
1731		goto out_put;
1732	}
1733
1734	lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1735	lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1736	if (!lsop->rqstbuf || !lsop->rspbuf) {
1737		nvme_fc_rcv_ls_req_err_msg(lport, w0);
1738		ret = -ENOMEM;
1739		goto out_free;
1740	}
1741
1742	lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1743					sizeof(*lsop->rspbuf),
1744					DMA_TO_DEVICE);
1745	if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1746		dev_info(lport->dev,
1747			"RCV %s LS failed: DMA mapping failure\n",
1748			(w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1749				nvmefc_ls_names[w0->ls_cmd] : "");
1750		ret = -EFAULT;
1751		goto out_free;
1752	}
1753
1754	lsop->rport = rport;
1755	lsop->lsrsp = lsrsp;
1756
1757	memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1758	lsop->rqstdatalen = lsreqbuf_len;
1759
1760	spin_lock_irqsave(&rport->lock, flags);
1761	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1762		spin_unlock_irqrestore(&rport->lock, flags);
1763		ret = -ENOTCONN;
1764		goto out_unmap;
1765	}
1766	list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1767	spin_unlock_irqrestore(&rport->lock, flags);
1768
1769	schedule_work(&rport->lsrcv_work);
1770
1771	return 0;
1772
1773out_unmap:
1774	fc_dma_unmap_single(lport->dev, lsop->rspdma,
1775			sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1776out_free:
1777	kfree(lsop->rspbuf);
1778	kfree(lsop->rqstbuf);
1779	kfree(lsop);
1780out_put:
1781	nvme_fc_rport_put(rport);
1782	return ret;
1783}
1784EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1785
1786
1787/* *********************** NVME Ctrl Routines **************************** */
1788
1789static void
1790__nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1791		struct nvme_fc_fcp_op *op)
1792{
1793	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1794				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1795	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1796				sizeof(op->cmd_iu), DMA_TO_DEVICE);
1797
1798	atomic_set(&op->state, FCPOP_STATE_UNINIT);
1799}
1800
1801static void
1802nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1803		unsigned int hctx_idx)
1804{
1805	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1806
1807	return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op);
1808}
1809
1810static int
1811__nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1812{
1813	unsigned long flags;
1814	int opstate;
1815
1816	spin_lock_irqsave(&ctrl->lock, flags);
1817	opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1818	if (opstate != FCPOP_STATE_ACTIVE)
1819		atomic_set(&op->state, opstate);
1820	else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1821		op->flags |= FCOP_FLAGS_TERMIO;
1822		ctrl->iocnt++;
1823	}
1824	spin_unlock_irqrestore(&ctrl->lock, flags);
1825
1826	if (opstate != FCPOP_STATE_ACTIVE)
1827		return -ECANCELED;
1828
1829	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1830					&ctrl->rport->remoteport,
1831					op->queue->lldd_handle,
1832					&op->fcp_req);
1833
1834	return 0;
1835}
1836
1837static void
1838nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1839{
1840	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1841	int i;
1842
1843	/* ensure we've initialized the ops once */
1844	if (!(aen_op->flags & FCOP_FLAGS_AEN))
1845		return;
1846
1847	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1848		__nvme_fc_abort_op(ctrl, aen_op);
1849}
1850
1851static inline void
1852__nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1853		struct nvme_fc_fcp_op *op, int opstate)
1854{
1855	unsigned long flags;
1856
1857	if (opstate == FCPOP_STATE_ABORTED) {
1858		spin_lock_irqsave(&ctrl->lock, flags);
1859		if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1860		    op->flags & FCOP_FLAGS_TERMIO) {
1861			if (!--ctrl->iocnt)
1862				wake_up(&ctrl->ioabort_wait);
1863		}
1864		spin_unlock_irqrestore(&ctrl->lock, flags);
1865	}
1866}
1867
1868static void
1869nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1870{
1871	struct nvme_fc_ctrl *ctrl =
1872			container_of(work, struct nvme_fc_ctrl, ioerr_work);
1873
1874	nvme_fc_error_recovery(ctrl, "transport detected io error");
1875}
1876
1877/*
1878 * nvme_fc_io_getuuid - Routine called to get the appid field
1879 * associated with request by the lldd
1880 * @req:IO request from nvme fc to driver
1881 * Returns: UUID if there is an appid associated with VM or
1882 * NULL if the user/libvirt has not set the appid to VM
1883 */
1884char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1885{
1886	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1887	struct request *rq = op->rq;
1888
1889	if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1890		return NULL;
1891	return blkcg_get_fc_appid(rq->bio);
1892}
1893EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1894
1895static void
1896nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1897{
1898	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1899	struct request *rq = op->rq;
1900	struct nvmefc_fcp_req *freq = &op->fcp_req;
1901	struct nvme_fc_ctrl *ctrl = op->ctrl;
1902	struct nvme_fc_queue *queue = op->queue;
1903	struct nvme_completion *cqe = &op->rsp_iu.cqe;
1904	struct nvme_command *sqe = &op->cmd_iu.sqe;
1905	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1906	union nvme_result result;
1907	bool terminate_assoc = true;
1908	int opstate;
1909
1910	/*
1911	 * WARNING:
1912	 * The current linux implementation of a nvme controller
1913	 * allocates a single tag set for all io queues and sizes
1914	 * the io queues to fully hold all possible tags. Thus, the
1915	 * implementation does not reference or care about the sqhd
1916	 * value as it never needs to use the sqhd/sqtail pointers
1917	 * for submission pacing.
1918	 *
1919	 * This affects the FC-NVME implementation in two ways:
1920	 * 1) As the value doesn't matter, we don't need to waste
1921	 *    cycles extracting it from ERSPs and stamping it in the
1922	 *    cases where the transport fabricates CQEs on successful
1923	 *    completions.
1924	 * 2) The FC-NVME implementation requires that delivery of
1925	 *    ERSP completions are to go back to the nvme layer in order
1926	 *    relative to the rsn, such that the sqhd value will always
1927	 *    be "in order" for the nvme layer. As the nvme layer in
1928	 *    linux doesn't care about sqhd, there's no need to return
1929	 *    them in order.
1930	 *
1931	 * Additionally:
1932	 * As the core nvme layer in linux currently does not look at
1933	 * every field in the cqe - in cases where the FC transport must
1934	 * fabricate a CQE, the following fields will not be set as they
1935	 * are not referenced:
1936	 *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1937	 *
1938	 * Failure or error of an individual i/o, in a transport
1939	 * detected fashion unrelated to the nvme completion status,
1940	 * potentially cause the initiator and target sides to get out
1941	 * of sync on SQ head/tail (aka outstanding io count allowed).
1942	 * Per FC-NVME spec, failure of an individual command requires
1943	 * the connection to be terminated, which in turn requires the
1944	 * association to be terminated.
1945	 */
1946
1947	opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1948
1949	fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1950				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1951
1952	if (opstate == FCPOP_STATE_ABORTED)
1953		status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1954	else if (freq->status) {
1955		status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1956		dev_info(ctrl->ctrl.device,
1957			"NVME-FC{%d}: io failed due to lldd error %d\n",
1958			ctrl->cnum, freq->status);
1959	}
1960
1961	/*
1962	 * For the linux implementation, if we have an unsuccessful
1963	 * status, the blk-mq layer can typically be called with the
1964	 * non-zero status and the content of the cqe isn't important.
1965	 */
1966	if (status)
1967		goto done;
1968
1969	/*
1970	 * command completed successfully relative to the wire
1971	 * protocol. However, validate anything received and
1972	 * extract the status and result from the cqe (create it
1973	 * where necessary).
1974	 */
1975
1976	switch (freq->rcv_rsplen) {
1977
1978	case 0:
1979	case NVME_FC_SIZEOF_ZEROS_RSP:
1980		/*
1981		 * No response payload or 12 bytes of payload (which
1982		 * should all be zeros) are considered successful and
1983		 * no payload in the CQE by the transport.
1984		 */
1985		if (freq->transferred_length !=
1986		    be32_to_cpu(op->cmd_iu.data_len)) {
1987			status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1988			dev_info(ctrl->ctrl.device,
1989				"NVME-FC{%d}: io failed due to bad transfer "
1990				"length: %d vs expected %d\n",
1991				ctrl->cnum, freq->transferred_length,
1992				be32_to_cpu(op->cmd_iu.data_len));
1993			goto done;
1994		}
1995		result.u64 = 0;
1996		break;
1997
1998	case sizeof(struct nvme_fc_ersp_iu):
1999		/*
2000		 * The ERSP IU contains a full completion with CQE.
2001		 * Validate ERSP IU and look at cqe.
2002		 */
2003		if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2004					(freq->rcv_rsplen / 4) ||
2005			     be32_to_cpu(op->rsp_iu.xfrd_len) !=
2006					freq->transferred_length ||
2007			     op->rsp_iu.ersp_result ||
2008			     sqe->common.command_id != cqe->command_id)) {
2009			status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2010			dev_info(ctrl->ctrl.device,
2011				"NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2012				"iu len %d, xfr len %d vs %d, status code "
2013				"%d, cmdid %d vs %d\n",
2014				ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2015				be32_to_cpu(op->rsp_iu.xfrd_len),
2016				freq->transferred_length,
2017				op->rsp_iu.ersp_result,
2018				sqe->common.command_id,
2019				cqe->command_id);
2020			goto done;
2021		}
2022		result = cqe->result;
2023		status = cqe->status;
2024		break;
2025
2026	default:
2027		status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2028		dev_info(ctrl->ctrl.device,
2029			"NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2030			"len %d\n",
2031			ctrl->cnum, freq->rcv_rsplen);
2032		goto done;
2033	}
2034
2035	terminate_assoc = false;
2036
2037done:
2038	if (op->flags & FCOP_FLAGS_AEN) {
2039		nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2040		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2041		atomic_set(&op->state, FCPOP_STATE_IDLE);
2042		op->flags = FCOP_FLAGS_AEN;	/* clear other flags */
2043		nvme_fc_ctrl_put(ctrl);
2044		goto check_error;
2045	}
2046
2047	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2048	if (!nvme_try_complete_req(rq, status, result))
2049		nvme_fc_complete_rq(rq);
2050
2051check_error:
2052	if (terminate_assoc &&
2053	    nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_RESETTING)
2054		queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2055}
2056
2057static int
2058__nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2059		struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2060		struct request *rq, u32 rqno)
2061{
2062	struct nvme_fcp_op_w_sgl *op_w_sgl =
2063		container_of(op, typeof(*op_w_sgl), op);
2064	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2065	int ret = 0;
2066
2067	memset(op, 0, sizeof(*op));
2068	op->fcp_req.cmdaddr = &op->cmd_iu;
2069	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2070	op->fcp_req.rspaddr = &op->rsp_iu;
2071	op->fcp_req.rsplen = sizeof(op->rsp_iu);
2072	op->fcp_req.done = nvme_fc_fcpio_done;
2073	op->ctrl = ctrl;
2074	op->queue = queue;
2075	op->rq = rq;
2076	op->rqno = rqno;
2077
2078	cmdiu->format_id = NVME_CMD_FORMAT_ID;
2079	cmdiu->fc_id = NVME_CMD_FC_ID;
2080	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2081	if (queue->qnum)
2082		cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2083					(NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2084	else
2085		cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2086
2087	op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2088				&op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2089	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2090		dev_err(ctrl->dev,
2091			"FCP Op failed - cmdiu dma mapping failed.\n");
2092		ret = -EFAULT;
2093		goto out_on_error;
2094	}
2095
2096	op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2097				&op->rsp_iu, sizeof(op->rsp_iu),
2098				DMA_FROM_DEVICE);
2099	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2100		dev_err(ctrl->dev,
2101			"FCP Op failed - rspiu dma mapping failed.\n");
2102		ret = -EFAULT;
2103	}
2104
2105	atomic_set(&op->state, FCPOP_STATE_IDLE);
2106out_on_error:
2107	return ret;
2108}
2109
2110static int
2111nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2112		unsigned int hctx_idx, unsigned int numa_node)
2113{
2114	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2115	struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2116	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2117	struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2118	int res;
2119
2120	res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2121	if (res)
2122		return res;
2123	op->op.fcp_req.first_sgl = op->sgl;
2124	op->op.fcp_req.private = &op->priv[0];
2125	nvme_req(rq)->ctrl = &ctrl->ctrl;
2126	nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2127	return res;
2128}
2129
2130static int
2131nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2132{
2133	struct nvme_fc_fcp_op *aen_op;
2134	struct nvme_fc_cmd_iu *cmdiu;
2135	struct nvme_command *sqe;
2136	void *private = NULL;
2137	int i, ret;
2138
2139	aen_op = ctrl->aen_ops;
2140	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2141		if (ctrl->lport->ops->fcprqst_priv_sz) {
2142			private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2143						GFP_KERNEL);
2144			if (!private)
2145				return -ENOMEM;
2146		}
2147
2148		cmdiu = &aen_op->cmd_iu;
2149		sqe = &cmdiu->sqe;
2150		ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2151				aen_op, (struct request *)NULL,
2152				(NVME_AQ_BLK_MQ_DEPTH + i));
2153		if (ret) {
2154			kfree(private);
2155			return ret;
2156		}
2157
2158		aen_op->flags = FCOP_FLAGS_AEN;
2159		aen_op->fcp_req.private = private;
2160
2161		memset(sqe, 0, sizeof(*sqe));
2162		sqe->common.opcode = nvme_admin_async_event;
2163		/* Note: core layer may overwrite the sqe.command_id value */
2164		sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2165	}
2166	return 0;
2167}
2168
2169static void
2170nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2171{
2172	struct nvme_fc_fcp_op *aen_op;
2173	int i;
2174
2175	cancel_work_sync(&ctrl->ctrl.async_event_work);
2176	aen_op = ctrl->aen_ops;
2177	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2178		__nvme_fc_exit_request(ctrl, aen_op);
2179
2180		kfree(aen_op->fcp_req.private);
2181		aen_op->fcp_req.private = NULL;
2182	}
2183}
2184
2185static inline int
2186__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2187{
2188	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data);
2189	struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2190
2191	hctx->driver_data = queue;
2192	queue->hctx = hctx;
2193	return 0;
2194}
2195
2196static int
2197nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2198{
2199	return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1);
2200}
2201
2202static int
2203nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2204		unsigned int hctx_idx)
2205{
2206	return __nvme_fc_init_hctx(hctx, data, hctx_idx);
2207}
2208
2209static void
2210nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2211{
2212	struct nvme_fc_queue *queue;
2213
2214	queue = &ctrl->queues[idx];
2215	memset(queue, 0, sizeof(*queue));
2216	queue->ctrl = ctrl;
2217	queue->qnum = idx;
2218	atomic_set(&queue->csn, 0);
2219	queue->dev = ctrl->dev;
2220
2221	if (idx > 0)
2222		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2223	else
2224		queue->cmnd_capsule_len = sizeof(struct nvme_command);
2225
2226	/*
2227	 * Considered whether we should allocate buffers for all SQEs
2228	 * and CQEs and dma map them - mapping their respective entries
2229	 * into the request structures (kernel vm addr and dma address)
2230	 * thus the driver could use the buffers/mappings directly.
2231	 * It only makes sense if the LLDD would use them for its
2232	 * messaging api. It's very unlikely most adapter api's would use
2233	 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2234	 * structures were used instead.
2235	 */
2236}
2237
2238/*
2239 * This routine terminates a queue at the transport level.
2240 * The transport has already ensured that all outstanding ios on
2241 * the queue have been terminated.
2242 * The transport will send a Disconnect LS request to terminate
2243 * the queue's connection. Termination of the admin queue will also
2244 * terminate the association at the target.
2245 */
2246static void
2247nvme_fc_free_queue(struct nvme_fc_queue *queue)
2248{
2249	if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2250		return;
2251
2252	clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2253	/*
2254	 * Current implementation never disconnects a single queue.
2255	 * It always terminates a whole association. So there is never
2256	 * a disconnect(queue) LS sent to the target.
2257	 */
2258
2259	queue->connection_id = 0;
2260	atomic_set(&queue->csn, 0);
2261}
2262
2263static void
2264__nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2265	struct nvme_fc_queue *queue, unsigned int qidx)
2266{
2267	if (ctrl->lport->ops->delete_queue)
2268		ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2269				queue->lldd_handle);
2270	queue->lldd_handle = NULL;
2271}
2272
2273static void
2274nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2275{
2276	int i;
2277
2278	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2279		nvme_fc_free_queue(&ctrl->queues[i]);
2280}
2281
2282static int
2283__nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2284	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2285{
2286	int ret = 0;
2287
2288	queue->lldd_handle = NULL;
2289	if (ctrl->lport->ops->create_queue)
2290		ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2291				qidx, qsize, &queue->lldd_handle);
2292
2293	return ret;
2294}
2295
2296static void
2297nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2298{
2299	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2300	int i;
2301
2302	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2303		__nvme_fc_delete_hw_queue(ctrl, queue, i);
2304}
2305
2306static int
2307nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2308{
2309	struct nvme_fc_queue *queue = &ctrl->queues[1];
2310	int i, ret;
2311
2312	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2313		ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2314		if (ret)
2315			goto delete_queues;
2316	}
2317
2318	return 0;
2319
2320delete_queues:
2321	for (; i > 0; i--)
2322		__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2323	return ret;
2324}
2325
2326static int
2327nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2328{
2329	int i, ret = 0;
2330
2331	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2332		ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2333					(qsize / 5));
2334		if (ret)
2335			break;
2336		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2337		if (ret)
2338			break;
2339
2340		set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2341	}
2342
2343	return ret;
2344}
2345
2346static void
2347nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2348{
2349	int i;
2350
2351	for (i = 1; i < ctrl->ctrl.queue_count; i++)
2352		nvme_fc_init_queue(ctrl, i);
2353}
2354
2355static void
2356nvme_fc_ctrl_free(struct kref *ref)
2357{
2358	struct nvme_fc_ctrl *ctrl =
2359		container_of(ref, struct nvme_fc_ctrl, ref);
2360	unsigned long flags;
2361
2362	/* remove from rport list */
2363	spin_lock_irqsave(&ctrl->rport->lock, flags);
2364	list_del(&ctrl->ctrl_list);
2365	spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2366
2367	kfree(ctrl->queues);
2368
2369	put_device(ctrl->dev);
2370	nvme_fc_rport_put(ctrl->rport);
2371
2372	ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2373	if (ctrl->ctrl.opts)
2374		nvmf_free_options(ctrl->ctrl.opts);
2375	kfree(ctrl);
2376}
2377
2378static void
2379nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2380{
2381	kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2382}
2383
2384static int
2385nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2386{
2387	return kref_get_unless_zero(&ctrl->ref);
2388}
2389
2390/*
2391 * All accesses from nvme core layer done - can now free the
2392 * controller. Called after last nvme_put_ctrl() call
2393 */
2394static void
2395nvme_fc_free_ctrl(struct nvme_ctrl *nctrl)
2396{
2397	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2398
2399	WARN_ON(nctrl != &ctrl->ctrl);
2400
2401	nvme_fc_ctrl_put(ctrl);
2402}
2403
2404/*
2405 * This routine is used by the transport when it needs to find active
2406 * io on a queue that is to be terminated. The transport uses
2407 * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2408 * this routine to kill them on a 1 by 1 basis.
2409 *
2410 * As FC allocates FC exchange for each io, the transport must contact
2411 * the LLDD to terminate the exchange, thus releasing the FC exchange.
2412 * After terminating the exchange the LLDD will call the transport's
2413 * normal io done path for the request, but it will have an aborted
2414 * status. The done path will return the io request back to the block
2415 * layer with an error status.
2416 */
2417static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2418{
2419	struct nvme_ctrl *nctrl = data;
2420	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2421	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2422
2423	op->nreq.flags |= NVME_REQ_CANCELLED;
2424	__nvme_fc_abort_op(ctrl, op);
2425	return true;
2426}
2427
2428/*
2429 * This routine runs through all outstanding commands on the association
2430 * and aborts them.  This routine is typically called by the
2431 * delete_association routine. It is also called due to an error during
2432 * reconnect. In that scenario, it is most likely a command that initializes
2433 * the controller, including fabric Connect commands on io queues, that
2434 * may have timed out or failed thus the io must be killed for the connect
2435 * thread to see the error.
2436 */
2437static void
2438__nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2439{
2440	int q;
2441
2442	/*
2443	 * if aborting io, the queues are no longer good, mark them
2444	 * all as not live.
2445	 */
2446	if (ctrl->ctrl.queue_count > 1) {
2447		for (q = 1; q < ctrl->ctrl.queue_count; q++)
2448			clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2449	}
2450	clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2451
2452	/*
2453	 * If io queues are present, stop them and terminate all outstanding
2454	 * ios on them. As FC allocates FC exchange for each io, the
2455	 * transport must contact the LLDD to terminate the exchange,
2456	 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2457	 * to tell us what io's are busy and invoke a transport routine
2458	 * to kill them with the LLDD.  After terminating the exchange
2459	 * the LLDD will call the transport's normal io done path, but it
2460	 * will have an aborted status. The done path will return the
2461	 * io requests back to the block layer as part of normal completions
2462	 * (but with error status).
2463	 */
2464	if (ctrl->ctrl.queue_count > 1) {
2465		nvme_quiesce_io_queues(&ctrl->ctrl);
2466		nvme_sync_io_queues(&ctrl->ctrl);
2467		blk_mq_tagset_busy_iter(&ctrl->tag_set,
2468				nvme_fc_terminate_exchange, &ctrl->ctrl);
2469		blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2470		if (start_queues)
2471			nvme_unquiesce_io_queues(&ctrl->ctrl);
2472	}
2473
2474	/*
2475	 * Other transports, which don't have link-level contexts bound
2476	 * to sqe's, would try to gracefully shutdown the controller by
2477	 * writing the registers for shutdown and polling (call
2478	 * nvme_disable_ctrl()). Given a bunch of i/o was potentially
2479	 * just aborted and we will wait on those contexts, and given
2480	 * there was no indication of how live the controller is on the
2481	 * link, don't send more io to create more contexts for the
2482	 * shutdown. Let the controller fail via keepalive failure if
2483	 * its still present.
2484	 */
2485
2486	/*
2487	 * clean up the admin queue. Same thing as above.
2488	 */
2489	nvme_quiesce_admin_queue(&ctrl->ctrl);
2490	blk_sync_queue(ctrl->ctrl.admin_q);
2491	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2492				nvme_fc_terminate_exchange, &ctrl->ctrl);
2493	blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2494	if (start_queues)
2495		nvme_unquiesce_admin_queue(&ctrl->ctrl);
2496}
2497
2498static void
2499nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2500{
2501	enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl);
2502
2503	/*
2504	 * if an error (io timeout, etc) while (re)connecting, the remote
2505	 * port requested terminating of the association (disconnect_ls)
2506	 * or an error (timeout or abort) occurred on an io while creating
2507	 * the controller.  Abort any ios on the association and let the
2508	 * create_association error path resolve things.
2509	 */
2510	if (state == NVME_CTRL_CONNECTING) {
2511		__nvme_fc_abort_outstanding_ios(ctrl, true);
2512		dev_warn(ctrl->ctrl.device,
2513			"NVME-FC{%d}: transport error during (re)connect\n",
2514			ctrl->cnum);
2515		return;
2516	}
2517
2518	/* Otherwise, only proceed if in LIVE state - e.g. on first error */
2519	if (state != NVME_CTRL_LIVE)
2520		return;
2521
2522	dev_warn(ctrl->ctrl.device,
2523		"NVME-FC{%d}: transport association event: %s\n",
2524		ctrl->cnum, errmsg);
2525	dev_warn(ctrl->ctrl.device,
2526		"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2527
2528	nvme_reset_ctrl(&ctrl->ctrl);
2529}
2530
2531static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2532{
2533	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2534	struct nvme_fc_ctrl *ctrl = op->ctrl;
2535	u16 qnum = op->queue->qnum;
2536	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2537	struct nvme_command *sqe = &cmdiu->sqe;
2538
2539	/*
2540	 * Attempt to abort the offending command. Command completion
2541	 * will detect the aborted io and will fail the connection.
2542	 */
2543	dev_info(ctrl->ctrl.device,
2544		"NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: "
2545		"x%08x/x%08x\n",
2546		ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype,
2547		nvme_fabrics_opcode_str(qnum, sqe),
2548		sqe->common.cdw10, sqe->common.cdw11);
2549	if (__nvme_fc_abort_op(ctrl, op))
2550		nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2551
2552	/*
2553	 * the io abort has been initiated. Have the reset timer
2554	 * restarted and the abort completion will complete the io
2555	 * shortly. Avoids a synchronous wait while the abort finishes.
2556	 */
2557	return BLK_EH_RESET_TIMER;
2558}
2559
2560static int
2561nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2562		struct nvme_fc_fcp_op *op)
2563{
2564	struct nvmefc_fcp_req *freq = &op->fcp_req;
2565	int ret;
2566
2567	freq->sg_cnt = 0;
2568
2569	if (!blk_rq_nr_phys_segments(rq))
2570		return 0;
2571
2572	freq->sg_table.sgl = freq->first_sgl;
2573	ret = sg_alloc_table_chained(&freq->sg_table,
2574			blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2575			NVME_INLINE_SG_CNT);
2576	if (ret)
2577		return -ENOMEM;
2578
2579	op->nents = blk_rq_map_sg(rq, freq->sg_table.sgl);
2580	WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2581	freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2582				op->nents, rq_dma_dir(rq));
2583	if (unlikely(freq->sg_cnt <= 0)) {
2584		sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2585		freq->sg_cnt = 0;
2586		return -EFAULT;
2587	}
2588
2589	/*
2590	 * TODO: blk_integrity_rq(rq)  for DIF
2591	 */
2592	return 0;
2593}
2594
2595static void
2596nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2597		struct nvme_fc_fcp_op *op)
2598{
2599	struct nvmefc_fcp_req *freq = &op->fcp_req;
2600
2601	if (!freq->sg_cnt)
2602		return;
2603
2604	fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2605			rq_dma_dir(rq));
2606
2607	sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2608
2609	freq->sg_cnt = 0;
2610}
2611
2612/*
2613 * In FC, the queue is a logical thing. At transport connect, the target
2614 * creates its "queue" and returns a handle that is to be given to the
2615 * target whenever it posts something to the corresponding SQ.  When an
2616 * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2617 * command contained within the SQE, an io, and assigns a FC exchange
2618 * to it. The SQE and the associated SQ handle are sent in the initial
2619 * CMD IU sents on the exchange. All transfers relative to the io occur
2620 * as part of the exchange.  The CQE is the last thing for the io,
2621 * which is transferred (explicitly or implicitly) with the RSP IU
2622 * sent on the exchange. After the CQE is received, the FC exchange is
2623 * terminated and the Exchange may be used on a different io.
2624 *
2625 * The transport to LLDD api has the transport making a request for a
2626 * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2627 * resource and transfers the command. The LLDD will then process all
2628 * steps to complete the io. Upon completion, the transport done routine
2629 * is called.
2630 *
2631 * So - while the operation is outstanding to the LLDD, there is a link
2632 * level FC exchange resource that is also outstanding. This must be
2633 * considered in all cleanup operations.
2634 */
2635static blk_status_t
2636nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2637	struct nvme_fc_fcp_op *op, u32 data_len,
2638	enum nvmefc_fcp_datadir	io_dir)
2639{
2640	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2641	struct nvme_command *sqe = &cmdiu->sqe;
2642	int ret, opstate;
2643
2644	/*
2645	 * before attempting to send the io, check to see if we believe
2646	 * the target device is present
2647	 */
2648	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2649		return BLK_STS_RESOURCE;
2650
2651	if (!nvme_fc_ctrl_get(ctrl))
2652		return BLK_STS_IOERR;
2653
2654	/* format the FC-NVME CMD IU and fcp_req */
2655	cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2656	cmdiu->data_len = cpu_to_be32(data_len);
2657	switch (io_dir) {
2658	case NVMEFC_FCP_WRITE:
2659		cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2660		break;
2661	case NVMEFC_FCP_READ:
2662		cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2663		break;
2664	case NVMEFC_FCP_NODATA:
2665		cmdiu->flags = 0;
2666		break;
2667	}
2668	op->fcp_req.payload_length = data_len;
2669	op->fcp_req.io_dir = io_dir;
2670	op->fcp_req.transferred_length = 0;
2671	op->fcp_req.rcv_rsplen = 0;
2672	op->fcp_req.status = NVME_SC_SUCCESS;
2673	op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2674
2675	/*
2676	 * validate per fabric rules, set fields mandated by fabric spec
2677	 * as well as those by FC-NVME spec.
2678	 */
2679	WARN_ON_ONCE(sqe->common.metadata);
2680	sqe->common.flags |= NVME_CMD_SGL_METABUF;
2681
2682	/*
2683	 * format SQE DPTR field per FC-NVME rules:
2684	 *    type=0x5     Transport SGL Data Block Descriptor
2685	 *    subtype=0xA  Transport-specific value
2686	 *    address=0
2687	 *    length=length of the data series
2688	 */
2689	sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2690					NVME_SGL_FMT_TRANSPORT_A;
2691	sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2692	sqe->rw.dptr.sgl.addr = 0;
2693
2694	if (!(op->flags & FCOP_FLAGS_AEN)) {
2695		ret = nvme_fc_map_data(ctrl, op->rq, op);
2696		if (ret < 0) {
2697			nvme_cleanup_cmd(op->rq);
2698			nvme_fc_ctrl_put(ctrl);
2699			if (ret == -ENOMEM || ret == -EAGAIN)
2700				return BLK_STS_RESOURCE;
2701			return BLK_STS_IOERR;
2702		}
2703	}
2704
2705	fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2706				  sizeof(op->cmd_iu), DMA_TO_DEVICE);
2707
2708	atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2709
2710	if (!(op->flags & FCOP_FLAGS_AEN))
2711		nvme_start_request(op->rq);
2712
2713	cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2714	ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2715					&ctrl->rport->remoteport,
2716					queue->lldd_handle, &op->fcp_req);
2717
2718	if (ret) {
2719		/*
2720		 * If the lld fails to send the command is there an issue with
2721		 * the csn value?  If the command that fails is the Connect,
2722		 * no - as the connection won't be live.  If it is a command
2723		 * post-connect, it's possible a gap in csn may be created.
2724		 * Does this matter?  As Linux initiators don't send fused
2725		 * commands, no.  The gap would exist, but as there's nothing
2726		 * that depends on csn order to be delivered on the target
2727		 * side, it shouldn't hurt.  It would be difficult for a
2728		 * target to even detect the csn gap as it has no idea when the
2729		 * cmd with the csn was supposed to arrive.
2730		 */
2731		opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2732		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2733
2734		if (!(op->flags & FCOP_FLAGS_AEN)) {
2735			nvme_fc_unmap_data(ctrl, op->rq, op);
2736			nvme_cleanup_cmd(op->rq);
2737		}
2738
2739		nvme_fc_ctrl_put(ctrl);
2740
2741		if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2742				ret != -EBUSY)
2743			return BLK_STS_IOERR;
2744
2745		return BLK_STS_RESOURCE;
2746	}
2747
2748	return BLK_STS_OK;
2749}
2750
2751static blk_status_t
2752nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2753			const struct blk_mq_queue_data *bd)
2754{
2755	struct nvme_ns *ns = hctx->queue->queuedata;
2756	struct nvme_fc_queue *queue = hctx->driver_data;
2757	struct nvme_fc_ctrl *ctrl = queue->ctrl;
2758	struct request *rq = bd->rq;
2759	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2760	enum nvmefc_fcp_datadir	io_dir;
2761	bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2762	u32 data_len;
2763	blk_status_t ret;
2764
2765	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2766	    !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2767		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2768
2769	ret = nvme_setup_cmd(ns, rq);
2770	if (ret)
2771		return ret;
2772
2773	/*
2774	 * nvme core doesn't quite treat the rq opaquely. Commands such
2775	 * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2776	 * there is no actual payload to be transferred.
2777	 * To get it right, key data transmission on there being 1 or
2778	 * more physical segments in the sg list. If there are no
2779	 * physical segments, there is no payload.
2780	 */
2781	if (blk_rq_nr_phys_segments(rq)) {
2782		data_len = blk_rq_payload_bytes(rq);
2783		io_dir = ((rq_data_dir(rq) == WRITE) ?
2784					NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2785	} else {
2786		data_len = 0;
2787		io_dir = NVMEFC_FCP_NODATA;
2788	}
2789
2790
2791	return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2792}
2793
2794static void
2795nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2796{
2797	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2798	struct nvme_fc_fcp_op *aen_op;
2799	blk_status_t ret;
2800
2801	if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2802		return;
2803
2804	aen_op = &ctrl->aen_ops[0];
2805
2806	ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2807					NVMEFC_FCP_NODATA);
2808	if (ret)
2809		dev_err(ctrl->ctrl.device,
2810			"failed async event work\n");
2811}
2812
2813static void
2814nvme_fc_complete_rq(struct request *rq)
2815{
2816	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2817	struct nvme_fc_ctrl *ctrl = op->ctrl;
2818
2819	atomic_set(&op->state, FCPOP_STATE_IDLE);
2820	op->flags &= ~FCOP_FLAGS_TERMIO;
2821
2822	nvme_fc_unmap_data(ctrl, rq, op);
2823	nvme_complete_rq(rq);
2824	nvme_fc_ctrl_put(ctrl);
2825}
2826
2827static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2828{
2829	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2830	int i;
2831
2832	for (i = 0; i < set->nr_maps; i++) {
2833		struct blk_mq_queue_map *map = &set->map[i];
2834
2835		if (!map->nr_queues) {
2836			WARN_ON(i == HCTX_TYPE_DEFAULT);
2837			continue;
2838		}
2839
2840		/* Call LLDD map queue functionality if defined */
2841		if (ctrl->lport->ops->map_queues)
2842			ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2843						     map);
2844		else
2845			blk_mq_map_queues(map);
2846	}
2847}
2848
2849static const struct blk_mq_ops nvme_fc_mq_ops = {
2850	.queue_rq	= nvme_fc_queue_rq,
2851	.complete	= nvme_fc_complete_rq,
2852	.init_request	= nvme_fc_init_request,
2853	.exit_request	= nvme_fc_exit_request,
2854	.init_hctx	= nvme_fc_init_hctx,
2855	.timeout	= nvme_fc_timeout,
2856	.map_queues	= nvme_fc_map_queues,
2857};
2858
2859static int
2860nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2861{
2862	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2863	unsigned int nr_io_queues;
2864	int ret;
2865
2866	nr_io_queues = min3(opts->nr_io_queues, num_online_cpus(),
2867				ctrl->lport->ops->max_hw_queues);
2868	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2869	if (ret) {
2870		dev_info(ctrl->ctrl.device,
2871			"set_queue_count failed: %d\n", ret);
2872		return ret;
2873	}
2874
2875	ctrl->ctrl.queue_count = nr_io_queues + 1;
2876	if (!nr_io_queues)
2877		return 0;
2878
2879	nvme_fc_init_io_queues(ctrl);
2880
2881	ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
2882			&nvme_fc_mq_ops, 1,
2883			struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2884				      ctrl->lport->ops->fcprqst_priv_sz));
2885	if (ret)
2886		return ret;
2887
2888	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2889	if (ret)
2890		goto out_cleanup_tagset;
2891
2892	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2893	if (ret)
2894		goto out_delete_hw_queues;
2895
2896	ctrl->ioq_live = true;
2897
2898	return 0;
2899
2900out_delete_hw_queues:
2901	nvme_fc_delete_hw_io_queues(ctrl);
2902out_cleanup_tagset:
2903	nvme_remove_io_tag_set(&ctrl->ctrl);
2904	nvme_fc_free_io_queues(ctrl);
2905
2906	/* force put free routine to ignore io queues */
2907	ctrl->ctrl.tagset = NULL;
2908
2909	return ret;
2910}
2911
2912static int
2913nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2914{
2915	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2916	u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2917	unsigned int nr_io_queues;
2918	int ret;
2919
2920	nr_io_queues = min3(opts->nr_io_queues, num_online_cpus(),
2921				ctrl->lport->ops->max_hw_queues);
2922	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2923	if (ret) {
2924		dev_info(ctrl->ctrl.device,
2925			"set_queue_count failed: %d\n", ret);
2926		return ret;
2927	}
2928
2929	if (!nr_io_queues && prior_ioq_cnt) {
2930		dev_info(ctrl->ctrl.device,
2931			"Fail Reconnect: At least 1 io queue "
2932			"required (was %d)\n", prior_ioq_cnt);
2933		return -ENOSPC;
2934	}
2935
2936	ctrl->ctrl.queue_count = nr_io_queues + 1;
2937	/* check for io queues existing */
2938	if (ctrl->ctrl.queue_count == 1)
2939		return 0;
2940
2941	if (prior_ioq_cnt != nr_io_queues) {
2942		dev_info(ctrl->ctrl.device,
2943			"reconnect: revising io queue count from %d to %d\n",
2944			prior_ioq_cnt, nr_io_queues);
2945		blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2946	}
2947
2948	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2949	if (ret)
2950		goto out_free_io_queues;
2951
2952	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2953	if (ret)
2954		goto out_delete_hw_queues;
2955
2956	return 0;
2957
2958out_delete_hw_queues:
2959	nvme_fc_delete_hw_io_queues(ctrl);
2960out_free_io_queues:
2961	nvme_fc_free_io_queues(ctrl);
2962	return ret;
2963}
2964
2965static void
2966nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
2967{
2968	struct nvme_fc_lport *lport = rport->lport;
2969
2970	atomic_inc(&lport->act_rport_cnt);
2971}
2972
2973static void
2974nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
2975{
2976	struct nvme_fc_lport *lport = rport->lport;
2977	u32 cnt;
2978
2979	cnt = atomic_dec_return(&lport->act_rport_cnt);
2980	if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
2981		lport->ops->localport_delete(&lport->localport);
2982}
2983
2984static int
2985nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
2986{
2987	struct nvme_fc_rport *rport = ctrl->rport;
2988	u32 cnt;
2989
2990	if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
2991		return 1;
2992
2993	cnt = atomic_inc_return(&rport->act_ctrl_cnt);
2994	if (cnt == 1)
2995		nvme_fc_rport_active_on_lport(rport);
2996
2997	return 0;
2998}
2999
3000static int
3001nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3002{
3003	struct nvme_fc_rport *rport = ctrl->rport;
3004	struct nvme_fc_lport *lport = rport->lport;
3005	u32 cnt;
3006
3007	/* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3008
3009	cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3010	if (cnt == 0) {
3011		if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3012			lport->ops->remoteport_delete(&rport->remoteport);
3013		nvme_fc_rport_inactive_on_lport(rport);
3014	}
3015
3016	return 0;
3017}
3018
3019/*
3020 * This routine restarts the controller on the host side, and
3021 * on the link side, recreates the controller association.
3022 */
3023static int
3024nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3025{
3026	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3027	struct nvmefc_ls_rcv_op *disls = NULL;
3028	unsigned long flags;
3029	int ret;
3030
3031	++ctrl->ctrl.nr_reconnects;
3032
3033	spin_lock_irqsave(&ctrl->rport->lock, flags);
3034	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
3035		spin_unlock_irqrestore(&ctrl->rport->lock, flags);
3036		return -ENODEV;
3037	}
3038
3039	if (nvme_fc_ctlr_active_on_rport(ctrl)) {
3040		spin_unlock_irqrestore(&ctrl->rport->lock, flags);
3041		return -ENOTUNIQ;
3042	}
3043	spin_unlock_irqrestore(&ctrl->rport->lock, flags);
3044
3045	dev_info(ctrl->ctrl.device,
3046		"NVME-FC{%d}: create association : host wwpn 0x%016llx "
3047		" rport wwpn 0x%016llx: NQN \"%s\"\n",
3048		ctrl->cnum, ctrl->lport->localport.port_name,
3049		ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3050
3051	clear_bit(ASSOC_FAILED, &ctrl->flags);
3052
3053	/*
3054	 * Create the admin queue
3055	 */
3056
3057	ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3058				NVME_AQ_DEPTH);
3059	if (ret)
3060		goto out_free_queue;
3061
3062	ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3063				NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3064	if (ret)
3065		goto out_delete_hw_queue;
3066
3067	ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3068	if (ret)
3069		goto out_disconnect_admin_queue;
3070
3071	set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3072
3073	/*
3074	 * Check controller capabilities
3075	 *
3076	 * todo:- add code to check if ctrl attributes changed from
3077	 * prior connection values
3078	 */
3079
3080	ret = nvme_enable_ctrl(&ctrl->ctrl);
3081	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3082		ret = -EIO;
3083	if (ret)
3084		goto out_disconnect_admin_queue;
3085
3086	ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3087	ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3088						(ilog2(SZ_4K) - 9);
3089
3090	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3091
3092	ret = nvme_init_ctrl_finish(&ctrl->ctrl, false);
3093	if (ret)
3094		goto out_disconnect_admin_queue;
3095	if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3096		ret = -EIO;
3097		goto out_stop_keep_alive;
3098	}
3099	/* sanity checks */
3100
3101	/* FC-NVME does not have other data in the capsule */
3102	if (ctrl->ctrl.icdoff) {
3103		dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3104				ctrl->ctrl.icdoff);
3105		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3106		goto out_stop_keep_alive;
3107	}
3108
3109	/* FC-NVME supports normal SGL Data Block Descriptors */
3110	if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3111		dev_err(ctrl->ctrl.device,
3112			"Mandatory sgls are not supported!\n");
3113		ret = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
3114		goto out_stop_keep_alive;
3115	}
3116
3117	if (opts->queue_size > ctrl->ctrl.maxcmd) {
3118		/* warn if maxcmd is lower than queue_size */
3119		dev_warn(ctrl->ctrl.device,
3120			"queue_size %zu > ctrl maxcmd %u, reducing "
3121			"to maxcmd\n",
3122			opts->queue_size, ctrl->ctrl.maxcmd);
3123		opts->queue_size = ctrl->ctrl.maxcmd;
3124		ctrl->ctrl.sqsize = opts->queue_size - 1;
3125	}
3126
3127	ret = nvme_fc_init_aen_ops(ctrl);
3128	if (ret)
3129		goto out_term_aen_ops;
3130
3131	/*
3132	 * Create the io queues
3133	 */
3134
3135	if (ctrl->ctrl.queue_count > 1) {
3136		if (!ctrl->ioq_live)
3137			ret = nvme_fc_create_io_queues(ctrl);
3138		else
3139			ret = nvme_fc_recreate_io_queues(ctrl);
3140	}
3141	if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3142		ret = -EIO;
3143	if (ret)
3144		goto out_term_aen_ops;
3145
3146	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE)) {
3147		ret = -EIO;
3148		goto out_term_aen_ops;
3149	}
3150
3151	ctrl->ctrl.nr_reconnects = 0;
3152	nvme_start_ctrl(&ctrl->ctrl);
3153
3154	return 0;	/* Success */
3155
3156out_term_aen_ops:
3157	nvme_fc_term_aen_ops(ctrl);
3158out_stop_keep_alive:
3159	nvme_stop_keep_alive(&ctrl->ctrl);
3160out_disconnect_admin_queue:
3161	dev_warn(ctrl->ctrl.device,
3162		"NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3163		ctrl->cnum, ctrl->association_id, ret);
3164	/* send a Disconnect(association) LS to fc-nvme target */
3165	nvme_fc_xmt_disconnect_assoc(ctrl);
3166	spin_lock_irqsave(&ctrl->lock, flags);
3167	ctrl->association_id = 0;
3168	disls = ctrl->rcv_disconn;
3169	ctrl->rcv_disconn = NULL;
3170	spin_unlock_irqrestore(&ctrl->lock, flags);
3171	if (disls)
3172		nvme_fc_xmt_ls_rsp(disls);
3173out_delete_hw_queue:
3174	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3175out_free_queue:
3176	nvme_fc_free_queue(&ctrl->queues[0]);
3177	clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3178	nvme_fc_ctlr_inactive_on_rport(ctrl);
3179
3180	return ret;
3181}
3182
3183
3184/*
3185 * This routine stops operation of the controller on the host side.
3186 * On the host os stack side: Admin and IO queues are stopped,
3187 *   outstanding ios on them terminated via FC ABTS.
3188 * On the link side: the association is terminated.
3189 */
3190static void
3191nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3192{
3193	struct nvmefc_ls_rcv_op *disls = NULL;
3194	unsigned long flags;
3195
3196	if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3197		return;
3198
3199	spin_lock_irqsave(&ctrl->lock, flags);
3200	set_bit(FCCTRL_TERMIO, &ctrl->flags);
3201	ctrl->iocnt = 0;
3202	spin_unlock_irqrestore(&ctrl->lock, flags);
3203
3204	__nvme_fc_abort_outstanding_ios(ctrl, false);
3205
3206	/* kill the aens as they are a separate path */
3207	nvme_fc_abort_aen_ops(ctrl);
3208
3209	/* wait for all io that had to be aborted */
3210	spin_lock_irq(&ctrl->lock);
3211	wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3212	clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3213	spin_unlock_irq(&ctrl->lock);
3214
3215	nvme_fc_term_aen_ops(ctrl);
3216
3217	/*
3218	 * send a Disconnect(association) LS to fc-nvme target
3219	 * Note: could have been sent at top of process, but
3220	 * cleaner on link traffic if after the aborts complete.
3221	 * Note: if association doesn't exist, association_id will be 0
3222	 */
3223	if (ctrl->association_id)
3224		nvme_fc_xmt_disconnect_assoc(ctrl);
3225
3226	spin_lock_irqsave(&ctrl->lock, flags);
3227	ctrl->association_id = 0;
3228	disls = ctrl->rcv_disconn;
3229	ctrl->rcv_disconn = NULL;
3230	spin_unlock_irqrestore(&ctrl->lock, flags);
3231	if (disls)
3232		/*
3233		 * if a Disconnect Request was waiting for a response, send
3234		 * now that all ABTS's have been issued (and are complete).
3235		 */
3236		nvme_fc_xmt_ls_rsp(disls);
3237
3238	if (ctrl->ctrl.tagset) {
3239		nvme_fc_delete_hw_io_queues(ctrl);
3240		nvme_fc_free_io_queues(ctrl);
3241	}
3242
3243	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3244	nvme_fc_free_queue(&ctrl->queues[0]);
3245
3246	/* re-enable the admin_q so anything new can fast fail */
3247	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3248
3249	/* resume the io queues so that things will fast fail */
3250	nvme_unquiesce_io_queues(&ctrl->ctrl);
3251
3252	nvme_fc_ctlr_inactive_on_rport(ctrl);
3253}
3254
3255static void
3256nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3257{
3258	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3259
3260	cancel_delayed_work_sync(&ctrl->connect_work);
3261
3262	/*
3263	 * kill the association on the link side.  this will block
3264	 * waiting for io to terminate
3265	 */
3266	nvme_fc_delete_association(ctrl);
3267	cancel_work_sync(&ctrl->ioerr_work);
3268
3269	if (ctrl->ctrl.tagset)
3270		nvme_remove_io_tag_set(&ctrl->ctrl);
3271
3272	nvme_unquiesce_admin_queue(&ctrl->ctrl);
3273	nvme_remove_admin_tag_set(&ctrl->ctrl);
3274}
3275
3276static void
3277nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3278{
3279	struct nvme_fc_rport *rport = ctrl->rport;
3280	struct nvme_fc_remote_port *portptr = &rport->remoteport;
3281	unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3282	bool recon = true;
3283
3284	if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_CONNECTING)
3285		return;
3286
3287	if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3288		dev_info(ctrl->ctrl.device,
3289			"NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3290			ctrl->cnum, status);
3291	} else if (time_after_eq(jiffies, rport->dev_loss_end))
3292		recon = false;
3293
3294	if (recon && nvmf_should_reconnect(&ctrl->ctrl, status)) {
3295		if (portptr->port_state == FC_OBJSTATE_ONLINE)
3296			dev_info(ctrl->ctrl.device,
3297				"NVME-FC{%d}: Reconnect attempt in %ld "
3298				"seconds\n",
3299				ctrl->cnum, recon_delay / HZ);
3300		else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3301			recon_delay = rport->dev_loss_end - jiffies;
3302
3303		queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3304	} else {
3305		if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3306			if (status > 0 && (status & NVME_STATUS_DNR))
3307				dev_warn(ctrl->ctrl.device,
3308					 "NVME-FC{%d}: reconnect failure\n",
3309					 ctrl->cnum);
3310			else
3311				dev_warn(ctrl->ctrl.device,
3312					 "NVME-FC{%d}: Max reconnect attempts "
3313					 "(%d) reached.\n",
3314					 ctrl->cnum, ctrl->ctrl.nr_reconnects);
3315		} else
3316			dev_warn(ctrl->ctrl.device,
3317				"NVME-FC{%d}: dev_loss_tmo (%d) expired "
3318				"while waiting for remoteport connectivity.\n",
3319				ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3320					(ctrl->ctrl.opts->max_reconnects *
3321					 ctrl->ctrl.opts->reconnect_delay)));
3322		WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3323	}
3324}
3325
3326static void
3327nvme_fc_reset_ctrl_work(struct work_struct *work)
3328{
3329	struct nvme_fc_ctrl *ctrl =
3330		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3331
3332	nvme_stop_ctrl(&ctrl->ctrl);
3333
3334	/* will block will waiting for io to terminate */
3335	nvme_fc_delete_association(ctrl);
3336
3337	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3338		dev_err(ctrl->ctrl.device,
3339			"NVME-FC{%d}: error_recovery: Couldn't change state "
3340			"to CONNECTING\n", ctrl->cnum);
3341
3342	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3343		if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3344			dev_err(ctrl->ctrl.device,
3345				"NVME-FC{%d}: failed to schedule connect "
3346				"after reset\n", ctrl->cnum);
3347		} else {
3348			flush_delayed_work(&ctrl->connect_work);
3349		}
3350	} else {
3351		nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3352	}
3353}
3354
3355
3356static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3357	.name			= "fc",
3358	.module			= THIS_MODULE,
3359	.flags			= NVME_F_FABRICS,
3360	.reg_read32		= nvmf_reg_read32,
3361	.reg_read64		= nvmf_reg_read64,
3362	.reg_write32		= nvmf_reg_write32,
3363	.subsystem_reset	= nvmf_subsystem_reset,
3364	.free_ctrl		= nvme_fc_free_ctrl,
3365	.submit_async_event	= nvme_fc_submit_async_event,
3366	.delete_ctrl		= nvme_fc_delete_ctrl,
3367	.get_address		= nvmf_get_address,
3368	.get_virt_boundary	= nvmf_get_virt_boundary,
3369};
3370
3371static void
3372nvme_fc_connect_ctrl_work(struct work_struct *work)
3373{
3374	int ret;
3375
3376	struct nvme_fc_ctrl *ctrl =
3377			container_of(to_delayed_work(work),
3378				struct nvme_fc_ctrl, connect_work);
3379
3380	ret = nvme_fc_create_association(ctrl);
3381	if (ret)
3382		nvme_fc_reconnect_or_delete(ctrl, ret);
3383	else
3384		dev_info(ctrl->ctrl.device,
3385			"NVME-FC{%d}: controller connect complete\n",
3386			ctrl->cnum);
3387}
3388
3389
3390static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3391	.queue_rq	= nvme_fc_queue_rq,
3392	.complete	= nvme_fc_complete_rq,
3393	.init_request	= nvme_fc_init_request,
3394	.exit_request	= nvme_fc_exit_request,
3395	.init_hctx	= nvme_fc_init_admin_hctx,
3396	.timeout	= nvme_fc_timeout,
3397};
3398
3399
3400/*
3401 * Fails a controller request if it matches an existing controller
3402 * (association) with the same tuple:
3403 * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3404 *
3405 * The ports don't need to be compared as they are intrinsically
3406 * already matched by the port pointers supplied.
3407 */
3408static bool
3409nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3410		struct nvmf_ctrl_options *opts)
3411{
3412	struct nvme_fc_ctrl *ctrl;
3413	unsigned long flags;
3414	bool found = false;
3415
3416	spin_lock_irqsave(&rport->lock, flags);
3417	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3418		found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3419		if (found)
3420			break;
3421	}
3422	spin_unlock_irqrestore(&rport->lock, flags);
3423
3424	return found;
3425}
3426
3427static struct nvme_fc_ctrl *
3428nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3429	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3430{
3431	struct nvme_fc_ctrl *ctrl;
3432	int ret, idx, ctrl_loss_tmo;
3433
3434	if (!(rport->remoteport.port_role &
3435	    (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3436		ret = -EBADR;
3437		goto out_fail;
3438	}
3439
3440	if (!opts->duplicate_connect &&
3441	    nvme_fc_existing_controller(rport, opts)) {
3442		ret = -EALREADY;
3443		goto out_fail;
3444	}
3445
3446	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3447	if (!ctrl) {
3448		ret = -ENOMEM;
3449		goto out_fail;
3450	}
3451
3452	idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3453	if (idx < 0) {
3454		ret = -ENOSPC;
3455		goto out_free_ctrl;
3456	}
3457
3458	/*
3459	 * if ctrl_loss_tmo is being enforced and the default reconnect delay
3460	 * is being used, change to a shorter reconnect delay for FC.
3461	 */
3462	if (opts->max_reconnects != -1 &&
3463	    opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3464	    opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3465		ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3466		opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3467		opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3468						opts->reconnect_delay);
3469	}
3470
3471	ctrl->ctrl.opts = opts;
3472	ctrl->ctrl.nr_reconnects = 0;
3473	INIT_LIST_HEAD(&ctrl->ctrl_list);
3474	ctrl->lport = lport;
3475	ctrl->rport = rport;
3476	ctrl->dev = lport->dev;
3477	ctrl->cnum = idx;
3478	ctrl->ioq_live = false;
3479	init_waitqueue_head(&ctrl->ioabort_wait);
3480
3481	get_device(ctrl->dev);
3482	kref_init(&ctrl->ref);
3483
3484	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3485	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3486	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3487	spin_lock_init(&ctrl->lock);
3488
3489	/* io queue count */
3490	ctrl->ctrl.queue_count = min_t(unsigned int,
3491				opts->nr_io_queues,
3492				lport->ops->max_hw_queues);
3493	ctrl->ctrl.queue_count++;	/* +1 for admin queue */
3494
3495	ctrl->ctrl.sqsize = opts->queue_size - 1;
3496	ctrl->ctrl.kato = opts->kato;
3497	ctrl->ctrl.cntlid = 0xffff;
3498
3499	ret = -ENOMEM;
3500	ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3501				sizeof(struct nvme_fc_queue), GFP_KERNEL);
3502	if (!ctrl->queues)
3503		goto out_free_ida;
3504
3505	nvme_fc_init_queue(ctrl, 0);
3506
3507	/*
3508	 * Would have been nice to init io queues tag set as well.
3509	 * However, we require interaction from the controller
3510	 * for max io queue count before we can do so.
3511	 * Defer this to the connect path.
3512	 */
3513
3514	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3515	if (ret)
3516		goto out_free_queues;
3517	if (lport->dev)
3518		ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3519
3520	return ctrl;
3521
3522out_free_queues:
3523	kfree(ctrl->queues);
3524out_free_ida:
3525	put_device(ctrl->dev);
3526	ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3527out_free_ctrl:
3528	kfree(ctrl);
3529out_fail:
3530	/* exit via here doesn't follow ctlr ref points */
3531	return ERR_PTR(ret);
3532}
3533
3534static struct nvme_ctrl *
3535nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3536	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3537{
3538	struct nvme_fc_ctrl *ctrl;
3539	unsigned long flags;
3540	int ret;
3541
3542	ctrl = nvme_fc_alloc_ctrl(dev, opts, lport, rport);
3543	if (IS_ERR(ctrl))
3544		return ERR_CAST(ctrl);
3545
3546	ret = nvme_add_ctrl(&ctrl->ctrl);
3547	if (ret)
3548		goto out_put_ctrl;
3549
3550	ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
3551			&nvme_fc_admin_mq_ops,
3552			struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3553				      ctrl->lport->ops->fcprqst_priv_sz));
3554	if (ret)
3555		goto fail_ctrl;
3556
3557	spin_lock_irqsave(&rport->lock, flags);
3558	list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3559	spin_unlock_irqrestore(&rport->lock, flags);
3560
3561	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3562		dev_err(ctrl->ctrl.device,
3563			"NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3564		goto fail_ctrl;
3565	}
3566
3567	if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3568		dev_err(ctrl->ctrl.device,
3569			"NVME-FC{%d}: failed to schedule initial connect\n",
3570			ctrl->cnum);
3571		goto fail_ctrl;
3572	}
3573
3574	flush_delayed_work(&ctrl->connect_work);
3575
3576	dev_info(ctrl->ctrl.device,
3577		"NVME-FC{%d}: new ctrl: NQN \"%s\", hostnqn: %s\n",
3578		ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl), opts->host->nqn);
3579
3580	return &ctrl->ctrl;
3581
3582fail_ctrl:
3583	nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3584	cancel_work_sync(&ctrl->ioerr_work);
3585	cancel_work_sync(&ctrl->ctrl.reset_work);
3586	cancel_delayed_work_sync(&ctrl->connect_work);
3587
3588	ctrl->ctrl.opts = NULL;
3589
3590	if (ctrl->ctrl.admin_tagset)
3591		nvme_remove_admin_tag_set(&ctrl->ctrl);
3592	/* initiate nvme ctrl ref counting teardown */
3593	nvme_uninit_ctrl(&ctrl->ctrl);
3594
3595out_put_ctrl:
3596	/* Remove core ctrl ref. */
3597	nvme_put_ctrl(&ctrl->ctrl);
3598
3599	/* as we're past the point where we transition to the ref
3600	 * counting teardown path, if we return a bad pointer here,
3601	 * the calling routine, thinking it's prior to the
3602	 * transition, will do an rport put. Since the teardown
3603	 * path also does a rport put, we do an extra get here to
3604	 * so proper order/teardown happens.
3605	 */
3606	nvme_fc_rport_get(rport);
3607
3608	return ERR_PTR(-EIO);
3609}
3610
3611struct nvmet_fc_traddr {
3612	u64	nn;
3613	u64	pn;
3614};
3615
3616static int
3617__nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3618{
3619	u64 token64;
3620
3621	if (match_u64(sstr, &token64))
3622		return -EINVAL;
3623	*val = token64;
3624
3625	return 0;
3626}
3627
3628/*
3629 * This routine validates and extracts the WWN's from the TRADDR string.
3630 * As kernel parsers need the 0x to determine number base, universally
3631 * build string to parse with 0x prefix before parsing name strings.
3632 */
3633static int
3634nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3635{
3636	char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3637	substring_t wwn = { name, &name[sizeof(name)-1] };
3638	int nnoffset, pnoffset;
3639
3640	/* validate if string is one of the 2 allowed formats */
3641	if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3642			!strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3643			!strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3644				"pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3645		nnoffset = NVME_FC_TRADDR_OXNNLEN;
3646		pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3647						NVME_FC_TRADDR_OXNNLEN;
3648	} else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3649			!strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3650			!strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3651				"pn-", NVME_FC_TRADDR_NNLEN))) {
3652		nnoffset = NVME_FC_TRADDR_NNLEN;
3653		pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3654	} else
3655		goto out_einval;
3656
3657	name[0] = '0';
3658	name[1] = 'x';
3659	name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3660
3661	memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3662	if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3663		goto out_einval;
3664
3665	memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3666	if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3667		goto out_einval;
3668
3669	return 0;
3670
3671out_einval:
3672	pr_warn("%s: bad traddr string\n", __func__);
3673	return -EINVAL;
3674}
3675
3676static struct nvme_ctrl *
3677nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3678{
3679	struct nvme_fc_lport *lport;
3680	struct nvme_fc_rport *rport;
3681	struct nvme_ctrl *ctrl;
3682	struct nvmet_fc_traddr laddr = { 0L, 0L };
3683	struct nvmet_fc_traddr raddr = { 0L, 0L };
3684	unsigned long flags;
3685	int ret;
3686
3687	ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3688	if (ret || !raddr.nn || !raddr.pn)
3689		return ERR_PTR(-EINVAL);
3690
3691	ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3692	if (ret || !laddr.nn || !laddr.pn)
3693		return ERR_PTR(-EINVAL);
3694
3695	/* find the host and remote ports to connect together */
3696	spin_lock_irqsave(&nvme_fc_lock, flags);
3697	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3698		if (lport->localport.node_name != laddr.nn ||
3699		    lport->localport.port_name != laddr.pn ||
3700		    lport->localport.port_state != FC_OBJSTATE_ONLINE)
3701			continue;
3702
3703		list_for_each_entry(rport, &lport->endp_list, endp_list) {
3704			if (rport->remoteport.node_name != raddr.nn ||
3705			    rport->remoteport.port_name != raddr.pn ||
3706			    rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3707				continue;
3708
3709			/* if fail to get reference fall through. Will error */
3710			if (!nvme_fc_rport_get(rport))
3711				break;
3712
3713			spin_unlock_irqrestore(&nvme_fc_lock, flags);
3714
3715			ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3716			if (IS_ERR(ctrl))
3717				nvme_fc_rport_put(rport);
3718			return ctrl;
3719		}
3720	}
3721	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3722
3723	pr_warn("%s: %s - %s combination not found\n",
3724		__func__, opts->traddr, opts->host_traddr);
3725	return ERR_PTR(-ENOENT);
3726}
3727
3728
3729static struct nvmf_transport_ops nvme_fc_transport = {
3730	.name		= "fc",
3731	.module		= THIS_MODULE,
3732	.required_opts	= NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3733	.allowed_opts	= NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3734	.create_ctrl	= nvme_fc_create_ctrl,
3735};
3736
3737/* Arbitrary successive failures max. With lots of subsystems could be high */
3738#define DISCOVERY_MAX_FAIL	20
3739
3740static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3741		struct device_attribute *attr, const char *buf, size_t count)
3742{
3743	unsigned long flags;
3744	LIST_HEAD(local_disc_list);
3745	struct nvme_fc_lport *lport;
3746	struct nvme_fc_rport *rport;
3747	int failcnt = 0;
3748
3749	spin_lock_irqsave(&nvme_fc_lock, flags);
3750restart:
3751	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3752		list_for_each_entry(rport, &lport->endp_list, endp_list) {
3753			if (!nvme_fc_lport_get(lport))
3754				continue;
3755			if (!nvme_fc_rport_get(rport)) {
3756				/*
3757				 * This is a temporary condition. Upon restart
3758				 * this rport will be gone from the list.
3759				 *
3760				 * Revert the lport put and retry.  Anything
3761				 * added to the list already will be skipped (as
3762				 * they are no longer list_empty).  Loops should
3763				 * resume at rports that were not yet seen.
3764				 */
3765				nvme_fc_lport_put(lport);
3766
3767				if (failcnt++ < DISCOVERY_MAX_FAIL)
3768					goto restart;
3769
3770				pr_err("nvme_discovery: too many reference "
3771				       "failures\n");
3772				goto process_local_list;
3773			}
3774			if (list_empty(&rport->disc_list))
3775				list_add_tail(&rport->disc_list,
3776					      &local_disc_list);
3777		}
3778	}
3779
3780process_local_list:
3781	while (!list_empty(&local_disc_list)) {
3782		rport = list_first_entry(&local_disc_list,
3783					 struct nvme_fc_rport, disc_list);
3784		list_del_init(&rport->disc_list);
3785		spin_unlock_irqrestore(&nvme_fc_lock, flags);
3786
3787		lport = rport->lport;
3788		/* signal discovery. Won't hurt if it repeats */
3789		nvme_fc_signal_discovery_scan(lport, rport);
3790		nvme_fc_rport_put(rport);
3791		nvme_fc_lport_put(lport);
3792
3793		spin_lock_irqsave(&nvme_fc_lock, flags);
3794	}
3795	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3796
3797	return count;
3798}
3799
3800static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3801
3802#ifdef CONFIG_BLK_CGROUP_FC_APPID
3803/* Parse the cgroup id from a buf and return the length of cgrpid */
3804static int fc_parse_cgrpid(const char *buf, u64 *id)
3805{
3806	char cgrp_id[16+1];
3807	int cgrpid_len, j;
3808
3809	memset(cgrp_id, 0x0, sizeof(cgrp_id));
3810	for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3811		if (buf[cgrpid_len] != ':')
3812			cgrp_id[cgrpid_len] = buf[cgrpid_len];
3813		else {
3814			j = 1;
3815			break;
3816		}
3817	}
3818	if (!j)
3819		return -EINVAL;
3820	if (kstrtou64(cgrp_id, 16, id) < 0)
3821		return -EINVAL;
3822	return cgrpid_len;
3823}
3824
3825/*
3826 * Parse and update the appid in the blkcg associated with the cgroupid.
3827 */
3828static ssize_t fc_appid_store(struct device *dev,
3829		struct device_attribute *attr, const char *buf, size_t count)
3830{
3831	size_t orig_count = count;
3832	u64 cgrp_id;
3833	int appid_len = 0;
3834	int cgrpid_len = 0;
3835	char app_id[FC_APPID_LEN];
3836	int ret = 0;
3837
3838	if (buf[count-1] == '\n')
3839		count--;
3840
3841	if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3842		return -EINVAL;
3843
3844	cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3845	if (cgrpid_len < 0)
3846		return -EINVAL;
3847	appid_len = count - cgrpid_len - 1;
3848	if (appid_len > FC_APPID_LEN)
3849		return -EINVAL;
3850
3851	memset(app_id, 0x0, sizeof(app_id));
3852	memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3853	ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3854	if (ret < 0)
3855		return ret;
3856	return orig_count;
3857}
3858static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3859#endif /* CONFIG_BLK_CGROUP_FC_APPID */
3860
3861static struct attribute *nvme_fc_attrs[] = {
3862	&dev_attr_nvme_discovery.attr,
3863#ifdef CONFIG_BLK_CGROUP_FC_APPID
3864	&dev_attr_appid_store.attr,
3865#endif
3866	NULL
3867};
3868
3869static const struct attribute_group nvme_fc_attr_group = {
3870	.attrs = nvme_fc_attrs,
3871};
3872
3873static const struct attribute_group *nvme_fc_attr_groups[] = {
3874	&nvme_fc_attr_group,
3875	NULL
3876};
3877
3878static struct class fc_class = {
3879	.name = "fc",
3880	.dev_groups = nvme_fc_attr_groups,
3881};
3882
3883static int __init nvme_fc_init_module(void)
3884{
3885	int ret;
3886
3887	/*
3888	 * NOTE:
3889	 * It is expected that in the future the kernel will combine
3890	 * the FC-isms that are currently under scsi and now being
3891	 * added to by NVME into a new standalone FC class. The SCSI
3892	 * and NVME protocols and their devices would be under this
3893	 * new FC class.
3894	 *
3895	 * As we need something to post FC-specific udev events to,
3896	 * specifically for nvme probe events, start by creating the
3897	 * new device class.  When the new standalone FC class is
3898	 * put in place, this code will move to a more generic
3899	 * location for the class.
3900	 */
3901	ret = class_register(&fc_class);
3902	if (ret) {
3903		pr_err("couldn't register class fc\n");
3904		return ret;
3905	}
3906
3907	/*
3908	 * Create a device for the FC-centric udev events
3909	 */
3910	fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3911				"fc_udev_device");
3912	if (IS_ERR(fc_udev_device)) {
3913		pr_err("couldn't create fc_udev device!\n");
3914		ret = PTR_ERR(fc_udev_device);
3915		goto out_destroy_class;
3916	}
3917
3918	ret = nvmf_register_transport(&nvme_fc_transport);
3919	if (ret)
3920		goto out_destroy_device;
3921
3922	return 0;
3923
3924out_destroy_device:
3925	device_destroy(&fc_class, MKDEV(0, 0));
3926out_destroy_class:
3927	class_unregister(&fc_class);
3928
3929	return ret;
3930}
3931
3932static void
3933nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3934{
3935	struct nvme_fc_ctrl *ctrl;
3936
3937	spin_lock(&rport->lock);
3938	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3939		dev_warn(ctrl->ctrl.device,
3940			"NVME-FC{%d}: transport unloading: deleting ctrl\n",
3941			ctrl->cnum);
3942		nvme_delete_ctrl(&ctrl->ctrl);
3943	}
3944	spin_unlock(&rport->lock);
3945}
3946
3947static void __exit nvme_fc_exit_module(void)
3948{
3949	struct nvme_fc_lport *lport;
3950	struct nvme_fc_rport *rport;
3951	unsigned long flags;
3952
3953	spin_lock_irqsave(&nvme_fc_lock, flags);
3954	list_for_each_entry(lport, &nvme_fc_lport_list, port_list)
3955		list_for_each_entry(rport, &lport->endp_list, endp_list)
3956			nvme_fc_delete_controllers(rport);
3957	spin_unlock_irqrestore(&nvme_fc_lock, flags);
3958	flush_workqueue(nvme_delete_wq);
3959
3960	nvmf_unregister_transport(&nvme_fc_transport);
3961
3962	device_destroy(&fc_class, MKDEV(0, 0));
3963	class_unregister(&fc_class);
3964}
3965
3966module_init(nvme_fc_init_module);
3967module_exit(nvme_fc_exit_module);
3968
3969MODULE_DESCRIPTION("NVMe host FC transport driver");
3970MODULE_LICENSE("GPL v2");