fabrics.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * NVMe over Fabrics common host code.
   4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   5 */
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7#include <linux/init.h>
   8#include <linux/miscdevice.h>
   9#include <linux/module.h>
  10#include <linux/mutex.h>
  11#include <linux/parser.h>
  12#include <linux/seq_file.h>
  13#include "nvme.h"
  14#include "fabrics.h"
  15#include <linux/nvme-keyring.h>
  16
  17static LIST_HEAD(nvmf_transports);
  18static DECLARE_RWSEM(nvmf_transports_rwsem);
  19
  20static LIST_HEAD(nvmf_hosts);
  21static DEFINE_MUTEX(nvmf_hosts_mutex);
  22
  23static struct nvmf_host *nvmf_default_host;
  24
  25static struct nvmf_host *nvmf_host_alloc(const char *hostnqn, uuid_t *id)
  26{
  27	struct nvmf_host *host;
  28
  29	host = kmalloc(sizeof(*host), GFP_KERNEL);
  30	if (!host)
  31		return NULL;
  32
  33	kref_init(&host->ref);
  34	uuid_copy(&host->id, id);
  35	strscpy(host->nqn, hostnqn, NVMF_NQN_SIZE);
  36
  37	return host;
  38}
  39
  40static struct nvmf_host *nvmf_host_add(const char *hostnqn, uuid_t *id)
  41{
  42	struct nvmf_host *host;
  43
  44	mutex_lock(&nvmf_hosts_mutex);
  45
  46	/*
  47	 * We have defined a host as how it is perceived by the target.
  48	 * Therefore, we don't allow different Host NQNs with the same Host ID.
  49	 * Similarly, we do not allow the usage of the same Host NQN with
  50	 * different Host IDs. This'll maintain unambiguous host identification.
  51	 */
  52	list_for_each_entry(host, &nvmf_hosts, list) {
  53		bool same_hostnqn = !strcmp(host->nqn, hostnqn);
  54		bool same_hostid = uuid_equal(&host->id, id);
  55
  56		if (same_hostnqn && same_hostid) {
  57			kref_get(&host->ref);
  58			goto out_unlock;
  59		}
  60		if (same_hostnqn) {
  61			pr_err("found same hostnqn %s but different hostid %pUb\n",
  62			       hostnqn, id);
  63			host = ERR_PTR(-EINVAL);
  64			goto out_unlock;
  65		}
  66		if (same_hostid) {
  67			pr_err("found same hostid %pUb but different hostnqn %s\n",
  68			       id, hostnqn);
  69			host = ERR_PTR(-EINVAL);
  70			goto out_unlock;
  71		}
  72	}
  73
  74	host = nvmf_host_alloc(hostnqn, id);
  75	if (!host) {
  76		host = ERR_PTR(-ENOMEM);
  77		goto out_unlock;
  78	}
  79
  80	list_add_tail(&host->list, &nvmf_hosts);
  81out_unlock:
  82	mutex_unlock(&nvmf_hosts_mutex);
  83	return host;
  84}
  85
  86static struct nvmf_host *nvmf_host_default(void)
  87{
  88	struct nvmf_host *host;
  89	char nqn[NVMF_NQN_SIZE];
  90	uuid_t id;
  91
  92	uuid_gen(&id);
  93	snprintf(nqn, NVMF_NQN_SIZE,
  94		"nqn.2014-08.org.nvmexpress:uuid:%pUb", &id);
  95
  96	host = nvmf_host_alloc(nqn, &id);
  97	if (!host)
  98		return NULL;
  99
 100	mutex_lock(&nvmf_hosts_mutex);
 101	list_add_tail(&host->list, &nvmf_hosts);
 102	mutex_unlock(&nvmf_hosts_mutex);
 103
 104	return host;
 105}
 106
 107static void nvmf_host_destroy(struct kref *ref)
 108{
 109	struct nvmf_host *host = container_of(ref, struct nvmf_host, ref);
 110
 111	mutex_lock(&nvmf_hosts_mutex);
 112	list_del(&host->list);
 113	mutex_unlock(&nvmf_hosts_mutex);
 114
 115	kfree(host);
 116}
 117
 118static void nvmf_host_put(struct nvmf_host *host)
 119{
 120	if (host)
 121		kref_put(&host->ref, nvmf_host_destroy);
 122}
 123
 124/**
 125 * nvmf_get_address() -  Get address/port
 126 * @ctrl:	Host NVMe controller instance which we got the address
 127 * @buf:	OUTPUT parameter that will contain the address/port
 128 * @size:	buffer size
 129 */
 130int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
 131{
 132	int len = 0;
 133
 134	if (ctrl->opts->mask & NVMF_OPT_TRADDR)
 135		len += scnprintf(buf, size, "traddr=%s", ctrl->opts->traddr);
 136	if (ctrl->opts->mask & NVMF_OPT_TRSVCID)
 137		len += scnprintf(buf + len, size - len, "%strsvcid=%s",
 138				(len) ? "," : "", ctrl->opts->trsvcid);
 139	if (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)
 140		len += scnprintf(buf + len, size - len, "%shost_traddr=%s",
 141				(len) ? "," : "", ctrl->opts->host_traddr);
 142	if (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)
 143		len += scnprintf(buf + len, size - len, "%shost_iface=%s",
 144				(len) ? "," : "", ctrl->opts->host_iface);
 145	len += scnprintf(buf + len, size - len, "\n");
 146
 147	return len;
 148}
 149EXPORT_SYMBOL_GPL(nvmf_get_address);
 150
 151/**
 152 * nvmf_reg_read32() -  NVMe Fabrics "Property Get" API function.
 153 * @ctrl:	Host NVMe controller instance maintaining the admin
 154 *		queue used to submit the property read command to
 155 *		the allocated NVMe controller resource on the target system.
 156 * @off:	Starting offset value of the targeted property
 157 *		register (see the fabrics section of the NVMe standard).
 158 * @val:	OUTPUT parameter that will contain the value of
 159 *		the property after a successful read.
 160 *
 161 * Used by the host system to retrieve a 32-bit capsule property value
 162 * from an NVMe controller on the target system.
 163 *
 164 * ("Capsule property" is an "PCIe register concept" applied to the
 165 * NVMe fabrics space.)
 166 *
 167 * Return:
 168 *	0: successful read
 169 *	> 0: NVMe error status code
 170 *	< 0: Linux errno error code
 171 */
 172int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 173{
 174	struct nvme_command cmd = { };
 175	union nvme_result res;
 176	int ret;
 177
 178	cmd.prop_get.opcode = nvme_fabrics_command;
 179	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
 180	cmd.prop_get.offset = cpu_to_le32(off);
 181
 182	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0,
 183			NVME_QID_ANY, NVME_SUBMIT_RESERVED);
 184
 185	if (ret >= 0)
 186		*val = le64_to_cpu(res.u64);
 187	if (unlikely(ret != 0))
 188		dev_err(ctrl->device,
 189			"Property Get error: %d, offset %#x\n",
 190			ret > 0 ? ret & ~NVME_STATUS_DNR : ret, off);
 191
 192	return ret;
 193}
 194EXPORT_SYMBOL_GPL(nvmf_reg_read32);
 195
 196/**
 197 * nvmf_reg_read64() -  NVMe Fabrics "Property Get" API function.
 198 * @ctrl:	Host NVMe controller instance maintaining the admin
 199 *		queue used to submit the property read command to
 200 *		the allocated controller resource on the target system.
 201 * @off:	Starting offset value of the targeted property
 202 *		register (see the fabrics section of the NVMe standard).
 203 * @val:	OUTPUT parameter that will contain the value of
 204 *		the property after a successful read.
 205 *
 206 * Used by the host system to retrieve a 64-bit capsule property value
 207 * from an NVMe controller on the target system.
 208 *
 209 * ("Capsule property" is an "PCIe register concept" applied to the
 210 * NVMe fabrics space.)
 211 *
 212 * Return:
 213 *	0: successful read
 214 *	> 0: NVMe error status code
 215 *	< 0: Linux errno error code
 216 */
 217int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
 218{
 219	struct nvme_command cmd = { };
 220	union nvme_result res;
 221	int ret;
 222
 223	cmd.prop_get.opcode = nvme_fabrics_command;
 224	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
 225	cmd.prop_get.attrib = 1;
 226	cmd.prop_get.offset = cpu_to_le32(off);
 227
 228	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res, NULL, 0,
 229			NVME_QID_ANY, NVME_SUBMIT_RESERVED);
 230
 231	if (ret >= 0)
 232		*val = le64_to_cpu(res.u64);
 233	if (unlikely(ret != 0))
 234		dev_err(ctrl->device,
 235			"Property Get error: %d, offset %#x\n",
 236			ret > 0 ? ret & ~NVME_STATUS_DNR : ret, off);
 237	return ret;
 238}
 239EXPORT_SYMBOL_GPL(nvmf_reg_read64);
 240
 241/**
 242 * nvmf_reg_write32() -  NVMe Fabrics "Property Write" API function.
 243 * @ctrl:	Host NVMe controller instance maintaining the admin
 244 *		queue used to submit the property read command to
 245 *		the allocated NVMe controller resource on the target system.
 246 * @off:	Starting offset value of the targeted property
 247 *		register (see the fabrics section of the NVMe standard).
 248 * @val:	Input parameter that contains the value to be
 249 *		written to the property.
 250 *
 251 * Used by the NVMe host system to write a 32-bit capsule property value
 252 * to an NVMe controller on the target system.
 253 *
 254 * ("Capsule property" is an "PCIe register concept" applied to the
 255 * NVMe fabrics space.)
 256 *
 257 * Return:
 258 *	0: successful write
 259 *	> 0: NVMe error status code
 260 *	< 0: Linux errno error code
 261 */
 262int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
 263{
 264	struct nvme_command cmd = { };
 265	int ret;
 266
 267	cmd.prop_set.opcode = nvme_fabrics_command;
 268	cmd.prop_set.fctype = nvme_fabrics_type_property_set;
 269	cmd.prop_set.attrib = 0;
 270	cmd.prop_set.offset = cpu_to_le32(off);
 271	cmd.prop_set.value = cpu_to_le64(val);
 272
 273	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, NULL, NULL, 0,
 274			NVME_QID_ANY, NVME_SUBMIT_RESERVED);
 275	if (unlikely(ret))
 276		dev_err(ctrl->device,
 277			"Property Set error: %d, offset %#x\n",
 278			ret > 0 ? ret & ~NVME_STATUS_DNR : ret, off);
 279	return ret;
 280}
 281EXPORT_SYMBOL_GPL(nvmf_reg_write32);
 282
 283int nvmf_subsystem_reset(struct nvme_ctrl *ctrl)
 284{
 285	int ret;
 286
 287	if (!nvme_wait_reset(ctrl))
 288		return -EBUSY;
 289
 290	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, NVME_SUBSYS_RESET);
 291	if (ret)
 292		return ret;
 293
 294	return nvme_try_sched_reset(ctrl);
 295}
 296EXPORT_SYMBOL_GPL(nvmf_subsystem_reset);
 297
 298/**
 299 * nvmf_log_connect_error() - Error-parsing-diagnostic print out function for
 300 * 				connect() errors.
 301 * @ctrl:	The specific /dev/nvmeX device that had the error.
 302 * @errval:	Error code to be decoded in a more human-friendly
 303 * 		printout.
 304 * @offset:	For use with the NVMe error code
 305 * 		NVME_SC_CONNECT_INVALID_PARAM.
 306 * @cmd:	This is the SQE portion of a submission capsule.
 307 * @data:	This is the "Data" portion of a submission capsule.
 308 */
 309static void nvmf_log_connect_error(struct nvme_ctrl *ctrl,
 310		int errval, int offset, struct nvme_command *cmd,
 311		struct nvmf_connect_data *data)
 312{
 313	int err_sctype = errval & ~NVME_STATUS_DNR;
 314
 315	if (errval < 0) {
 316		dev_err(ctrl->device,
 317			"Connect command failed, errno: %d\n", errval);
 318		return;
 319	}
 320
 321	switch (err_sctype) {
 322	case NVME_SC_CONNECT_INVALID_PARAM:
 323		if (offset >> 16) {
 324			char *inv_data = "Connect Invalid Data Parameter";
 325
 326			switch (offset & 0xffff) {
 327			case (offsetof(struct nvmf_connect_data, cntlid)):
 328				dev_err(ctrl->device,
 329					"%s, cntlid: %d\n",
 330					inv_data, data->cntlid);
 331				break;
 332			case (offsetof(struct nvmf_connect_data, hostnqn)):
 333				dev_err(ctrl->device,
 334					"%s, hostnqn \"%s\"\n",
 335					inv_data, data->hostnqn);
 336				break;
 337			case (offsetof(struct nvmf_connect_data, subsysnqn)):
 338				dev_err(ctrl->device,
 339					"%s, subsysnqn \"%s\"\n",
 340					inv_data, data->subsysnqn);
 341				break;
 342			default:
 343				dev_err(ctrl->device,
 344					"%s, starting byte offset: %d\n",
 345				       inv_data, offset & 0xffff);
 346				break;
 347			}
 348		} else {
 349			char *inv_sqe = "Connect Invalid SQE Parameter";
 350
 351			switch (offset) {
 352			case (offsetof(struct nvmf_connect_command, qid)):
 353				dev_err(ctrl->device,
 354				       "%s, qid %d\n",
 355					inv_sqe, cmd->connect.qid);
 356				break;
 357			default:
 358				dev_err(ctrl->device,
 359					"%s, starting byte offset: %d\n",
 360					inv_sqe, offset);
 361			}
 362		}
 363		break;
 364	case NVME_SC_CONNECT_INVALID_HOST:
 365		dev_err(ctrl->device,
 366			"Connect for subsystem %s is not allowed, hostnqn: %s\n",
 367			data->subsysnqn, data->hostnqn);
 368		break;
 369	case NVME_SC_CONNECT_CTRL_BUSY:
 370		dev_err(ctrl->device,
 371			"Connect command failed: controller is busy or not available\n");
 372		break;
 373	case NVME_SC_CONNECT_FORMAT:
 374		dev_err(ctrl->device,
 375			"Connect incompatible format: %d",
 376			cmd->connect.recfmt);
 377		break;
 378	case NVME_SC_HOST_PATH_ERROR:
 379		dev_err(ctrl->device,
 380			"Connect command failed: host path error\n");
 381		break;
 382	case NVME_SC_AUTH_REQUIRED:
 383		dev_err(ctrl->device,
 384			"Connect command failed: authentication required\n");
 385		break;
 386	default:
 387		dev_err(ctrl->device,
 388			"Connect command failed, error wo/DNR bit: %d\n",
 389			err_sctype);
 390		break;
 391	}
 392}
 393
 394static struct nvmf_connect_data *nvmf_connect_data_prep(struct nvme_ctrl *ctrl,
 395		u16 cntlid)
 396{
 397	struct nvmf_connect_data *data;
 398
 399	data = kzalloc(sizeof(*data), GFP_KERNEL);
 400	if (!data)
 401		return NULL;
 402
 403	uuid_copy(&data->hostid, &ctrl->opts->host->id);
 404	data->cntlid = cpu_to_le16(cntlid);
 405	strscpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
 406	strscpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
 407
 408	return data;
 409}
 410
 411static void nvmf_connect_cmd_prep(struct nvme_ctrl *ctrl, u16 qid,
 412		struct nvme_command *cmd)
 413{
 414	cmd->connect.opcode = nvme_fabrics_command;
 415	cmd->connect.fctype = nvme_fabrics_type_connect;
 416	cmd->connect.qid = cpu_to_le16(qid);
 417
 418	if (qid) {
 419		cmd->connect.sqsize = cpu_to_le16(ctrl->sqsize);
 420	} else {
 421		cmd->connect.sqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
 422
 423		/*
 424		 * set keep-alive timeout in seconds granularity (ms * 1000)
 425		 */
 426		cmd->connect.kato = cpu_to_le32(ctrl->kato * 1000);
 427	}
 428
 429	if (ctrl->opts->disable_sqflow)
 430		cmd->connect.cattr |= NVME_CONNECT_DISABLE_SQFLOW;
 431}
 432
 433/**
 434 * nvmf_connect_admin_queue() - NVMe Fabrics Admin Queue "Connect"
 435 *				API function.
 436 * @ctrl:	Host nvme controller instance used to request
 437 *              a new NVMe controller allocation on the target
 438 *              system and  establish an NVMe Admin connection to
 439 *              that controller.
 440 *
 441 * This function enables an NVMe host device to request a new allocation of
 442 * an NVMe controller resource on a target system as well establish a
 443 * fabrics-protocol connection of the NVMe Admin queue between the
 444 * host system device and the allocated NVMe controller on the
 445 * target system via a NVMe Fabrics "Connect" command.
 446 */
 447int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 448{
 449	struct nvme_command cmd = { };
 450	union nvme_result res;
 451	struct nvmf_connect_data *data;
 452	int ret;
 453	u32 result;
 454
 455	nvmf_connect_cmd_prep(ctrl, 0, &cmd);
 456
 457	data = nvmf_connect_data_prep(ctrl, 0xffff);
 458	if (!data)
 459		return -ENOMEM;
 460
 461	ret = __nvme_submit_sync_cmd(ctrl->fabrics_q, &cmd, &res,
 462			data, sizeof(*data), NVME_QID_ANY,
 463			NVME_SUBMIT_AT_HEAD |
 464			NVME_SUBMIT_NOWAIT |
 465			NVME_SUBMIT_RESERVED);
 466	if (ret) {
 467		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 468				       &cmd, data);
 469		goto out_free_data;
 470	}
 471
 472	result = le32_to_cpu(res.u32);
 473	ctrl->cntlid = result & 0xFFFF;
 474	if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) {
 475		/* Check for secure concatenation */
 476		if ((result & NVME_CONNECT_AUTHREQ_ASCR) &&
 477		    !ctrl->opts->concat) {
 478			dev_warn(ctrl->device,
 479				 "qid 0: secure concatenation is not supported\n");
 480			ret = -EOPNOTSUPP;
 481			goto out_free_data;
 482		}
 483		/* Authentication required */
 484		ret = nvme_auth_negotiate(ctrl, 0);
 485		if (ret) {
 486			dev_warn(ctrl->device,
 487				 "qid 0: authentication setup failed\n");
 488			goto out_free_data;
 489		}
 490		ret = nvme_auth_wait(ctrl, 0);
 491		if (ret) {
 492			dev_warn(ctrl->device,
 493				 "qid 0: authentication failed, error %d\n",
 494				 ret);
 495		} else
 496			dev_info(ctrl->device,
 497				 "qid 0: authenticated\n");
 498	}
 499out_free_data:
 500	kfree(data);
 501	return ret;
 502}
 503EXPORT_SYMBOL_GPL(nvmf_connect_admin_queue);
 504
 505/**
 506 * nvmf_connect_io_queue() - NVMe Fabrics I/O Queue "Connect"
 507 *			     API function.
 508 * @ctrl:	Host nvme controller instance used to establish an
 509 *		NVMe I/O queue connection to the already allocated NVMe
 510 *		controller on the target system.
 511 * @qid:	NVMe I/O queue number for the new I/O connection between
 512 *		host and target (note qid == 0 is illegal as this is
 513 *		the Admin queue, per NVMe standard).
 514 *
 515 * This function issues a fabrics-protocol connection
 516 * of a NVMe I/O queue (via NVMe Fabrics "Connect" command)
 517 * between the host system device and the allocated NVMe controller
 518 * on the target system.
 519 *
 520 * Return:
 521 *	0: success
 522 *	> 0: NVMe error status code
 523 *	< 0: Linux errno error code
 524 */
 525int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 526{
 527	struct nvme_command cmd = { };
 528	struct nvmf_connect_data *data;
 529	union nvme_result res;
 530	int ret;
 531	u32 result;
 532
 533	nvmf_connect_cmd_prep(ctrl, qid, &cmd);
 534
 535	data = nvmf_connect_data_prep(ctrl, ctrl->cntlid);
 536	if (!data)
 537		return -ENOMEM;
 538
 539	ret = __nvme_submit_sync_cmd(ctrl->connect_q, &cmd, &res,
 540			data, sizeof(*data), qid,
 541			NVME_SUBMIT_AT_HEAD |
 542			NVME_SUBMIT_RESERVED |
 543			NVME_SUBMIT_NOWAIT);
 544	if (ret) {
 545		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 546				       &cmd, data);
 547		goto out_free_data;
 548	}
 549	result = le32_to_cpu(res.u32);
 550	if (result & (NVME_CONNECT_AUTHREQ_ATR | NVME_CONNECT_AUTHREQ_ASCR)) {
 551		/* Secure concatenation is not implemented */
 552		if (result & NVME_CONNECT_AUTHREQ_ASCR) {
 553			dev_warn(ctrl->device,
 554				 "qid %d: secure concatenation is not supported\n", qid);
 555			ret = -EOPNOTSUPP;
 556			goto out_free_data;
 557		}
 558		/* Authentication required */
 559		ret = nvme_auth_negotiate(ctrl, qid);
 560		if (ret) {
 561			dev_warn(ctrl->device,
 562				 "qid %d: authentication setup failed\n", qid);
 563			goto out_free_data;
 564		}
 565		ret = nvme_auth_wait(ctrl, qid);
 566		if (ret) {
 567			dev_warn(ctrl->device,
 568				 "qid %u: authentication failed, error %d\n",
 569				 qid, ret);
 570		}
 571	}
 572out_free_data:
 573	kfree(data);
 574	return ret;
 575}
 576EXPORT_SYMBOL_GPL(nvmf_connect_io_queue);
 577
 578/*
 579 * Evaluate the status information returned by the transport in order to decided
 580 * if a reconnect attempt should be scheduled.
 581 *
 582 * Do not retry when:
 583 *
 584 * - the DNR bit is set and the specification states no further connect
 585 *   attempts with the same set of parameters should be attempted.
 586 *
 587 * - when the authentication attempt fails, because the key was invalid.
 588 *   This error code is set on the host side.
 589 */
 590bool nvmf_should_reconnect(struct nvme_ctrl *ctrl, int status)
 591{
 592	if (status > 0 && (status & NVME_STATUS_DNR))
 593		return false;
 594
 595	if (status == -EKEYREJECTED || status == -ENOKEY)
 596		return false;
 597
 598	if (ctrl->opts->max_reconnects == -1 ||
 599	    ctrl->nr_reconnects < ctrl->opts->max_reconnects)
 600		return true;
 601
 602	return false;
 603}
 604EXPORT_SYMBOL_GPL(nvmf_should_reconnect);
 605
 606/**
 607 * nvmf_register_transport() - NVMe Fabrics Library registration function.
 608 * @ops:	Transport ops instance to be registered to the
 609 *		common fabrics library.
 610 *
 611 * API function that registers the type of specific transport fabric
 612 * being implemented to the common NVMe fabrics library. Part of
 613 * the overall init sequence of starting up a fabrics driver.
 614 */
 615int nvmf_register_transport(struct nvmf_transport_ops *ops)
 616{
 617	if (!ops->create_ctrl)
 618		return -EINVAL;
 619
 620	down_write(&nvmf_transports_rwsem);
 621	list_add_tail(&ops->entry, &nvmf_transports);
 622	up_write(&nvmf_transports_rwsem);
 623
 624	return 0;
 625}
 626EXPORT_SYMBOL_GPL(nvmf_register_transport);
 627
 628/**
 629 * nvmf_unregister_transport() - NVMe Fabrics Library unregistration function.
 630 * @ops:	Transport ops instance to be unregistered from the
 631 *		common fabrics library.
 632 *
 633 * Fabrics API function that unregisters the type of specific transport
 634 * fabric being implemented from the common NVMe fabrics library.
 635 * Part of the overall exit sequence of unloading the implemented driver.
 636 */
 637void nvmf_unregister_transport(struct nvmf_transport_ops *ops)
 638{
 639	down_write(&nvmf_transports_rwsem);
 640	list_del(&ops->entry);
 641	up_write(&nvmf_transports_rwsem);
 642}
 643EXPORT_SYMBOL_GPL(nvmf_unregister_transport);
 644
 645static struct nvmf_transport_ops *nvmf_lookup_transport(
 646		struct nvmf_ctrl_options *opts)
 647{
 648	struct nvmf_transport_ops *ops;
 649
 650	lockdep_assert_held(&nvmf_transports_rwsem);
 651
 652	list_for_each_entry(ops, &nvmf_transports, entry) {
 653		if (strcmp(ops->name, opts->transport) == 0)
 654			return ops;
 655	}
 656
 657	return NULL;
 658}
 659
 660static struct key *nvmf_parse_key(int key_id)
 661{
 662	struct key *key;
 663
 664	if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) {
 665		pr_err("TLS is not supported\n");
 666		return ERR_PTR(-EINVAL);
 667	}
 668
 669	key = nvme_tls_key_lookup(key_id);
 670	if (IS_ERR(key))
 671		pr_err("key id %08x not found\n", key_id);
 672	else
 673		pr_debug("Using key id %08x\n", key_id);
 674	return key;
 675}
 676
 677static const match_table_t opt_tokens = {
 678	{ NVMF_OPT_TRANSPORT,		"transport=%s"		},
 679	{ NVMF_OPT_TRADDR,		"traddr=%s"		},
 680	{ NVMF_OPT_TRSVCID,		"trsvcid=%s"		},
 681	{ NVMF_OPT_NQN,			"nqn=%s"		},
 682	{ NVMF_OPT_QUEUE_SIZE,		"queue_size=%d"		},
 683	{ NVMF_OPT_NR_IO_QUEUES,	"nr_io_queues=%d"	},
 684	{ NVMF_OPT_RECONNECT_DELAY,	"reconnect_delay=%d"	},
 685	{ NVMF_OPT_CTRL_LOSS_TMO,	"ctrl_loss_tmo=%d"	},
 686	{ NVMF_OPT_KATO,		"keep_alive_tmo=%d"	},
 687	{ NVMF_OPT_HOSTNQN,		"hostnqn=%s"		},
 688	{ NVMF_OPT_HOST_TRADDR,		"host_traddr=%s"	},
 689	{ NVMF_OPT_HOST_IFACE,		"host_iface=%s"		},
 690	{ NVMF_OPT_HOST_ID,		"hostid=%s"		},
 691	{ NVMF_OPT_DUP_CONNECT,		"duplicate_connect"	},
 692	{ NVMF_OPT_DISABLE_SQFLOW,	"disable_sqflow"	},
 693	{ NVMF_OPT_HDR_DIGEST,		"hdr_digest"		},
 694	{ NVMF_OPT_DATA_DIGEST,		"data_digest"		},
 695	{ NVMF_OPT_NR_WRITE_QUEUES,	"nr_write_queues=%d"	},
 696	{ NVMF_OPT_NR_POLL_QUEUES,	"nr_poll_queues=%d"	},
 697	{ NVMF_OPT_TOS,			"tos=%d"		},
 698#ifdef CONFIG_NVME_TCP_TLS
 699	{ NVMF_OPT_KEYRING,		"keyring=%d"		},
 700	{ NVMF_OPT_TLS_KEY,		"tls_key=%d"		},
 701#endif
 702	{ NVMF_OPT_FAIL_FAST_TMO,	"fast_io_fail_tmo=%d"	},
 703	{ NVMF_OPT_DISCOVERY,		"discovery"		},
 704#ifdef CONFIG_NVME_HOST_AUTH
 705	{ NVMF_OPT_DHCHAP_SECRET,	"dhchap_secret=%s"	},
 706	{ NVMF_OPT_DHCHAP_CTRL_SECRET,	"dhchap_ctrl_secret=%s"	},
 707#endif
 708#ifdef CONFIG_NVME_TCP_TLS
 709	{ NVMF_OPT_TLS,			"tls"			},
 710	{ NVMF_OPT_CONCAT,		"concat"		},
 711#endif
 712	{ NVMF_OPT_ERR,			NULL			}
 713};
 714
 715static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 716		const char *buf)
 717{
 718	substring_t args[MAX_OPT_ARGS];
 719	char *options, *o, *p;
 720	int token, ret = 0;
 721	size_t nqnlen  = 0;
 722	int ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO, key_id;
 723	uuid_t hostid;
 724	char hostnqn[NVMF_NQN_SIZE];
 725	struct key *key;
 726
 727	/* Set defaults */
 728	opts->queue_size = NVMF_DEF_QUEUE_SIZE;
 729	opts->nr_io_queues = num_online_cpus();
 730	opts->reconnect_delay = NVMF_DEF_RECONNECT_DELAY;
 731	opts->kato = 0;
 732	opts->duplicate_connect = false;
 733	opts->fast_io_fail_tmo = NVMF_DEF_FAIL_FAST_TMO;
 734	opts->hdr_digest = false;
 735	opts->data_digest = false;
 736	opts->tos = -1; /* < 0 == use transport default */
 737	opts->tls = false;
 738	opts->tls_key = NULL;
 739	opts->keyring = NULL;
 740	opts->concat = false;
 741
 742	options = o = kstrdup(buf, GFP_KERNEL);
 743	if (!options)
 744		return -ENOMEM;
 745
 746	/* use default host if not given by user space */
 747	uuid_copy(&hostid, &nvmf_default_host->id);
 748	strscpy(hostnqn, nvmf_default_host->nqn, NVMF_NQN_SIZE);
 749
 750	while ((p = strsep(&o, ",\n")) != NULL) {
 751		if (!*p)
 752			continue;
 753
 754		token = match_token(p, opt_tokens, args);
 755		opts->mask |= token;
 756		switch (token) {
 757		case NVMF_OPT_TRANSPORT:
 758			p = match_strdup(args);
 759			if (!p) {
 760				ret = -ENOMEM;
 761				goto out;
 762			}
 763			kfree(opts->transport);
 764			opts->transport = p;
 765			break;
 766		case NVMF_OPT_NQN:
 767			p = match_strdup(args);
 768			if (!p) {
 769				ret = -ENOMEM;
 770				goto out;
 771			}
 772			kfree(opts->subsysnqn);
 773			opts->subsysnqn = p;
 774			nqnlen = strlen(opts->subsysnqn);
 775			if (nqnlen >= NVMF_NQN_SIZE) {
 776				pr_err("%s needs to be < %d bytes\n",
 777					opts->subsysnqn, NVMF_NQN_SIZE);
 778				ret = -EINVAL;
 779				goto out;
 780			}
 781			opts->discovery_nqn =
 782				!(strcmp(opts->subsysnqn,
 783					 NVME_DISC_SUBSYS_NAME));
 784			break;
 785		case NVMF_OPT_TRADDR:
 786			p = match_strdup(args);
 787			if (!p) {
 788				ret = -ENOMEM;
 789				goto out;
 790			}
 791			kfree(opts->traddr);
 792			opts->traddr = p;
 793			break;
 794		case NVMF_OPT_TRSVCID:
 795			p = match_strdup(args);
 796			if (!p) {
 797				ret = -ENOMEM;
 798				goto out;
 799			}
 800			kfree(opts->trsvcid);
 801			opts->trsvcid = p;
 802			break;
 803		case NVMF_OPT_QUEUE_SIZE:
 804			if (match_int(args, &token)) {
 805				ret = -EINVAL;
 806				goto out;
 807			}
 808			if (token < NVMF_MIN_QUEUE_SIZE ||
 809			    token > NVMF_MAX_QUEUE_SIZE) {
 810				pr_err("Invalid queue_size %d\n", token);
 811				ret = -EINVAL;
 812				goto out;
 813			}
 814			opts->queue_size = token;
 815			break;
 816		case NVMF_OPT_NR_IO_QUEUES:
 817			if (match_int(args, &token)) {
 818				ret = -EINVAL;
 819				goto out;
 820			}
 821			if (token <= 0) {
 822				pr_err("Invalid number of IOQs %d\n", token);
 823				ret = -EINVAL;
 824				goto out;
 825			}
 826			if (opts->discovery_nqn) {
 827				pr_debug("Ignoring nr_io_queues value for discovery controller\n");
 828				break;
 829			}
 830
 831			opts->nr_io_queues = min_t(unsigned int,
 832					num_online_cpus(), token);
 833			break;
 834		case NVMF_OPT_KATO:
 835			if (match_int(args, &token)) {
 836				ret = -EINVAL;
 837				goto out;
 838			}
 839
 840			if (token < 0) {
 841				pr_err("Invalid keep_alive_tmo %d\n", token);
 842				ret = -EINVAL;
 843				goto out;
 844			} else if (token == 0 && !opts->discovery_nqn) {
 845				/* Allowed for debug */
 846				pr_warn("keep_alive_tmo 0 won't execute keep alives!!!\n");
 847			}
 848			opts->kato = token;
 849			break;
 850		case NVMF_OPT_CTRL_LOSS_TMO:
 851			if (match_int(args, &token)) {
 852				ret = -EINVAL;
 853				goto out;
 854			}
 855
 856			if (token < 0)
 857				pr_warn("ctrl_loss_tmo < 0 will reconnect forever\n");
 858			ctrl_loss_tmo = token;
 859			break;
 860		case NVMF_OPT_FAIL_FAST_TMO:
 861			if (match_int(args, &token)) {
 862				ret = -EINVAL;
 863				goto out;
 864			}
 865
 866			if (token >= 0)
 867				pr_warn("I/O fail on reconnect controller after %d sec\n",
 868					token);
 869			else
 870				token = -1;
 871
 872			opts->fast_io_fail_tmo = token;
 873			break;
 874		case NVMF_OPT_HOSTNQN:
 875			if (opts->host) {
 876				pr_err("hostnqn already user-assigned: %s\n",
 877				       opts->host->nqn);
 878				ret = -EADDRINUSE;
 879				goto out;
 880			}
 881			p = match_strdup(args);
 882			if (!p) {
 883				ret = -ENOMEM;
 884				goto out;
 885			}
 886			nqnlen = strlen(p);
 887			if (nqnlen >= NVMF_NQN_SIZE) {
 888				pr_err("%s needs to be < %d bytes\n",
 889					p, NVMF_NQN_SIZE);
 890				kfree(p);
 891				ret = -EINVAL;
 892				goto out;
 893			}
 894			strscpy(hostnqn, p, NVMF_NQN_SIZE);
 895			kfree(p);
 896			break;
 897		case NVMF_OPT_RECONNECT_DELAY:
 898			if (match_int(args, &token)) {
 899				ret = -EINVAL;
 900				goto out;
 901			}
 902			if (token <= 0) {
 903				pr_err("Invalid reconnect_delay %d\n", token);
 904				ret = -EINVAL;
 905				goto out;
 906			}
 907			opts->reconnect_delay = token;
 908			break;
 909		case NVMF_OPT_HOST_TRADDR:
 910			p = match_strdup(args);
 911			if (!p) {
 912				ret = -ENOMEM;
 913				goto out;
 914			}
 915			kfree(opts->host_traddr);
 916			opts->host_traddr = p;
 917			break;
 918		case NVMF_OPT_HOST_IFACE:
 919			p = match_strdup(args);
 920			if (!p) {
 921				ret = -ENOMEM;
 922				goto out;
 923			}
 924			kfree(opts->host_iface);
 925			opts->host_iface = p;
 926			break;
 927		case NVMF_OPT_HOST_ID:
 928			p = match_strdup(args);
 929			if (!p) {
 930				ret = -ENOMEM;
 931				goto out;
 932			}
 933			ret = uuid_parse(p, &hostid);
 934			if (ret) {
 935				pr_err("Invalid hostid %s\n", p);
 936				ret = -EINVAL;
 937				kfree(p);
 938				goto out;
 939			}
 940			kfree(p);
 941			break;
 942		case NVMF_OPT_DUP_CONNECT:
 943			opts->duplicate_connect = true;
 944			break;
 945		case NVMF_OPT_DISABLE_SQFLOW:
 946			opts->disable_sqflow = true;
 947			break;
 948		case NVMF_OPT_HDR_DIGEST:
 949			opts->hdr_digest = true;
 950			break;
 951		case NVMF_OPT_DATA_DIGEST:
 952			opts->data_digest = true;
 953			break;
 954		case NVMF_OPT_NR_WRITE_QUEUES:
 955			if (match_int(args, &token)) {
 956				ret = -EINVAL;
 957				goto out;
 958			}
 959			if (token <= 0) {
 960				pr_err("Invalid nr_write_queues %d\n", token);
 961				ret = -EINVAL;
 962				goto out;
 963			}
 964			opts->nr_write_queues = token;
 965			break;
 966		case NVMF_OPT_NR_POLL_QUEUES:
 967			if (match_int(args, &token)) {
 968				ret = -EINVAL;
 969				goto out;
 970			}
 971			if (token <= 0) {
 972				pr_err("Invalid nr_poll_queues %d\n", token);
 973				ret = -EINVAL;
 974				goto out;
 975			}
 976			opts->nr_poll_queues = token;
 977			break;
 978		case NVMF_OPT_TOS:
 979			if (match_int(args, &token)) {
 980				ret = -EINVAL;
 981				goto out;
 982			}
 983			if (token < 0) {
 984				pr_err("Invalid type of service %d\n", token);
 985				ret = -EINVAL;
 986				goto out;
 987			}
 988			if (token > 255) {
 989				pr_warn("Clamping type of service to 255\n");
 990				token = 255;
 991			}
 992			opts->tos = token;
 993			break;
 994		case NVMF_OPT_KEYRING:
 995			if (match_int(args, &key_id) || key_id <= 0) {
 996				ret = -EINVAL;
 997				goto out;
 998			}
 999			key = nvmf_parse_key(key_id);
1000			if (IS_ERR(key)) {
1001				ret = PTR_ERR(key);
1002				goto out;
1003			}
1004			key_put(opts->keyring);
1005			opts->keyring = key;
1006			break;
1007		case NVMF_OPT_TLS_KEY:
1008			if (match_int(args, &key_id) || key_id <= 0) {
1009				ret = -EINVAL;
1010				goto out;
1011			}
1012			key = nvmf_parse_key(key_id);
1013			if (IS_ERR(key)) {
1014				ret = PTR_ERR(key);
1015				goto out;
1016			}
1017			key_put(opts->tls_key);
1018			opts->tls_key = key;
1019			break;
1020		case NVMF_OPT_DISCOVERY:
1021			opts->discovery_nqn = true;
1022			break;
1023		case NVMF_OPT_DHCHAP_SECRET:
1024			p = match_strdup(args);
1025			if (!p) {
1026				ret = -ENOMEM;
1027				goto out;
1028			}
1029			if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) {
1030				pr_err("Invalid DH-CHAP secret %s\n", p);
1031				ret = -EINVAL;
1032				goto out;
1033			}
1034			kfree(opts->dhchap_secret);
1035			opts->dhchap_secret = p;
1036			break;
1037		case NVMF_OPT_DHCHAP_CTRL_SECRET:
1038			p = match_strdup(args);
1039			if (!p) {
1040				ret = -ENOMEM;
1041				goto out;
1042			}
1043			if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) {
1044				pr_err("Invalid DH-CHAP secret %s\n", p);
1045				ret = -EINVAL;
1046				goto out;
1047			}
1048			kfree(opts->dhchap_ctrl_secret);
1049			opts->dhchap_ctrl_secret = p;
1050			break;
1051		case NVMF_OPT_TLS:
1052			if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) {
1053				pr_err("TLS is not supported\n");
1054				ret = -EINVAL;
1055				goto out;
1056			}
1057			opts->tls = true;
1058			break;
1059		case NVMF_OPT_CONCAT:
1060			if (!IS_ENABLED(CONFIG_NVME_TCP_TLS)) {
1061				pr_err("TLS is not supported\n");
1062				ret = -EINVAL;
1063				goto out;
1064			}
1065			opts->concat = true;
1066			break;
1067		default:
1068			pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n",
1069				p);
1070			ret = -EINVAL;
1071			goto out;
1072		}
1073	}
1074
1075	if (opts->discovery_nqn) {
1076		opts->nr_io_queues = 0;
1077		opts->nr_write_queues = 0;
1078		opts->nr_poll_queues = 0;
1079		opts->duplicate_connect = true;
1080	} else {
1081		if (!opts->kato)
1082			opts->kato = NVME_DEFAULT_KATO;
1083	}
1084	if (ctrl_loss_tmo < 0) {
1085		opts->max_reconnects = -1;
1086	} else {
1087		opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
1088						opts->reconnect_delay);
1089		if (ctrl_loss_tmo < opts->fast_io_fail_tmo)
1090			pr_warn("failfast tmo (%d) larger than controller loss tmo (%d)\n",
1091				opts->fast_io_fail_tmo, ctrl_loss_tmo);
1092	}
1093	if (opts->concat) {
1094		if (opts->tls) {
1095			pr_err("Secure concatenation over TLS is not supported\n");
1096			ret = -EINVAL;
1097			goto out;
1098		}
1099		if (opts->tls_key) {
1100			pr_err("Cannot specify a TLS key for secure concatenation\n");
1101			ret = -EINVAL;
1102			goto out;
1103		}
1104		if (!opts->dhchap_secret) {
1105			pr_err("Need to enable DH-CHAP for secure concatenation\n");
1106			ret = -EINVAL;
1107			goto out;
1108		}
1109	}
1110
1111	opts->host = nvmf_host_add(hostnqn, &hostid);
1112	if (IS_ERR(opts->host)) {
1113		ret = PTR_ERR(opts->host);
1114		opts->host = NULL;
1115		goto out;
1116	}
1117
1118out:
1119	kfree(options);
1120	return ret;
1121}
1122
1123void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues,
1124			u32 io_queues[HCTX_MAX_TYPES])
1125{
1126	if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1127		/*
1128		 * separate read/write queues
1129		 * hand out dedicated default queues only after we have
1130		 * sufficient read queues.
1131		 */
1132		io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1133		nr_io_queues -= io_queues[HCTX_TYPE_READ];
1134		io_queues[HCTX_TYPE_DEFAULT] =
1135			min(opts->nr_write_queues, nr_io_queues);
1136		nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
1137	} else {
1138		/*
1139		 * shared read/write queues
1140		 * either no write queues were requested, or we don't have
1141		 * sufficient queue count to have dedicated default queues.
1142		 */
1143		io_queues[HCTX_TYPE_DEFAULT] =
1144			min(opts->nr_io_queues, nr_io_queues);
1145		nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
1146	}
1147
1148	if (opts->nr_poll_queues && nr_io_queues) {
1149		/* map dedicated poll queues only if we have queues left */
1150		io_queues[HCTX_TYPE_POLL] =
1151			min(opts->nr_poll_queues, nr_io_queues);
1152	}
1153}
1154EXPORT_SYMBOL_GPL(nvmf_set_io_queues);
1155
1156void nvmf_map_queues(struct blk_mq_tag_set *set, struct nvme_ctrl *ctrl,
1157		     u32 io_queues[HCTX_MAX_TYPES])
1158{
1159	struct nvmf_ctrl_options *opts = ctrl->opts;
1160
1161	if (opts->nr_write_queues && io_queues[HCTX_TYPE_READ]) {
1162		/* separate read/write queues */
1163		set->map[HCTX_TYPE_DEFAULT].nr_queues =
1164			io_queues[HCTX_TYPE_DEFAULT];
1165		set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1166		set->map[HCTX_TYPE_READ].nr_queues =
1167			io_queues[HCTX_TYPE_READ];
1168		set->map[HCTX_TYPE_READ].queue_offset =
1169			io_queues[HCTX_TYPE_DEFAULT];
1170	} else {
1171		/* shared read/write queues */
1172		set->map[HCTX_TYPE_DEFAULT].nr_queues =
1173			io_queues[HCTX_TYPE_DEFAULT];
1174		set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
1175		set->map[HCTX_TYPE_READ].nr_queues =
1176			io_queues[HCTX_TYPE_DEFAULT];
1177		set->map[HCTX_TYPE_READ].queue_offset = 0;
1178	}
1179
1180	blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
1181	blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
1182	if (opts->nr_poll_queues && io_queues[HCTX_TYPE_POLL]) {
1183		/* map dedicated poll queues only if we have queues left */
1184		set->map[HCTX_TYPE_POLL].nr_queues = io_queues[HCTX_TYPE_POLL];
1185		set->map[HCTX_TYPE_POLL].queue_offset =
1186			io_queues[HCTX_TYPE_DEFAULT] +
1187			io_queues[HCTX_TYPE_READ];
1188		blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
1189	}
1190
1191	dev_info(ctrl->device,
1192		"mapped %d/%d/%d default/read/poll queues.\n",
1193		io_queues[HCTX_TYPE_DEFAULT],
1194		io_queues[HCTX_TYPE_READ],
1195		io_queues[HCTX_TYPE_POLL]);
1196}
1197EXPORT_SYMBOL_GPL(nvmf_map_queues);
1198
1199static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
1200		unsigned int required_opts)
1201{
1202	if ((opts->mask & required_opts) != required_opts) {
1203		unsigned int i;
1204
1205		for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
1206			if ((opt_tokens[i].token & required_opts) &&
1207			    !(opt_tokens[i].token & opts->mask)) {
1208				pr_warn("missing parameter '%s'\n",
1209					opt_tokens[i].pattern);
1210			}
1211		}
1212
1213		return -EINVAL;
1214	}
1215
1216	return 0;
1217}
1218
1219bool nvmf_ip_options_match(struct nvme_ctrl *ctrl,
1220		struct nvmf_ctrl_options *opts)
1221{
1222	if (!nvmf_ctlr_matches_baseopts(ctrl, opts) ||
1223	    strcmp(opts->traddr, ctrl->opts->traddr) ||
1224	    strcmp(opts->trsvcid, ctrl->opts->trsvcid))
1225		return false;
1226
1227	/*
1228	 * Checking the local address or host interfaces is rough.
1229	 *
1230	 * In most cases, none is specified and the host port or
1231	 * host interface is selected by the stack.
1232	 *
1233	 * Assume no match if:
1234	 * -  local address or host interface is specified and address
1235	 *    or host interface is not the same
1236	 * -  local address or host interface is not specified but
1237	 *    remote is, or vice versa (admin using specific
1238	 *    host_traddr/host_iface when it matters).
1239	 */
1240	if ((opts->mask & NVMF_OPT_HOST_TRADDR) &&
1241	    (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
1242		if (strcmp(opts->host_traddr, ctrl->opts->host_traddr))
1243			return false;
1244	} else if ((opts->mask & NVMF_OPT_HOST_TRADDR) ||
1245		   (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) {
1246		return false;
1247	}
1248
1249	if ((opts->mask & NVMF_OPT_HOST_IFACE) &&
1250	    (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) {
1251		if (strcmp(opts->host_iface, ctrl->opts->host_iface))
1252			return false;
1253	} else if ((opts->mask & NVMF_OPT_HOST_IFACE) ||
1254		   (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) {
1255		return false;
1256	}
1257
1258	return true;
1259}
1260EXPORT_SYMBOL_GPL(nvmf_ip_options_match);
1261
1262static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts,
1263		unsigned int allowed_opts)
1264{
1265	if (opts->mask & ~allowed_opts) {
1266		unsigned int i;
1267
1268		for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
1269			if ((opt_tokens[i].token & opts->mask) &&
1270			    (opt_tokens[i].token & ~allowed_opts)) {
1271				pr_warn("invalid parameter '%s'\n",
1272					opt_tokens[i].pattern);
1273			}
1274		}
1275
1276		return -EINVAL;
1277	}
1278
1279	return 0;
1280}
1281
1282void nvmf_free_options(struct nvmf_ctrl_options *opts)
1283{
1284	nvmf_host_put(opts->host);
1285	key_put(opts->keyring);
1286	key_put(opts->tls_key);
1287	kfree(opts->transport);
1288	kfree(opts->traddr);
1289	kfree(opts->trsvcid);
1290	kfree(opts->subsysnqn);
1291	kfree(opts->host_traddr);
1292	kfree(opts->host_iface);
1293	kfree_sensitive(opts->dhchap_secret);
1294	kfree_sensitive(opts->dhchap_ctrl_secret);
1295	kfree(opts);
1296}
1297EXPORT_SYMBOL_GPL(nvmf_free_options);
1298
1299#define NVMF_REQUIRED_OPTS	(NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
1300#define NVMF_ALLOWED_OPTS	(NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
1301				 NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
1302				 NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\
1303				 NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_DISCOVERY |\
1304				 NVMF_OPT_FAIL_FAST_TMO | NVMF_OPT_DHCHAP_SECRET |\
1305				 NVMF_OPT_DHCHAP_CTRL_SECRET)
1306
1307static struct nvme_ctrl *
1308nvmf_create_ctrl(struct device *dev, const char *buf)
1309{
1310	struct nvmf_ctrl_options *opts;
1311	struct nvmf_transport_ops *ops;
1312	struct nvme_ctrl *ctrl;
1313	int ret;
1314
1315	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
1316	if (!opts)
1317		return ERR_PTR(-ENOMEM);
1318
1319	ret = nvmf_parse_options(opts, buf);
1320	if (ret)
1321		goto out_free_opts;
1322
1323
1324	request_module("nvme-%s", opts->transport);
1325
1326	/*
1327	 * Check the generic options first as we need a valid transport for
1328	 * the lookup below.  Then clear the generic flags so that transport
1329	 * drivers don't have to care about them.
1330	 */
1331	ret = nvmf_check_required_opts(opts, NVMF_REQUIRED_OPTS);
1332	if (ret)
1333		goto out_free_opts;
1334	opts->mask &= ~NVMF_REQUIRED_OPTS;
1335
1336	down_read(&nvmf_transports_rwsem);
1337	ops = nvmf_lookup_transport(opts);
1338	if (!ops) {
1339		pr_info("no handler found for transport %s.\n",
1340			opts->transport);
1341		ret = -EINVAL;
1342		goto out_unlock;
1343	}
1344
1345	if (!try_module_get(ops->module)) {
1346		ret = -EBUSY;
1347		goto out_unlock;
1348	}
1349	up_read(&nvmf_transports_rwsem);
1350
1351	ret = nvmf_check_required_opts(opts, ops->required_opts);
1352	if (ret)
1353		goto out_module_put;
1354	ret = nvmf_check_allowed_opts(opts, NVMF_ALLOWED_OPTS |
1355				ops->allowed_opts | ops->required_opts);
1356	if (ret)
1357		goto out_module_put;
1358
1359	ctrl = ops->create_ctrl(dev, opts);
1360	if (IS_ERR(ctrl)) {
1361		ret = PTR_ERR(ctrl);
1362		goto out_module_put;
1363	}
1364
1365	module_put(ops->module);
1366	return ctrl;
1367
1368out_module_put:
1369	module_put(ops->module);
1370	goto out_free_opts;
1371out_unlock:
1372	up_read(&nvmf_transports_rwsem);
1373out_free_opts:
1374	nvmf_free_options(opts);
1375	return ERR_PTR(ret);
1376}
1377
1378static const struct class nvmf_class = {
1379	.name = "nvme-fabrics",
1380};
1381
1382static struct device *nvmf_device;
1383static DEFINE_MUTEX(nvmf_dev_mutex);
1384
1385static ssize_t nvmf_dev_write(struct file *file, const char __user *ubuf,
1386		size_t count, loff_t *pos)
1387{
1388	struct seq_file *seq_file = file->private_data;
1389	struct nvme_ctrl *ctrl;
1390	const char *buf;
1391	int ret = 0;
1392
1393	if (count > PAGE_SIZE)
1394		return -ENOMEM;
1395
1396	buf = memdup_user_nul(ubuf, count);
1397	if (IS_ERR(buf))
1398		return PTR_ERR(buf);
1399
1400	mutex_lock(&nvmf_dev_mutex);
1401	if (seq_file->private) {
1402		ret = -EINVAL;
1403		goto out_unlock;
1404	}
1405
1406	ctrl = nvmf_create_ctrl(nvmf_device, buf);
1407	if (IS_ERR(ctrl)) {
1408		ret = PTR_ERR(ctrl);
1409		goto out_unlock;
1410	}
1411
1412	seq_file->private = ctrl;
1413
1414out_unlock:
1415	mutex_unlock(&nvmf_dev_mutex);
1416	kfree(buf);
1417	return ret ? ret : count;
1418}
1419
1420static void __nvmf_concat_opt_tokens(struct seq_file *seq_file)
1421{
1422	const struct match_token *tok;
1423	int idx;
1424
1425	/*
1426	 * Add dummy entries for instance and cntlid to
1427	 * signal an invalid/non-existing controller
1428	 */
1429	seq_puts(seq_file, "instance=-1,cntlid=-1");
1430	for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) {
1431		tok = &opt_tokens[idx];
1432		if (tok->token == NVMF_OPT_ERR)
1433			continue;
1434		seq_putc(seq_file, ',');
1435		seq_puts(seq_file, tok->pattern);
1436	}
1437	seq_putc(seq_file, '\n');
1438}
1439
1440static int nvmf_dev_show(struct seq_file *seq_file, void *private)
1441{
1442	struct nvme_ctrl *ctrl;
1443
1444	mutex_lock(&nvmf_dev_mutex);
1445	ctrl = seq_file->private;
1446	if (!ctrl) {
1447		__nvmf_concat_opt_tokens(seq_file);
1448		goto out_unlock;
1449	}
1450
1451	seq_printf(seq_file, "instance=%d,cntlid=%d\n",
1452			ctrl->instance, ctrl->cntlid);
1453
1454out_unlock:
1455	mutex_unlock(&nvmf_dev_mutex);
1456	return 0;
1457}
1458
1459static int nvmf_dev_open(struct inode *inode, struct file *file)
1460{
1461	/*
1462	 * The miscdevice code initializes file->private_data, but doesn't
1463	 * make use of it later.
1464	 */
1465	file->private_data = NULL;
1466	return single_open(file, nvmf_dev_show, NULL);
1467}
1468
1469static int nvmf_dev_release(struct inode *inode, struct file *file)
1470{
1471	struct seq_file *seq_file = file->private_data;
1472	struct nvme_ctrl *ctrl = seq_file->private;
1473
1474	if (ctrl)
1475		nvme_put_ctrl(ctrl);
1476	return single_release(inode, file);
1477}
1478
1479static const struct file_operations nvmf_dev_fops = {
1480	.owner		= THIS_MODULE,
1481	.write		= nvmf_dev_write,
1482	.read		= seq_read,
1483	.open		= nvmf_dev_open,
1484	.release	= nvmf_dev_release,
1485};
1486
1487static struct miscdevice nvmf_misc = {
1488	.minor		= MISC_DYNAMIC_MINOR,
1489	.name           = "nvme-fabrics",
1490	.fops		= &nvmf_dev_fops,
1491};
1492
1493static int __init nvmf_init(void)
1494{
1495	int ret;
1496
1497	nvmf_default_host = nvmf_host_default();
1498	if (!nvmf_default_host)
1499		return -ENOMEM;
1500
1501	ret = class_register(&nvmf_class);
1502	if (ret) {
1503		pr_err("couldn't register class nvme-fabrics\n");
1504		goto out_free_host;
1505	}
1506
1507	nvmf_device =
1508		device_create(&nvmf_class, NULL, MKDEV(0, 0), NULL, "ctl");
1509	if (IS_ERR(nvmf_device)) {
1510		pr_err("couldn't create nvme-fabrics device!\n");
1511		ret = PTR_ERR(nvmf_device);
1512		goto out_destroy_class;
1513	}
1514
1515	ret = misc_register(&nvmf_misc);
1516	if (ret) {
1517		pr_err("couldn't register misc device: %d\n", ret);
1518		goto out_destroy_device;
1519	}
1520
1521	return 0;
1522
1523out_destroy_device:
1524	device_destroy(&nvmf_class, MKDEV(0, 0));
1525out_destroy_class:
1526	class_unregister(&nvmf_class);
1527out_free_host:
1528	nvmf_host_put(nvmf_default_host);
1529	return ret;
1530}
1531
1532static void __exit nvmf_exit(void)
1533{
1534	misc_deregister(&nvmf_misc);
1535	device_destroy(&nvmf_class, MKDEV(0, 0));
1536	class_unregister(&nvmf_class);
1537	nvmf_host_put(nvmf_default_host);
1538
1539	BUILD_BUG_ON(sizeof(struct nvmf_common_command) != 64);
1540	BUILD_BUG_ON(sizeof(struct nvmf_connect_command) != 64);
1541	BUILD_BUG_ON(sizeof(struct nvmf_property_get_command) != 64);
1542	BUILD_BUG_ON(sizeof(struct nvmf_property_set_command) != 64);
1543	BUILD_BUG_ON(sizeof(struct nvmf_auth_send_command) != 64);
1544	BUILD_BUG_ON(sizeof(struct nvmf_auth_receive_command) != 64);
1545	BUILD_BUG_ON(sizeof(struct nvmf_connect_data) != 1024);
1546	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_negotiate_data) != 8);
1547	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_challenge_data) != 16);
1548	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_reply_data) != 16);
1549	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success1_data) != 16);
1550	BUILD_BUG_ON(sizeof(struct nvmf_auth_dhchap_success2_data) != 16);
1551}
1552
1553MODULE_LICENSE("GPL v2");
1554MODULE_DESCRIPTION("NVMe host fabrics library");
1555
1556module_init(nvmf_init);
1557module_exit(nvmf_exit);