passthru.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * NVMe Over Fabrics Target Passthrough command implementation.
  4 *
  5 * Copyright (c) 2017-2018 Western Digital Corporation or its
  6 * affiliates.
  7 * Copyright (c) 2019-2020, Eideticom Inc.
  8 *
  9 */
 10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 11#include <linux/module.h>
 12
 13#include "../host/nvme.h"
 14#include "nvmet.h"
 15
 16MODULE_IMPORT_NS("NVME_TARGET_PASSTHRU");
 17
 18/*
 19 * xarray to maintain one passthru subsystem per nvme controller.
 20 */
 21static DEFINE_XARRAY(passthru_subsystems);
 22
 23void nvmet_passthrough_override_cap(struct nvmet_ctrl *ctrl)
 24{
 25	/*
 26	 * Multiple command set support can only be declared if the underlying
 27	 * controller actually supports it.
 28	 */
 29	if (!nvme_multi_css(ctrl->subsys->passthru_ctrl))
 30		ctrl->cap &= ~(1ULL << 43);
 31}
 32
 33static u16 nvmet_passthru_override_id_descs(struct nvmet_req *req)
 34{
 35	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 36	u16 status = NVME_SC_SUCCESS;
 37	int pos, len;
 38	bool csi_seen = false;
 39	void *data;
 40	u8 csi;
 41
 42	if (!ctrl->subsys->clear_ids)
 43		return status;
 44
 45	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
 46	if (!data)
 47		return NVME_SC_INTERNAL;
 48
 49	status = nvmet_copy_from_sgl(req, 0, data, NVME_IDENTIFY_DATA_SIZE);
 50	if (status)
 51		goto out_free;
 52
 53	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
 54		struct nvme_ns_id_desc *cur = data + pos;
 55
 56		if (cur->nidl == 0)
 57			break;
 58		if (cur->nidt == NVME_NIDT_CSI) {
 59			memcpy(&csi, cur + 1, NVME_NIDT_CSI_LEN);
 60			csi_seen = true;
 61			break;
 62		}
 63		len = sizeof(struct nvme_ns_id_desc) + cur->nidl;
 64	}
 65
 66	memset(data, 0, NVME_IDENTIFY_DATA_SIZE);
 67	if (csi_seen) {
 68		struct nvme_ns_id_desc *cur = data;
 69
 70		cur->nidt = NVME_NIDT_CSI;
 71		cur->nidl = NVME_NIDT_CSI_LEN;
 72		memcpy(cur + 1, &csi, NVME_NIDT_CSI_LEN);
 73	}
 74	status = nvmet_copy_to_sgl(req, 0, data, NVME_IDENTIFY_DATA_SIZE);
 75out_free:
 76	kfree(data);
 77	return status;
 78}
 79
 80static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req)
 81{
 82	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 83	struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl;
 84	u16 status = NVME_SC_SUCCESS;
 85	struct nvme_id_ctrl *id;
 86	unsigned int max_hw_sectors;
 87	int page_shift;
 88
 89	id = kzalloc(sizeof(*id), GFP_KERNEL);
 90	if (!id)
 91		return NVME_SC_INTERNAL;
 92
 93	status = nvmet_copy_from_sgl(req, 0, id, sizeof(*id));
 94	if (status)
 95		goto out_free;
 96
 97	id->cntlid = cpu_to_le16(ctrl->cntlid);
 98	id->ver = cpu_to_le32(ctrl->subsys->ver);
 99
100	/*
101	 * The passthru NVMe driver may have a limit on the number of segments
102	 * which depends on the host's memory fragmentation. To solve this,
103	 * ensure mdts is limited to the pages equal to the number of segments.
104	 */
105	max_hw_sectors = min_not_zero(pctrl->max_segments << PAGE_SECTORS_SHIFT,
106				      pctrl->max_hw_sectors);
107
108	/*
109	 * nvmet_passthru_map_sg is limited to using a single bio so limit
110	 * the mdts based on BIO_MAX_VECS as well
111	 */
112	max_hw_sectors = min_not_zero(BIO_MAX_VECS << PAGE_SECTORS_SHIFT,
113				      max_hw_sectors);
114
115	page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
116
117	id->mdts = ilog2(max_hw_sectors) + 9 - page_shift;
118
119	id->acl = 3;
120	/*
121	 * We export aerl limit for the fabrics controller, update this when
122	 * passthru based aerl support is added.
123	 */
124	id->aerl = NVMET_ASYNC_EVENTS - 1;
125
126	/* emulate kas as most of the PCIe ctrl don't have a support for kas */
127	id->kas = cpu_to_le16(NVMET_KAS);
128
129	/* don't support host memory buffer */
130	id->hmpre = 0;
131	id->hmmin = 0;
132
133	id->sqes = min_t(__u8, ((0x6 << 4) | 0x6), id->sqes);
134	id->cqes = min_t(__u8, ((0x4 << 4) | 0x4), id->cqes);
135	id->maxcmd = cpu_to_le16(NVMET_MAX_CMD(ctrl));
136
137	/* don't support fuse commands */
138	id->fuses = 0;
139
140	id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */
141	if (ctrl->ops->flags & NVMF_KEYED_SGLS)
142		id->sgls |= cpu_to_le32(1 << 2);
143	if (req->port->inline_data_size)
144		id->sgls |= cpu_to_le32(1 << 20);
145
146	/*
147	 * When passthru controller is setup using nvme-loop transport it will
148	 * export the passthru ctrl subsysnqn (PCIe NVMe ctrl) and will fail in
149	 * the nvme/host/core.c in the nvme_init_subsystem()->nvme_active_ctrl()
150	 * code path with duplicate ctrl subsysnqn. In order to prevent that we
151	 * mask the passthru-ctrl subsysnqn with the target ctrl subsysnqn.
152	 */
153	strscpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn));
154
155	/* use fabric id-ctrl values */
156	id->ioccsz = cpu_to_le32((sizeof(struct nvme_command) +
157				req->port->inline_data_size) / 16);
158	id->iorcsz = cpu_to_le32(sizeof(struct nvme_completion) / 16);
159
160	id->msdbd = ctrl->ops->msdbd;
161
162	/* Support multipath connections with fabrics */
163	id->cmic |= 1 << 1;
164
165	/* Disable reservations, see nvmet_parse_passthru_io_cmd() */
166	id->oncs &= cpu_to_le16(~NVME_CTRL_ONCS_RESERVATIONS);
167
168	status = nvmet_copy_to_sgl(req, 0, id, sizeof(struct nvme_id_ctrl));
169
170out_free:
171	kfree(id);
172	return status;
173}
174
175static u16 nvmet_passthru_override_id_ns(struct nvmet_req *req)
176{
177	u16 status = NVME_SC_SUCCESS;
178	struct nvme_id_ns *id;
179	int i;
180
181	id = kzalloc(sizeof(*id), GFP_KERNEL);
182	if (!id)
183		return NVME_SC_INTERNAL;
184
185	status = nvmet_copy_from_sgl(req, 0, id, sizeof(struct nvme_id_ns));
186	if (status)
187		goto out_free;
188
189	for (i = 0; i < (id->nlbaf + 1); i++)
190		if (id->lbaf[i].ms)
191			memset(&id->lbaf[i], 0, sizeof(id->lbaf[i]));
192
193	id->flbas = id->flbas & ~(1 << 4);
194
195	/*
196	 * Presently the NVMEof target code does not support sending
197	 * metadata, so we must disable it here. This should be updated
198	 * once target starts supporting metadata.
199	 */
200	id->mc = 0;
201
202	if (req->sq->ctrl->subsys->clear_ids) {
203		memset(id->nguid, 0, NVME_NIDT_NGUID_LEN);
204		memset(id->eui64, 0, NVME_NIDT_EUI64_LEN);
205	}
206
207	status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));
208
209out_free:
210	kfree(id);
211	return status;
212}
213
214static void nvmet_passthru_execute_cmd_work(struct work_struct *w)
215{
216	struct nvmet_req *req = container_of(w, struct nvmet_req, p.work);
217	struct request *rq = req->p.rq;
218	struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
219	struct nvme_ns *ns = rq->q->queuedata;
220	u32 effects;
221	int status;
222
223	effects = nvme_passthru_start(ctrl, ns, req->cmd->common.opcode);
224	status = nvme_execute_rq(rq, false);
225	if (status == NVME_SC_SUCCESS &&
226	    req->cmd->common.opcode == nvme_admin_identify) {
227		switch (req->cmd->identify.cns) {
228		case NVME_ID_CNS_CTRL:
229			status = nvmet_passthru_override_id_ctrl(req);
230			break;
231		case NVME_ID_CNS_NS:
232			status = nvmet_passthru_override_id_ns(req);
233			break;
234		case NVME_ID_CNS_NS_DESC_LIST:
235			status = nvmet_passthru_override_id_descs(req);
236			break;
237		}
238	} else if (status < 0)
239		status = NVME_SC_INTERNAL;
240
241	req->cqe->result = nvme_req(rq)->result;
242	nvmet_req_complete(req, status);
243	blk_mq_free_request(rq);
244
245	if (effects)
246		nvme_passthru_end(ctrl, ns, effects, req->cmd, status);
247}
248
249static enum rq_end_io_ret nvmet_passthru_req_done(struct request *rq,
250						  blk_status_t blk_status)
251{
252	struct nvmet_req *req = rq->end_io_data;
253
254	req->cqe->result = nvme_req(rq)->result;
255	nvmet_req_complete(req, nvme_req(rq)->status);
256	blk_mq_free_request(rq);
257	return RQ_END_IO_NONE;
258}
259
260static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq)
261{
262	struct scatterlist *sg;
263	struct bio *bio;
264	int ret = -EINVAL;
265	int i;
266
267	if (req->sg_cnt > BIO_MAX_VECS)
268		return -EINVAL;
269
270	if (nvmet_use_inline_bvec(req)) {
271		bio = &req->p.inline_bio;
272		bio_init(bio, NULL, req->inline_bvec,
273			 ARRAY_SIZE(req->inline_bvec), req_op(rq));
274	} else {
275		bio = bio_alloc(NULL, bio_max_segs(req->sg_cnt), req_op(rq),
276				GFP_KERNEL);
277		bio->bi_end_io = bio_put;
278	}
279
280	for_each_sg(req->sg, sg, req->sg_cnt, i) {
281		if (bio_add_page(bio, sg_page(sg), sg->length, sg->offset) <
282				sg->length)
283			goto out_bio_put;
284	}
285
286	ret = blk_rq_append_bio(rq, bio);
287	if (ret)
288		goto out_bio_put;
289	return 0;
290
291out_bio_put:
292	nvmet_req_bio_put(req, bio);
293	return ret;
294}
295
296static void nvmet_passthru_execute_cmd(struct nvmet_req *req)
297{
298	struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru_ctrl;
299	struct request_queue *q = ctrl->admin_q;
300	struct nvme_ns *ns = NULL;
301	struct request *rq = NULL;
302	unsigned int timeout;
303	u32 effects;
304	u16 status;
305	int ret;
306
307	if (likely(req->sq->qid != 0)) {
308		u32 nsid = le32_to_cpu(req->cmd->common.nsid);
309
310		ns = nvme_find_get_ns(ctrl, nsid);
311		if (unlikely(!ns)) {
312			pr_err("failed to get passthru ns nsid:%u\n", nsid);
313			status = NVME_SC_INVALID_NS | NVME_STATUS_DNR;
314			goto out;
315		}
316
317		q = ns->queue;
318		timeout = nvmet_req_subsys(req)->io_timeout;
319	} else {
320		timeout = nvmet_req_subsys(req)->admin_timeout;
321	}
322
323	rq = blk_mq_alloc_request(q, nvme_req_op(req->cmd), 0);
324	if (IS_ERR(rq)) {
325		status = NVME_SC_INTERNAL;
326		goto out_put_ns;
327	}
328	nvme_init_request(rq, req->cmd);
329
330	if (timeout)
331		rq->timeout = timeout;
332
333	if (req->sg_cnt) {
334		ret = nvmet_passthru_map_sg(req, rq);
335		if (unlikely(ret)) {
336			status = NVME_SC_INTERNAL;
337			goto out_put_req;
338		}
339	}
340
341	/*
342	 * If a command needs post-execution fixups, or there are any
343	 * non-trivial effects, make sure to execute the command synchronously
344	 * in a workqueue so that nvme_passthru_end gets called.
345	 */
346	effects = nvme_command_effects(ctrl, ns, req->cmd->common.opcode);
347	if (req->p.use_workqueue ||
348	    (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))) {
349		INIT_WORK(&req->p.work, nvmet_passthru_execute_cmd_work);
350		req->p.rq = rq;
351		queue_work(nvmet_wq, &req->p.work);
352	} else {
353		rq->end_io = nvmet_passthru_req_done;
354		rq->end_io_data = req;
355		blk_execute_rq_nowait(rq, false);
356	}
357
358	if (ns)
359		nvme_put_ns(ns);
360
361	return;
362
363out_put_req:
364	blk_mq_free_request(rq);
365out_put_ns:
366	if (ns)
367		nvme_put_ns(ns);
368out:
369	nvmet_req_complete(req, status);
370}
371
372/*
373 * We need to emulate set host behaviour to ensure that any requested
374 * behaviour of the target's host matches the requested behaviour
375 * of the device's host and fail otherwise.
376 */
377static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req)
378{
379	struct nvme_ctrl *ctrl = nvmet_req_subsys(req)->passthru_ctrl;
380	struct nvme_feat_host_behavior *host;
381	u16 status = NVME_SC_INTERNAL;
382	int ret;
383
384	host = kzalloc(sizeof(*host) * 2, GFP_KERNEL);
385	if (!host)
386		goto out_complete_req;
387
388	ret = nvme_get_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
389				host, sizeof(*host), NULL);
390	if (ret)
391		goto out_free_host;
392
393	status = nvmet_copy_from_sgl(req, 0, &host[1], sizeof(*host));
394	if (status)
395		goto out_free_host;
396
397	if (memcmp(&host[0], &host[1], sizeof(host[0]))) {
398		pr_warn("target host has requested different behaviour from the local host\n");
399		status = NVME_SC_INTERNAL;
400	}
401
402out_free_host:
403	kfree(host);
404out_complete_req:
405	nvmet_req_complete(req, status);
406}
407
408static u16 nvmet_setup_passthru_command(struct nvmet_req *req)
409{
410	req->p.use_workqueue = false;
411	req->execute = nvmet_passthru_execute_cmd;
412	return NVME_SC_SUCCESS;
413}
414
415u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req)
416{
417	/* Reject any commands with non-sgl flags set (ie. fused commands) */
418	if (req->cmd->common.flags & ~NVME_CMD_SGL_ALL)
419		return NVME_SC_INVALID_FIELD;
420
421	switch (req->cmd->common.opcode) {
422	case nvme_cmd_resv_register:
423	case nvme_cmd_resv_report:
424	case nvme_cmd_resv_acquire:
425	case nvme_cmd_resv_release:
426		/*
427		 * Reservations cannot be supported properly because the
428		 * underlying device has no way of differentiating different
429		 * hosts that connect via fabrics. This could potentially be
430		 * emulated in the future if regular targets grow support for
431		 * this feature.
432		 */
433		return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
434	}
435
436	return nvmet_setup_passthru_command(req);
437}
438
439/*
440 * Only features that are emulated or specifically allowed in the list  are
441 * passed down to the controller. This function implements the allow list for
442 * both get and set features.
443 */
444static u16 nvmet_passthru_get_set_features(struct nvmet_req *req)
445{
446	switch (le32_to_cpu(req->cmd->features.fid)) {
447	case NVME_FEAT_ARBITRATION:
448	case NVME_FEAT_POWER_MGMT:
449	case NVME_FEAT_LBA_RANGE:
450	case NVME_FEAT_TEMP_THRESH:
451	case NVME_FEAT_ERR_RECOVERY:
452	case NVME_FEAT_VOLATILE_WC:
453	case NVME_FEAT_WRITE_ATOMIC:
454	case NVME_FEAT_AUTO_PST:
455	case NVME_FEAT_TIMESTAMP:
456	case NVME_FEAT_HCTM:
457	case NVME_FEAT_NOPSC:
458	case NVME_FEAT_RRL:
459	case NVME_FEAT_PLM_CONFIG:
460	case NVME_FEAT_PLM_WINDOW:
461	case NVME_FEAT_HOST_BEHAVIOR:
462	case NVME_FEAT_SANITIZE:
463	case NVME_FEAT_VENDOR_START ... NVME_FEAT_VENDOR_END:
464		return nvmet_setup_passthru_command(req);
465
466	case NVME_FEAT_ASYNC_EVENT:
467		/* There is no support for forwarding ASYNC events */
468	case NVME_FEAT_IRQ_COALESCE:
469	case NVME_FEAT_IRQ_CONFIG:
470		/* The IRQ settings will not apply to the target controller */
471	case NVME_FEAT_HOST_MEM_BUF:
472		/*
473		 * Any HMB that's set will not be passed through and will
474		 * not work as expected
475		 */
476	case NVME_FEAT_SW_PROGRESS:
477		/*
478		 * The Pre-Boot Software Load Count doesn't make much
479		 * sense for a target to export
480		 */
481	case NVME_FEAT_RESV_MASK:
482	case NVME_FEAT_RESV_PERSIST:
483		/* No reservations, see nvmet_parse_passthru_io_cmd() */
484	default:
485		return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
486	}
487}
488
489u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
490{
491	/* Reject any commands with non-sgl flags set (ie. fused commands) */
492	if (req->cmd->common.flags & ~NVME_CMD_SGL_ALL)
493		return NVME_SC_INVALID_FIELD;
494
495	/*
496	 * Passthru all vendor specific commands
497	 */
498	if (req->cmd->common.opcode >= nvme_admin_vendor_start)
499		return nvmet_setup_passthru_command(req);
500
501	switch (req->cmd->common.opcode) {
502	case nvme_admin_async_event:
503		req->execute = nvmet_execute_async_event;
504		return NVME_SC_SUCCESS;
505	case nvme_admin_keep_alive:
506		/*
507		 * Most PCIe ctrls don't support keep alive cmd, we route keep
508		 * alive to the non-passthru mode. In future please change this
509		 * code when PCIe ctrls with keep alive support available.
510		 */
511		req->execute = nvmet_execute_keep_alive;
512		return NVME_SC_SUCCESS;
513	case nvme_admin_set_features:
514		switch (le32_to_cpu(req->cmd->features.fid)) {
515		case NVME_FEAT_ASYNC_EVENT:
516		case NVME_FEAT_KATO:
517		case NVME_FEAT_NUM_QUEUES:
518		case NVME_FEAT_HOST_ID:
519			req->execute = nvmet_execute_set_features;
520			return NVME_SC_SUCCESS;
521		case NVME_FEAT_HOST_BEHAVIOR:
522			req->execute = nvmet_passthru_set_host_behaviour;
523			return NVME_SC_SUCCESS;
524		default:
525			return nvmet_passthru_get_set_features(req);
526		}
527		break;
528	case nvme_admin_get_features:
529		switch (le32_to_cpu(req->cmd->features.fid)) {
530		case NVME_FEAT_ASYNC_EVENT:
531		case NVME_FEAT_KATO:
532		case NVME_FEAT_NUM_QUEUES:
533		case NVME_FEAT_HOST_ID:
534			req->execute = nvmet_execute_get_features;
535			return NVME_SC_SUCCESS;
536		case NVME_FEAT_FDP:
537			return nvmet_setup_passthru_command(req);
538		default:
539			return nvmet_passthru_get_set_features(req);
540		}
541		break;
542	case nvme_admin_identify:
543		switch (req->cmd->identify.cns) {
544		case NVME_ID_CNS_CS_CTRL:
545			switch (req->cmd->identify.csi) {
546			case NVME_CSI_ZNS:
547				req->execute = nvmet_passthru_execute_cmd;
548				req->p.use_workqueue = true;
549				return NVME_SC_SUCCESS;
550			}
551			return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
552		case NVME_ID_CNS_CTRL:
553		case NVME_ID_CNS_NS:
554		case NVME_ID_CNS_NS_DESC_LIST:
555			req->execute = nvmet_passthru_execute_cmd;
556			req->p.use_workqueue = true;
557			return NVME_SC_SUCCESS;
558		case NVME_ID_CNS_CS_NS:
559			switch (req->cmd->identify.csi) {
560			case NVME_CSI_ZNS:
561				req->execute = nvmet_passthru_execute_cmd;
562				req->p.use_workqueue = true;
563				return NVME_SC_SUCCESS;
564			}
565			return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
566		default:
567			return nvmet_setup_passthru_command(req);
568		}
569	case nvme_admin_get_log_page:
570		return nvmet_setup_passthru_command(req);
571	default:
572		/* Reject commands not in the allowlist above */
573		return nvmet_report_invalid_opcode(req);
574	}
575}
576
577int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys)
578{
579	struct nvme_ctrl *ctrl;
580	struct file *file;
581	int ret = -EINVAL;
582	void *old;
583
584	mutex_lock(&subsys->lock);
585	if (!subsys->passthru_ctrl_path)
586		goto out_unlock;
587	if (subsys->passthru_ctrl)
588		goto out_unlock;
589
590	if (subsys->nr_namespaces) {
591		pr_info("cannot enable both passthru and regular namespaces for a single subsystem");
592		goto out_unlock;
593	}
594
595	file = filp_open(subsys->passthru_ctrl_path, O_RDWR, 0);
596	if (IS_ERR(file)) {
597		ret = PTR_ERR(file);
598		goto out_unlock;
599	}
600
601	ctrl = nvme_ctrl_from_file(file);
602	if (!ctrl) {
603		pr_err("failed to open nvme controller %s\n",
604		       subsys->passthru_ctrl_path);
605
606		goto out_put_file;
607	}
608
609	old = xa_cmpxchg(&passthru_subsystems, ctrl->instance, NULL,
610			 subsys, GFP_KERNEL);
611	if (xa_is_err(old)) {
612		ret = xa_err(old);
613		goto out_put_file;
614	}
615
616	if (old)
617		goto out_put_file;
618
619	subsys->passthru_ctrl = ctrl;
620	subsys->ver = ctrl->vs;
621
622	if (subsys->ver < NVME_VS(1, 2, 1)) {
623		pr_warn("nvme controller version is too old: %llu.%llu.%llu, advertising 1.2.1\n",
624			NVME_MAJOR(subsys->ver), NVME_MINOR(subsys->ver),
625			NVME_TERTIARY(subsys->ver));
626		subsys->ver = NVME_VS(1, 2, 1);
627	}
628	nvme_get_ctrl(ctrl);
629	__module_get(subsys->passthru_ctrl->ops->module);
630	ret = 0;
631
632out_put_file:
633	filp_close(file, NULL);
634out_unlock:
635	mutex_unlock(&subsys->lock);
636	return ret;
637}
638
639static void __nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
640{
641	if (subsys->passthru_ctrl) {
642		xa_erase(&passthru_subsystems, subsys->passthru_ctrl->instance);
643		module_put(subsys->passthru_ctrl->ops->module);
644		nvme_put_ctrl(subsys->passthru_ctrl);
645	}
646	subsys->passthru_ctrl = NULL;
647	subsys->ver = NVMET_DEFAULT_VS;
648}
649
650void nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys)
651{
652	mutex_lock(&subsys->lock);
653	__nvmet_passthru_ctrl_disable(subsys);
654	mutex_unlock(&subsys->lock);
655}
656
657void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
658{
659	mutex_lock(&subsys->lock);
660	__nvmet_passthru_ctrl_disable(subsys);
661	mutex_unlock(&subsys->lock);
662	kfree(subsys->passthru_ctrl_path);
663}