zns.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * NVMe ZNS-ZBD command implementation.
  4 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  5 */
  6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7#include <linux/nvme.h>
  8#include <linux/blkdev.h>
  9#include "nvmet.h"
 10
 11/*
 12 * We set the Memory Page Size Minimum (MPSMIN) for target controller to 0
 13 * which gets added by 12 in the nvme_enable_ctrl() which results in 2^12 = 4k
 14 * as page_shift value. When calculating the ZASL use shift by 12.
 15 */
 16#define NVMET_MPSMIN_SHIFT	12
 17
 18static inline u8 nvmet_zasl(unsigned int zone_append_sects)
 19{
 20	/*
 21	 * Zone Append Size Limit (zasl) is expressed as a power of 2 value
 22	 * with the minimum memory page size (i.e. 12) as unit.
 23	 */
 24	return ilog2(zone_append_sects >> (NVMET_MPSMIN_SHIFT - 9));
 25}
 26
 27static int validate_conv_zones_cb(struct blk_zone *z,
 28				  unsigned int i, void *data)
 29{
 30	if (z->type == BLK_ZONE_TYPE_CONVENTIONAL)
 31		return -EOPNOTSUPP;
 32	return 0;
 33}
 34
 35bool nvmet_bdev_zns_enable(struct nvmet_ns *ns)
 36{
 37	u8 zasl = nvmet_zasl(bdev_max_zone_append_sectors(ns->bdev));
 38	struct gendisk *bd_disk = ns->bdev->bd_disk;
 39	int ret;
 40
 41	if (ns->subsys->zasl) {
 42		if (ns->subsys->zasl > zasl)
 43			return false;
 44	}
 45	ns->subsys->zasl = zasl;
 46
 47	/*
 48	 * Generic zoned block devices may have a smaller last zone which is
 49	 * not supported by ZNS. Exclude zoned drives that have such smaller
 50	 * last zone.
 51	 */
 52	if (get_capacity(bd_disk) & (bdev_zone_sectors(ns->bdev) - 1))
 53		return false;
 54	/*
 55	 * ZNS does not define a conventional zone type. Use report zones
 56	 * to detect if the device has conventional zones and reject it if
 57	 * it does.
 58	 */
 59	ret = blkdev_report_zones(ns->bdev, 0, bdev_nr_zones(ns->bdev),
 60				  validate_conv_zones_cb, NULL);
 61	if (ret < 0)
 62		return false;
 63
 64	ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev));
 65
 66	return true;
 67}
 68
 69void nvmet_execute_identify_ctrl_zns(struct nvmet_req *req)
 70{
 71	u8 zasl = req->sq->ctrl->subsys->zasl;
 72	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 73	struct nvme_id_ctrl_zns *id;
 74	u16 status;
 75
 76	id = kzalloc(sizeof(*id), GFP_KERNEL);
 77	if (!id) {
 78		status = NVME_SC_INTERNAL;
 79		goto out;
 80	}
 81
 82	if (ctrl->ops->get_mdts)
 83		id->zasl = min_t(u8, ctrl->ops->get_mdts(ctrl), zasl);
 84	else
 85		id->zasl = zasl;
 86
 87	status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id));
 88
 89	kfree(id);
 90out:
 91	nvmet_req_complete(req, status);
 92}
 93
 94void nvmet_execute_identify_ns_zns(struct nvmet_req *req)
 95{
 96	struct nvme_id_ns_zns *id_zns = NULL;
 97	u64 zsze;
 98	u16 status;
 99	u32 mar, mor;
100
101	if (le32_to_cpu(req->cmd->identify.nsid) == NVME_NSID_ALL) {
102		req->error_loc = offsetof(struct nvme_identify, nsid);
103		status = NVME_SC_INVALID_NS | NVME_STATUS_DNR;
104		goto out;
105	}
106
107	id_zns = kzalloc(sizeof(*id_zns), GFP_KERNEL);
108	if (!id_zns) {
109		status = NVME_SC_INTERNAL;
110		goto out;
111	}
112
113	status = nvmet_req_find_ns(req);
114	if (status)
115		goto done;
116
117	if (nvmet_ns_revalidate(req->ns)) {
118		mutex_lock(&req->ns->subsys->lock);
119		nvmet_ns_changed(req->ns->subsys, req->ns->nsid);
120		mutex_unlock(&req->ns->subsys->lock);
121	}
122
123	if (!bdev_is_zoned(req->ns->bdev)) {
124		status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
125		req->error_loc = offsetof(struct nvme_identify, nsid);
126		goto out;
127	}
128
129	zsze = (bdev_zone_sectors(req->ns->bdev) << 9) >>
130					req->ns->blksize_shift;
131	id_zns->lbafe[0].zsze = cpu_to_le64(zsze);
132
133	mor = bdev_max_open_zones(req->ns->bdev);
134	if (!mor)
135		mor = U32_MAX;
136	else
137		mor--;
138	id_zns->mor = cpu_to_le32(mor);
139
140	mar = bdev_max_active_zones(req->ns->bdev);
141	if (!mar)
142		mar = U32_MAX;
143	else
144		mar--;
145	id_zns->mar = cpu_to_le32(mar);
146
147done:
148	status = nvmet_copy_to_sgl(req, 0, id_zns, sizeof(*id_zns));
149out:
150	kfree(id_zns);
151	nvmet_req_complete(req, status);
152}
153
154static u16 nvmet_bdev_validate_zone_mgmt_recv(struct nvmet_req *req)
155{
156	sector_t sect = nvmet_lba_to_sect(req->ns, req->cmd->zmr.slba);
157	u32 out_bufsize = (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2;
158
159	if (sect >= get_capacity(req->ns->bdev->bd_disk)) {
160		req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, slba);
161		return NVME_SC_LBA_RANGE | NVME_STATUS_DNR;
162	}
163
164	if (out_bufsize < sizeof(struct nvme_zone_report)) {
165		req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, numd);
166		return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
167	}
168
169	if (req->cmd->zmr.zra != NVME_ZRA_ZONE_REPORT) {
170		req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, zra);
171		return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
172	}
173
174	switch (req->cmd->zmr.pr) {
175	case 0:
176	case 1:
177		break;
178	default:
179		req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, pr);
180		return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
181	}
182
183	switch (req->cmd->zmr.zrasf) {
184	case NVME_ZRASF_ZONE_REPORT_ALL:
185	case NVME_ZRASF_ZONE_STATE_EMPTY:
186	case NVME_ZRASF_ZONE_STATE_IMP_OPEN:
187	case NVME_ZRASF_ZONE_STATE_EXP_OPEN:
188	case NVME_ZRASF_ZONE_STATE_CLOSED:
189	case NVME_ZRASF_ZONE_STATE_FULL:
190	case NVME_ZRASF_ZONE_STATE_READONLY:
191	case NVME_ZRASF_ZONE_STATE_OFFLINE:
192		break;
193	default:
194		req->error_loc =
195			offsetof(struct nvme_zone_mgmt_recv_cmd, zrasf);
196		return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
197	}
198
199	return NVME_SC_SUCCESS;
200}
201
202struct nvmet_report_zone_data {
203	struct nvmet_req *req;
204	u64 out_buf_offset;
205	u64 out_nr_zones;
206	u64 nr_zones;
207	u8 zrasf;
208};
209
210static int nvmet_bdev_report_zone_cb(struct blk_zone *z, unsigned i, void *d)
211{
212	static const unsigned int nvme_zrasf_to_blk_zcond[] = {
213		[NVME_ZRASF_ZONE_STATE_EMPTY]	 = BLK_ZONE_COND_EMPTY,
214		[NVME_ZRASF_ZONE_STATE_IMP_OPEN] = BLK_ZONE_COND_IMP_OPEN,
215		[NVME_ZRASF_ZONE_STATE_EXP_OPEN] = BLK_ZONE_COND_EXP_OPEN,
216		[NVME_ZRASF_ZONE_STATE_CLOSED]	 = BLK_ZONE_COND_CLOSED,
217		[NVME_ZRASF_ZONE_STATE_READONLY] = BLK_ZONE_COND_READONLY,
218		[NVME_ZRASF_ZONE_STATE_FULL]	 = BLK_ZONE_COND_FULL,
219		[NVME_ZRASF_ZONE_STATE_OFFLINE]	 = BLK_ZONE_COND_OFFLINE,
220	};
221	struct nvmet_report_zone_data *rz = d;
222
223	if (rz->zrasf != NVME_ZRASF_ZONE_REPORT_ALL &&
224	    z->cond != nvme_zrasf_to_blk_zcond[rz->zrasf])
225		return 0;
226
227	if (rz->nr_zones < rz->out_nr_zones) {
228		struct nvme_zone_descriptor zdesc = { };
229		u16 status;
230
231		zdesc.zcap = nvmet_sect_to_lba(rz->req->ns, z->capacity);
232		zdesc.zslba = nvmet_sect_to_lba(rz->req->ns, z->start);
233		zdesc.wp = nvmet_sect_to_lba(rz->req->ns, z->wp);
234		zdesc.za = z->reset ? 1 << 2 : 0;
235		zdesc.zs = z->cond << 4;
236		zdesc.zt = z->type;
237
238		status = nvmet_copy_to_sgl(rz->req, rz->out_buf_offset, &zdesc,
239					   sizeof(zdesc));
240		if (status)
241			return -EINVAL;
242
243		rz->out_buf_offset += sizeof(zdesc);
244	}
245
246	rz->nr_zones++;
247
248	return 0;
249}
250
251static unsigned long nvmet_req_nr_zones_from_slba(struct nvmet_req *req)
252{
253	unsigned int sect = nvmet_lba_to_sect(req->ns, req->cmd->zmr.slba);
254
255	return bdev_nr_zones(req->ns->bdev) - bdev_zone_no(req->ns->bdev, sect);
256}
257
258static unsigned long get_nr_zones_from_buf(struct nvmet_req *req, u32 bufsize)
259{
260	if (bufsize <= sizeof(struct nvme_zone_report))
261		return 0;
262
263	return (bufsize - sizeof(struct nvme_zone_report)) /
264		sizeof(struct nvme_zone_descriptor);
265}
266
267static void nvmet_bdev_zone_zmgmt_recv_work(struct work_struct *w)
268{
269	struct nvmet_req *req = container_of(w, struct nvmet_req, z.zmgmt_work);
270	sector_t start_sect = nvmet_lba_to_sect(req->ns, req->cmd->zmr.slba);
271	unsigned long req_slba_nr_zones = nvmet_req_nr_zones_from_slba(req);
272	u32 out_bufsize = (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2;
273	__le64 nr_zones;
274	u16 status;
275	int ret;
276	struct nvmet_report_zone_data rz_data = {
277		.out_nr_zones = get_nr_zones_from_buf(req, out_bufsize),
278		/* leave the place for report zone header */
279		.out_buf_offset = sizeof(struct nvme_zone_report),
280		.zrasf = req->cmd->zmr.zrasf,
281		.nr_zones = 0,
282		.req = req,
283	};
284
285	status = nvmet_bdev_validate_zone_mgmt_recv(req);
286	if (status)
287		goto out;
288
289	if (!req_slba_nr_zones) {
290		status = NVME_SC_SUCCESS;
291		goto out;
292	}
293
294	ret = blkdev_report_zones(req->ns->bdev, start_sect, req_slba_nr_zones,
295				 nvmet_bdev_report_zone_cb, &rz_data);
296	if (ret < 0) {
297		status = NVME_SC_INTERNAL;
298		goto out;
299	}
300
301	/*
302	 * When partial bit is set nr_zones must indicate the number of zone
303	 * descriptors actually transferred.
304	 */
305	if (req->cmd->zmr.pr)
306		rz_data.nr_zones = min(rz_data.nr_zones, rz_data.out_nr_zones);
307
308	nr_zones = cpu_to_le64(rz_data.nr_zones);
309	status = nvmet_copy_to_sgl(req, 0, &nr_zones, sizeof(nr_zones));
310
311out:
312	nvmet_req_complete(req, status);
313}
314
315void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req)
316{
317	INIT_WORK(&req->z.zmgmt_work, nvmet_bdev_zone_zmgmt_recv_work);
318	queue_work(zbd_wq, &req->z.zmgmt_work);
319}
320
321static inline enum req_op zsa_req_op(u8 zsa)
322{
323	switch (zsa) {
324	case NVME_ZONE_OPEN:
325		return REQ_OP_ZONE_OPEN;
326	case NVME_ZONE_CLOSE:
327		return REQ_OP_ZONE_CLOSE;
328	case NVME_ZONE_FINISH:
329		return REQ_OP_ZONE_FINISH;
330	case NVME_ZONE_RESET:
331		return REQ_OP_ZONE_RESET;
332	default:
333		return REQ_OP_LAST;
334	}
335}
336
337static u16 blkdev_zone_mgmt_errno_to_nvme_status(int ret)
338{
339	switch (ret) {
340	case 0:
341		return NVME_SC_SUCCESS;
342	case -EINVAL:
343	case -EIO:
344		return NVME_SC_ZONE_INVALID_TRANSITION | NVME_STATUS_DNR;
345	default:
346		return NVME_SC_INTERNAL;
347	}
348}
349
350struct nvmet_zone_mgmt_send_all_data {
351	unsigned long *zbitmap;
352	struct nvmet_req *req;
353};
354
355static int zmgmt_send_scan_cb(struct blk_zone *z, unsigned i, void *d)
356{
357	struct nvmet_zone_mgmt_send_all_data *data = d;
358
359	switch (zsa_req_op(data->req->cmd->zms.zsa)) {
360	case REQ_OP_ZONE_OPEN:
361		switch (z->cond) {
362		case BLK_ZONE_COND_CLOSED:
363			break;
364		default:
365			return 0;
366		}
367		break;
368	case REQ_OP_ZONE_CLOSE:
369		switch (z->cond) {
370		case BLK_ZONE_COND_IMP_OPEN:
371		case BLK_ZONE_COND_EXP_OPEN:
372			break;
373		default:
374			return 0;
375		}
376		break;
377	case REQ_OP_ZONE_FINISH:
378		switch (z->cond) {
379		case BLK_ZONE_COND_IMP_OPEN:
380		case BLK_ZONE_COND_EXP_OPEN:
381		case BLK_ZONE_COND_CLOSED:
382			break;
383		default:
384			return 0;
385		}
386		break;
387	default:
388		return -EINVAL;
389	}
390
391	set_bit(i, data->zbitmap);
392
393	return 0;
394}
395
396static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req)
397{
398	struct block_device *bdev = req->ns->bdev;
399	unsigned int nr_zones = bdev_nr_zones(bdev);
400	struct bio *bio = NULL;
401	sector_t sector = 0;
402	int ret;
403	struct nvmet_zone_mgmt_send_all_data d = {
404		.req = req,
405	};
406
407	d.zbitmap = kcalloc_node(BITS_TO_LONGS(nr_zones), sizeof(*(d.zbitmap)),
408				 GFP_NOIO, bdev->bd_disk->node_id);
409	if (!d.zbitmap) {
410		ret = -ENOMEM;
411		goto out;
412	}
413
414	/* Scan and build bitmap of the eligible zones */
415	ret = blkdev_report_zones(bdev, 0, nr_zones, zmgmt_send_scan_cb, &d);
416	if (ret != nr_zones) {
417		if (ret > 0)
418			ret = -EIO;
419		goto out;
420	} else {
421		/* We scanned all the zones */
422		ret = 0;
423	}
424
425	while (sector < bdev_nr_sectors(bdev)) {
426		if (test_bit(disk_zone_no(bdev->bd_disk, sector), d.zbitmap)) {
427			bio = blk_next_bio(bio, bdev, 0,
428				zsa_req_op(req->cmd->zms.zsa) | REQ_SYNC,
429				GFP_KERNEL);
430			bio->bi_iter.bi_sector = sector;
431			/* This may take a while, so be nice to others */
432			cond_resched();
433		}
434		sector += bdev_zone_sectors(bdev);
435	}
436
437	if (bio) {
438		ret = submit_bio_wait(bio);
439		bio_put(bio);
440	}
441
442out:
443	kfree(d.zbitmap);
444
445	return blkdev_zone_mgmt_errno_to_nvme_status(ret);
446}
447
448static u16 nvmet_bdev_execute_zmgmt_send_all(struct nvmet_req *req)
449{
450	int ret;
451
452	switch (zsa_req_op(req->cmd->zms.zsa)) {
453	case REQ_OP_ZONE_RESET:
454		ret = blkdev_zone_mgmt(req->ns->bdev, REQ_OP_ZONE_RESET, 0,
455				       get_capacity(req->ns->bdev->bd_disk));
456		if (ret < 0)
457			return blkdev_zone_mgmt_errno_to_nvme_status(ret);
458		break;
459	case REQ_OP_ZONE_OPEN:
460	case REQ_OP_ZONE_CLOSE:
461	case REQ_OP_ZONE_FINISH:
462		return nvmet_bdev_zone_mgmt_emulate_all(req);
463	default:
464		/* this is needed to quiet compiler warning */
465		req->error_loc = offsetof(struct nvme_zone_mgmt_send_cmd, zsa);
466		return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
467	}
468
469	return NVME_SC_SUCCESS;
470}
471
472static void nvmet_bdev_zmgmt_send_work(struct work_struct *w)
473{
474	struct nvmet_req *req = container_of(w, struct nvmet_req, z.zmgmt_work);
475	sector_t sect = nvmet_lba_to_sect(req->ns, req->cmd->zms.slba);
476	enum req_op op = zsa_req_op(req->cmd->zms.zsa);
477	struct block_device *bdev = req->ns->bdev;
478	sector_t zone_sectors = bdev_zone_sectors(bdev);
479	u16 status = NVME_SC_SUCCESS;
480	int ret;
481
482	if (op == REQ_OP_LAST) {
483		req->error_loc = offsetof(struct nvme_zone_mgmt_send_cmd, zsa);
484		status = NVME_SC_ZONE_INVALID_TRANSITION | NVME_STATUS_DNR;
485		goto out;
486	}
487
488	/* when select all bit is set slba field is ignored */
489	if (req->cmd->zms.select_all) {
490		status = nvmet_bdev_execute_zmgmt_send_all(req);
491		goto out;
492	}
493
494	if (sect >= get_capacity(bdev->bd_disk)) {
495		req->error_loc = offsetof(struct nvme_zone_mgmt_send_cmd, slba);
496		status = NVME_SC_LBA_RANGE | NVME_STATUS_DNR;
497		goto out;
498	}
499
500	if (sect & (zone_sectors - 1)) {
501		req->error_loc = offsetof(struct nvme_zone_mgmt_send_cmd, slba);
502		status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
503		goto out;
504	}
505
506	ret = blkdev_zone_mgmt(bdev, op, sect, zone_sectors);
507	if (ret < 0)
508		status = blkdev_zone_mgmt_errno_to_nvme_status(ret);
509
510out:
511	nvmet_req_complete(req, status);
512}
513
514void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req)
515{
516	INIT_WORK(&req->z.zmgmt_work, nvmet_bdev_zmgmt_send_work);
517	queue_work(zbd_wq, &req->z.zmgmt_work);
518}
519
520static void nvmet_bdev_zone_append_bio_done(struct bio *bio)
521{
522	struct nvmet_req *req = bio->bi_private;
523
524	if (bio->bi_status == BLK_STS_OK) {
525		req->cqe->result.u64 =
526			nvmet_sect_to_lba(req->ns, bio->bi_iter.bi_sector);
527	}
528
529	nvmet_req_complete(req, blk_to_nvme_status(req, bio->bi_status));
530	nvmet_req_bio_put(req, bio);
531}
532
533void nvmet_bdev_execute_zone_append(struct nvmet_req *req)
534{
535	sector_t sect = nvmet_lba_to_sect(req->ns, req->cmd->rw.slba);
536	const blk_opf_t opf = REQ_OP_ZONE_APPEND | REQ_SYNC | REQ_IDLE;
537	u16 status = NVME_SC_SUCCESS;
538	unsigned int total_len = 0;
539	struct scatterlist *sg;
540	u32 data_len = nvmet_rw_data_len(req);
541	struct bio *bio;
542	int sg_cnt;
543
544	/* Request is completed on len mismatch in nvmet_check_transfer_len() */
545	if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req)))
546		return;
547
548	if (data_len >
549	    bdev_max_zone_append_sectors(req->ns->bdev) << SECTOR_SHIFT) {
550		req->error_loc = offsetof(struct nvme_rw_command, length);
551		status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
552		goto out;
553	}
554
555	if (!req->sg_cnt) {
556		nvmet_req_complete(req, 0);
557		return;
558	}
559
560	if (sect >= get_capacity(req->ns->bdev->bd_disk)) {
561		req->error_loc = offsetof(struct nvme_rw_command, slba);
562		status = NVME_SC_LBA_RANGE | NVME_STATUS_DNR;
563		goto out;
564	}
565
566	if (sect & (bdev_zone_sectors(req->ns->bdev) - 1)) {
567		req->error_loc = offsetof(struct nvme_rw_command, slba);
568		status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
569		goto out;
570	}
571
572	if (nvmet_use_inline_bvec(req)) {
573		bio = &req->z.inline_bio;
574		bio_init(bio, req->ns->bdev, req->inline_bvec,
575			 ARRAY_SIZE(req->inline_bvec), opf);
576	} else {
577		bio = bio_alloc(req->ns->bdev, req->sg_cnt, opf, GFP_KERNEL);
578	}
579
580	bio->bi_end_io = nvmet_bdev_zone_append_bio_done;
581	bio->bi_iter.bi_sector = sect;
582	bio->bi_private = req;
583	if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
584		bio->bi_opf |= REQ_FUA;
585
586	for_each_sg(req->sg, sg, req->sg_cnt, sg_cnt) {
587		unsigned int len = sg->length;
588
589		if (bio_add_page(bio, sg_page(sg), len, sg->offset) != len) {
590			status = NVME_SC_INTERNAL;
591			goto out_put_bio;
592		}
593		total_len += len;
594	}
595
596	if (total_len != data_len) {
597		status = NVME_SC_INTERNAL | NVME_STATUS_DNR;
598		goto out_put_bio;
599	}
600
601	submit_bio(bio);
602	return;
603
604out_put_bio:
605	nvmet_req_bio_put(req, bio);
606out:
607	nvmet_req_complete(req, status);
608}
609
610u16 nvmet_bdev_zns_parse_io_cmd(struct nvmet_req *req)
611{
612	struct nvme_command *cmd = req->cmd;
613
614	switch (cmd->common.opcode) {
615	case nvme_cmd_zone_append:
616		req->execute = nvmet_bdev_execute_zone_append;
617		return 0;
618	case nvme_cmd_zone_mgmt_recv:
619		req->execute = nvmet_bdev_execute_zone_mgmt_recv;
620		return 0;
621	case nvme_cmd_zone_mgmt_send:
622		req->execute = nvmet_bdev_execute_zone_mgmt_send;
623		return 0;
624	default:
625		return nvmet_bdev_parse_io_cmd(req);
626	}
627}