target_core_iblock.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*******************************************************************************
   3 * Filename:  target_core_iblock.c
   4 *
   5 * This file contains the Storage Engine  <-> Linux BlockIO transport
   6 * specific functions.
   7 *
   8 * (c) Copyright 2003-2013 Datera, Inc.
   9 *
  10 * Nicholas A. Bellinger <nab@kernel.org>
  11 *
  12 ******************************************************************************/
  13
  14#include <linux/string.h>
  15#include <linux/parser.h>
  16#include <linux/timer.h>
  17#include <linux/fs.h>
  18#include <linux/blkdev.h>
  19#include <linux/blk-integrity.h>
  20#include <linux/slab.h>
  21#include <linux/spinlock.h>
  22#include <linux/bio.h>
  23#include <linux/file.h>
  24#include <linux/module.h>
  25#include <linux/scatterlist.h>
  26#include <linux/pr.h>
  27#include <scsi/scsi_proto.h>
  28#include <scsi/scsi_common.h>
  29#include <linux/unaligned.h>
  30
  31#include <target/target_core_base.h>
  32#include <target/target_core_backend.h>
  33
  34#include "target_core_iblock.h"
  35#include "target_core_pr.h"
  36
  37#define IBLOCK_MAX_BIO_PER_TASK	 32	/* max # of bios to submit at a time */
  38#define IBLOCK_BIO_POOL_SIZE	128
  39
  40static inline struct iblock_dev *IBLOCK_DEV(struct se_device *dev)
  41{
  42	return container_of(dev, struct iblock_dev, dev);
  43}
  44
  45
  46static int iblock_attach_hba(struct se_hba *hba, u32 host_id)
  47{
  48	pr_debug("CORE_HBA[%d] - TCM iBlock HBA Driver %s on"
  49		" Generic Target Core Stack %s\n", hba->hba_id,
  50		IBLOCK_VERSION, TARGET_CORE_VERSION);
  51	return 0;
  52}
  53
  54static void iblock_detach_hba(struct se_hba *hba)
  55{
  56}
  57
  58static struct se_device *iblock_alloc_device(struct se_hba *hba, const char *name)
  59{
  60	struct iblock_dev *ib_dev = NULL;
  61
  62	ib_dev = kzalloc(sizeof(struct iblock_dev), GFP_KERNEL);
  63	if (!ib_dev) {
  64		pr_err("Unable to allocate struct iblock_dev\n");
  65		return NULL;
  66	}
  67	ib_dev->ibd_exclusive = true;
  68
  69	ib_dev->ibd_plug = kcalloc(nr_cpu_ids, sizeof(*ib_dev->ibd_plug),
  70				   GFP_KERNEL);
  71	if (!ib_dev->ibd_plug)
  72		goto free_dev;
  73
  74	pr_debug( "IBLOCK: Allocated ib_dev for %s\n", name);
  75
  76	return &ib_dev->dev;
  77
  78free_dev:
  79	kfree(ib_dev);
  80	return NULL;
  81}
  82
  83static bool iblock_configure_unmap(struct se_device *dev)
  84{
  85	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
  86
  87	return target_configure_unmap_from_bdev(&dev->dev_attrib,
  88						ib_dev->ibd_bd);
  89}
  90
  91static int iblock_configure_device(struct se_device *dev)
  92{
  93	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
  94	struct request_queue *q;
  95	struct file *bdev_file;
  96	struct block_device *bd;
  97	struct blk_integrity *bi;
  98	blk_mode_t mode = BLK_OPEN_READ;
  99	void *holder = ib_dev;
 100	unsigned int max_write_zeroes_sectors;
 101	int ret;
 102
 103	if (!(ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)) {
 104		pr_err("Missing udev_path= parameters for IBLOCK\n");
 105		return -EINVAL;
 106	}
 107
 108	ret = bioset_init(&ib_dev->ibd_bio_set, IBLOCK_BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
 109	if (ret) {
 110		pr_err("IBLOCK: Unable to create bioset\n");
 111		goto out;
 112	}
 113
 114	pr_debug("IBLOCK: Claiming struct block_device: %s: %d\n",
 115		 ib_dev->ibd_udev_path, ib_dev->ibd_exclusive);
 116
 117	if (!ib_dev->ibd_readonly)
 118		mode |= BLK_OPEN_WRITE;
 119	else
 120		dev->dev_flags |= DF_READ_ONLY;
 121
 122	if (!ib_dev->ibd_exclusive)
 123		holder = NULL;
 124
 125	bdev_file = bdev_file_open_by_path(ib_dev->ibd_udev_path, mode, holder,
 126					NULL);
 127	if (IS_ERR(bdev_file)) {
 128		ret = PTR_ERR(bdev_file);
 129		goto out_free_bioset;
 130	}
 131	ib_dev->ibd_bdev_file = bdev_file;
 132	ib_dev->ibd_bd = bd = file_bdev(bdev_file);
 133
 134	q = bdev_get_queue(bd);
 135
 136	dev->dev_attrib.hw_block_size = bdev_logical_block_size(bd);
 137	dev->dev_attrib.hw_max_sectors = mult_frac(queue_max_hw_sectors(q),
 138			SECTOR_SIZE,
 139			dev->dev_attrib.hw_block_size);
 140	dev->dev_attrib.hw_queue_depth = q->nr_requests;
 141
 142	/*
 143	 * Enable write same emulation for IBLOCK and use 0xFFFF as
 144	 * the smaller WRITE_SAME(10) only has a two-byte block count.
 145	 */
 146	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bd);
 147	if (max_write_zeroes_sectors)
 148		dev->dev_attrib.max_write_same_len = max_write_zeroes_sectors;
 149	else
 150		dev->dev_attrib.max_write_same_len = 0xFFFF;
 151
 152	if (bdev_nonrot(bd))
 153		dev->dev_attrib.is_nonrot = 1;
 154
 155	target_configure_write_atomic_from_bdev(&dev->dev_attrib, bd);
 156
 157	bi = bdev_get_integrity(bd);
 158	if (!bi)
 159		return 0;
 160
 161	switch (bi->csum_type) {
 162	case BLK_INTEGRITY_CSUM_IP:
 163		pr_err("IBLOCK export of blk_integrity: %s not supported\n",
 164			blk_integrity_profile_name(bi));
 165		ret = -ENOSYS;
 166		goto out_blkdev_put;
 167	case BLK_INTEGRITY_CSUM_CRC:
 168		if (bi->flags & BLK_INTEGRITY_REF_TAG)
 169			dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE1_PROT;
 170		else
 171			dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE3_PROT;
 172		break;
 173	default:
 174		break;
 175	}
 176
 177	dev->dev_attrib.hw_pi_prot_type = dev->dev_attrib.pi_prot_type;
 178	return 0;
 179
 180out_blkdev_put:
 181	fput(ib_dev->ibd_bdev_file);
 182out_free_bioset:
 183	bioset_exit(&ib_dev->ibd_bio_set);
 184out:
 185	return ret;
 186}
 187
 188static void iblock_dev_call_rcu(struct rcu_head *p)
 189{
 190	struct se_device *dev = container_of(p, struct se_device, rcu_head);
 191	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 192
 193	kfree(ib_dev->ibd_plug);
 194	kfree(ib_dev);
 195}
 196
 197static void iblock_free_device(struct se_device *dev)
 198{
 199	call_rcu(&dev->rcu_head, iblock_dev_call_rcu);
 200}
 201
 202static void iblock_destroy_device(struct se_device *dev)
 203{
 204	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 205
 206	if (ib_dev->ibd_bdev_file)
 207		fput(ib_dev->ibd_bdev_file);
 208	bioset_exit(&ib_dev->ibd_bio_set);
 209}
 210
 211static struct se_dev_plug *iblock_plug_device(struct se_device *se_dev)
 212{
 213	struct iblock_dev *ib_dev = IBLOCK_DEV(se_dev);
 214	struct iblock_dev_plug *ib_dev_plug;
 215
 216	/*
 217	 * Each se_device has a per cpu work this can be run from. We
 218	 * shouldn't have multiple threads on the same cpu calling this
 219	 * at the same time.
 220	 */
 221	ib_dev_plug = &ib_dev->ibd_plug[raw_smp_processor_id()];
 222	if (test_and_set_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags))
 223		return NULL;
 224
 225	blk_start_plug(&ib_dev_plug->blk_plug);
 226	return &ib_dev_plug->se_plug;
 227}
 228
 229static void iblock_unplug_device(struct se_dev_plug *se_plug)
 230{
 231	struct iblock_dev_plug *ib_dev_plug = container_of(se_plug,
 232					struct iblock_dev_plug, se_plug);
 233
 234	blk_finish_plug(&ib_dev_plug->blk_plug);
 235	clear_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags);
 236}
 237
 238static sector_t iblock_get_blocks(struct se_device *dev)
 239{
 240	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 241	u32 block_size = bdev_logical_block_size(ib_dev->ibd_bd);
 242	unsigned long long blocks_long =
 243		div_u64(bdev_nr_bytes(ib_dev->ibd_bd), block_size) - 1;
 244
 245	if (block_size == dev->dev_attrib.block_size)
 246		return blocks_long;
 247
 248	switch (block_size) {
 249	case 4096:
 250		switch (dev->dev_attrib.block_size) {
 251		case 2048:
 252			blocks_long <<= 1;
 253			break;
 254		case 1024:
 255			blocks_long <<= 2;
 256			break;
 257		case 512:
 258			blocks_long <<= 3;
 259			break;
 260		default:
 261			break;
 262		}
 263		break;
 264	case 2048:
 265		switch (dev->dev_attrib.block_size) {
 266		case 4096:
 267			blocks_long >>= 1;
 268			break;
 269		case 1024:
 270			blocks_long <<= 1;
 271			break;
 272		case 512:
 273			blocks_long <<= 2;
 274			break;
 275		default:
 276			break;
 277		}
 278		break;
 279	case 1024:
 280		switch (dev->dev_attrib.block_size) {
 281		case 4096:
 282			blocks_long >>= 2;
 283			break;
 284		case 2048:
 285			blocks_long >>= 1;
 286			break;
 287		case 512:
 288			blocks_long <<= 1;
 289			break;
 290		default:
 291			break;
 292		}
 293		break;
 294	case 512:
 295		switch (dev->dev_attrib.block_size) {
 296		case 4096:
 297			blocks_long >>= 3;
 298			break;
 299		case 2048:
 300			blocks_long >>= 2;
 301			break;
 302		case 1024:
 303			blocks_long >>= 1;
 304			break;
 305		default:
 306			break;
 307		}
 308		break;
 309	default:
 310		break;
 311	}
 312
 313	return blocks_long;
 314}
 315
 316static void iblock_complete_cmd(struct se_cmd *cmd, blk_status_t blk_status)
 317{
 318	struct iblock_req *ibr = cmd->priv;
 319	u8 status;
 320
 321	if (!refcount_dec_and_test(&ibr->pending))
 322		return;
 323
 324	if (blk_status == BLK_STS_RESV_CONFLICT)
 325		status = SAM_STAT_RESERVATION_CONFLICT;
 326	else if (atomic_read(&ibr->ib_bio_err_cnt))
 327		status = SAM_STAT_CHECK_CONDITION;
 328	else
 329		status = SAM_STAT_GOOD;
 330
 331	target_complete_cmd(cmd, status);
 332	kfree(ibr);
 333}
 334
 335static void iblock_bio_done(struct bio *bio)
 336{
 337	struct se_cmd *cmd = bio->bi_private;
 338	struct iblock_req *ibr = cmd->priv;
 339	blk_status_t blk_status = bio->bi_status;
 340
 341	if (bio->bi_status) {
 342		pr_err("bio error: %p,  err: %d\n", bio, bio->bi_status);
 343		/*
 344		 * Bump the ib_bio_err_cnt and release bio.
 345		 */
 346		atomic_inc(&ibr->ib_bio_err_cnt);
 347		smp_mb__after_atomic();
 348	}
 349
 350	bio_put(bio);
 351
 352	iblock_complete_cmd(cmd, blk_status);
 353}
 354
 355static struct bio *iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num,
 356				  blk_opf_t opf)
 357{
 358	struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
 359	struct bio *bio;
 360
 361	/*
 362	 * Only allocate as many vector entries as the bio code allows us to,
 363	 * we'll loop later on until we have handled the whole request.
 364	 */
 365	bio = bio_alloc_bioset(ib_dev->ibd_bd, bio_max_segs(sg_num), opf,
 366			       GFP_NOIO, &ib_dev->ibd_bio_set);
 367	if (!bio) {
 368		pr_err("Unable to allocate memory for bio\n");
 369		return NULL;
 370	}
 371
 372	bio->bi_private = cmd;
 373	bio->bi_end_io = &iblock_bio_done;
 374	bio->bi_iter.bi_sector = lba;
 375
 376	return bio;
 377}
 378
 379static void iblock_submit_bios(struct bio_list *list)
 380{
 381	struct blk_plug plug;
 382	struct bio *bio;
 383	/*
 384	 * The block layer handles nested plugs, so just plug/unplug to handle
 385	 * fabric drivers that didn't support batching and multi bio cmds.
 386	 */
 387	blk_start_plug(&plug);
 388	while ((bio = bio_list_pop(list)))
 389		submit_bio(bio);
 390	blk_finish_plug(&plug);
 391}
 392
 393static void iblock_end_io_flush(struct bio *bio)
 394{
 395	struct se_cmd *cmd = bio->bi_private;
 396
 397	if (bio->bi_status)
 398		pr_err("IBLOCK: cache flush failed: %d\n", bio->bi_status);
 399
 400	if (cmd) {
 401		if (bio->bi_status)
 402			target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION);
 403		else
 404			target_complete_cmd(cmd, SAM_STAT_GOOD);
 405	}
 406
 407	bio_put(bio);
 408}
 409
 410/*
 411 * Implement SYCHRONIZE CACHE.  Note that we can't handle lba ranges and must
 412 * always flush the whole cache.
 413 */
 414static sense_reason_t
 415iblock_execute_sync_cache(struct se_cmd *cmd)
 416{
 417	struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
 418	int immed = (cmd->t_task_cdb[1] & 0x2);
 419	struct bio *bio;
 420
 421	/*
 422	 * If the Immediate bit is set, queue up the GOOD response
 423	 * for this SYNCHRONIZE_CACHE op.
 424	 */
 425	if (immed)
 426		target_complete_cmd(cmd, SAM_STAT_GOOD);
 427
 428	bio = bio_alloc(ib_dev->ibd_bd, 0, REQ_OP_WRITE | REQ_PREFLUSH,
 429			GFP_KERNEL);
 430	bio->bi_end_io = iblock_end_io_flush;
 431	if (!immed)
 432		bio->bi_private = cmd;
 433	submit_bio(bio);
 434	return 0;
 435}
 436
 437static sense_reason_t
 438iblock_execute_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
 439{
 440	struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
 441	struct se_device *dev = cmd->se_dev;
 442	int ret;
 443
 444	ret = blkdev_issue_discard(bdev,
 445				   target_to_linux_sector(dev, lba),
 446				   target_to_linux_sector(dev,  nolb),
 447				   GFP_KERNEL);
 448	if (ret < 0) {
 449		pr_err("blkdev_issue_discard() failed: %d\n", ret);
 450		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 451	}
 452
 453	return 0;
 454}
 455
 456static sense_reason_t
 457iblock_execute_zero_out(struct block_device *bdev, struct se_cmd *cmd)
 458{
 459	struct se_device *dev = cmd->se_dev;
 460	struct scatterlist *sg = &cmd->t_data_sg[0];
 461	unsigned char *buf, *not_zero;
 462	int ret;
 463
 464	buf = kmap(sg_page(sg)) + sg->offset;
 465	if (!buf)
 466		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 467	/*
 468	 * Fall back to block_execute_write_same() slow-path if
 469	 * incoming WRITE_SAME payload does not contain zeros.
 470	 */
 471	not_zero = memchr_inv(buf, 0x00, cmd->data_length);
 472	kunmap(sg_page(sg));
 473
 474	if (not_zero)
 475		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 476
 477	ret = blkdev_issue_zeroout(bdev,
 478				target_to_linux_sector(dev, cmd->t_task_lba),
 479				target_to_linux_sector(dev,
 480					sbc_get_write_same_sectors(cmd)),
 481				GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
 482	if (ret)
 483		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 484
 485	target_complete_cmd(cmd, SAM_STAT_GOOD);
 486	return 0;
 487}
 488
 489static sense_reason_t
 490iblock_execute_write_same(struct se_cmd *cmd)
 491{
 492	struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
 493	struct iblock_req *ibr;
 494	struct scatterlist *sg;
 495	struct bio *bio;
 496	struct bio_list list;
 497	struct se_device *dev = cmd->se_dev;
 498	sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
 499	sector_t sectors = target_to_linux_sector(dev,
 500					sbc_get_write_same_sectors(cmd));
 501
 502	if (cmd->prot_op) {
 503		pr_err("WRITE_SAME: Protection information with IBLOCK"
 504		       " backends not supported\n");
 505		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 506	}
 507
 508	if (!cmd->t_data_nents)
 509		return TCM_INVALID_CDB_FIELD;
 510
 511	sg = &cmd->t_data_sg[0];
 512
 513	if (cmd->t_data_nents > 1 ||
 514	    sg->length != cmd->se_dev->dev_attrib.block_size) {
 515		pr_err("WRITE_SAME: Illegal SGL t_data_nents: %u length: %u"
 516			" block_size: %u\n", cmd->t_data_nents, sg->length,
 517			cmd->se_dev->dev_attrib.block_size);
 518		return TCM_INVALID_CDB_FIELD;
 519	}
 520
 521	if (bdev_write_zeroes_sectors(bdev)) {
 522		if (!iblock_execute_zero_out(bdev, cmd))
 523			return 0;
 524	}
 525
 526	ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
 527	if (!ibr)
 528		goto fail;
 529	cmd->priv = ibr;
 530
 531	bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE);
 532	if (!bio)
 533		goto fail_free_ibr;
 534
 535	bio_list_init(&list);
 536	bio_list_add(&list, bio);
 537
 538	refcount_set(&ibr->pending, 1);
 539
 540	while (sectors) {
 541		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
 542				!= sg->length) {
 543
 544			bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE);
 545			if (!bio)
 546				goto fail_put_bios;
 547
 548			refcount_inc(&ibr->pending);
 549			bio_list_add(&list, bio);
 550		}
 551
 552		/* Always in 512 byte units for Linux/Block */
 553		block_lba += sg->length >> SECTOR_SHIFT;
 554		sectors -= sg->length >> SECTOR_SHIFT;
 555	}
 556
 557	iblock_submit_bios(&list);
 558	return 0;
 559
 560fail_put_bios:
 561	while ((bio = bio_list_pop(&list)))
 562		bio_put(bio);
 563fail_free_ibr:
 564	kfree(ibr);
 565fail:
 566	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 567}
 568
 569enum {
 570	Opt_udev_path, Opt_readonly, Opt_force, Opt_exclusive, Opt_err,
 571};
 572
 573static match_table_t tokens = {
 574	{Opt_udev_path, "udev_path=%s"},
 575	{Opt_readonly, "readonly=%d"},
 576	{Opt_force, "force=%d"},
 577	{Opt_exclusive, "exclusive=%d"},
 578	{Opt_err, NULL}
 579};
 580
 581static ssize_t iblock_set_configfs_dev_params(struct se_device *dev,
 582		const char *page, ssize_t count)
 583{
 584	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 585	char *orig, *ptr, *arg_p, *opts;
 586	substring_t args[MAX_OPT_ARGS];
 587	int ret = 0, token, tmp_exclusive;
 588	unsigned long tmp_readonly;
 589
 590	opts = kstrdup(page, GFP_KERNEL);
 591	if (!opts)
 592		return -ENOMEM;
 593
 594	orig = opts;
 595
 596	while ((ptr = strsep(&opts, ",\n")) != NULL) {
 597		if (!*ptr)
 598			continue;
 599
 600		token = match_token(ptr, tokens, args);
 601		switch (token) {
 602		case Opt_udev_path:
 603			if (ib_dev->ibd_bd) {
 604				pr_err("Unable to set udev_path= while"
 605					" ib_dev->ibd_bd exists\n");
 606				ret = -EEXIST;
 607				goto out;
 608			}
 609			if (match_strlcpy(ib_dev->ibd_udev_path, &args[0],
 610				SE_UDEV_PATH_LEN) == 0) {
 611				ret = -EINVAL;
 612				break;
 613			}
 614			pr_debug("IBLOCK: Referencing UDEV path: %s\n",
 615					ib_dev->ibd_udev_path);
 616			ib_dev->ibd_flags |= IBDF_HAS_UDEV_PATH;
 617			break;
 618		case Opt_readonly:
 619			arg_p = match_strdup(&args[0]);
 620			if (!arg_p) {
 621				ret = -ENOMEM;
 622				break;
 623			}
 624			ret = kstrtoul(arg_p, 0, &tmp_readonly);
 625			kfree(arg_p);
 626			if (ret < 0) {
 627				pr_err("kstrtoul() failed for"
 628						" readonly=\n");
 629				goto out;
 630			}
 631			ib_dev->ibd_readonly = tmp_readonly;
 632			pr_debug("IBLOCK: readonly: %d\n", ib_dev->ibd_readonly);
 633			break;
 634		case Opt_exclusive:
 635			arg_p = match_strdup(&args[0]);
 636			if (!arg_p) {
 637				ret = -ENOMEM;
 638				break;
 639			}
 640			ret = kstrtoint(arg_p, 0, &tmp_exclusive);
 641			kfree(arg_p);
 642			if (ret < 0) {
 643				pr_err("kstrtoul() failed for exclusive=\n");
 644				goto out;
 645			}
 646			ib_dev->ibd_exclusive = tmp_exclusive;
 647			pr_debug("IBLOCK: exclusive: %d\n",
 648				 ib_dev->ibd_exclusive);
 649			break;
 650		case Opt_force:
 651			break;
 652		default:
 653			break;
 654		}
 655	}
 656
 657out:
 658	kfree(orig);
 659	return (!ret) ? count : ret;
 660}
 661
 662static ssize_t iblock_show_configfs_dev_params(struct se_device *dev, char *b)
 663{
 664	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 665	struct block_device *bd = ib_dev->ibd_bd;
 666	ssize_t bl = 0;
 667
 668	if (bd)
 669		bl += sprintf(b + bl, "iBlock device: %pg", bd);
 670	if (ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)
 671		bl += sprintf(b + bl, "  UDEV PATH: %s",
 672				ib_dev->ibd_udev_path);
 673	bl += sprintf(b + bl, "  readonly: %d\n", ib_dev->ibd_readonly);
 674	bl += sprintf(b + bl, "  exclusive: %d\n", ib_dev->ibd_exclusive);
 675
 676	bl += sprintf(b + bl, "        ");
 677	if (bd) {
 678		bl += sprintf(b + bl, "Major: %d Minor: %d  %s\n",
 679			MAJOR(bd->bd_dev), MINOR(bd->bd_dev),
 680			"CLAIMED: IBLOCK");
 681	} else {
 682		bl += sprintf(b + bl, "Major: 0 Minor: 0\n");
 683	}
 684
 685	return bl;
 686}
 687
 688static int
 689iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio,
 690		 struct sg_mapping_iter *miter)
 691{
 692	struct se_device *dev = cmd->se_dev;
 693	struct blk_integrity *bi;
 694	struct bio_integrity_payload *bip;
 695	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 696	int rc;
 697	size_t resid, len;
 698
 699	bi = bdev_get_integrity(ib_dev->ibd_bd);
 700	if (!bi) {
 701		pr_err("Unable to locate bio_integrity\n");
 702		return -ENODEV;
 703	}
 704
 705	bip = bio_integrity_alloc(bio, GFP_NOIO, bio_max_segs(cmd->t_prot_nents));
 706	if (IS_ERR(bip)) {
 707		pr_err("Unable to allocate bio_integrity_payload\n");
 708		return PTR_ERR(bip);
 709	}
 710
 711	/* virtual start sector must be in integrity interval units */
 712	bip_set_seed(bip, bio->bi_iter.bi_sector >>
 713				  (bi->interval_exp - SECTOR_SHIFT));
 714
 715	pr_debug("IBLOCK BIP Size: %u Sector: %llu\n", bip->bip_iter.bi_size,
 716		 (unsigned long long)bip->bip_iter.bi_sector);
 717
 718	resid = bio_integrity_bytes(bi, bio_sectors(bio));
 719	while (resid > 0 && sg_miter_next(miter)) {
 720
 721		len = min_t(size_t, miter->length, resid);
 722		rc = bio_integrity_add_page(bio, miter->page, len,
 723					    offset_in_page(miter->addr));
 724		if (rc != len) {
 725			pr_err("bio_integrity_add_page() failed; %d\n", rc);
 726			sg_miter_stop(miter);
 727			return -ENOMEM;
 728		}
 729
 730		pr_debug("Added bio integrity page: %p length: %zu offset: %lu\n",
 731			  miter->page, len, offset_in_page(miter->addr));
 732
 733		resid -= len;
 734		if (len < miter->length)
 735			miter->consumed -= miter->length - len;
 736	}
 737	sg_miter_stop(miter);
 738
 739	return 0;
 740}
 741
 742static sense_reason_t
 743iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 744		  enum dma_data_direction data_direction)
 745{
 746	struct se_device *dev = cmd->se_dev;
 747	sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
 748	struct iblock_req *ibr;
 749	struct bio *bio;
 750	struct bio_list list;
 751	struct scatterlist *sg;
 752	u32 sg_num = sgl_nents;
 753	blk_opf_t opf;
 754	unsigned bio_cnt;
 755	int i, rc;
 756	struct sg_mapping_iter prot_miter;
 757	unsigned int miter_dir;
 758
 759	if (data_direction == DMA_TO_DEVICE) {
 760		struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 761
 762		/*
 763		 * Set bits to indicate WRITE_ODIRECT so we are not throttled
 764		 * by WBT.
 765		 */
 766		opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
 767		/*
 768		 * Force writethrough using REQ_FUA if a volatile write cache
 769		 * is not enabled, or if initiator set the Force Unit Access bit.
 770		 */
 771		miter_dir = SG_MITER_TO_SG;
 772		if (bdev_fua(ib_dev->ibd_bd)) {
 773			if (cmd->se_cmd_flags & SCF_FUA)
 774				opf |= REQ_FUA;
 775			else if (!bdev_write_cache(ib_dev->ibd_bd))
 776				opf |= REQ_FUA;
 777		}
 778
 779		if (cmd->se_cmd_flags & SCF_ATOMIC)
 780			opf |= REQ_ATOMIC;
 781	} else {
 782		opf = REQ_OP_READ;
 783		miter_dir = SG_MITER_FROM_SG;
 784	}
 785
 786	ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
 787	if (!ibr)
 788		goto fail;
 789	cmd->priv = ibr;
 790
 791	if (!sgl_nents) {
 792		refcount_set(&ibr->pending, 1);
 793		iblock_complete_cmd(cmd, BLK_STS_OK);
 794		return 0;
 795	}
 796
 797	bio = iblock_get_bio(cmd, block_lba, sgl_nents, opf);
 798	if (!bio)
 799		goto fail_free_ibr;
 800
 801	bio_list_init(&list);
 802	bio_list_add(&list, bio);
 803
 804	refcount_set(&ibr->pending, 2);
 805	bio_cnt = 1;
 806
 807	if (cmd->prot_type && dev->dev_attrib.pi_prot_type)
 808		sg_miter_start(&prot_miter, cmd->t_prot_sg, cmd->t_prot_nents,
 809			       miter_dir);
 810
 811	for_each_sg(sgl, sg, sgl_nents, i) {
 812		/*
 813		 * XXX: if the length the device accepts is shorter than the
 814		 *	length of the S/G list entry this will cause and
 815		 *	endless loop.  Better hope no driver uses huge pages.
 816		 */
 817		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
 818				!= sg->length) {
 819			if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
 820				rc = iblock_alloc_bip(cmd, bio, &prot_miter);
 821				if (rc)
 822					goto fail_put_bios;
 823			}
 824
 825			if (bio_cnt >= IBLOCK_MAX_BIO_PER_TASK) {
 826				iblock_submit_bios(&list);
 827				bio_cnt = 0;
 828			}
 829
 830			bio = iblock_get_bio(cmd, block_lba, sg_num, opf);
 831			if (!bio)
 832				goto fail_put_bios;
 833
 834			refcount_inc(&ibr->pending);
 835			bio_list_add(&list, bio);
 836			bio_cnt++;
 837		}
 838
 839		/* Always in 512 byte units for Linux/Block */
 840		block_lba += sg->length >> SECTOR_SHIFT;
 841		sg_num--;
 842	}
 843
 844	if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
 845		rc = iblock_alloc_bip(cmd, bio, &prot_miter);
 846		if (rc)
 847			goto fail_put_bios;
 848	}
 849
 850	iblock_submit_bios(&list);
 851	iblock_complete_cmd(cmd, BLK_STS_OK);
 852	return 0;
 853
 854fail_put_bios:
 855	while ((bio = bio_list_pop(&list)))
 856		bio_put(bio);
 857fail_free_ibr:
 858	kfree(ibr);
 859fail:
 860	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 861}
 862
 863static sense_reason_t iblock_execute_pr_out(struct se_cmd *cmd, u8 sa, u64 key,
 864					    u64 sa_key, u8 type, bool aptpl)
 865{
 866	struct se_device *dev = cmd->se_dev;
 867	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 868	struct block_device *bdev = ib_dev->ibd_bd;
 869	const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops;
 870	int ret;
 871
 872	if (!ops) {
 873		pr_err("Block device does not support pr_ops but iblock device has been configured for PR passthrough.\n");
 874		return TCM_UNSUPPORTED_SCSI_OPCODE;
 875	}
 876
 877	switch (sa) {
 878	case PRO_REGISTER:
 879	case PRO_REGISTER_AND_IGNORE_EXISTING_KEY:
 880		if (!ops->pr_register) {
 881			pr_err("block device does not support pr_register.\n");
 882			return TCM_UNSUPPORTED_SCSI_OPCODE;
 883		}
 884
 885		/* The block layer pr ops always enables aptpl */
 886		if (!aptpl)
 887			pr_info("APTPL not set by initiator, but will be used.\n");
 888
 889		ret = ops->pr_register(bdev, key, sa_key,
 890				sa == PRO_REGISTER ? 0 : PR_FL_IGNORE_KEY);
 891		break;
 892	case PRO_RESERVE:
 893		if (!ops->pr_reserve) {
 894			pr_err("block_device does not support pr_reserve.\n");
 895			return TCM_UNSUPPORTED_SCSI_OPCODE;
 896		}
 897
 898		ret = ops->pr_reserve(bdev, key, scsi_pr_type_to_block(type), 0);
 899		break;
 900	case PRO_CLEAR:
 901		if (!ops->pr_clear) {
 902			pr_err("block_device does not support pr_clear.\n");
 903			return TCM_UNSUPPORTED_SCSI_OPCODE;
 904		}
 905
 906		ret = ops->pr_clear(bdev, key);
 907		break;
 908	case PRO_PREEMPT:
 909	case PRO_PREEMPT_AND_ABORT:
 910		if (!ops->pr_clear) {
 911			pr_err("block_device does not support pr_preempt.\n");
 912			return TCM_UNSUPPORTED_SCSI_OPCODE;
 913		}
 914
 915		ret = ops->pr_preempt(bdev, key, sa_key,
 916				      scsi_pr_type_to_block(type),
 917				      sa == PRO_PREEMPT_AND_ABORT);
 918		break;
 919	case PRO_RELEASE:
 920		if (!ops->pr_clear) {
 921			pr_err("block_device does not support pr_pclear.\n");
 922			return TCM_UNSUPPORTED_SCSI_OPCODE;
 923		}
 924
 925		ret = ops->pr_release(bdev, key, scsi_pr_type_to_block(type));
 926		break;
 927	default:
 928		pr_err("Unknown PERSISTENT_RESERVE_OUT SA: 0x%02x\n", sa);
 929		return TCM_UNSUPPORTED_SCSI_OPCODE;
 930	}
 931
 932	if (!ret)
 933		return TCM_NO_SENSE;
 934	else if (ret == PR_STS_RESERVATION_CONFLICT)
 935		return TCM_RESERVATION_CONFLICT;
 936	else
 937		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 938}
 939
 940static void iblock_pr_report_caps(unsigned char *param_data)
 941{
 942	u16 len = 8;
 943
 944	put_unaligned_be16(len, &param_data[0]);
 945	/*
 946	 * When using the pr_ops passthrough method we only support exporting
 947	 * the device through one target port because from the backend module
 948	 * level we can't see the target port config. As a result we only
 949	 * support registration directly from the I_T nexus the cmd is sent
 950	 * through and do not set ATP_C here.
 951	 *
 952	 * The block layer pr_ops do not support passing in initiators so
 953	 * we don't set SIP_C here.
 954	 */
 955	/* PTPL_C: Persistence across Target Power Loss bit */
 956	param_data[2] |= 0x01;
 957	/*
 958	 * We are filling in the PERSISTENT RESERVATION TYPE MASK below, so
 959	 * set the TMV: Task Mask Valid bit.
 960	 */
 961	param_data[3] |= 0x80;
 962	/*
 963	 * Change ALLOW COMMANDs to 0x20 or 0x40 later from Table 166
 964	 */
 965	param_data[3] |= 0x10; /* ALLOW COMMANDs field 001b */
 966	/*
 967	 * PTPL_A: Persistence across Target Power Loss Active bit. The block
 968	 * layer pr ops always enables this so report it active.
 969	 */
 970	param_data[3] |= 0x01;
 971	/*
 972	 * Setup the PERSISTENT RESERVATION TYPE MASK from Table 212 spc4r37.
 973	 */
 974	param_data[4] |= 0x80; /* PR_TYPE_EXCLUSIVE_ACCESS_ALLREG */
 975	param_data[4] |= 0x40; /* PR_TYPE_EXCLUSIVE_ACCESS_REGONLY */
 976	param_data[4] |= 0x20; /* PR_TYPE_WRITE_EXCLUSIVE_REGONLY */
 977	param_data[4] |= 0x08; /* PR_TYPE_EXCLUSIVE_ACCESS */
 978	param_data[4] |= 0x02; /* PR_TYPE_WRITE_EXCLUSIVE */
 979	param_data[5] |= 0x01; /* PR_TYPE_EXCLUSIVE_ACCESS_ALLREG */
 980}
 981
 982static sense_reason_t iblock_pr_read_keys(struct se_cmd *cmd,
 983					  unsigned char *param_data)
 984{
 985	struct se_device *dev = cmd->se_dev;
 986	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 987	struct block_device *bdev = ib_dev->ibd_bd;
 988	const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops;
 989	int i, len, paths, data_offset;
 990	struct pr_keys *keys;
 991	sense_reason_t ret;
 992
 993	if (!ops) {
 994		pr_err("Block device does not support pr_ops but iblock device has been configured for PR passthrough.\n");
 995		return TCM_UNSUPPORTED_SCSI_OPCODE;
 996	}
 997
 998	if (!ops->pr_read_keys) {
 999		pr_err("Block device does not support read_keys.\n");
1000		return TCM_UNSUPPORTED_SCSI_OPCODE;
1001	}
1002
1003	/*
1004	 * We don't know what's under us, but dm-multipath will register every
1005	 * path with the same key, so start off with enough space for 16 paths.
1006	 * which is not a lot of memory and should normally be enough.
1007	 */
1008	paths = 16;
1009retry:
1010	len = 8 * paths;
1011	keys = kzalloc(sizeof(*keys) + len, GFP_KERNEL);
1012	if (!keys)
1013		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1014
1015	keys->num_keys = paths;
1016	if (!ops->pr_read_keys(bdev, keys)) {
1017		if (keys->num_keys > paths) {
1018			kfree(keys);
1019			paths *= 2;
1020			goto retry;
1021		}
1022	} else {
1023		ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1024		goto free_keys;
1025	}
1026
1027	ret = TCM_NO_SENSE;
1028
1029	put_unaligned_be32(keys->generation, &param_data[0]);
1030	if (!keys->num_keys) {
1031		put_unaligned_be32(0, &param_data[4]);
1032		goto free_keys;
1033	}
1034
1035	put_unaligned_be32(8 * keys->num_keys, &param_data[4]);
1036
1037	data_offset = 8;
1038	for (i = 0; i < keys->num_keys; i++) {
1039		if (data_offset + 8 > cmd->data_length)
1040			break;
1041
1042		put_unaligned_be64(keys->keys[i], &param_data[data_offset]);
1043		data_offset += 8;
1044	}
1045
1046free_keys:
1047	kfree(keys);
1048	return ret;
1049}
1050
1051static sense_reason_t iblock_pr_read_reservation(struct se_cmd *cmd,
1052						 unsigned char *param_data)
1053{
1054	struct se_device *dev = cmd->se_dev;
1055	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1056	struct block_device *bdev = ib_dev->ibd_bd;
1057	const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops;
1058	struct pr_held_reservation rsv = { };
1059
1060	if (!ops) {
1061		pr_err("Block device does not support pr_ops but iblock device has been configured for PR passthrough.\n");
1062		return TCM_UNSUPPORTED_SCSI_OPCODE;
1063	}
1064
1065	if (!ops->pr_read_reservation) {
1066		pr_err("Block device does not support read_keys.\n");
1067		return TCM_UNSUPPORTED_SCSI_OPCODE;
1068	}
1069
1070	if (ops->pr_read_reservation(bdev, &rsv))
1071		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1072
1073	put_unaligned_be32(rsv.generation, &param_data[0]);
1074	if (!block_pr_type_to_scsi(rsv.type)) {
1075		put_unaligned_be32(0, &param_data[4]);
1076		return TCM_NO_SENSE;
1077	}
1078
1079	put_unaligned_be32(16, &param_data[4]);
1080
1081	if (cmd->data_length < 16)
1082		return TCM_NO_SENSE;
1083	put_unaligned_be64(rsv.key, &param_data[8]);
1084
1085	if (cmd->data_length < 22)
1086		return TCM_NO_SENSE;
1087	param_data[21] = block_pr_type_to_scsi(rsv.type);
1088
1089	return TCM_NO_SENSE;
1090}
1091
1092static sense_reason_t iblock_execute_pr_in(struct se_cmd *cmd, u8 sa,
1093					   unsigned char *param_data)
1094{
1095	sense_reason_t ret = TCM_NO_SENSE;
1096
1097	switch (sa) {
1098	case PRI_REPORT_CAPABILITIES:
1099		iblock_pr_report_caps(param_data);
1100		break;
1101	case PRI_READ_KEYS:
1102		ret = iblock_pr_read_keys(cmd, param_data);
1103		break;
1104	case PRI_READ_RESERVATION:
1105		ret = iblock_pr_read_reservation(cmd, param_data);
1106		break;
1107	default:
1108		pr_err("Unknown PERSISTENT_RESERVE_IN SA: 0x%02x\n", sa);
1109		return TCM_UNSUPPORTED_SCSI_OPCODE;
1110	}
1111
1112	return ret;
1113}
1114
1115static sector_t iblock_get_alignment_offset_lbas(struct se_device *dev)
1116{
1117	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1118	struct block_device *bd = ib_dev->ibd_bd;
1119	int ret;
1120
1121	ret = bdev_alignment_offset(bd);
1122	if (ret == -1)
1123		return 0;
1124
1125	/* convert offset-bytes to offset-lbas */
1126	return ret / bdev_logical_block_size(bd);
1127}
1128
1129static unsigned int iblock_get_lbppbe(struct se_device *dev)
1130{
1131	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1132	struct block_device *bd = ib_dev->ibd_bd;
1133	unsigned int logs_per_phys =
1134		bdev_physical_block_size(bd) / bdev_logical_block_size(bd);
1135
1136	return ilog2(logs_per_phys);
1137}
1138
1139static unsigned int iblock_get_io_min(struct se_device *dev)
1140{
1141	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1142	struct block_device *bd = ib_dev->ibd_bd;
1143
1144	return bdev_io_min(bd);
1145}
1146
1147static unsigned int iblock_get_io_opt(struct se_device *dev)
1148{
1149	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
1150	struct block_device *bd = ib_dev->ibd_bd;
1151
1152	return bdev_io_opt(bd);
1153}
1154
1155static struct exec_cmd_ops iblock_exec_cmd_ops = {
1156	.execute_rw		= iblock_execute_rw,
1157	.execute_sync_cache	= iblock_execute_sync_cache,
1158	.execute_write_same	= iblock_execute_write_same,
1159	.execute_unmap		= iblock_execute_unmap,
1160	.execute_pr_out		= iblock_execute_pr_out,
1161	.execute_pr_in		= iblock_execute_pr_in,
1162};
1163
1164static sense_reason_t
1165iblock_parse_cdb(struct se_cmd *cmd)
1166{
1167	return sbc_parse_cdb(cmd, &iblock_exec_cmd_ops);
1168}
1169
1170static bool iblock_get_write_cache(struct se_device *dev)
1171{
1172	return bdev_write_cache(IBLOCK_DEV(dev)->ibd_bd);
1173}
1174
1175static const struct target_backend_ops iblock_ops = {
1176	.name			= "iblock",
1177	.inquiry_prod		= "IBLOCK",
1178	.transport_flags_changeable = TRANSPORT_FLAG_PASSTHROUGH_PGR,
1179	.inquiry_rev		= IBLOCK_VERSION,
1180	.owner			= THIS_MODULE,
1181	.attach_hba		= iblock_attach_hba,
1182	.detach_hba		= iblock_detach_hba,
1183	.alloc_device		= iblock_alloc_device,
1184	.configure_device	= iblock_configure_device,
1185	.destroy_device		= iblock_destroy_device,
1186	.free_device		= iblock_free_device,
1187	.configure_unmap	= iblock_configure_unmap,
1188	.plug_device		= iblock_plug_device,
1189	.unplug_device		= iblock_unplug_device,
1190	.parse_cdb		= iblock_parse_cdb,
1191	.set_configfs_dev_params = iblock_set_configfs_dev_params,
1192	.show_configfs_dev_params = iblock_show_configfs_dev_params,
1193	.get_device_type	= sbc_get_device_type,
1194	.get_blocks		= iblock_get_blocks,
1195	.get_alignment_offset_lbas = iblock_get_alignment_offset_lbas,
1196	.get_lbppbe		= iblock_get_lbppbe,
1197	.get_io_min		= iblock_get_io_min,
1198	.get_io_opt		= iblock_get_io_opt,
1199	.get_write_cache	= iblock_get_write_cache,
1200	.tb_dev_attrib_attrs	= sbc_attrib_attrs,
1201};
1202
1203static int __init iblock_module_init(void)
1204{
1205	return transport_backend_register(&iblock_ops);
1206}
1207
1208static void __exit iblock_module_exit(void)
1209{
1210	target_backend_unregister(&iblock_ops);
1211}
1212
1213MODULE_DESCRIPTION("TCM IBLOCK subsystem plugin");
1214MODULE_AUTHOR("nab@Linux-iSCSI.org");
1215MODULE_LICENSE("GPL");
1216
1217module_init(iblock_module_init);
1218module_exit(iblock_module_exit);