main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
   4 * Shaohua Li <shli@fb.com>
   5 */
   6#include <linux/module.h>
   7
   8#include <linux/moduleparam.h>
   9#include <linux/sched.h>
  10#include <linux/fs.h>
  11#include <linux/init.h>
  12#include "null_blk.h"
  13
  14#undef pr_fmt
  15#define pr_fmt(fmt)	"null_blk: " fmt
  16
  17#define FREE_BATCH		16
  18
  19#define TICKS_PER_SEC		50ULL
  20#define TIMER_INTERVAL		(NSEC_PER_SEC / TICKS_PER_SEC)
  21
  22#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  23static DECLARE_FAULT_ATTR(null_timeout_attr);
  24static DECLARE_FAULT_ATTR(null_requeue_attr);
  25static DECLARE_FAULT_ATTR(null_init_hctx_attr);
  26#endif
  27
  28static inline u64 mb_per_tick(int mbps)
  29{
  30	return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
  31}
  32
  33/*
  34 * Status flags for nullb_device.
  35 *
  36 * CONFIGURED:	Device has been configured and turned on. Cannot reconfigure.
  37 * UP:		Device is currently on and visible in userspace.
  38 * THROTTLED:	Device is being throttled.
  39 * CACHE:	Device is using a write-back cache.
  40 */
  41enum nullb_device_flags {
  42	NULLB_DEV_FL_CONFIGURED	= 0,
  43	NULLB_DEV_FL_UP		= 1,
  44	NULLB_DEV_FL_THROTTLED	= 2,
  45	NULLB_DEV_FL_CACHE	= 3,
  46};
  47
  48#define MAP_SZ		((PAGE_SIZE >> SECTOR_SHIFT) + 2)
  49/*
  50 * nullb_page is a page in memory for nullb devices.
  51 *
  52 * @page:	The page holding the data.
  53 * @bitmap:	The bitmap represents which sector in the page has data.
  54 *		Each bit represents one block size. For example, sector 8
  55 *		will use the 7th bit
  56 * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
  57 * page is being flushing to storage. FREE means the cache page is freed and
  58 * should be skipped from flushing to storage. Please see
  59 * null_make_cache_space
  60 */
  61struct nullb_page {
  62	struct page *page;
  63	DECLARE_BITMAP(bitmap, MAP_SZ);
  64};
  65#define NULLB_PAGE_LOCK (MAP_SZ - 1)
  66#define NULLB_PAGE_FREE (MAP_SZ - 2)
  67
  68static LIST_HEAD(nullb_list);
  69static struct mutex lock;
  70static int null_major;
  71static DEFINE_IDA(nullb_indexes);
  72static struct blk_mq_tag_set tag_set;
  73
  74enum {
  75	NULL_IRQ_NONE		= 0,
  76	NULL_IRQ_SOFTIRQ	= 1,
  77	NULL_IRQ_TIMER		= 2,
  78};
  79
  80static bool g_virt_boundary;
  81module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
  82MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
  83
  84static int g_no_sched;
  85module_param_named(no_sched, g_no_sched, int, 0444);
  86MODULE_PARM_DESC(no_sched, "No io scheduler");
  87
  88static int g_submit_queues = 1;
  89module_param_named(submit_queues, g_submit_queues, int, 0444);
  90MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  91
  92static int g_poll_queues = 1;
  93module_param_named(poll_queues, g_poll_queues, int, 0444);
  94MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
  95
  96static int g_home_node = NUMA_NO_NODE;
  97module_param_named(home_node, g_home_node, int, 0444);
  98MODULE_PARM_DESC(home_node, "Home node for the device");
  99
 100#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 101/*
 102 * For more details about fault injection, please refer to
 103 * Documentation/fault-injection/fault-injection.rst.
 104 */
 105static char g_timeout_str[80];
 106module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
 107MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
 108
 109static char g_requeue_str[80];
 110module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
 111MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
 112
 113static char g_init_hctx_str[80];
 114module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
 115MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
 116#endif
 117
 118/*
 119 * Historic queue modes.
 120 *
 121 * These days nothing but NULL_Q_MQ is actually supported, but we keep it the
 122 * enum for error reporting.
 123 */
 124enum {
 125	NULL_Q_BIO	= 0,
 126	NULL_Q_RQ	= 1,
 127	NULL_Q_MQ	= 2,
 128};
 129
 130static int g_queue_mode = NULL_Q_MQ;
 131
 132static int null_param_store_val(const char *str, int *val, int min, int max)
 133{
 134	int ret, new_val;
 135
 136	ret = kstrtoint(str, 10, &new_val);
 137	if (ret)
 138		return -EINVAL;
 139
 140	if (new_val < min || new_val > max)
 141		return -EINVAL;
 142
 143	*val = new_val;
 144	return 0;
 145}
 146
 147static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
 148{
 149	return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
 150}
 151
 152static const struct kernel_param_ops null_queue_mode_param_ops = {
 153	.set	= null_set_queue_mode,
 154	.get	= param_get_int,
 155};
 156
 157device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
 158MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 159
 160static int g_gb = 250;
 161module_param_named(gb, g_gb, int, 0444);
 162MODULE_PARM_DESC(gb, "Size in GB");
 163
 164static int g_bs = 512;
 165module_param_named(bs, g_bs, int, 0444);
 166MODULE_PARM_DESC(bs, "Block size (in bytes)");
 167
 168static int g_max_sectors;
 169module_param_named(max_sectors, g_max_sectors, int, 0444);
 170MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
 171
 172static unsigned int nr_devices = 1;
 173module_param(nr_devices, uint, 0444);
 174MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 175
 176static bool g_blocking;
 177module_param_named(blocking, g_blocking, bool, 0444);
 178MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 179
 180static bool g_shared_tags;
 181module_param_named(shared_tags, g_shared_tags, bool, 0444);
 182MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 183
 184static bool g_shared_tag_bitmap;
 185module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
 186MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
 187
 188static int g_irqmode = NULL_IRQ_SOFTIRQ;
 189
 190static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 191{
 192	return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
 193					NULL_IRQ_TIMER);
 194}
 195
 196static const struct kernel_param_ops null_irqmode_param_ops = {
 197	.set	= null_set_irqmode,
 198	.get	= param_get_int,
 199};
 200
 201device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
 202MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 203
 204static unsigned long g_completion_nsec = 10000;
 205module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
 206MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 207
 208static int g_hw_queue_depth = 64;
 209module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
 210MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 211
 212static bool g_use_per_node_hctx;
 213module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
 214MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 215
 216static bool g_memory_backed;
 217module_param_named(memory_backed, g_memory_backed, bool, 0444);
 218MODULE_PARM_DESC(memory_backed, "Create a memory-backed block device. Default: false");
 219
 220static bool g_discard;
 221module_param_named(discard, g_discard, bool, 0444);
 222MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed null_blk device). Default: false");
 223
 224static unsigned long g_cache_size;
 225module_param_named(cache_size, g_cache_size, ulong, 0444);
 226MODULE_PARM_DESC(cache_size, "Cache size in MiB for memory-backed device. Default: 0 (none)");
 227
 228static bool g_fua = true;
 229module_param_named(fua, g_fua, bool, 0444);
 230MODULE_PARM_DESC(fua, "Enable/disable FUA support when cache_size is used. Default: true");
 231
 232static unsigned int g_mbps;
 233module_param_named(mbps, g_mbps, uint, 0444);
 234MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)");
 235
 236static bool g_zoned;
 237module_param_named(zoned, g_zoned, bool, S_IRUGO);
 238MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
 239
 240static unsigned long g_zone_size = 256;
 241module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
 242MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
 243
 244static unsigned long g_zone_capacity;
 245module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
 246MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
 247
 248static unsigned int g_zone_nr_conv;
 249module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
 250MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
 251
 252static unsigned int g_zone_max_open;
 253module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
 254MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
 255
 256static unsigned int g_zone_max_active;
 257module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
 258MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
 259
 260static int g_zone_append_max_sectors = INT_MAX;
 261module_param_named(zone_append_max_sectors, g_zone_append_max_sectors, int, 0444);
 262MODULE_PARM_DESC(zone_append_max_sectors,
 263		 "Maximum size of a zone append command (in 512B sectors). Specify 0 for zone append emulation");
 264
 265static bool g_zone_full;
 266module_param_named(zone_full, g_zone_full, bool, S_IRUGO);
 267MODULE_PARM_DESC(zone_full, "Initialize the sequential write required zones of a zoned device to be full. Default: false");
 268
 269static bool g_rotational;
 270module_param_named(rotational, g_rotational, bool, S_IRUGO);
 271MODULE_PARM_DESC(rotational, "Set the rotational feature for the device. Default: false");
 272
 273static struct nullb_device *null_alloc_dev(void);
 274static void null_free_dev(struct nullb_device *dev);
 275static void null_del_dev(struct nullb *nullb);
 276static int null_add_dev(struct nullb_device *dev);
 277static struct nullb *null_find_dev_by_name(const char *name);
 278static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
 279
 280static inline struct nullb_device *to_nullb_device(struct config_item *item)
 281{
 282	return item ? container_of(to_config_group(item), struct nullb_device, group) : NULL;
 283}
 284
 285static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
 286{
 287	return snprintf(page, PAGE_SIZE, "%u\n", val);
 288}
 289
 290static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
 291	char *page)
 292{
 293	return snprintf(page, PAGE_SIZE, "%lu\n", val);
 294}
 295
 296static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
 297{
 298	return snprintf(page, PAGE_SIZE, "%u\n", val);
 299}
 300
 301static ssize_t nullb_device_uint_attr_store(unsigned int *val,
 302	const char *page, size_t count)
 303{
 304	unsigned int tmp;
 305	int result;
 306
 307	result = kstrtouint(page, 0, &tmp);
 308	if (result < 0)
 309		return result;
 310
 311	*val = tmp;
 312	return count;
 313}
 314
 315static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
 316	const char *page, size_t count)
 317{
 318	int result;
 319	unsigned long tmp;
 320
 321	result = kstrtoul(page, 0, &tmp);
 322	if (result < 0)
 323		return result;
 324
 325	*val = tmp;
 326	return count;
 327}
 328
 329static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
 330	size_t count)
 331{
 332	bool tmp;
 333	int result;
 334
 335	result = kstrtobool(page,  &tmp);
 336	if (result < 0)
 337		return result;
 338
 339	*val = tmp;
 340	return count;
 341}
 342
 343/* The following macro should only be used with TYPE = {uint, ulong, bool}. */
 344#define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY)				\
 345static ssize_t								\
 346nullb_device_##NAME##_show(struct config_item *item, char *page)	\
 347{									\
 348	return nullb_device_##TYPE##_attr_show(				\
 349				to_nullb_device(item)->NAME, page);	\
 350}									\
 351static ssize_t								\
 352nullb_device_##NAME##_store(struct config_item *item, const char *page,	\
 353			    size_t count)				\
 354{									\
 355	int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
 356	struct nullb_device *dev = to_nullb_device(item);		\
 357	TYPE new_value = 0;						\
 358	int ret;							\
 359									\
 360	ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
 361	if (ret < 0)							\
 362		return ret;						\
 363	if (apply_fn)							\
 364		ret = apply_fn(dev, new_value);				\
 365	else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) 	\
 366		ret = -EBUSY;						\
 367	if (ret < 0)							\
 368		return ret;						\
 369	dev->NAME = new_value;						\
 370	return count;							\
 371}									\
 372CONFIGFS_ATTR(nullb_device_, NAME);
 373
 374static int nullb_update_nr_hw_queues(struct nullb_device *dev,
 375				     unsigned int submit_queues,
 376				     unsigned int poll_queues)
 377
 378{
 379	struct blk_mq_tag_set *set;
 380	int ret, nr_hw_queues;
 381
 382	if (!dev->nullb)
 383		return 0;
 384
 385	/*
 386	 * Make sure at least one submit queue exists.
 387	 */
 388	if (!submit_queues)
 389		return -EINVAL;
 390
 391	/*
 392	 * Make sure that null_init_hctx() does not access nullb->queues[] past
 393	 * the end of that array.
 394	 */
 395	if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues)
 396		return -EINVAL;
 397
 398	/*
 399	 * Keep previous and new queue numbers in nullb_device for reference in
 400	 * the call back function null_map_queues().
 401	 */
 402	dev->prev_submit_queues = dev->submit_queues;
 403	dev->prev_poll_queues = dev->poll_queues;
 404	dev->submit_queues = submit_queues;
 405	dev->poll_queues = poll_queues;
 406
 407	set = dev->nullb->tag_set;
 408	nr_hw_queues = submit_queues + poll_queues;
 409	blk_mq_update_nr_hw_queues(set, nr_hw_queues);
 410	ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM;
 411
 412	if (ret) {
 413		/* on error, revert the queue numbers */
 414		dev->submit_queues = dev->prev_submit_queues;
 415		dev->poll_queues = dev->prev_poll_queues;
 416	}
 417
 418	return ret;
 419}
 420
 421static int nullb_apply_submit_queues(struct nullb_device *dev,
 422				     unsigned int submit_queues)
 423{
 424	int ret;
 425
 426	mutex_lock(&lock);
 427	ret = nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues);
 428	mutex_unlock(&lock);
 429
 430	return ret;
 431}
 432
 433static int nullb_apply_poll_queues(struct nullb_device *dev,
 434				   unsigned int poll_queues)
 435{
 436	int ret;
 437
 438	mutex_lock(&lock);
 439	ret = nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues);
 440	mutex_unlock(&lock);
 441
 442	return ret;
 443}
 444
 445NULLB_DEVICE_ATTR(size, ulong, NULL);
 446NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
 447NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
 448NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues);
 449NULLB_DEVICE_ATTR(home_node, uint, NULL);
 450NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
 451NULLB_DEVICE_ATTR(blocksize, uint, NULL);
 452NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
 453NULLB_DEVICE_ATTR(irqmode, uint, NULL);
 454NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
 455NULLB_DEVICE_ATTR(index, uint, NULL);
 456NULLB_DEVICE_ATTR(blocking, bool, NULL);
 457NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
 458NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
 459NULLB_DEVICE_ATTR(discard, bool, NULL);
 460NULLB_DEVICE_ATTR(mbps, uint, NULL);
 461NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
 462NULLB_DEVICE_ATTR(zoned, bool, NULL);
 463NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
 464NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
 465NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
 466NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
 467NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
 468NULLB_DEVICE_ATTR(zone_append_max_sectors, uint, NULL);
 469NULLB_DEVICE_ATTR(zone_full, bool, NULL);
 470NULLB_DEVICE_ATTR(virt_boundary, bool, NULL);
 471NULLB_DEVICE_ATTR(no_sched, bool, NULL);
 472NULLB_DEVICE_ATTR(shared_tags, bool, NULL);
 473NULLB_DEVICE_ATTR(shared_tag_bitmap, bool, NULL);
 474NULLB_DEVICE_ATTR(fua, bool, NULL);
 475NULLB_DEVICE_ATTR(rotational, bool, NULL);
 476NULLB_DEVICE_ATTR(badblocks_once, bool, NULL);
 477NULLB_DEVICE_ATTR(badblocks_partial_io, bool, NULL);
 478
 479static ssize_t nullb_device_power_show(struct config_item *item, char *page)
 480{
 481	return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
 482}
 483
 484static ssize_t nullb_device_power_store(struct config_item *item,
 485				     const char *page, size_t count)
 486{
 487	struct nullb_device *dev = to_nullb_device(item);
 488	bool newp = false;
 489	ssize_t ret;
 490
 491	ret = nullb_device_bool_attr_store(&newp, page, count);
 492	if (ret < 0)
 493		return ret;
 494
 495	ret = count;
 496	mutex_lock(&lock);
 497	if (!dev->power && newp) {
 498		if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
 499			goto out;
 500
 501		ret = null_add_dev(dev);
 502		if (ret) {
 503			clear_bit(NULLB_DEV_FL_UP, &dev->flags);
 504			goto out;
 505		}
 506
 507		set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 508		dev->power = newp;
 509		ret = count;
 510	} else if (dev->power && !newp) {
 511		if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 512			dev->power = newp;
 513			null_del_dev(dev->nullb);
 514		}
 515		clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 516	}
 517
 518out:
 519	mutex_unlock(&lock);
 520	return ret;
 521}
 522
 523CONFIGFS_ATTR(nullb_device_, power);
 524
 525static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
 526{
 527	struct nullb_device *t_dev = to_nullb_device(item);
 528
 529	return badblocks_show(&t_dev->badblocks, page, 0);
 530}
 531
 532static ssize_t nullb_device_badblocks_store(struct config_item *item,
 533				     const char *page, size_t count)
 534{
 535	struct nullb_device *t_dev = to_nullb_device(item);
 536	char *orig, *buf, *tmp;
 537	u64 start, end;
 538	int ret;
 539
 540	orig = kstrndup(page, count, GFP_KERNEL);
 541	if (!orig)
 542		return -ENOMEM;
 543
 544	buf = strstrip(orig);
 545
 546	ret = -EINVAL;
 547	if (buf[0] != '+' && buf[0] != '-')
 548		goto out;
 549	tmp = strchr(&buf[1], '-');
 550	if (!tmp)
 551		goto out;
 552	*tmp = '\0';
 553	ret = kstrtoull(buf + 1, 0, &start);
 554	if (ret)
 555		goto out;
 556	ret = kstrtoull(tmp + 1, 0, &end);
 557	if (ret)
 558		goto out;
 559	ret = -EINVAL;
 560	if (start > end)
 561		goto out;
 562	/* enable badblocks */
 563	cmpxchg(&t_dev->badblocks.shift, -1, 0);
 564	if (buf[0] == '+') {
 565		if (badblocks_set(&t_dev->badblocks, start,
 566				  end - start + 1, 1))
 567			ret = count;
 568	} else if (badblocks_clear(&t_dev->badblocks, start,
 569				   end - start + 1)) {
 570		ret = count;
 571	}
 572out:
 573	kfree(orig);
 574	return ret;
 575}
 576CONFIGFS_ATTR(nullb_device_, badblocks);
 577
 578static ssize_t nullb_device_zone_readonly_store(struct config_item *item,
 579						const char *page, size_t count)
 580{
 581	struct nullb_device *dev = to_nullb_device(item);
 582
 583	return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY);
 584}
 585CONFIGFS_ATTR_WO(nullb_device_, zone_readonly);
 586
 587static ssize_t nullb_device_zone_offline_store(struct config_item *item,
 588					       const char *page, size_t count)
 589{
 590	struct nullb_device *dev = to_nullb_device(item);
 591
 592	return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE);
 593}
 594CONFIGFS_ATTR_WO(nullb_device_, zone_offline);
 595
 596static struct configfs_attribute *nullb_device_attrs[] = {
 597	&nullb_device_attr_badblocks,
 598	&nullb_device_attr_badblocks_once,
 599	&nullb_device_attr_badblocks_partial_io,
 600	&nullb_device_attr_blocking,
 601	&nullb_device_attr_blocksize,
 602	&nullb_device_attr_cache_size,
 603	&nullb_device_attr_completion_nsec,
 604	&nullb_device_attr_discard,
 605	&nullb_device_attr_fua,
 606	&nullb_device_attr_home_node,
 607	&nullb_device_attr_hw_queue_depth,
 608	&nullb_device_attr_index,
 609	&nullb_device_attr_irqmode,
 610	&nullb_device_attr_max_sectors,
 611	&nullb_device_attr_mbps,
 612	&nullb_device_attr_memory_backed,
 613	&nullb_device_attr_no_sched,
 614	&nullb_device_attr_poll_queues,
 615	&nullb_device_attr_power,
 616	&nullb_device_attr_queue_mode,
 617	&nullb_device_attr_rotational,
 618	&nullb_device_attr_shared_tag_bitmap,
 619	&nullb_device_attr_shared_tags,
 620	&nullb_device_attr_size,
 621	&nullb_device_attr_submit_queues,
 622	&nullb_device_attr_use_per_node_hctx,
 623	&nullb_device_attr_virt_boundary,
 624	&nullb_device_attr_zone_append_max_sectors,
 625	&nullb_device_attr_zone_capacity,
 626	&nullb_device_attr_zone_full,
 627	&nullb_device_attr_zone_max_active,
 628	&nullb_device_attr_zone_max_open,
 629	&nullb_device_attr_zone_nr_conv,
 630	&nullb_device_attr_zone_offline,
 631	&nullb_device_attr_zone_readonly,
 632	&nullb_device_attr_zone_size,
 633	&nullb_device_attr_zoned,
 634	NULL,
 635};
 636
 637static void nullb_device_release(struct config_item *item)
 638{
 639	struct nullb_device *dev = to_nullb_device(item);
 640
 641	null_free_device_storage(dev, false);
 642	null_free_dev(dev);
 643}
 644
 645static struct configfs_item_operations nullb_device_ops = {
 646	.release	= nullb_device_release,
 647};
 648
 649static const struct config_item_type nullb_device_type = {
 650	.ct_item_ops	= &nullb_device_ops,
 651	.ct_attrs	= nullb_device_attrs,
 652	.ct_owner	= THIS_MODULE,
 653};
 654
 655#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 656
 657static void nullb_add_fault_config(struct nullb_device *dev)
 658{
 659	fault_config_init(&dev->timeout_config, "timeout_inject");
 660	fault_config_init(&dev->requeue_config, "requeue_inject");
 661	fault_config_init(&dev->init_hctx_fault_config, "init_hctx_fault_inject");
 662
 663	configfs_add_default_group(&dev->timeout_config.group, &dev->group);
 664	configfs_add_default_group(&dev->requeue_config.group, &dev->group);
 665	configfs_add_default_group(&dev->init_hctx_fault_config.group, &dev->group);
 666}
 667
 668static void nullb_del_fault_config(struct nullb_device *dev)
 669{
 670	config_item_put(&dev->init_hctx_fault_config.group.cg_item);
 671	config_item_put(&dev->requeue_config.group.cg_item);
 672	config_item_put(&dev->timeout_config.group.cg_item);
 673}
 674
 675#else
 676
 677static void nullb_add_fault_config(struct nullb_device *dev)
 678{
 679}
 680
 681static void nullb_del_fault_config(struct nullb_device *dev)
 682{
 683}
 684#endif
 685
 686static struct
 687config_group *nullb_group_make_group(struct config_group *group, const char *name)
 688{
 689	struct nullb_device *dev;
 690
 691	if (null_find_dev_by_name(name))
 692		return ERR_PTR(-EEXIST);
 693
 694	dev = null_alloc_dev();
 695	if (!dev)
 696		return ERR_PTR(-ENOMEM);
 697
 698	config_group_init_type_name(&dev->group, name, &nullb_device_type);
 699	nullb_add_fault_config(dev);
 700
 701	return &dev->group;
 702}
 703
 704static void
 705nullb_group_drop_item(struct config_group *group, struct config_item *item)
 706{
 707	struct nullb_device *dev = to_nullb_device(item);
 708
 709	if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 710		mutex_lock(&lock);
 711		dev->power = false;
 712		null_del_dev(dev->nullb);
 713		mutex_unlock(&lock);
 714	}
 715	nullb_del_fault_config(dev);
 716	config_item_put(item);
 717}
 718
 719static ssize_t memb_group_features_show(struct config_item *item, char *page)
 720{
 721
 722	struct configfs_attribute **entry;
 723	char delimiter = ',';
 724	size_t left = PAGE_SIZE;
 725	size_t written = 0;
 726	int ret;
 727
 728	for (entry = &nullb_device_attrs[0]; *entry && left > 0; entry++) {
 729		if (!*(entry + 1))
 730			delimiter = '\n';
 731		ret = snprintf(page + written, left, "%s%c", (*entry)->ca_name,
 732			       delimiter);
 733		if (ret >= left) {
 734			WARN_ONCE(1, "Too many null_blk features to print\n");
 735			memzero_explicit(page, PAGE_SIZE);
 736			return -ENOBUFS;
 737		}
 738		left -= ret;
 739		written += ret;
 740	}
 741
 742	return written;
 743}
 744
 745CONFIGFS_ATTR_RO(memb_group_, features);
 746
 747static struct configfs_attribute *nullb_group_attrs[] = {
 748	&memb_group_attr_features,
 749	NULL,
 750};
 751
 752static struct configfs_group_operations nullb_group_ops = {
 753	.make_group	= nullb_group_make_group,
 754	.drop_item	= nullb_group_drop_item,
 755};
 756
 757static const struct config_item_type nullb_group_type = {
 758	.ct_group_ops	= &nullb_group_ops,
 759	.ct_attrs	= nullb_group_attrs,
 760	.ct_owner	= THIS_MODULE,
 761};
 762
 763static struct configfs_subsystem nullb_subsys = {
 764	.su_group = {
 765		.cg_item = {
 766			.ci_namebuf = "nullb",
 767			.ci_type = &nullb_group_type,
 768		},
 769	},
 770};
 771
 772static inline int null_cache_active(struct nullb *nullb)
 773{
 774	return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
 775}
 776
 777static struct nullb_device *null_alloc_dev(void)
 778{
 779	struct nullb_device *dev;
 780
 781	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 782	if (!dev)
 783		return NULL;
 784
 785#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 786	dev->timeout_config.attr = null_timeout_attr;
 787	dev->requeue_config.attr = null_requeue_attr;
 788	dev->init_hctx_fault_config.attr = null_init_hctx_attr;
 789#endif
 790
 791	INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
 792	INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
 793	if (badblocks_init(&dev->badblocks, 0)) {
 794		kfree(dev);
 795		return NULL;
 796	}
 797
 798	dev->size = g_gb * 1024;
 799	dev->completion_nsec = g_completion_nsec;
 800	dev->submit_queues = g_submit_queues;
 801	dev->prev_submit_queues = g_submit_queues;
 802	dev->poll_queues = g_poll_queues;
 803	dev->prev_poll_queues = g_poll_queues;
 804	dev->home_node = g_home_node;
 805	dev->queue_mode = g_queue_mode;
 806	dev->blocksize = g_bs;
 807	dev->max_sectors = g_max_sectors;
 808	dev->irqmode = g_irqmode;
 809	dev->hw_queue_depth = g_hw_queue_depth;
 810	dev->blocking = g_blocking;
 811	dev->memory_backed = g_memory_backed;
 812	dev->discard = g_discard;
 813	dev->cache_size = g_cache_size;
 814	dev->mbps = g_mbps;
 815	dev->use_per_node_hctx = g_use_per_node_hctx;
 816	dev->zoned = g_zoned;
 817	dev->zone_size = g_zone_size;
 818	dev->zone_capacity = g_zone_capacity;
 819	dev->zone_nr_conv = g_zone_nr_conv;
 820	dev->zone_max_open = g_zone_max_open;
 821	dev->zone_max_active = g_zone_max_active;
 822	dev->zone_append_max_sectors = g_zone_append_max_sectors;
 823	dev->zone_full = g_zone_full;
 824	dev->virt_boundary = g_virt_boundary;
 825	dev->no_sched = g_no_sched;
 826	dev->shared_tags = g_shared_tags;
 827	dev->shared_tag_bitmap = g_shared_tag_bitmap;
 828	dev->fua = g_fua;
 829	dev->rotational = g_rotational;
 830
 831	return dev;
 832}
 833
 834static void null_free_dev(struct nullb_device *dev)
 835{
 836	if (!dev)
 837		return;
 838
 839	null_free_zoned_dev(dev);
 840	badblocks_exit(&dev->badblocks);
 841	kfree(dev);
 842}
 843
 844static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 845{
 846	struct nullb_cmd *cmd = container_of(timer, struct nullb_cmd, timer);
 847
 848	blk_mq_end_request(blk_mq_rq_from_pdu(cmd), cmd->error);
 849	return HRTIMER_NORESTART;
 850}
 851
 852static void null_cmd_end_timer(struct nullb_cmd *cmd)
 853{
 854	ktime_t kt = cmd->nq->dev->completion_nsec;
 855
 856	hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 857}
 858
 859static void null_complete_rq(struct request *rq)
 860{
 861	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
 862
 863	blk_mq_end_request(rq, cmd->error);
 864}
 865
 866static struct nullb_page *null_alloc_page(void)
 867{
 868	struct nullb_page *t_page;
 869
 870	t_page = kmalloc(sizeof(struct nullb_page), GFP_NOIO);
 871	if (!t_page)
 872		return NULL;
 873
 874	t_page->page = alloc_pages(GFP_NOIO, 0);
 875	if (!t_page->page) {
 876		kfree(t_page);
 877		return NULL;
 878	}
 879
 880	memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
 881	return t_page;
 882}
 883
 884static void null_free_page(struct nullb_page *t_page)
 885{
 886	__set_bit(NULLB_PAGE_FREE, t_page->bitmap);
 887	if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
 888		return;
 889	__free_page(t_page->page);
 890	kfree(t_page);
 891}
 892
 893static bool null_page_empty(struct nullb_page *page)
 894{
 895	int size = MAP_SZ - 2;
 896
 897	return find_first_bit(page->bitmap, size) == size;
 898}
 899
 900static void null_free_sector(struct nullb *nullb, sector_t sector,
 901	bool is_cache)
 902{
 903	unsigned int sector_bit;
 904	u64 idx;
 905	struct nullb_page *t_page, *ret;
 906	struct radix_tree_root *root;
 907
 908	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 909	idx = sector >> PAGE_SECTORS_SHIFT;
 910	sector_bit = (sector & SECTOR_MASK);
 911
 912	t_page = radix_tree_lookup(root, idx);
 913	if (t_page) {
 914		__clear_bit(sector_bit, t_page->bitmap);
 915
 916		if (null_page_empty(t_page)) {
 917			ret = radix_tree_delete_item(root, idx, t_page);
 918			WARN_ON(ret != t_page);
 919			null_free_page(ret);
 920			if (is_cache)
 921				nullb->dev->curr_cache -= PAGE_SIZE;
 922		}
 923	}
 924}
 925
 926static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
 927	struct nullb_page *t_page, bool is_cache)
 928{
 929	struct radix_tree_root *root;
 930
 931	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 932
 933	if (radix_tree_insert(root, idx, t_page)) {
 934		null_free_page(t_page);
 935		t_page = radix_tree_lookup(root, idx);
 936		WARN_ON(!t_page || t_page->page->private != idx);
 937	} else if (is_cache)
 938		nullb->dev->curr_cache += PAGE_SIZE;
 939
 940	return t_page;
 941}
 942
 943static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
 944{
 945	unsigned long pos = 0;
 946	int nr_pages;
 947	struct nullb_page *ret, *t_pages[FREE_BATCH];
 948	struct radix_tree_root *root;
 949
 950	root = is_cache ? &dev->cache : &dev->data;
 951
 952	do {
 953		int i;
 954
 955		nr_pages = radix_tree_gang_lookup(root,
 956				(void **)t_pages, pos, FREE_BATCH);
 957
 958		for (i = 0; i < nr_pages; i++) {
 959			pos = t_pages[i]->page->private;
 960			ret = radix_tree_delete_item(root, pos, t_pages[i]);
 961			WARN_ON(ret != t_pages[i]);
 962			null_free_page(ret);
 963		}
 964
 965		pos++;
 966	} while (nr_pages == FREE_BATCH);
 967
 968	if (is_cache)
 969		dev->curr_cache = 0;
 970}
 971
 972static struct nullb_page *__null_lookup_page(struct nullb *nullb,
 973	sector_t sector, bool for_write, bool is_cache)
 974{
 975	unsigned int sector_bit;
 976	u64 idx;
 977	struct nullb_page *t_page;
 978	struct radix_tree_root *root;
 979
 980	idx = sector >> PAGE_SECTORS_SHIFT;
 981	sector_bit = (sector & SECTOR_MASK);
 982
 983	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 984	t_page = radix_tree_lookup(root, idx);
 985	WARN_ON(t_page && t_page->page->private != idx);
 986
 987	if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
 988		return t_page;
 989
 990	return NULL;
 991}
 992
 993static struct nullb_page *null_lookup_page(struct nullb *nullb,
 994	sector_t sector, bool for_write, bool ignore_cache)
 995{
 996	struct nullb_page *page = NULL;
 997
 998	if (!ignore_cache)
 999		page = __null_lookup_page(nullb, sector, for_write, true);
1000	if (page)
1001		return page;
1002	return __null_lookup_page(nullb, sector, for_write, false);
1003}
1004
1005static struct nullb_page *null_insert_page(struct nullb *nullb,
1006					   sector_t sector, bool ignore_cache)
1007	__releases(&nullb->lock)
1008	__acquires(&nullb->lock)
1009{
1010	u64 idx;
1011	struct nullb_page *t_page;
1012
1013	t_page = null_lookup_page(nullb, sector, true, ignore_cache);
1014	if (t_page)
1015		return t_page;
1016
1017	spin_unlock_irq(&nullb->lock);
1018
1019	t_page = null_alloc_page();
1020	if (!t_page)
1021		goto out_lock;
1022
1023	if (radix_tree_preload(GFP_NOIO))
1024		goto out_freepage;
1025
1026	spin_lock_irq(&nullb->lock);
1027	idx = sector >> PAGE_SECTORS_SHIFT;
1028	t_page->page->private = idx;
1029	t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
1030	radix_tree_preload_end();
1031
1032	return t_page;
1033out_freepage:
1034	null_free_page(t_page);
1035out_lock:
1036	spin_lock_irq(&nullb->lock);
1037	return null_lookup_page(nullb, sector, true, ignore_cache);
1038}
1039
1040static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
1041{
1042	int i;
1043	unsigned int offset;
1044	u64 idx;
1045	struct nullb_page *t_page, *ret;
1046	void *dst, *src;
1047
1048	idx = c_page->page->private;
1049
1050	t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
1051
1052	__clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
1053	if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
1054		null_free_page(c_page);
1055		if (t_page && null_page_empty(t_page)) {
1056			ret = radix_tree_delete_item(&nullb->dev->data,
1057				idx, t_page);
1058			null_free_page(t_page);
1059		}
1060		return 0;
1061	}
1062
1063	if (!t_page)
1064		return -ENOMEM;
1065
1066	src = kmap_local_page(c_page->page);
1067	dst = kmap_local_page(t_page->page);
1068
1069	for (i = 0; i < PAGE_SECTORS;
1070			i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
1071		if (test_bit(i, c_page->bitmap)) {
1072			offset = (i << SECTOR_SHIFT);
1073			memcpy(dst + offset, src + offset,
1074				nullb->dev->blocksize);
1075			__set_bit(i, t_page->bitmap);
1076		}
1077	}
1078
1079	kunmap_local(dst);
1080	kunmap_local(src);
1081
1082	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
1083	null_free_page(ret);
1084	nullb->dev->curr_cache -= PAGE_SIZE;
1085
1086	return 0;
1087}
1088
1089static int null_make_cache_space(struct nullb *nullb, unsigned long n)
1090{
1091	int i, err, nr_pages;
1092	struct nullb_page *c_pages[FREE_BATCH];
1093	unsigned long flushed = 0, one_round;
1094
1095again:
1096	if ((nullb->dev->cache_size * 1024 * 1024) >
1097	     nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
1098		return 0;
1099
1100	nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
1101			(void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
1102	/*
1103	 * nullb_flush_cache_page could unlock before using the c_pages. To
1104	 * avoid race, we don't allow page free
1105	 */
1106	for (i = 0; i < nr_pages; i++) {
1107		nullb->cache_flush_pos = c_pages[i]->page->private;
1108		/*
1109		 * We found the page which is being flushed to disk by other
1110		 * threads
1111		 */
1112		if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
1113			c_pages[i] = NULL;
1114		else
1115			__set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
1116	}
1117
1118	one_round = 0;
1119	for (i = 0; i < nr_pages; i++) {
1120		if (c_pages[i] == NULL)
1121			continue;
1122		err = null_flush_cache_page(nullb, c_pages[i]);
1123		if (err)
1124			return err;
1125		one_round++;
1126	}
1127	flushed += one_round << PAGE_SHIFT;
1128
1129	if (n > flushed) {
1130		if (nr_pages == 0)
1131			nullb->cache_flush_pos = 0;
1132		if (one_round == 0) {
1133			/* give other threads a chance */
1134			spin_unlock_irq(&nullb->lock);
1135			spin_lock_irq(&nullb->lock);
1136		}
1137		goto again;
1138	}
1139	return 0;
1140}
1141
1142static blk_status_t copy_to_nullb(struct nullb *nullb, void *source,
1143				  loff_t pos, size_t n, bool is_fua)
1144{
1145	size_t temp, count = 0;
1146	struct nullb_page *t_page;
1147	sector_t sector;
1148
1149	while (count < n) {
1150		temp = min3(nullb->dev->blocksize, n - count,
1151			    PAGE_SIZE - offset_in_page(pos));
1152		sector = pos >> SECTOR_SHIFT;
1153
1154		if (null_cache_active(nullb) && !is_fua)
1155			null_make_cache_space(nullb, PAGE_SIZE);
1156
1157		t_page = null_insert_page(nullb, sector,
1158			!null_cache_active(nullb) || is_fua);
1159		if (!t_page)
1160			return BLK_STS_NOSPC;
1161
1162		memcpy_to_page(t_page->page, offset_in_page(pos),
1163			       source + count, temp);
1164
1165		__set_bit(sector & SECTOR_MASK, t_page->bitmap);
1166
1167		if (is_fua)
1168			null_free_sector(nullb, sector, true);
1169
1170		count += temp;
1171		pos += temp;
1172	}
1173	return BLK_STS_OK;
1174}
1175
1176static void copy_from_nullb(struct nullb *nullb, void *dest, loff_t pos,
1177			    size_t n)
1178{
1179	size_t temp, count = 0;
1180	struct nullb_page *t_page;
1181	sector_t sector;
1182
1183	while (count < n) {
1184		temp = min3(nullb->dev->blocksize, n - count,
1185			    PAGE_SIZE - offset_in_page(pos));
1186		sector = pos >> SECTOR_SHIFT;
1187
1188		t_page = null_lookup_page(nullb, sector, false,
1189			!null_cache_active(nullb));
1190		if (t_page)
1191			memcpy_from_page(dest + count, t_page->page,
1192					 offset_in_page(pos), temp);
1193		else
1194			memset(dest + count, 0, temp);
1195
1196		count += temp;
1197		pos += temp;
1198	}
1199}
1200
1201blk_status_t null_handle_discard(struct nullb_device *dev,
1202				 sector_t sector, sector_t nr_sectors)
1203{
1204	struct nullb *nullb = dev->nullb;
1205	size_t n = nr_sectors << SECTOR_SHIFT;
1206	size_t temp;
1207
1208	spin_lock_irq(&nullb->lock);
1209	while (n > 0) {
1210		temp = min_t(size_t, n, dev->blocksize);
1211		null_free_sector(nullb, sector, false);
1212		if (null_cache_active(nullb))
1213			null_free_sector(nullb, sector, true);
1214		sector += temp >> SECTOR_SHIFT;
1215		n -= temp;
1216	}
1217	spin_unlock_irq(&nullb->lock);
1218
1219	return BLK_STS_OK;
1220}
1221
1222static blk_status_t null_handle_flush(struct nullb *nullb)
1223{
1224	int err;
1225
1226	if (!null_cache_active(nullb))
1227		return 0;
1228
1229	spin_lock_irq(&nullb->lock);
1230	while (true) {
1231		err = null_make_cache_space(nullb,
1232			nullb->dev->cache_size * 1024 * 1024);
1233		if (err || nullb->dev->curr_cache == 0)
1234			break;
1235	}
1236
1237	WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1238	spin_unlock_irq(&nullb->lock);
1239	return errno_to_blk_status(err);
1240}
1241
1242static blk_status_t null_transfer(struct nullb *nullb, struct page *page,
1243	unsigned int len, unsigned int off, bool is_write, loff_t pos,
1244	bool is_fua)
1245{
1246	struct nullb_device *dev = nullb->dev;
1247	blk_status_t err = BLK_STS_OK;
1248	unsigned int valid_len = len;
1249	void *p;
1250
1251	p = kmap_local_page(page) + off;
1252	if (!is_write) {
1253		if (dev->zoned) {
1254			valid_len = null_zone_valid_read_len(nullb,
1255				pos >> SECTOR_SHIFT, len);
1256			if (valid_len && valid_len != len)
1257				valid_len -= pos & (SECTOR_SIZE - 1);
1258		}
1259
1260		if (valid_len) {
1261			copy_from_nullb(nullb, p, pos, valid_len);
1262			off += valid_len;
1263			len -= valid_len;
1264		}
1265
1266		if (len)
1267			memset(p + valid_len, 0xff, len);
1268		flush_dcache_page(page);
1269	} else {
1270		flush_dcache_page(page);
1271		err = copy_to_nullb(nullb, p, pos, len, is_fua);
1272	}
1273
1274	kunmap_local(p);
1275	return err;
1276}
1277
1278/*
1279 * Transfer data for the given request. The transfer size is capped with the
1280 * nr_sectors argument.
1281 */
1282static blk_status_t null_handle_data_transfer(struct nullb_cmd *cmd,
1283					      sector_t nr_sectors)
1284{
1285	struct request *rq = blk_mq_rq_from_pdu(cmd);
1286	struct nullb *nullb = cmd->nq->dev->nullb;
1287	blk_status_t err = BLK_STS_OK;
1288	unsigned int len;
1289	loff_t pos = blk_rq_pos(rq) << SECTOR_SHIFT;
1290	unsigned int max_bytes = nr_sectors << SECTOR_SHIFT;
1291	unsigned int transferred_bytes = 0;
1292	struct req_iterator iter;
1293	struct bio_vec bvec;
1294
1295	spin_lock_irq(&nullb->lock);
1296	rq_for_each_segment(bvec, rq, iter) {
1297		len = bvec.bv_len;
1298		if (transferred_bytes + len > max_bytes)
1299			len = max_bytes - transferred_bytes;
1300		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1301				     op_is_write(req_op(rq)), pos,
1302				     rq->cmd_flags & REQ_FUA);
1303		if (err)
1304			break;
1305		pos += len;
1306		transferred_bytes += len;
1307		if (transferred_bytes >= max_bytes)
1308			break;
1309	}
1310	spin_unlock_irq(&nullb->lock);
1311
1312	return err;
1313}
1314
1315static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1316{
1317	struct nullb_device *dev = cmd->nq->dev;
1318	struct nullb *nullb = dev->nullb;
1319	blk_status_t sts = BLK_STS_OK;
1320	struct request *rq = blk_mq_rq_from_pdu(cmd);
1321
1322	if (!hrtimer_active(&nullb->bw_timer))
1323		hrtimer_restart(&nullb->bw_timer);
1324
1325	if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
1326		blk_mq_stop_hw_queues(nullb->q);
1327		/* race with timer */
1328		if (atomic_long_read(&nullb->cur_bytes) > 0)
1329			blk_mq_start_stopped_hw_queues(nullb->q, true);
1330		/* requeue request */
1331		sts = BLK_STS_DEV_RESOURCE;
1332	}
1333	return sts;
1334}
1335
1336/*
1337 * Check if the command should fail for the badblocks. If so, return
1338 * BLK_STS_IOERR and return number of partial I/O sectors to be written or read,
1339 * which may be less than the requested number of sectors.
1340 *
1341 * @cmd:        The command to handle.
1342 * @sector:     The start sector for I/O.
1343 * @nr_sectors: Specifies number of sectors to write or read, and returns the
1344 *              number of sectors to be written or read.
1345 */
1346blk_status_t null_handle_badblocks(struct nullb_cmd *cmd, sector_t sector,
1347				   unsigned int *nr_sectors)
1348{
1349	struct badblocks *bb = &cmd->nq->dev->badblocks;
1350	struct nullb_device *dev = cmd->nq->dev;
1351	unsigned int block_sectors = dev->blocksize >> SECTOR_SHIFT;
1352	sector_t first_bad, bad_sectors;
1353	unsigned int partial_io_sectors = 0;
1354
1355	if (!badblocks_check(bb, sector, *nr_sectors, &first_bad, &bad_sectors))
1356		return BLK_STS_OK;
1357
1358	if (cmd->nq->dev->badblocks_once)
1359		badblocks_clear(bb, first_bad, bad_sectors);
1360
1361	if (cmd->nq->dev->badblocks_partial_io) {
1362		if (!IS_ALIGNED(first_bad, block_sectors))
1363			first_bad = ALIGN_DOWN(first_bad, block_sectors);
1364		if (sector < first_bad)
1365			partial_io_sectors = first_bad - sector;
1366	}
1367	*nr_sectors = partial_io_sectors;
1368
1369	return BLK_STS_IOERR;
1370}
1371
1372blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd, enum req_op op,
1373				       sector_t sector, sector_t nr_sectors)
1374{
1375	struct nullb_device *dev = cmd->nq->dev;
1376
1377	if (op == REQ_OP_DISCARD)
1378		return null_handle_discard(dev, sector, nr_sectors);
1379
1380	return null_handle_data_transfer(cmd, nr_sectors);
1381}
1382
1383static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1384{
1385	struct request *rq = blk_mq_rq_from_pdu(cmd);
1386	struct nullb_device *dev = cmd->nq->dev;
1387	struct bio *bio;
1388
1389	if (!dev->memory_backed && req_op(rq) == REQ_OP_READ) {
1390		__rq_for_each_bio(bio, rq)
1391			zero_fill_bio(bio);
1392	}
1393}
1394
1395static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1396{
1397	struct request *rq = blk_mq_rq_from_pdu(cmd);
1398
1399	/*
1400	 * Since root privileges are required to configure the null_blk
1401	 * driver, it is fine that this driver does not initialize the
1402	 * data buffers of read commands. Zero-initialize these buffers
1403	 * anyway if KMSAN is enabled to prevent that KMSAN complains
1404	 * about null_blk not initializing read data buffers.
1405	 */
1406	if (IS_ENABLED(CONFIG_KMSAN))
1407		nullb_zero_read_cmd_buffer(cmd);
1408
1409	/* Complete IO by inline, softirq or timer */
1410	switch (cmd->nq->dev->irqmode) {
1411	case NULL_IRQ_SOFTIRQ:
1412		blk_mq_complete_request(rq);
1413		break;
1414	case NULL_IRQ_NONE:
1415		blk_mq_end_request(rq, cmd->error);
1416		break;
1417	case NULL_IRQ_TIMER:
1418		null_cmd_end_timer(cmd);
1419		break;
1420	}
1421}
1422
1423blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1424			      sector_t sector, unsigned int nr_sectors)
1425{
1426	struct nullb_device *dev = cmd->nq->dev;
1427	blk_status_t badblocks_ret = BLK_STS_OK;
1428	blk_status_t ret;
1429
1430	if (dev->badblocks.shift != -1)
1431		badblocks_ret = null_handle_badblocks(cmd, sector, &nr_sectors);
1432
1433	if (dev->memory_backed && nr_sectors) {
1434		ret = null_handle_memory_backed(cmd, op, sector, nr_sectors);
1435		if (ret != BLK_STS_OK)
1436			return ret;
1437	}
1438
1439	return badblocks_ret;
1440}
1441
1442static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1443			    sector_t nr_sectors, enum req_op op)
1444{
1445	struct nullb_device *dev = cmd->nq->dev;
1446	struct nullb *nullb = dev->nullb;
1447	blk_status_t sts;
1448
1449	if (op == REQ_OP_FLUSH) {
1450		cmd->error = null_handle_flush(nullb);
1451		goto out;
1452	}
1453
1454	if (dev->zoned)
1455		sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1456	else
1457		sts = null_process_cmd(cmd, op, sector, nr_sectors);
1458
1459	/* Do not overwrite errors (e.g. timeout errors) */
1460	if (cmd->error == BLK_STS_OK)
1461		cmd->error = sts;
1462
1463out:
1464	nullb_complete_cmd(cmd);
1465}
1466
1467static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1468{
1469	struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1470	ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1471	unsigned int mbps = nullb->dev->mbps;
1472
1473	if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1474		return HRTIMER_NORESTART;
1475
1476	atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1477	blk_mq_start_stopped_hw_queues(nullb->q, true);
1478
1479	hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1480
1481	return HRTIMER_RESTART;
1482}
1483
1484static void nullb_setup_bwtimer(struct nullb *nullb)
1485{
1486	ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1487
1488	hrtimer_setup(&nullb->bw_timer, nullb_bwtimer_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1489	atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1490	hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1491}
1492
1493#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1494
1495static bool should_timeout_request(struct request *rq)
1496{
1497	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1498	struct nullb_device *dev = cmd->nq->dev;
1499
1500	return should_fail(&dev->timeout_config.attr, 1);
1501}
1502
1503static bool should_requeue_request(struct request *rq)
1504{
1505	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1506	struct nullb_device *dev = cmd->nq->dev;
1507
1508	return should_fail(&dev->requeue_config.attr, 1);
1509}
1510
1511static bool should_init_hctx_fail(struct nullb_device *dev)
1512{
1513	return should_fail(&dev->init_hctx_fault_config.attr, 1);
1514}
1515
1516#else
1517
1518static bool should_timeout_request(struct request *rq)
1519{
1520	return false;
1521}
1522
1523static bool should_requeue_request(struct request *rq)
1524{
1525	return false;
1526}
1527
1528static bool should_init_hctx_fail(struct nullb_device *dev)
1529{
1530	return false;
1531}
1532
1533#endif
1534
1535static void null_map_queues(struct blk_mq_tag_set *set)
1536{
1537	struct nullb *nullb = set->driver_data;
1538	int i, qoff;
1539	unsigned int submit_queues = g_submit_queues;
1540	unsigned int poll_queues = g_poll_queues;
1541
1542	if (nullb) {
1543		struct nullb_device *dev = nullb->dev;
1544
1545		/*
1546		 * Refer nr_hw_queues of the tag set to check if the expected
1547		 * number of hardware queues are prepared. If block layer failed
1548		 * to prepare them, use previous numbers of submit queues and
1549		 * poll queues to map queues.
1550		 */
1551		if (set->nr_hw_queues ==
1552		    dev->submit_queues + dev->poll_queues) {
1553			submit_queues = dev->submit_queues;
1554			poll_queues = dev->poll_queues;
1555		} else if (set->nr_hw_queues ==
1556			   dev->prev_submit_queues + dev->prev_poll_queues) {
1557			submit_queues = dev->prev_submit_queues;
1558			poll_queues = dev->prev_poll_queues;
1559		} else {
1560			pr_warn("tag set has unexpected nr_hw_queues: %d\n",
1561				set->nr_hw_queues);
1562			WARN_ON_ONCE(true);
1563			submit_queues = 1;
1564			poll_queues = 0;
1565		}
1566	}
1567
1568	for (i = 0, qoff = 0; i < set->nr_maps; i++) {
1569		struct blk_mq_queue_map *map = &set->map[i];
1570
1571		switch (i) {
1572		case HCTX_TYPE_DEFAULT:
1573			map->nr_queues = submit_queues;
1574			break;
1575		case HCTX_TYPE_READ:
1576			map->nr_queues = 0;
1577			continue;
1578		case HCTX_TYPE_POLL:
1579			map->nr_queues = poll_queues;
1580			break;
1581		}
1582		map->queue_offset = qoff;
1583		qoff += map->nr_queues;
1584		blk_mq_map_queues(map);
1585	}
1586}
1587
1588static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1589{
1590	struct nullb_queue *nq = hctx->driver_data;
1591	LIST_HEAD(list);
1592	int nr = 0;
1593	struct request *rq;
1594
1595	spin_lock(&nq->poll_lock);
1596	list_splice_init(&nq->poll_list, &list);
1597	list_for_each_entry(rq, &list, queuelist)
1598		blk_mq_set_request_complete(rq);
1599	spin_unlock(&nq->poll_lock);
1600
1601	while (!list_empty(&list)) {
1602		struct nullb_cmd *cmd;
1603		struct request *req;
1604
1605		req = list_first_entry(&list, struct request, queuelist);
1606		list_del_init(&req->queuelist);
1607		cmd = blk_mq_rq_to_pdu(req);
1608		cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
1609						blk_rq_sectors(req));
1610		if (!blk_mq_add_to_batch(req, iob, cmd->error != BLK_STS_OK,
1611					 blk_mq_end_request_batch))
1612			blk_mq_end_request(req, cmd->error);
1613		nr++;
1614	}
1615
1616	return nr;
1617}
1618
1619static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
1620{
1621	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1622	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1623
1624	if (hctx->type == HCTX_TYPE_POLL) {
1625		struct nullb_queue *nq = hctx->driver_data;
1626
1627		spin_lock(&nq->poll_lock);
1628		/* The request may have completed meanwhile. */
1629		if (blk_mq_request_completed(rq)) {
1630			spin_unlock(&nq->poll_lock);
1631			return BLK_EH_DONE;
1632		}
1633		list_del_init(&rq->queuelist);
1634		spin_unlock(&nq->poll_lock);
1635	}
1636
1637	pr_info("rq %p timed out\n", rq);
1638
1639	/*
1640	 * If the device is marked as blocking (i.e. memory backed or zoned
1641	 * device), the submission path may be blocked waiting for resources
1642	 * and cause real timeouts. For these real timeouts, the submission
1643	 * path will complete the request using blk_mq_complete_request().
1644	 * Only fake timeouts need to execute blk_mq_complete_request() here.
1645	 */
1646	cmd->error = BLK_STS_TIMEOUT;
1647	if (cmd->fake_timeout || hctx->type == HCTX_TYPE_POLL)
1648		blk_mq_complete_request(rq);
1649	return BLK_EH_DONE;
1650}
1651
1652static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1653				  const struct blk_mq_queue_data *bd)
1654{
1655	struct request *rq = bd->rq;
1656	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1657	struct nullb_queue *nq = hctx->driver_data;
1658	sector_t nr_sectors = blk_rq_sectors(rq);
1659	sector_t sector = blk_rq_pos(rq);
1660	const bool is_poll = hctx->type == HCTX_TYPE_POLL;
1661
1662	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1663
1664	if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
1665		hrtimer_setup(&cmd->timer, null_cmd_timer_expired, CLOCK_MONOTONIC,
1666			      HRTIMER_MODE_REL);
1667	}
1668	cmd->error = BLK_STS_OK;
1669	cmd->nq = nq;
1670	cmd->fake_timeout = should_timeout_request(rq) ||
1671		blk_should_fake_timeout(rq->q);
1672
1673	if (should_requeue_request(rq)) {
1674		/*
1675		 * Alternate between hitting the core BUSY path, and the
1676		 * driver driven requeue path
1677		 */
1678		nq->requeue_selection++;
1679		if (nq->requeue_selection & 1)
1680			return BLK_STS_RESOURCE;
1681		blk_mq_requeue_request(rq, true);
1682		return BLK_STS_OK;
1683	}
1684
1685	if (test_bit(NULLB_DEV_FL_THROTTLED, &nq->dev->flags)) {
1686		blk_status_t sts = null_handle_throttled(cmd);
1687
1688		if (sts != BLK_STS_OK)
1689			return sts;
1690	}
1691
1692	blk_mq_start_request(rq);
1693
1694	if (is_poll) {
1695		spin_lock(&nq->poll_lock);
1696		list_add_tail(&rq->queuelist, &nq->poll_list);
1697		spin_unlock(&nq->poll_lock);
1698		return BLK_STS_OK;
1699	}
1700	if (cmd->fake_timeout)
1701		return BLK_STS_OK;
1702
1703	null_handle_cmd(cmd, sector, nr_sectors, req_op(rq));
1704	return BLK_STS_OK;
1705}
1706
1707static void null_queue_rqs(struct rq_list *rqlist)
1708{
1709	struct rq_list requeue_list = {};
1710	struct blk_mq_queue_data bd = { };
1711	blk_status_t ret;
1712
1713	do {
1714		struct request *rq = rq_list_pop(rqlist);
1715
1716		bd.rq = rq;
1717		ret = null_queue_rq(rq->mq_hctx, &bd);
1718		if (ret != BLK_STS_OK)
1719			rq_list_add_tail(&requeue_list, rq);
1720	} while (!rq_list_empty(rqlist));
1721
1722	*rqlist = requeue_list;
1723}
1724
1725static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1726{
1727	nq->dev = nullb->dev;
1728	INIT_LIST_HEAD(&nq->poll_list);
1729	spin_lock_init(&nq->poll_lock);
1730}
1731
1732static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1733			  unsigned int hctx_idx)
1734{
1735	struct nullb *nullb = hctx->queue->queuedata;
1736	struct nullb_queue *nq;
1737
1738	if (should_init_hctx_fail(nullb->dev))
1739		return -EFAULT;
1740
1741	nq = &nullb->queues[hctx_idx];
1742	hctx->driver_data = nq;
1743	null_init_queue(nullb, nq);
1744
1745	return 0;
1746}
1747
1748static const struct blk_mq_ops null_mq_ops = {
1749	.queue_rq       = null_queue_rq,
1750	.queue_rqs	= null_queue_rqs,
1751	.complete	= null_complete_rq,
1752	.timeout	= null_timeout_rq,
1753	.poll		= null_poll,
1754	.map_queues	= null_map_queues,
1755	.init_hctx	= null_init_hctx,
1756};
1757
1758static void null_del_dev(struct nullb *nullb)
1759{
1760	struct nullb_device *dev;
1761
1762	if (!nullb)
1763		return;
1764
1765	dev = nullb->dev;
1766
1767	ida_free(&nullb_indexes, nullb->index);
1768
1769	list_del_init(&nullb->list);
1770
1771	del_gendisk(nullb->disk);
1772
1773	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1774		hrtimer_cancel(&nullb->bw_timer);
1775		atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1776		blk_mq_start_stopped_hw_queues(nullb->q, true);
1777	}
1778
1779	put_disk(nullb->disk);
1780	if (nullb->tag_set == &nullb->__tag_set)
1781		blk_mq_free_tag_set(nullb->tag_set);
1782	kfree(nullb->queues);
1783	if (null_cache_active(nullb))
1784		null_free_device_storage(nullb->dev, true);
1785	kfree(nullb);
1786	dev->nullb = NULL;
1787}
1788
1789static void null_config_discard(struct nullb *nullb, struct queue_limits *lim)
1790{
1791	if (nullb->dev->discard == false)
1792		return;
1793
1794	if (!nullb->dev->memory_backed) {
1795		nullb->dev->discard = false;
1796		pr_info("discard option is ignored without memory backing\n");
1797		return;
1798	}
1799
1800	if (nullb->dev->zoned) {
1801		nullb->dev->discard = false;
1802		pr_info("discard option is ignored in zoned mode\n");
1803		return;
1804	}
1805
1806	lim->max_hw_discard_sectors = UINT_MAX >> 9;
1807}
1808
1809static const struct block_device_operations null_ops = {
1810	.owner		= THIS_MODULE,
1811	.report_zones	= null_report_zones,
1812};
1813
1814static int setup_queues(struct nullb *nullb)
1815{
1816	int nqueues = nr_cpu_ids;
1817
1818	if (g_poll_queues)
1819		nqueues += g_poll_queues;
1820
1821	nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue),
1822				GFP_KERNEL);
1823	if (!nullb->queues)
1824		return -ENOMEM;
1825
1826	return 0;
1827}
1828
1829static int null_init_tag_set(struct blk_mq_tag_set *set, int poll_queues)
1830{
1831	set->ops = &null_mq_ops;
1832	set->cmd_size = sizeof(struct nullb_cmd);
1833	set->timeout = 5 * HZ;
1834	set->nr_maps = 1;
1835	if (poll_queues) {
1836		set->nr_hw_queues += poll_queues;
1837		set->nr_maps += 2;
1838	}
1839	return blk_mq_alloc_tag_set(set);
1840}
1841
1842static int null_init_global_tag_set(void)
1843{
1844	int error;
1845
1846	if (tag_set.ops)
1847		return 0;
1848
1849	tag_set.nr_hw_queues = g_submit_queues;
1850	tag_set.queue_depth = g_hw_queue_depth;
1851	tag_set.numa_node = g_home_node;
1852	if (g_no_sched)
1853		tag_set.flags |= BLK_MQ_F_NO_SCHED_BY_DEFAULT;
1854	if (g_shared_tag_bitmap)
1855		tag_set.flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1856	if (g_blocking)
1857		tag_set.flags |= BLK_MQ_F_BLOCKING;
1858
1859	error = null_init_tag_set(&tag_set, g_poll_queues);
1860	if (error)
1861		tag_set.ops = NULL;
1862	return error;
1863}
1864
1865static int null_setup_tagset(struct nullb *nullb)
1866{
1867	if (nullb->dev->shared_tags) {
1868		nullb->tag_set = &tag_set;
1869		return null_init_global_tag_set();
1870	}
1871
1872	nullb->tag_set = &nullb->__tag_set;
1873	nullb->tag_set->driver_data = nullb;
1874	nullb->tag_set->nr_hw_queues = nullb->dev->submit_queues;
1875	nullb->tag_set->queue_depth = nullb->dev->hw_queue_depth;
1876	nullb->tag_set->numa_node = nullb->dev->home_node;
1877	if (nullb->dev->no_sched)
1878		nullb->tag_set->flags |= BLK_MQ_F_NO_SCHED_BY_DEFAULT;
1879	if (nullb->dev->shared_tag_bitmap)
1880		nullb->tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1881	if (nullb->dev->blocking)
1882		nullb->tag_set->flags |= BLK_MQ_F_BLOCKING;
1883	return null_init_tag_set(nullb->tag_set, nullb->dev->poll_queues);
1884}
1885
1886static int null_validate_conf(struct nullb_device *dev)
1887{
1888	if (dev->queue_mode == NULL_Q_RQ) {
1889		pr_err("legacy IO path is no longer available\n");
1890		return -EINVAL;
1891	}
1892	if (dev->queue_mode == NULL_Q_BIO) {
1893		pr_err("BIO-based IO path is no longer available, using blk-mq instead.\n");
1894		dev->queue_mode = NULL_Q_MQ;
1895	}
1896
1897	if (dev->use_per_node_hctx) {
1898		if (dev->submit_queues != nr_online_nodes)
1899			dev->submit_queues = nr_online_nodes;
1900	} else if (dev->submit_queues > nr_cpu_ids)
1901		dev->submit_queues = nr_cpu_ids;
1902	else if (dev->submit_queues == 0)
1903		dev->submit_queues = 1;
1904	dev->prev_submit_queues = dev->submit_queues;
1905
1906	if (dev->poll_queues > g_poll_queues)
1907		dev->poll_queues = g_poll_queues;
1908	dev->prev_poll_queues = dev->poll_queues;
1909	dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1910
1911	/* Do memory allocation, so set blocking */
1912	if (dev->memory_backed)
1913		dev->blocking = true;
1914	else /* cache is meaningless */
1915		dev->cache_size = 0;
1916	dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1917						dev->cache_size);
1918	dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1919
1920	if (dev->zoned &&
1921	    (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
1922		pr_err("zone_size must be power-of-two\n");
1923		return -EINVAL;
1924	}
1925
1926	return 0;
1927}
1928
1929#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1930static bool __null_setup_fault(struct fault_attr *attr, char *str)
1931{
1932	if (!str[0])
1933		return true;
1934
1935	if (!setup_fault_attr(attr, str))
1936		return false;
1937
1938	attr->verbose = 0;
1939	return true;
1940}
1941#endif
1942
1943static bool null_setup_fault(void)
1944{
1945#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1946	if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
1947		return false;
1948	if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
1949		return false;
1950	if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str))
1951		return false;
1952#endif
1953	return true;
1954}
1955
1956static int null_add_dev(struct nullb_device *dev)
1957{
1958	struct queue_limits lim = {
1959		.logical_block_size	= dev->blocksize,
1960		.physical_block_size	= dev->blocksize,
1961		.max_hw_sectors		= dev->max_sectors,
1962		.dma_alignment		= 1,
1963	};
1964
1965	struct nullb *nullb;
1966	int rv;
1967
1968	rv = null_validate_conf(dev);
1969	if (rv)
1970		return rv;
1971
1972	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1973	if (!nullb) {
1974		rv = -ENOMEM;
1975		goto out;
1976	}
1977	nullb->dev = dev;
1978	dev->nullb = nullb;
1979
1980	spin_lock_init(&nullb->lock);
1981
1982	rv = setup_queues(nullb);
1983	if (rv)
1984		goto out_free_nullb;
1985
1986	rv = null_setup_tagset(nullb);
1987	if (rv)
1988		goto out_cleanup_queues;
1989
1990	if (dev->virt_boundary)
1991		lim.virt_boundary_mask = PAGE_SIZE - 1;
1992	null_config_discard(nullb, &lim);
1993	if (dev->zoned) {
1994		rv = null_init_zoned_dev(dev, &lim);
1995		if (rv)
1996			goto out_cleanup_tags;
1997	}
1998
1999	if (dev->cache_size > 0) {
2000		set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
2001		lim.features |= BLK_FEAT_WRITE_CACHE;
2002		if (dev->fua)
2003			lim.features |= BLK_FEAT_FUA;
2004	}
2005
2006	if (dev->rotational)
2007		lim.features |= BLK_FEAT_ROTATIONAL;
2008
2009	nullb->disk = blk_mq_alloc_disk(nullb->tag_set, &lim, nullb);
2010	if (IS_ERR(nullb->disk)) {
2011		rv = PTR_ERR(nullb->disk);
2012		goto out_cleanup_zone;
2013	}
2014	nullb->q = nullb->disk->queue;
2015
2016	if (dev->mbps) {
2017		set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
2018		nullb_setup_bwtimer(nullb);
2019	}
2020
2021	nullb->q->queuedata = nullb;
2022
2023	rv = ida_alloc(&nullb_indexes, GFP_KERNEL);
2024	if (rv < 0)
2025		goto out_cleanup_disk;
2026
2027	nullb->index = rv;
2028	dev->index = rv;
2029
2030	if (config_item_name(&dev->group.cg_item)) {
2031		/* Use configfs dir name as the device name */
2032		snprintf(nullb->disk_name, sizeof(nullb->disk_name),
2033			 "%s", config_item_name(&dev->group.cg_item));
2034	} else {
2035		sprintf(nullb->disk_name, "nullb%d", nullb->index);
2036	}
2037
2038	set_capacity(nullb->disk,
2039		((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT);
2040	nullb->disk->major = null_major;
2041	nullb->disk->first_minor = nullb->index;
2042	nullb->disk->minors = 1;
2043	nullb->disk->fops = &null_ops;
2044	nullb->disk->private_data = nullb;
2045	strscpy(nullb->disk->disk_name, nullb->disk_name);
2046
2047	if (nullb->dev->zoned) {
2048		rv = null_register_zoned_dev(nullb);
2049		if (rv)
2050			goto out_ida_free;
2051	}
2052
2053	rv = add_disk(nullb->disk);
2054	if (rv)
2055		goto out_ida_free;
2056
2057	list_add_tail(&nullb->list, &nullb_list);
2058
2059	pr_info("disk %s created\n", nullb->disk_name);
2060
2061	return 0;
2062
2063out_ida_free:
2064	ida_free(&nullb_indexes, nullb->index);
2065out_cleanup_disk:
2066	put_disk(nullb->disk);
2067out_cleanup_zone:
2068	null_free_zoned_dev(dev);
2069out_cleanup_tags:
2070	if (nullb->tag_set == &nullb->__tag_set)
2071		blk_mq_free_tag_set(nullb->tag_set);
2072out_cleanup_queues:
2073	kfree(nullb->queues);
2074out_free_nullb:
2075	kfree(nullb);
2076	dev->nullb = NULL;
2077out:
2078	return rv;
2079}
2080
2081static struct nullb *null_find_dev_by_name(const char *name)
2082{
2083	struct nullb *nullb = NULL, *nb;
2084
2085	mutex_lock(&lock);
2086	list_for_each_entry(nb, &nullb_list, list) {
2087		if (strcmp(nb->disk_name, name) == 0) {
2088			nullb = nb;
2089			break;
2090		}
2091	}
2092	mutex_unlock(&lock);
2093
2094	return nullb;
2095}
2096
2097static int null_create_dev(void)
2098{
2099	struct nullb_device *dev;
2100	int ret;
2101
2102	dev = null_alloc_dev();
2103	if (!dev)
2104		return -ENOMEM;
2105
2106	mutex_lock(&lock);
2107	ret = null_add_dev(dev);
2108	mutex_unlock(&lock);
2109	if (ret) {
2110		null_free_dev(dev);
2111		return ret;
2112	}
2113
2114	return 0;
2115}
2116
2117static void null_destroy_dev(struct nullb *nullb)
2118{
2119	struct nullb_device *dev = nullb->dev;
2120
2121	null_del_dev(nullb);
2122	null_free_device_storage(dev, false);
2123	null_free_dev(dev);
2124}
2125
2126static int __init null_init(void)
2127{
2128	int ret = 0;
2129	unsigned int i;
2130	struct nullb *nullb;
2131
2132	if (g_bs > PAGE_SIZE) {
2133		pr_warn("invalid block size\n");
2134		pr_warn("defaults block size to %lu\n", PAGE_SIZE);
2135		g_bs = PAGE_SIZE;
2136	}
2137
2138	if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
2139		pr_err("invalid home_node value\n");
2140		g_home_node = NUMA_NO_NODE;
2141	}
2142
2143	if (!null_setup_fault())
2144		return -EINVAL;
2145
2146	if (g_queue_mode == NULL_Q_RQ) {
2147		pr_err("legacy IO path is no longer available\n");
2148		return -EINVAL;
2149	}
2150
2151	if (g_use_per_node_hctx) {
2152		if (g_submit_queues != nr_online_nodes) {
2153			pr_warn("submit_queues param is set to %u.\n",
2154				nr_online_nodes);
2155			g_submit_queues = nr_online_nodes;
2156		}
2157	} else if (g_submit_queues > nr_cpu_ids) {
2158		g_submit_queues = nr_cpu_ids;
2159	} else if (g_submit_queues <= 0) {
2160		g_submit_queues = 1;
2161	}
2162
2163	config_group_init(&nullb_subsys.su_group);
2164	mutex_init(&nullb_subsys.su_mutex);
2165
2166	ret = configfs_register_subsystem(&nullb_subsys);
2167	if (ret)
2168		return ret;
2169
2170	mutex_init(&lock);
2171
2172	null_major = register_blkdev(0, "nullb");
2173	if (null_major < 0) {
2174		ret = null_major;
2175		goto err_conf;
2176	}
2177
2178	for (i = 0; i < nr_devices; i++) {
2179		ret = null_create_dev();
2180		if (ret)
2181			goto err_dev;
2182	}
2183
2184	pr_info("module loaded\n");
2185	return 0;
2186
2187err_dev:
2188	while (!list_empty(&nullb_list)) {
2189		nullb = list_entry(nullb_list.next, struct nullb, list);
2190		null_destroy_dev(nullb);
2191	}
2192	unregister_blkdev(null_major, "nullb");
2193err_conf:
2194	configfs_unregister_subsystem(&nullb_subsys);
2195	return ret;
2196}
2197
2198static void __exit null_exit(void)
2199{
2200	struct nullb *nullb;
2201
2202	configfs_unregister_subsystem(&nullb_subsys);
2203
2204	unregister_blkdev(null_major, "nullb");
2205
2206	mutex_lock(&lock);
2207	while (!list_empty(&nullb_list)) {
2208		nullb = list_entry(nullb_list.next, struct nullb, list);
2209		null_destroy_dev(nullb);
2210	}
2211	mutex_unlock(&lock);
2212
2213	if (tag_set.ops)
2214		blk_mq_free_tag_set(&tag_set);
2215
2216	mutex_destroy(&lock);
2217}
2218
2219module_init(null_init);
2220module_exit(null_exit);
2221
2222MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2223MODULE_DESCRIPTION("multi queue aware block test driver");
2224MODULE_LICENSE("GPL");