blk-sysfs.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Functions related to sysfs handling
   4 */
   5#include <linux/kernel.h>
   6#include <linux/slab.h>
   7#include <linux/module.h>
   8#include <linux/bio.h>
   9#include <linux/blkdev.h>
  10#include <linux/backing-dev.h>
  11#include <linux/blktrace_api.h>
  12#include <linux/debugfs.h>
  13
  14#include "blk.h"
  15#include "blk-mq.h"
  16#include "blk-mq-debugfs.h"
  17#include "blk-mq-sched.h"
  18#include "blk-rq-qos.h"
  19#include "blk-wbt.h"
  20#include "blk-cgroup.h"
  21#include "blk-throttle.h"
  22
  23struct queue_sysfs_entry {
  24	struct attribute attr;
  25	ssize_t (*show)(struct gendisk *disk, char *page);
  26	ssize_t (*show_limit)(struct gendisk *disk, char *page);
  27
  28	ssize_t (*store)(struct gendisk *disk, const char *page, size_t count);
  29	int (*store_limit)(struct gendisk *disk, const char *page,
  30			size_t count, struct queue_limits *lim);
  31};
  32
  33static ssize_t
  34queue_var_show(unsigned long var, char *page)
  35{
  36	return sysfs_emit(page, "%lu\n", var);
  37}
  38
  39static ssize_t
  40queue_var_store(unsigned long *var, const char *page, size_t count)
  41{
  42	int err;
  43	unsigned long v;
  44
  45	err = kstrtoul(page, 10, &v);
  46	if (err || v > UINT_MAX)
  47		return -EINVAL;
  48
  49	*var = v;
  50
  51	return count;
  52}
  53
  54static ssize_t queue_requests_show(struct gendisk *disk, char *page)
  55{
  56	ssize_t ret;
  57
  58	mutex_lock(&disk->queue->elevator_lock);
  59	ret = queue_var_show(disk->queue->nr_requests, page);
  60	mutex_unlock(&disk->queue->elevator_lock);
  61	return ret;
  62}
  63
  64static ssize_t
  65queue_requests_store(struct gendisk *disk, const char *page, size_t count)
  66{
  67	struct request_queue *q = disk->queue;
  68	struct blk_mq_tag_set *set = q->tag_set;
  69	struct elevator_tags *et = NULL;
  70	unsigned int memflags;
  71	unsigned long nr;
  72	int ret;
  73
  74	ret = queue_var_store(&nr, page, count);
  75	if (ret < 0)
  76		return ret;
  77
  78	/*
  79	 * Serialize updating nr_requests with concurrent queue_requests_store()
  80	 * and switching elevator.
  81	 *
  82	 * Use trylock to avoid circular lock dependency with kernfs active
  83	 * reference during concurrent disk deletion:
  84	 *   update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del)
  85	 *   kn->active -> update_nr_hwq_lock (via this sysfs write path)
  86	 */
  87	if (!down_write_trylock(&set->update_nr_hwq_lock))
  88		return -EBUSY;
  89
  90	if (nr == q->nr_requests)
  91		goto unlock;
  92
  93	if (nr < BLKDEV_MIN_RQ)
  94		nr = BLKDEV_MIN_RQ;
  95
  96	/*
  97	 * Switching elevator is protected by update_nr_hwq_lock:
  98	 *  - read lock is held from elevator sysfs attribute;
  99	 *  - write lock is held from updating nr_hw_queues;
 100	 * Hence it's safe to access q->elevator here with write lock held.
 101	 */
 102	if (nr <= set->reserved_tags ||
 103	    (q->elevator && nr > MAX_SCHED_RQ) ||
 104	    (!q->elevator && nr > set->queue_depth)) {
 105		ret = -EINVAL;
 106		goto unlock;
 107	}
 108
 109	if (!blk_mq_is_shared_tags(set->flags) && q->elevator &&
 110	    nr > q->elevator->et->nr_requests) {
 111		/*
 112		 * Tags will grow, allocate memory before freezing queue to
 113		 * prevent deadlock.
 114		 */
 115		et = blk_mq_alloc_sched_tags(set, q->nr_hw_queues, nr);
 116		if (!et) {
 117			ret = -ENOMEM;
 118			goto unlock;
 119		}
 120	}
 121
 122	memflags = blk_mq_freeze_queue(q);
 123	mutex_lock(&q->elevator_lock);
 124	et = blk_mq_update_nr_requests(q, et, nr);
 125	mutex_unlock(&q->elevator_lock);
 126	blk_mq_unfreeze_queue(q, memflags);
 127
 128	if (et)
 129		blk_mq_free_sched_tags(et, set);
 130
 131unlock:
 132	up_write(&set->update_nr_hwq_lock);
 133	return ret;
 134}
 135
 136static ssize_t queue_ra_show(struct gendisk *disk, char *page)
 137{
 138	ssize_t ret;
 139
 140	mutex_lock(&disk->queue->limits_lock);
 141	ret = queue_var_show(disk->bdi->ra_pages << (PAGE_SHIFT - 10), page);
 142	mutex_unlock(&disk->queue->limits_lock);
 143
 144	return ret;
 145}
 146
 147static ssize_t
 148queue_ra_store(struct gendisk *disk, const char *page, size_t count)
 149{
 150	unsigned long ra_kb;
 151	ssize_t ret;
 152	struct request_queue *q = disk->queue;
 153
 154	ret = queue_var_store(&ra_kb, page, count);
 155	if (ret < 0)
 156		return ret;
 157	/*
 158	 * The ->ra_pages change below is protected by ->limits_lock because it
 159	 * is usually calculated from the queue limits by
 160	 * queue_limits_commit_update().
 161	 *
 162	 * bdi->ra_pages reads are not serialized against bdi->ra_pages writes.
 163	 * Use WRITE_ONCE() to write bdi->ra_pages once.
 164	 */
 165	mutex_lock(&q->limits_lock);
 166	WRITE_ONCE(disk->bdi->ra_pages, ra_kb >> (PAGE_SHIFT - 10));
 167	mutex_unlock(&q->limits_lock);
 168
 169	return ret;
 170}
 171
 172#define QUEUE_SYSFS_LIMIT_SHOW(_field)					\
 173static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)	\
 174{									\
 175	return queue_var_show(disk->queue->limits._field, page);	\
 176}
 177
 178QUEUE_SYSFS_LIMIT_SHOW(max_segments)
 179QUEUE_SYSFS_LIMIT_SHOW(max_discard_segments)
 180QUEUE_SYSFS_LIMIT_SHOW(max_integrity_segments)
 181QUEUE_SYSFS_LIMIT_SHOW(max_segment_size)
 182QUEUE_SYSFS_LIMIT_SHOW(max_write_streams)
 183QUEUE_SYSFS_LIMIT_SHOW(write_stream_granularity)
 184QUEUE_SYSFS_LIMIT_SHOW(logical_block_size)
 185QUEUE_SYSFS_LIMIT_SHOW(physical_block_size)
 186QUEUE_SYSFS_LIMIT_SHOW(chunk_sectors)
 187QUEUE_SYSFS_LIMIT_SHOW(io_min)
 188QUEUE_SYSFS_LIMIT_SHOW(io_opt)
 189QUEUE_SYSFS_LIMIT_SHOW(discard_granularity)
 190QUEUE_SYSFS_LIMIT_SHOW(zone_write_granularity)
 191QUEUE_SYSFS_LIMIT_SHOW(virt_boundary_mask)
 192QUEUE_SYSFS_LIMIT_SHOW(dma_alignment)
 193QUEUE_SYSFS_LIMIT_SHOW(max_open_zones)
 194QUEUE_SYSFS_LIMIT_SHOW(max_active_zones)
 195QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_min)
 196QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_max)
 197
 198#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(_field)			\
 199static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)	\
 200{									\
 201	return sysfs_emit(page, "%llu\n",				\
 202		(unsigned long long)disk->queue->limits._field <<	\
 203			SECTOR_SHIFT);					\
 204}
 205
 206QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_discard_sectors)
 207QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_discard_sectors)
 208QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_write_zeroes_sectors)
 209QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_wzeroes_unmap_sectors)
 210QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_wzeroes_unmap_sectors)
 211QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_max_sectors)
 212QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_boundary_sectors)
 213QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_zone_append_sectors)
 214
 215#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(_field)			\
 216static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)	\
 217{									\
 218	return queue_var_show(disk->queue->limits._field >> 1, page);	\
 219}
 220
 221QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_sectors)
 222QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_hw_sectors)
 223
 224#define QUEUE_SYSFS_SHOW_CONST(_name, _val)				\
 225static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)	\
 226{									\
 227	return sysfs_emit(page, "%d\n", _val);				\
 228}
 229
 230/* deprecated fields */
 231QUEUE_SYSFS_SHOW_CONST(discard_zeroes_data, 0)
 232QUEUE_SYSFS_SHOW_CONST(write_same_max, 0)
 233QUEUE_SYSFS_SHOW_CONST(poll_delay, -1)
 234
 235static int queue_max_discard_sectors_store(struct gendisk *disk,
 236		const char *page, size_t count, struct queue_limits *lim)
 237{
 238	unsigned long max_discard_bytes;
 239	ssize_t ret;
 240
 241	ret = queue_var_store(&max_discard_bytes, page, count);
 242	if (ret < 0)
 243		return ret;
 244
 245	if (max_discard_bytes & (disk->queue->limits.discard_granularity - 1))
 246		return -EINVAL;
 247
 248	if ((max_discard_bytes >> SECTOR_SHIFT) > UINT_MAX)
 249		return -EINVAL;
 250
 251	lim->max_user_discard_sectors = max_discard_bytes >> SECTOR_SHIFT;
 252	return 0;
 253}
 254
 255static int queue_max_wzeroes_unmap_sectors_store(struct gendisk *disk,
 256		const char *page, size_t count, struct queue_limits *lim)
 257{
 258	unsigned long max_zeroes_bytes, max_hw_zeroes_bytes;
 259	ssize_t ret;
 260
 261	ret = queue_var_store(&max_zeroes_bytes, page, count);
 262	if (ret < 0)
 263		return ret;
 264
 265	max_hw_zeroes_bytes = lim->max_hw_wzeroes_unmap_sectors << SECTOR_SHIFT;
 266	if (max_zeroes_bytes != 0 && max_zeroes_bytes != max_hw_zeroes_bytes)
 267		return -EINVAL;
 268
 269	lim->max_user_wzeroes_unmap_sectors = max_zeroes_bytes >> SECTOR_SHIFT;
 270	return 0;
 271}
 272
 273static int
 274queue_max_sectors_store(struct gendisk *disk, const char *page, size_t count,
 275		struct queue_limits *lim)
 276{
 277	unsigned long max_sectors_kb;
 278	ssize_t ret;
 279
 280	ret = queue_var_store(&max_sectors_kb, page, count);
 281	if (ret < 0)
 282		return ret;
 283
 284	lim->max_user_sectors = max_sectors_kb << 1;
 285	return 0;
 286}
 287
 288static ssize_t queue_feature_store(struct gendisk *disk, const char *page,
 289		size_t count, struct queue_limits *lim, blk_features_t feature)
 290{
 291	unsigned long val;
 292	ssize_t ret;
 293
 294	ret = queue_var_store(&val, page, count);
 295	if (ret < 0)
 296		return ret;
 297
 298	if (val)
 299		lim->features |= feature;
 300	else
 301		lim->features &= ~feature;
 302	return 0;
 303}
 304
 305#define QUEUE_SYSFS_FEATURE(_name, _feature)				\
 306static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)	\
 307{									\
 308	return sysfs_emit(page, "%u\n",					\
 309		!!(disk->queue->limits.features & _feature));		\
 310}									\
 311static int queue_##_name##_store(struct gendisk *disk,			\
 312		const char *page, size_t count, struct queue_limits *lim) \
 313{									\
 314	return queue_feature_store(disk, page, count, lim, _feature);	\
 315}
 316
 317QUEUE_SYSFS_FEATURE(rotational, BLK_FEAT_ROTATIONAL)
 318QUEUE_SYSFS_FEATURE(add_random, BLK_FEAT_ADD_RANDOM)
 319QUEUE_SYSFS_FEATURE(iostats, BLK_FEAT_IO_STAT)
 320QUEUE_SYSFS_FEATURE(stable_writes, BLK_FEAT_STABLE_WRITES);
 321
 322#define QUEUE_SYSFS_FEATURE_SHOW(_name, _feature)			\
 323static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)	\
 324{									\
 325	return sysfs_emit(page, "%u\n",					\
 326		!!(disk->queue->limits.features & _feature));		\
 327}
 328
 329QUEUE_SYSFS_FEATURE_SHOW(fua, BLK_FEAT_FUA);
 330QUEUE_SYSFS_FEATURE_SHOW(dax, BLK_FEAT_DAX);
 331
 332static ssize_t queue_poll_show(struct gendisk *disk, char *page)
 333{
 334	if (queue_is_mq(disk->queue))
 335		return sysfs_emit(page, "%u\n", blk_mq_can_poll(disk->queue));
 336
 337	return sysfs_emit(page, "%u\n",
 338			!!(disk->queue->limits.features & BLK_FEAT_POLL));
 339}
 340
 341static ssize_t queue_zoned_show(struct gendisk *disk, char *page)
 342{
 343	if (blk_queue_is_zoned(disk->queue))
 344		return sysfs_emit(page, "host-managed\n");
 345	return sysfs_emit(page, "none\n");
 346}
 347
 348static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page)
 349{
 350	return queue_var_show(disk_nr_zones(disk), page);
 351}
 352
 353static ssize_t queue_iostats_passthrough_show(struct gendisk *disk, char *page)
 354{
 355	return queue_var_show(!!blk_queue_passthrough_stat(disk->queue), page);
 356}
 357
 358static int queue_iostats_passthrough_store(struct gendisk *disk,
 359		const char *page, size_t count, struct queue_limits *lim)
 360{
 361	unsigned long ios;
 362	ssize_t ret;
 363
 364	ret = queue_var_store(&ios, page, count);
 365	if (ret < 0)
 366		return ret;
 367
 368	if (ios)
 369		lim->flags |= BLK_FLAG_IOSTATS_PASSTHROUGH;
 370	else
 371		lim->flags &= ~BLK_FLAG_IOSTATS_PASSTHROUGH;
 372	return 0;
 373}
 374
 375static ssize_t queue_nomerges_show(struct gendisk *disk, char *page)
 376{
 377	return queue_var_show((blk_queue_nomerges(disk->queue) << 1) |
 378			       blk_queue_noxmerges(disk->queue), page);
 379}
 380
 381static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page,
 382				    size_t count)
 383{
 384	unsigned long nm;
 385	struct request_queue *q = disk->queue;
 386	ssize_t ret = queue_var_store(&nm, page, count);
 387
 388	if (ret < 0)
 389		return ret;
 390
 391	blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
 392	blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
 393	if (nm == 2)
 394		blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
 395	else if (nm)
 396		blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
 397
 398	return ret;
 399}
 400
 401static ssize_t queue_rq_affinity_show(struct gendisk *disk, char *page)
 402{
 403	bool set = test_bit(QUEUE_FLAG_SAME_COMP, &disk->queue->queue_flags);
 404	bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &disk->queue->queue_flags);
 405
 406	return queue_var_show(set << force, page);
 407}
 408
 409static ssize_t
 410queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count)
 411{
 412	ssize_t ret = -EINVAL;
 413#ifdef CONFIG_SMP
 414	struct request_queue *q = disk->queue;
 415	unsigned long val;
 416
 417	ret = queue_var_store(&val, page, count);
 418	if (ret < 0)
 419		return ret;
 420
 421	/*
 422	 * Here we update two queue flags each using atomic bitops, although
 423	 * updating two flags isn't atomic it should be harmless as those flags
 424	 * are accessed individually using atomic test_bit operation. So we
 425	 * don't grab any lock while updating these flags.
 426	 */
 427	if (val == 2) {
 428		blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
 429		blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
 430	} else if (val == 1) {
 431		blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
 432		blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
 433	} else if (val == 0) {
 434		blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
 435		blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
 436	}
 437#endif
 438	return ret;
 439}
 440
 441static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page,
 442				size_t count)
 443{
 444	return count;
 445}
 446
 447static ssize_t queue_poll_store(struct gendisk *disk, const char *page,
 448				size_t count)
 449{
 450	ssize_t ret = count;
 451	struct request_queue *q = disk->queue;
 452
 453	if (!(q->limits.features & BLK_FEAT_POLL)) {
 454		ret = -EINVAL;
 455		goto out;
 456	}
 457
 458	pr_info_ratelimited("writes to the poll attribute are ignored.\n");
 459	pr_info_ratelimited("please use driver specific parameters instead.\n");
 460out:
 461	return ret;
 462}
 463
 464static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page)
 465{
 466	return sysfs_emit(page, "%u\n",
 467			jiffies_to_msecs(READ_ONCE(disk->queue->rq_timeout)));
 468}
 469
 470static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page,
 471				  size_t count)
 472{
 473	unsigned int val;
 474	int err;
 475	struct request_queue *q = disk->queue;
 476
 477	err = kstrtou32(page, 10, &val);
 478	if (err || val == 0)
 479		return -EINVAL;
 480
 481	blk_queue_rq_timeout(q, msecs_to_jiffies(val));
 482
 483	return count;
 484}
 485
 486static ssize_t queue_wc_show(struct gendisk *disk, char *page)
 487{
 488	if (blk_queue_write_cache(disk->queue))
 489		return sysfs_emit(page, "write back\n");
 490	return sysfs_emit(page, "write through\n");
 491}
 492
 493static int queue_wc_store(struct gendisk *disk, const char *page,
 494		size_t count, struct queue_limits *lim)
 495{
 496	bool disable;
 497
 498	if (!strncmp(page, "write back", 10)) {
 499		disable = false;
 500	} else if (!strncmp(page, "write through", 13) ||
 501		   !strncmp(page, "none", 4)) {
 502		disable = true;
 503	} else {
 504		return -EINVAL;
 505	}
 506
 507	if (disable)
 508		lim->flags |= BLK_FLAG_WRITE_CACHE_DISABLED;
 509	else
 510		lim->flags &= ~BLK_FLAG_WRITE_CACHE_DISABLED;
 511	return 0;
 512}
 513
 514#define QUEUE_RO_ENTRY(_prefix, _name)			\
 515static struct queue_sysfs_entry _prefix##_entry = {	\
 516	.attr	= { .name = _name, .mode = 0444 },	\
 517	.show	= _prefix##_show,			\
 518};
 519
 520#define QUEUE_RW_ENTRY(_prefix, _name)			\
 521static struct queue_sysfs_entry _prefix##_entry = {	\
 522	.attr	= { .name = _name, .mode = 0644 },	\
 523	.show	= _prefix##_show,			\
 524	.store	= _prefix##_store,			\
 525};
 526
 527#define QUEUE_LIM_RO_ENTRY(_prefix, _name)			\
 528static struct queue_sysfs_entry _prefix##_entry = {	\
 529	.attr		= { .name = _name, .mode = 0444 },	\
 530	.show_limit	= _prefix##_show,			\
 531}
 532
 533#define QUEUE_LIM_RW_ENTRY(_prefix, _name)			\
 534static struct queue_sysfs_entry _prefix##_entry = {	\
 535	.attr		= { .name = _name, .mode = 0644 },	\
 536	.show_limit	= _prefix##_show,			\
 537	.store_limit	= _prefix##_store,			\
 538}
 539
 540QUEUE_RW_ENTRY(queue_requests, "nr_requests");
 541QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
 542QUEUE_LIM_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
 543QUEUE_LIM_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
 544QUEUE_LIM_RO_ENTRY(queue_max_segments, "max_segments");
 545QUEUE_LIM_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
 546QUEUE_LIM_RO_ENTRY(queue_max_segment_size, "max_segment_size");
 547QUEUE_LIM_RO_ENTRY(queue_max_write_streams, "max_write_streams");
 548QUEUE_LIM_RO_ENTRY(queue_write_stream_granularity, "write_stream_granularity");
 549QUEUE_RW_ENTRY(elv_iosched, "scheduler");
 550
 551QUEUE_LIM_RO_ENTRY(queue_logical_block_size, "logical_block_size");
 552QUEUE_LIM_RO_ENTRY(queue_physical_block_size, "physical_block_size");
 553QUEUE_LIM_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
 554QUEUE_LIM_RO_ENTRY(queue_io_min, "minimum_io_size");
 555QUEUE_LIM_RO_ENTRY(queue_io_opt, "optimal_io_size");
 556
 557QUEUE_LIM_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
 558QUEUE_LIM_RO_ENTRY(queue_discard_granularity, "discard_granularity");
 559QUEUE_LIM_RO_ENTRY(queue_max_hw_discard_sectors, "discard_max_hw_bytes");
 560QUEUE_LIM_RW_ENTRY(queue_max_discard_sectors, "discard_max_bytes");
 561QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
 562
 563QUEUE_LIM_RO_ENTRY(queue_atomic_write_max_sectors, "atomic_write_max_bytes");
 564QUEUE_LIM_RO_ENTRY(queue_atomic_write_boundary_sectors,
 565		"atomic_write_boundary_bytes");
 566QUEUE_LIM_RO_ENTRY(queue_atomic_write_unit_max, "atomic_write_unit_max_bytes");
 567QUEUE_LIM_RO_ENTRY(queue_atomic_write_unit_min, "atomic_write_unit_min_bytes");
 568
 569QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
 570QUEUE_LIM_RO_ENTRY(queue_max_write_zeroes_sectors, "write_zeroes_max_bytes");
 571QUEUE_LIM_RO_ENTRY(queue_max_hw_wzeroes_unmap_sectors,
 572		"write_zeroes_unmap_max_hw_bytes");
 573QUEUE_LIM_RW_ENTRY(queue_max_wzeroes_unmap_sectors,
 574		"write_zeroes_unmap_max_bytes");
 575QUEUE_LIM_RO_ENTRY(queue_max_zone_append_sectors, "zone_append_max_bytes");
 576QUEUE_LIM_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
 577
 578QUEUE_LIM_RO_ENTRY(queue_zoned, "zoned");
 579QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
 580QUEUE_LIM_RO_ENTRY(queue_max_open_zones, "max_open_zones");
 581QUEUE_LIM_RO_ENTRY(queue_max_active_zones, "max_active_zones");
 582
 583QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
 584QUEUE_LIM_RW_ENTRY(queue_iostats_passthrough, "iostats_passthrough");
 585QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
 586QUEUE_RW_ENTRY(queue_poll, "io_poll");
 587QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
 588QUEUE_LIM_RW_ENTRY(queue_wc, "write_cache");
 589QUEUE_LIM_RO_ENTRY(queue_fua, "fua");
 590QUEUE_LIM_RO_ENTRY(queue_dax, "dax");
 591QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
 592QUEUE_LIM_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
 593QUEUE_LIM_RO_ENTRY(queue_dma_alignment, "dma_alignment");
 594
 595/* legacy alias for logical_block_size: */
 596static struct queue_sysfs_entry queue_hw_sector_size_entry = {
 597	.attr		= {.name = "hw_sector_size", .mode = 0444 },
 598	.show_limit	= queue_logical_block_size_show,
 599};
 600
 601QUEUE_LIM_RW_ENTRY(queue_rotational, "rotational");
 602QUEUE_LIM_RW_ENTRY(queue_iostats, "iostats");
 603QUEUE_LIM_RW_ENTRY(queue_add_random, "add_random");
 604QUEUE_LIM_RW_ENTRY(queue_stable_writes, "stable_writes");
 605
 606#ifdef CONFIG_BLK_WBT
 607static ssize_t queue_var_store64(s64 *var, const char *page)
 608{
 609	int err;
 610	s64 v;
 611
 612	err = kstrtos64(page, 10, &v);
 613	if (err < 0)
 614		return err;
 615
 616	*var = v;
 617	return 0;
 618}
 619
 620static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)
 621{
 622	ssize_t ret;
 623	struct request_queue *q = disk->queue;
 624
 625	mutex_lock(&disk->rqos_state_mutex);
 626	if (!wbt_rq_qos(q)) {
 627		ret = -EINVAL;
 628		goto out;
 629	}
 630
 631	if (wbt_disabled(q)) {
 632		ret = sysfs_emit(page, "0\n");
 633		goto out;
 634	}
 635
 636	ret = sysfs_emit(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
 637out:
 638	mutex_unlock(&disk->rqos_state_mutex);
 639	return ret;
 640}
 641
 642static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,
 643				  size_t count)
 644{
 645	struct request_queue *q = disk->queue;
 646	struct rq_qos *rqos;
 647	ssize_t ret;
 648	s64 val;
 649	unsigned int memflags;
 650
 651	ret = queue_var_store64(&val, page);
 652	if (ret < 0)
 653		return ret;
 654	if (val < -1)
 655		return -EINVAL;
 656
 657	/*
 658	 * Ensure that the queue is idled, in case the latency update
 659	 * ends up either enabling or disabling wbt completely. We can't
 660	 * have IO inflight if that happens.
 661	 */
 662	memflags = blk_mq_freeze_queue(q);
 663
 664	rqos = wbt_rq_qos(q);
 665	if (!rqos) {
 666		ret = wbt_init(disk);
 667		if (ret)
 668			goto out;
 669	}
 670
 671	ret = count;
 672	if (val == -1)
 673		val = wbt_default_latency_nsec(q);
 674	else if (val >= 0)
 675		val *= 1000ULL;
 676
 677	if (wbt_get_min_lat(q) == val)
 678		goto out;
 679
 680	blk_mq_quiesce_queue(q);
 681
 682	mutex_lock(&disk->rqos_state_mutex);
 683	wbt_set_min_lat(q, val);
 684	mutex_unlock(&disk->rqos_state_mutex);
 685
 686	blk_mq_unquiesce_queue(q);
 687out:
 688	blk_mq_unfreeze_queue(q, memflags);
 689
 690	return ret;
 691}
 692
 693QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
 694#endif
 695
 696/* Common attributes for bio-based and request-based queues. */
 697static struct attribute *queue_attrs[] = {
 698	/*
 699	 * Attributes which are protected with q->limits_lock.
 700	 */
 701	&queue_max_hw_sectors_entry.attr,
 702	&queue_max_sectors_entry.attr,
 703	&queue_max_segments_entry.attr,
 704	&queue_max_discard_segments_entry.attr,
 705	&queue_max_integrity_segments_entry.attr,
 706	&queue_max_segment_size_entry.attr,
 707	&queue_max_write_streams_entry.attr,
 708	&queue_write_stream_granularity_entry.attr,
 709	&queue_hw_sector_size_entry.attr,
 710	&queue_logical_block_size_entry.attr,
 711	&queue_physical_block_size_entry.attr,
 712	&queue_chunk_sectors_entry.attr,
 713	&queue_io_min_entry.attr,
 714	&queue_io_opt_entry.attr,
 715	&queue_discard_granularity_entry.attr,
 716	&queue_max_discard_sectors_entry.attr,
 717	&queue_max_hw_discard_sectors_entry.attr,
 718	&queue_atomic_write_max_sectors_entry.attr,
 719	&queue_atomic_write_boundary_sectors_entry.attr,
 720	&queue_atomic_write_unit_min_entry.attr,
 721	&queue_atomic_write_unit_max_entry.attr,
 722	&queue_max_write_zeroes_sectors_entry.attr,
 723	&queue_max_hw_wzeroes_unmap_sectors_entry.attr,
 724	&queue_max_wzeroes_unmap_sectors_entry.attr,
 725	&queue_max_zone_append_sectors_entry.attr,
 726	&queue_zone_write_granularity_entry.attr,
 727	&queue_rotational_entry.attr,
 728	&queue_zoned_entry.attr,
 729	&queue_max_open_zones_entry.attr,
 730	&queue_max_active_zones_entry.attr,
 731	&queue_iostats_passthrough_entry.attr,
 732	&queue_iostats_entry.attr,
 733	&queue_stable_writes_entry.attr,
 734	&queue_add_random_entry.attr,
 735	&queue_wc_entry.attr,
 736	&queue_fua_entry.attr,
 737	&queue_dax_entry.attr,
 738	&queue_virt_boundary_mask_entry.attr,
 739	&queue_dma_alignment_entry.attr,
 740	&queue_ra_entry.attr,
 741
 742	/*
 743	 * Attributes which don't require locking.
 744	 */
 745	&queue_discard_zeroes_data_entry.attr,
 746	&queue_write_same_max_entry.attr,
 747	&queue_nr_zones_entry.attr,
 748	&queue_nomerges_entry.attr,
 749	&queue_poll_entry.attr,
 750	&queue_poll_delay_entry.attr,
 751
 752	NULL,
 753};
 754
 755/* Request-based queue attributes that are not relevant for bio-based queues. */
 756static struct attribute *blk_mq_queue_attrs[] = {
 757	/*
 758	 * Attributes which require some form of locking other than
 759	 * q->sysfs_lock.
 760	 */
 761	&elv_iosched_entry.attr,
 762	&queue_requests_entry.attr,
 763#ifdef CONFIG_BLK_WBT
 764	&queue_wb_lat_entry.attr,
 765#endif
 766	/*
 767	 * Attributes which don't require locking.
 768	 */
 769	&queue_rq_affinity_entry.attr,
 770	&queue_io_timeout_entry.attr,
 771
 772	NULL,
 773};
 774
 775static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
 776				int n)
 777{
 778	struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 779	struct request_queue *q = disk->queue;
 780
 781	if ((attr == &queue_max_open_zones_entry.attr ||
 782	     attr == &queue_max_active_zones_entry.attr) &&
 783	    !blk_queue_is_zoned(q))
 784		return 0;
 785
 786	return attr->mode;
 787}
 788
 789static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,
 790					 struct attribute *attr, int n)
 791{
 792	struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 793	struct request_queue *q = disk->queue;
 794
 795	if (!queue_is_mq(q))
 796		return 0;
 797
 798	if (attr == &queue_io_timeout_entry.attr && !q->mq_ops->timeout)
 799		return 0;
 800
 801	return attr->mode;
 802}
 803
 804static struct attribute_group queue_attr_group = {
 805	.attrs = queue_attrs,
 806	.is_visible = queue_attr_visible,
 807};
 808
 809static struct attribute_group blk_mq_queue_attr_group = {
 810	.attrs = blk_mq_queue_attrs,
 811	.is_visible = blk_mq_queue_attr_visible,
 812};
 813
 814#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
 815
 816static ssize_t
 817queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 818{
 819	struct queue_sysfs_entry *entry = to_queue(attr);
 820	struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 821
 822	if (!entry->show && !entry->show_limit)
 823		return -EIO;
 824
 825	if (entry->show_limit) {
 826		ssize_t res;
 827
 828		mutex_lock(&disk->queue->limits_lock);
 829		res = entry->show_limit(disk, page);
 830		mutex_unlock(&disk->queue->limits_lock);
 831		return res;
 832	}
 833
 834	return entry->show(disk, page);
 835}
 836
 837static ssize_t
 838queue_attr_store(struct kobject *kobj, struct attribute *attr,
 839		    const char *page, size_t length)
 840{
 841	struct queue_sysfs_entry *entry = to_queue(attr);
 842	struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
 843	struct request_queue *q = disk->queue;
 844
 845	if (!entry->store_limit && !entry->store)
 846		return -EIO;
 847
 848	if (entry->store_limit) {
 849		ssize_t res;
 850
 851		struct queue_limits lim = queue_limits_start_update(q);
 852
 853		res = entry->store_limit(disk, page, length, &lim);
 854		if (res < 0) {
 855			queue_limits_cancel_update(q);
 856			return res;
 857		}
 858
 859		res = queue_limits_commit_update_frozen(q, &lim);
 860		if (res)
 861			return res;
 862		return length;
 863	}
 864
 865	return entry->store(disk, page, length);
 866}
 867
 868static const struct sysfs_ops queue_sysfs_ops = {
 869	.show	= queue_attr_show,
 870	.store	= queue_attr_store,
 871};
 872
 873static const struct attribute_group *blk_queue_attr_groups[] = {
 874	&queue_attr_group,
 875	&blk_mq_queue_attr_group,
 876	NULL
 877};
 878
 879static void blk_queue_release(struct kobject *kobj)
 880{
 881	/* nothing to do here, all data is associated with the parent gendisk */
 882}
 883
 884const struct kobj_type blk_queue_ktype = {
 885	.default_groups = blk_queue_attr_groups,
 886	.sysfs_ops	= &queue_sysfs_ops,
 887	.release	= blk_queue_release,
 888};
 889
 890static void blk_debugfs_remove(struct gendisk *disk)
 891{
 892	struct request_queue *q = disk->queue;
 893
 894	mutex_lock(&q->debugfs_mutex);
 895	blk_trace_shutdown(q);
 896	debugfs_remove_recursive(q->debugfs_dir);
 897	q->debugfs_dir = NULL;
 898	q->sched_debugfs_dir = NULL;
 899	q->rqos_debugfs_dir = NULL;
 900	mutex_unlock(&q->debugfs_mutex);
 901}
 902
 903/**
 904 * blk_register_queue - register a block layer queue with sysfs
 905 * @disk: Disk of which the request queue should be registered with sysfs.
 906 */
 907int blk_register_queue(struct gendisk *disk)
 908{
 909	struct request_queue *q = disk->queue;
 910	int ret;
 911
 912	ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue");
 913	if (ret < 0)
 914		return ret;
 915
 916	if (queue_is_mq(q)) {
 917		ret = blk_mq_sysfs_register(disk);
 918		if (ret)
 919			goto out_del_queue_kobj;
 920	}
 921	mutex_lock(&q->sysfs_lock);
 922
 923	mutex_lock(&q->debugfs_mutex);
 924	q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root);
 925	if (queue_is_mq(q))
 926		blk_mq_debugfs_register(q);
 927	mutex_unlock(&q->debugfs_mutex);
 928
 929	ret = disk_register_independent_access_ranges(disk);
 930	if (ret)
 931		goto out_debugfs_remove;
 932
 933	ret = blk_crypto_sysfs_register(disk);
 934	if (ret)
 935		goto out_unregister_ia_ranges;
 936
 937	if (queue_is_mq(q))
 938		elevator_set_default(q);
 939
 940	blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
 941	wbt_init_enable_default(disk);
 942
 943	/* Now everything is ready and send out KOBJ_ADD uevent */
 944	kobject_uevent(&disk->queue_kobj, KOBJ_ADD);
 945	if (q->elevator)
 946		kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
 947	mutex_unlock(&q->sysfs_lock);
 948
 949	/*
 950	 * SCSI probing may synchronously create and destroy a lot of
 951	 * request_queues for non-existent devices.  Shutting down a fully
 952	 * functional queue takes measureable wallclock time as RCU grace
 953	 * periods are involved.  To avoid excessive latency in these
 954	 * cases, a request_queue starts out in a degraded mode which is
 955	 * faster to shut down and is made fully functional here as
 956	 * request_queues for non-existent devices never get registered.
 957	 */
 958	blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
 959	percpu_ref_switch_to_percpu(&q->q_usage_counter);
 960
 961	return ret;
 962
 963out_unregister_ia_ranges:
 964	disk_unregister_independent_access_ranges(disk);
 965out_debugfs_remove:
 966	blk_debugfs_remove(disk);
 967	mutex_unlock(&q->sysfs_lock);
 968	if (queue_is_mq(q))
 969		blk_mq_sysfs_unregister(disk);
 970out_del_queue_kobj:
 971	kobject_del(&disk->queue_kobj);
 972	return ret;
 973}
 974
 975/**
 976 * blk_unregister_queue - counterpart of blk_register_queue()
 977 * @disk: Disk of which the request queue should be unregistered from sysfs.
 978 *
 979 * Note: the caller is responsible for guaranteeing that this function is called
 980 * after blk_register_queue() has finished.
 981 */
 982void blk_unregister_queue(struct gendisk *disk)
 983{
 984	struct request_queue *q = disk->queue;
 985
 986	if (WARN_ON(!q))
 987		return;
 988
 989	/* Return early if disk->queue was never registered. */
 990	if (!blk_queue_registered(q))
 991		return;
 992
 993	/*
 994	 * Since sysfs_remove_dir() prevents adding new directory entries
 995	 * before removal of existing entries starts, protect against
 996	 * concurrent elv_iosched_store() calls.
 997	 */
 998	mutex_lock(&q->sysfs_lock);
 999	blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
1000	mutex_unlock(&q->sysfs_lock);
1001
1002	/*
1003	 * Remove the sysfs attributes before unregistering the queue data
1004	 * structures that can be modified through sysfs.
1005	 */
1006	if (queue_is_mq(q))
1007		blk_mq_sysfs_unregister(disk);
1008	blk_crypto_sysfs_unregister(disk);
1009
1010	mutex_lock(&q->sysfs_lock);
1011	disk_unregister_independent_access_ranges(disk);
1012	mutex_unlock(&q->sysfs_lock);
1013
1014	/* Now that we've deleted all child objects, we can delete the queue. */
1015	kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE);
1016	kobject_del(&disk->queue_kobj);
1017
1018	if (queue_is_mq(q))
1019		elevator_set_none(q);
1020
1021	blk_debugfs_remove(disk);
1022}