super.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * bcache setup/teardown code, and some metadata io - read a superblock and
   4 * figure out what to do with it.
   5 *
   6 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
   7 * Copyright 2012 Google, Inc.
   8 */
   9
  10#include "bcache.h"
  11#include "btree.h"
  12#include "debug.h"
  13#include "extents.h"
  14#include "request.h"
  15#include "writeback.h"
  16#include "features.h"
  17
  18#include <linux/blkdev.h>
  19#include <linux/pagemap.h>
  20#include <linux/debugfs.h>
  21#include <linux/idr.h>
  22#include <linux/kthread.h>
  23#include <linux/workqueue.h>
  24#include <linux/module.h>
  25#include <linux/random.h>
  26#include <linux/reboot.h>
  27#include <linux/sysfs.h>
  28
  29unsigned int bch_cutoff_writeback;
  30unsigned int bch_cutoff_writeback_sync;
  31
  32static const char bcache_magic[] = {
  33	0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
  34	0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
  35};
  36
  37static const char invalid_uuid[] = {
  38	0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
  39	0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
  40};
  41
  42static struct kobject *bcache_kobj;
  43struct mutex bch_register_lock;
  44bool bcache_is_reboot;
  45LIST_HEAD(bch_cache_sets);
  46static LIST_HEAD(uncached_devices);
  47
  48static int bcache_major;
  49static DEFINE_IDA(bcache_device_idx);
  50static wait_queue_head_t unregister_wait;
  51struct workqueue_struct *bcache_wq;
  52struct workqueue_struct *bch_flush_wq;
  53struct workqueue_struct *bch_journal_wq;
  54
  55
  56#define BTREE_MAX_PAGES		(256 * 1024 / PAGE_SIZE)
  57/* limitation of partitions number on single bcache device */
  58#define BCACHE_MINORS		128
  59/* limitation of bcache devices number on single system */
  60#define BCACHE_DEVICE_IDX_MAX	((1U << MINORBITS)/BCACHE_MINORS)
  61
  62/* Superblock */
  63
  64static unsigned int get_bucket_size(struct cache_sb *sb, struct cache_sb_disk *s)
  65{
  66	unsigned int bucket_size = le16_to_cpu(s->bucket_size);
  67
  68	if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) {
  69		if (bch_has_feature_large_bucket(sb)) {
  70			unsigned int max, order;
  71
  72			max = sizeof(unsigned int) * BITS_PER_BYTE - 1;
  73			order = le16_to_cpu(s->bucket_size);
  74			/*
  75			 * bcache tool will make sure the overflow won't
  76			 * happen, an error message here is enough.
  77			 */
  78			if (order > max)
  79				pr_err("Bucket size (1 << %u) overflows\n",
  80					order);
  81			bucket_size = 1 << order;
  82		} else if (bch_has_feature_obso_large_bucket(sb)) {
  83			bucket_size +=
  84				le16_to_cpu(s->obso_bucket_size_hi) << 16;
  85		}
  86	}
  87
  88	return bucket_size;
  89}
  90
  91static const char *read_super_common(struct cache_sb *sb,  struct block_device *bdev,
  92				     struct cache_sb_disk *s)
  93{
  94	const char *err;
  95	unsigned int i;
  96
  97	sb->first_bucket= le16_to_cpu(s->first_bucket);
  98	sb->nbuckets	= le64_to_cpu(s->nbuckets);
  99	sb->bucket_size	= get_bucket_size(sb, s);
 100
 101	sb->nr_in_set	= le16_to_cpu(s->nr_in_set);
 102	sb->nr_this_dev	= le16_to_cpu(s->nr_this_dev);
 103
 104	err = "Too many journal buckets";
 105	if (sb->keys > SB_JOURNAL_BUCKETS)
 106		goto err;
 107
 108	err = "Too many buckets";
 109	if (sb->nbuckets > LONG_MAX)
 110		goto err;
 111
 112	err = "Not enough buckets";
 113	if (sb->nbuckets < 1 << 7)
 114		goto err;
 115
 116	err = "Bad block size (not power of 2)";
 117	if (!is_power_of_2(sb->block_size))
 118		goto err;
 119
 120	err = "Bad block size (larger than page size)";
 121	if (sb->block_size > PAGE_SECTORS)
 122		goto err;
 123
 124	err = "Bad bucket size (not power of 2)";
 125	if (!is_power_of_2(sb->bucket_size))
 126		goto err;
 127
 128	err = "Bad bucket size (smaller than page size)";
 129	if (sb->bucket_size < PAGE_SECTORS)
 130		goto err;
 131
 132	err = "Invalid superblock: device too small";
 133	if (get_capacity(bdev->bd_disk) <
 134	    sb->bucket_size * sb->nbuckets)
 135		goto err;
 136
 137	err = "Bad UUID";
 138	if (bch_is_zero(sb->set_uuid, 16))
 139		goto err;
 140
 141	err = "Bad cache device number in set";
 142	if (!sb->nr_in_set ||
 143	    sb->nr_in_set <= sb->nr_this_dev ||
 144	    sb->nr_in_set > MAX_CACHES_PER_SET)
 145		goto err;
 146
 147	err = "Journal buckets not sequential";
 148	for (i = 0; i < sb->keys; i++)
 149		if (sb->d[i] != sb->first_bucket + i)
 150			goto err;
 151
 152	err = "Too many journal buckets";
 153	if (sb->first_bucket + sb->keys > sb->nbuckets)
 154		goto err;
 155
 156	err = "Invalid superblock: first bucket comes before end of super";
 157	if (sb->first_bucket * sb->bucket_size < 16)
 158		goto err;
 159
 160	err = NULL;
 161err:
 162	return err;
 163}
 164
 165
 166static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
 167			      struct cache_sb_disk **res)
 168{
 169	const char *err;
 170	struct cache_sb_disk *s;
 171	struct folio *folio;
 172	unsigned int i;
 173
 174	folio = mapping_read_folio_gfp(bdev->bd_mapping,
 175			SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL);
 176	if (IS_ERR(folio))
 177		return "IO error";
 178	s = folio_address(folio) + offset_in_folio(folio, SB_OFFSET);
 179
 180	sb->offset		= le64_to_cpu(s->offset);
 181	sb->version		= le64_to_cpu(s->version);
 182
 183	memcpy(sb->magic,	s->magic, 16);
 184	memcpy(sb->uuid,	s->uuid, 16);
 185	memcpy(sb->set_uuid,	s->set_uuid, 16);
 186	memcpy(sb->label,	s->label, SB_LABEL_SIZE);
 187
 188	sb->flags		= le64_to_cpu(s->flags);
 189	sb->seq			= le64_to_cpu(s->seq);
 190	sb->last_mount		= le32_to_cpu(s->last_mount);
 191	sb->keys		= le16_to_cpu(s->keys);
 192
 193	for (i = 0; i < SB_JOURNAL_BUCKETS; i++)
 194		sb->d[i] = le64_to_cpu(s->d[i]);
 195
 196	pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u\n",
 197		 sb->version, sb->flags, sb->seq, sb->keys);
 198
 199	err = "Not a bcache superblock (bad offset)";
 200	if (sb->offset != SB_SECTOR)
 201		goto err;
 202
 203	err = "Not a bcache superblock (bad magic)";
 204	if (memcmp(sb->magic, bcache_magic, 16))
 205		goto err;
 206
 207	err = "Bad checksum";
 208	if (s->csum != csum_set(s))
 209		goto err;
 210
 211	err = "Bad UUID";
 212	if (bch_is_zero(sb->uuid, 16))
 213		goto err;
 214
 215	sb->block_size	= le16_to_cpu(s->block_size);
 216
 217	err = "Superblock block size smaller than device block size";
 218	if (sb->block_size << 9 < bdev_logical_block_size(bdev))
 219		goto err;
 220
 221	switch (sb->version) {
 222	case BCACHE_SB_VERSION_BDEV:
 223		sb->data_offset	= BDEV_DATA_START_DEFAULT;
 224		break;
 225	case BCACHE_SB_VERSION_BDEV_WITH_OFFSET:
 226	case BCACHE_SB_VERSION_BDEV_WITH_FEATURES:
 227		sb->data_offset	= le64_to_cpu(s->data_offset);
 228
 229		err = "Bad data offset";
 230		if (sb->data_offset < BDEV_DATA_START_DEFAULT)
 231			goto err;
 232
 233		break;
 234	case BCACHE_SB_VERSION_CDEV:
 235	case BCACHE_SB_VERSION_CDEV_WITH_UUID:
 236		err = read_super_common(sb, bdev, s);
 237		if (err)
 238			goto err;
 239		break;
 240	case BCACHE_SB_VERSION_CDEV_WITH_FEATURES:
 241		/*
 242		 * Feature bits are needed in read_super_common(),
 243		 * convert them firstly.
 244		 */
 245		sb->feature_compat = le64_to_cpu(s->feature_compat);
 246		sb->feature_incompat = le64_to_cpu(s->feature_incompat);
 247		sb->feature_ro_compat = le64_to_cpu(s->feature_ro_compat);
 248
 249		/* Check incompatible features */
 250		err = "Unsupported compatible feature found";
 251		if (bch_has_unknown_compat_features(sb))
 252			goto err;
 253
 254		err = "Unsupported read-only compatible feature found";
 255		if (bch_has_unknown_ro_compat_features(sb))
 256			goto err;
 257
 258		err = "Unsupported incompatible feature found";
 259		if (bch_has_unknown_incompat_features(sb))
 260			goto err;
 261
 262		err = read_super_common(sb, bdev, s);
 263		if (err)
 264			goto err;
 265		break;
 266	default:
 267		err = "Unsupported superblock version";
 268		goto err;
 269	}
 270
 271	sb->last_mount = (u32)ktime_get_real_seconds();
 272	*res = s;
 273	return NULL;
 274err:
 275	folio_put(folio);
 276	return err;
 277}
 278
 279static void write_bdev_super_endio(struct bio *bio)
 280{
 281	struct cached_dev *dc = bio->bi_private;
 282
 283	if (bio->bi_status)
 284		bch_count_backing_io_errors(dc, bio);
 285
 286	closure_put(&dc->sb_write);
 287}
 288
 289static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out,
 290		struct bio *bio)
 291{
 292	unsigned int i;
 293
 294	bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_META;
 295	bio->bi_iter.bi_sector	= SB_SECTOR;
 296	bio_add_virt_nofail(bio, out, SB_SIZE);
 297
 298	out->offset		= cpu_to_le64(sb->offset);
 299
 300	memcpy(out->uuid,	sb->uuid, 16);
 301	memcpy(out->set_uuid,	sb->set_uuid, 16);
 302	memcpy(out->label,	sb->label, SB_LABEL_SIZE);
 303
 304	out->flags		= cpu_to_le64(sb->flags);
 305	out->seq		= cpu_to_le64(sb->seq);
 306
 307	out->last_mount		= cpu_to_le32(sb->last_mount);
 308	out->first_bucket	= cpu_to_le16(sb->first_bucket);
 309	out->keys		= cpu_to_le16(sb->keys);
 310
 311	for (i = 0; i < sb->keys; i++)
 312		out->d[i] = cpu_to_le64(sb->d[i]);
 313
 314	if (sb->version >= BCACHE_SB_VERSION_CDEV_WITH_FEATURES) {
 315		out->feature_compat    = cpu_to_le64(sb->feature_compat);
 316		out->feature_incompat  = cpu_to_le64(sb->feature_incompat);
 317		out->feature_ro_compat = cpu_to_le64(sb->feature_ro_compat);
 318	}
 319
 320	out->version		= cpu_to_le64(sb->version);
 321	out->csum = csum_set(out);
 322
 323	pr_debug("ver %llu, flags %llu, seq %llu\n",
 324		 sb->version, sb->flags, sb->seq);
 325
 326	submit_bio(bio);
 327}
 328
 329static CLOSURE_CALLBACK(bch_write_bdev_super_unlock)
 330{
 331	closure_type(dc, struct cached_dev, sb_write);
 332
 333	up(&dc->sb_write_mutex);
 334}
 335
 336void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
 337{
 338	struct closure *cl = &dc->sb_write;
 339	struct bio *bio = &dc->sb_bio;
 340
 341	down(&dc->sb_write_mutex);
 342	closure_init(cl, parent);
 343
 344	bio_init(bio, dc->bdev, dc->sb_bv, 1, 0);
 345	bio->bi_end_io	= write_bdev_super_endio;
 346	bio->bi_private = dc;
 347
 348	closure_get(cl);
 349	/* I/O request sent to backing device */
 350	__write_super(&dc->sb, dc->sb_disk, bio);
 351
 352	closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
 353}
 354
 355static void write_super_endio(struct bio *bio)
 356{
 357	struct cache *ca = bio->bi_private;
 358
 359	/* is_read = 0 */
 360	bch_count_io_errors(ca, bio->bi_status, 0,
 361			    "writing superblock");
 362	closure_put(&ca->set->sb_write);
 363}
 364
 365static CLOSURE_CALLBACK(bcache_write_super_unlock)
 366{
 367	closure_type(c, struct cache_set, sb_write);
 368
 369	up(&c->sb_write_mutex);
 370}
 371
 372void bcache_write_super(struct cache_set *c)
 373{
 374	struct closure *cl = &c->sb_write;
 375	struct cache *ca = c->cache;
 376	struct bio *bio = &ca->sb_bio;
 377	unsigned int version = BCACHE_SB_VERSION_CDEV_WITH_UUID;
 378
 379	down(&c->sb_write_mutex);
 380	closure_init(cl, &c->cl);
 381
 382	ca->sb.seq++;
 383
 384	if (ca->sb.version < version)
 385		ca->sb.version = version;
 386
 387	bio_init(bio, ca->bdev, ca->sb_bv, 1, 0);
 388	bio->bi_end_io	= write_super_endio;
 389	bio->bi_private = ca;
 390
 391	closure_get(cl);
 392	__write_super(&ca->sb, ca->sb_disk, bio);
 393
 394	closure_return_with_destructor(cl, bcache_write_super_unlock);
 395}
 396
 397/* UUID io */
 398
 399static void uuid_endio(struct bio *bio)
 400{
 401	struct closure *cl = bio->bi_private;
 402	struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
 403
 404	cache_set_err_on(bio->bi_status, c, "accessing uuids");
 405	bch_bbio_free(bio, c);
 406	closure_put(cl);
 407}
 408
 409static CLOSURE_CALLBACK(uuid_io_unlock)
 410{
 411	closure_type(c, struct cache_set, uuid_write);
 412
 413	up(&c->uuid_write_mutex);
 414}
 415
 416static void uuid_io(struct cache_set *c, blk_opf_t opf, struct bkey *k,
 417		    struct closure *parent)
 418{
 419	struct closure *cl = &c->uuid_write;
 420	struct uuid_entry *u;
 421	unsigned int i;
 422	char buf[80];
 423
 424	BUG_ON(!parent);
 425	down(&c->uuid_write_mutex);
 426	closure_init(cl, parent);
 427
 428	for (i = 0; i < KEY_PTRS(k); i++) {
 429		struct bio *bio = bch_bbio_alloc(c);
 430
 431		bio->bi_opf = opf | REQ_SYNC | REQ_META;
 432		bio->bi_iter.bi_size = KEY_SIZE(k) << 9;
 433
 434		bio->bi_end_io	= uuid_endio;
 435		bio->bi_private = cl;
 436		bch_bio_map(bio, c->uuids);
 437
 438		bch_submit_bbio(bio, c, k, i);
 439
 440		if ((opf & REQ_OP_MASK) != REQ_OP_WRITE)
 441			break;
 442	}
 443
 444	bch_extent_to_text(buf, sizeof(buf), k);
 445	pr_debug("%s UUIDs at %s\n", (opf & REQ_OP_MASK) == REQ_OP_WRITE ?
 446		 "wrote" : "read", buf);
 447
 448	for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)
 449		if (!bch_is_zero(u->uuid, 16))
 450			pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u\n",
 451				 u - c->uuids, u->uuid, u->label,
 452				 u->first_reg, u->last_reg, u->invalidated);
 453
 454	closure_return_with_destructor(cl, uuid_io_unlock);
 455}
 456
 457static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
 458{
 459	struct bkey *k = &j->uuid_bucket;
 460
 461	if (__bch_btree_ptr_invalid(c, k))
 462		return "bad uuid pointer";
 463
 464	bkey_copy(&c->uuid_bucket, k);
 465	uuid_io(c, REQ_OP_READ, k, cl);
 466
 467	if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
 468		struct uuid_entry_v0	*u0 = (void *) c->uuids;
 469		struct uuid_entry	*u1 = (void *) c->uuids;
 470		int i;
 471
 472		closure_sync(cl);
 473
 474		/*
 475		 * Since the new uuid entry is bigger than the old, we have to
 476		 * convert starting at the highest memory address and work down
 477		 * in order to do it in place
 478		 */
 479
 480		for (i = c->nr_uuids - 1;
 481		     i >= 0;
 482		     --i) {
 483			memcpy(u1[i].uuid,	u0[i].uuid, 16);
 484			memcpy(u1[i].label,	u0[i].label, 32);
 485
 486			u1[i].first_reg		= u0[i].first_reg;
 487			u1[i].last_reg		= u0[i].last_reg;
 488			u1[i].invalidated	= u0[i].invalidated;
 489
 490			u1[i].flags	= 0;
 491			u1[i].sectors	= 0;
 492		}
 493	}
 494
 495	return NULL;
 496}
 497
 498static int __uuid_write(struct cache_set *c)
 499{
 500	BKEY_PADDED(key) k;
 501	struct closure cl;
 502	struct cache *ca = c->cache;
 503	unsigned int size;
 504
 505	closure_init_stack(&cl);
 506	lockdep_assert_held(&bch_register_lock);
 507
 508	if (bch_bucket_alloc_set(c, RESERVE_BTREE, &k.key, true))
 509		return 1;
 510
 511	size =  meta_bucket_pages(&ca->sb) * PAGE_SECTORS;
 512	SET_KEY_SIZE(&k.key, size);
 513	uuid_io(c, REQ_OP_WRITE, &k.key, &cl);
 514	closure_sync(&cl);
 515
 516	/* Only one bucket used for uuid write */
 517	atomic_long_add(ca->sb.bucket_size, &ca->meta_sectors_written);
 518
 519	bkey_copy(&c->uuid_bucket, &k.key);
 520	bkey_put(c, &k.key);
 521	return 0;
 522}
 523
 524int bch_uuid_write(struct cache_set *c)
 525{
 526	int ret = __uuid_write(c);
 527
 528	if (!ret)
 529		bch_journal_meta(c, NULL);
 530
 531	return ret;
 532}
 533
 534static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid)
 535{
 536	struct uuid_entry *u;
 537
 538	for (u = c->uuids;
 539	     u < c->uuids + c->nr_uuids; u++)
 540		if (!memcmp(u->uuid, uuid, 16))
 541			return u;
 542
 543	return NULL;
 544}
 545
 546static struct uuid_entry *uuid_find_empty(struct cache_set *c)
 547{
 548	static const char zero_uuid[16] __nonstring =
 549		{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 550
 551	return uuid_find(c, zero_uuid);
 552}
 553
 554/*
 555 * Bucket priorities/gens:
 556 *
 557 * For each bucket, we store on disk its
 558 *   8 bit gen
 559 *  16 bit priority
 560 *
 561 * See alloc.c for an explanation of the gen. The priority is used to implement
 562 * lru (and in the future other) cache replacement policies; for most purposes
 563 * it's just an opaque integer.
 564 *
 565 * The gens and the priorities don't have a whole lot to do with each other, and
 566 * it's actually the gens that must be written out at specific times - it's no
 567 * big deal if the priorities don't get written, if we lose them we just reuse
 568 * buckets in suboptimal order.
 569 *
 570 * On disk they're stored in a packed array, and in as many buckets are required
 571 * to fit them all. The buckets we use to store them form a list; the journal
 572 * header points to the first bucket, the first bucket points to the second
 573 * bucket, et cetera.
 574 *
 575 * This code is used by the allocation code; periodically (whenever it runs out
 576 * of buckets to allocate from) the allocation code will invalidate some
 577 * buckets, but it can't use those buckets until their new gens are safely on
 578 * disk.
 579 */
 580
 581static void prio_endio(struct bio *bio)
 582{
 583	struct cache *ca = bio->bi_private;
 584
 585	cache_set_err_on(bio->bi_status, ca->set, "accessing priorities");
 586	bch_bbio_free(bio, ca->set);
 587	closure_put(&ca->prio);
 588}
 589
 590static void prio_io(struct cache *ca, uint64_t bucket, blk_opf_t opf)
 591{
 592	struct closure *cl = &ca->prio;
 593	struct bio *bio = bch_bbio_alloc(ca->set);
 594
 595	closure_init_stack(cl);
 596
 597	bio->bi_iter.bi_sector	= bucket * ca->sb.bucket_size;
 598	bio_set_dev(bio, ca->bdev);
 599	bio->bi_iter.bi_size	= meta_bucket_bytes(&ca->sb);
 600
 601	bio->bi_end_io	= prio_endio;
 602	bio->bi_private = ca;
 603	bio->bi_opf = opf | REQ_SYNC | REQ_META;
 604	bch_bio_map(bio, ca->disk_buckets);
 605
 606	closure_bio_submit(ca->set, bio, &ca->prio);
 607	closure_sync(cl);
 608}
 609
 610int bch_prio_write(struct cache *ca, bool wait)
 611{
 612	int i;
 613	struct bucket *b;
 614	struct closure cl;
 615
 616	pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu\n",
 617		 fifo_used(&ca->free[RESERVE_PRIO]),
 618		 fifo_used(&ca->free[RESERVE_NONE]),
 619		 fifo_used(&ca->free_inc));
 620
 621	/*
 622	 * Pre-check if there are enough free buckets. In the non-blocking
 623	 * scenario it's better to fail early rather than starting to allocate
 624	 * buckets and do a cleanup later in case of failure.
 625	 */
 626	if (!wait) {
 627		size_t avail = fifo_used(&ca->free[RESERVE_PRIO]) +
 628			       fifo_used(&ca->free[RESERVE_NONE]);
 629		if (prio_buckets(ca) > avail)
 630			return -ENOMEM;
 631	}
 632
 633	closure_init_stack(&cl);
 634
 635	lockdep_assert_held(&ca->set->bucket_lock);
 636
 637	ca->disk_buckets->seq++;
 638
 639	atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),
 640			&ca->meta_sectors_written);
 641
 642	for (i = prio_buckets(ca) - 1; i >= 0; --i) {
 643		long bucket;
 644		struct prio_set *p = ca->disk_buckets;
 645		struct bucket_disk *d = p->data;
 646		struct bucket_disk *end = d + prios_per_bucket(ca);
 647
 648		for (b = ca->buckets + i * prios_per_bucket(ca);
 649		     b < ca->buckets + ca->sb.nbuckets && d < end;
 650		     b++, d++) {
 651			d->prio = cpu_to_le16(b->prio);
 652			d->gen = b->gen;
 653		}
 654
 655		p->next_bucket	= ca->prio_buckets[i + 1];
 656		p->magic	= pset_magic(&ca->sb);
 657		p->csum		= bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8);
 658
 659		bucket = bch_bucket_alloc(ca, RESERVE_PRIO, wait);
 660		BUG_ON(bucket == -1);
 661
 662		mutex_unlock(&ca->set->bucket_lock);
 663		prio_io(ca, bucket, REQ_OP_WRITE);
 664		mutex_lock(&ca->set->bucket_lock);
 665
 666		ca->prio_buckets[i] = bucket;
 667		atomic_dec_bug(&ca->buckets[bucket].pin);
 668	}
 669
 670	mutex_unlock(&ca->set->bucket_lock);
 671
 672	bch_journal_meta(ca->set, &cl);
 673	closure_sync(&cl);
 674
 675	mutex_lock(&ca->set->bucket_lock);
 676
 677	/*
 678	 * Don't want the old priorities to get garbage collected until after we
 679	 * finish writing the new ones, and they're journalled
 680	 */
 681	for (i = 0; i < prio_buckets(ca); i++) {
 682		if (ca->prio_last_buckets[i])
 683			__bch_bucket_free(ca,
 684				&ca->buckets[ca->prio_last_buckets[i]]);
 685
 686		ca->prio_last_buckets[i] = ca->prio_buckets[i];
 687	}
 688	return 0;
 689}
 690
 691static int prio_read(struct cache *ca, uint64_t bucket)
 692{
 693	struct prio_set *p = ca->disk_buckets;
 694	struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;
 695	struct bucket *b;
 696	unsigned int bucket_nr = 0;
 697	int ret = -EIO;
 698
 699	for (b = ca->buckets;
 700	     b < ca->buckets + ca->sb.nbuckets;
 701	     b++, d++) {
 702		if (d == end) {
 703			ca->prio_buckets[bucket_nr] = bucket;
 704			ca->prio_last_buckets[bucket_nr] = bucket;
 705			bucket_nr++;
 706
 707			prio_io(ca, bucket, REQ_OP_READ);
 708
 709			if (p->csum !=
 710			    bch_crc64(&p->magic, meta_bucket_bytes(&ca->sb) - 8)) {
 711				pr_warn("bad csum reading priorities\n");
 712				goto out;
 713			}
 714
 715			if (p->magic != pset_magic(&ca->sb)) {
 716				pr_warn("bad magic reading priorities\n");
 717				goto out;
 718			}
 719
 720			bucket = p->next_bucket;
 721			d = p->data;
 722		}
 723
 724		b->prio = le16_to_cpu(d->prio);
 725		b->gen = b->last_gc = d->gen;
 726	}
 727
 728	ret = 0;
 729out:
 730	return ret;
 731}
 732
 733/* Bcache device */
 734
 735static int open_dev(struct gendisk *disk, blk_mode_t mode)
 736{
 737	struct bcache_device *d = disk->private_data;
 738
 739	if (test_bit(BCACHE_DEV_CLOSING, &d->flags))
 740		return -ENXIO;
 741
 742	closure_get(&d->cl);
 743	return 0;
 744}
 745
 746static void release_dev(struct gendisk *b)
 747{
 748	struct bcache_device *d = b->private_data;
 749
 750	closure_put(&d->cl);
 751}
 752
 753static int ioctl_dev(struct block_device *b, blk_mode_t mode,
 754		     unsigned int cmd, unsigned long arg)
 755{
 756	struct bcache_device *d = b->bd_disk->private_data;
 757
 758	return d->ioctl(d, mode, cmd, arg);
 759}
 760
 761static const struct block_device_operations bcache_cached_ops = {
 762	.submit_bio	= cached_dev_submit_bio,
 763	.open		= open_dev,
 764	.release	= release_dev,
 765	.ioctl		= ioctl_dev,
 766	.owner		= THIS_MODULE,
 767};
 768
 769static const struct block_device_operations bcache_flash_ops = {
 770	.submit_bio	= flash_dev_submit_bio,
 771	.open		= open_dev,
 772	.release	= release_dev,
 773	.ioctl		= ioctl_dev,
 774	.owner		= THIS_MODULE,
 775};
 776
 777void bcache_device_stop(struct bcache_device *d)
 778{
 779	if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags))
 780		/*
 781		 * closure_fn set to
 782		 * - cached device: cached_dev_flush()
 783		 * - flash dev: flash_dev_flush()
 784		 */
 785		closure_queue(&d->cl);
 786}
 787
 788static void bcache_device_unlink(struct bcache_device *d)
 789{
 790	lockdep_assert_held(&bch_register_lock);
 791
 792	if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) {
 793		struct cache *ca = d->c->cache;
 794
 795		sysfs_remove_link(&d->c->kobj, d->name);
 796		sysfs_remove_link(&d->kobj, "cache");
 797
 798		bd_unlink_disk_holder(ca->bdev, d->disk);
 799	}
 800}
 801
 802static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
 803			       const char *name)
 804{
 805	struct cache *ca = c->cache;
 806	int ret;
 807
 808	bd_link_disk_holder(ca->bdev, d->disk);
 809
 810	snprintf(d->name, BCACHEDEVNAME_SIZE,
 811		 "%s%u", name, d->id);
 812
 813	ret = sysfs_create_link(&d->kobj, &c->kobj, "cache");
 814	if (ret < 0)
 815		pr_err("Couldn't create device -> cache set symlink\n");
 816
 817	ret = sysfs_create_link(&c->kobj, &d->kobj, d->name);
 818	if (ret < 0)
 819		pr_err("Couldn't create cache set -> device symlink\n");
 820
 821	clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags);
 822}
 823
 824static void bcache_device_detach(struct bcache_device *d)
 825{
 826	lockdep_assert_held(&bch_register_lock);
 827
 828	atomic_dec(&d->c->attached_dev_nr);
 829
 830	if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) {
 831		struct uuid_entry *u = d->c->uuids + d->id;
 832
 833		SET_UUID_FLASH_ONLY(u, 0);
 834		memcpy(u->uuid, invalid_uuid, 16);
 835		u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds());
 836		bch_uuid_write(d->c);
 837	}
 838
 839	bcache_device_unlink(d);
 840
 841	d->c->devices[d->id] = NULL;
 842	closure_put(&d->c->caching);
 843	d->c = NULL;
 844}
 845
 846static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,
 847				 unsigned int id)
 848{
 849	d->id = id;
 850	d->c = c;
 851	c->devices[id] = d;
 852
 853	if (id >= c->devices_max_used)
 854		c->devices_max_used = id + 1;
 855
 856	closure_get(&c->caching);
 857}
 858
 859static inline int first_minor_to_idx(int first_minor)
 860{
 861	return (first_minor/BCACHE_MINORS);
 862}
 863
 864static inline int idx_to_first_minor(int idx)
 865{
 866	return (idx * BCACHE_MINORS);
 867}
 868
 869static void bcache_device_free(struct bcache_device *d)
 870{
 871	struct gendisk *disk = d->disk;
 872
 873	lockdep_assert_held(&bch_register_lock);
 874
 875	if (disk)
 876		pr_info("%s stopped\n", disk->disk_name);
 877	else
 878		pr_err("bcache device (NULL gendisk) stopped\n");
 879
 880	if (d->c)
 881		bcache_device_detach(d);
 882
 883	if (disk) {
 884		ida_free(&bcache_device_idx,
 885			 first_minor_to_idx(disk->first_minor));
 886		put_disk(disk);
 887	}
 888
 889	bioset_exit(&d->bio_split);
 890	bioset_exit(&d->bio_detached);
 891	kvfree(d->full_dirty_stripes);
 892	kvfree(d->stripe_sectors_dirty);
 893
 894	closure_debug_destroy(&d->cl);
 895}
 896
 897static int bcache_device_init(struct bcache_device *d, unsigned int block_size,
 898		sector_t sectors, struct block_device *cached_bdev,
 899		const struct block_device_operations *ops)
 900{
 901	const size_t max_stripes = min_t(size_t, INT_MAX,
 902					 SIZE_MAX / sizeof(atomic_t));
 903	struct queue_limits lim = {
 904		.max_hw_sectors		= UINT_MAX,
 905		.max_sectors		= UINT_MAX,
 906		.max_segment_size	= UINT_MAX,
 907		.max_segments		= BIO_MAX_VECS,
 908		.max_hw_discard_sectors	= UINT_MAX,
 909		.io_min			= block_size,
 910		.logical_block_size	= block_size,
 911		.physical_block_size	= block_size,
 912		.features		= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA,
 913	};
 914	uint64_t n;
 915	int idx;
 916
 917	if (cached_bdev) {
 918		d->stripe_size = bdev_io_opt(cached_bdev) >> SECTOR_SHIFT;
 919		lim.io_opt = umax(block_size, bdev_io_opt(cached_bdev));
 920	}
 921	if (!d->stripe_size)
 922		d->stripe_size = 1 << 31;
 923	else if (d->stripe_size < BCH_MIN_STRIPE_SZ)
 924		d->stripe_size = roundup(BCH_MIN_STRIPE_SZ, d->stripe_size);
 925
 926	n = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
 927	if (!n || n > max_stripes) {
 928		pr_err("nr_stripes too large or invalid: %llu (start sector beyond end of disk?)\n",
 929			n);
 930		return -ENOMEM;
 931	}
 932	d->nr_stripes = n;
 933
 934	n = d->nr_stripes * sizeof(atomic_t);
 935	d->stripe_sectors_dirty = kvzalloc(n, GFP_KERNEL);
 936	if (!d->stripe_sectors_dirty)
 937		return -ENOMEM;
 938
 939	n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);
 940	d->full_dirty_stripes = kvzalloc(n, GFP_KERNEL);
 941	if (!d->full_dirty_stripes)
 942		goto out_free_stripe_sectors_dirty;
 943
 944	idx = ida_alloc_max(&bcache_device_idx, BCACHE_DEVICE_IDX_MAX - 1,
 945			    GFP_KERNEL);
 946	if (idx < 0)
 947		goto out_free_full_dirty_stripes;
 948
 949	if (bioset_init(&d->bio_split, 4, offsetof(struct bbio, bio),
 950			BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER))
 951		goto out_ida_remove;
 952
 953	if (bioset_init(&d->bio_detached, 4,
 954			offsetof(struct detached_dev_io_private, bio),
 955			BIOSET_NEED_BVECS|BIOSET_NEED_RESCUER))
 956		goto out_bioset_split_exit;
 957
 958	if (lim.logical_block_size > PAGE_SIZE && cached_bdev) {
 959		/*
 960		 * This should only happen with BCACHE_SB_VERSION_BDEV.
 961		 * Block/page size is checked for BCACHE_SB_VERSION_CDEV.
 962		 */
 963		pr_info("bcache%i: sb/logical block size (%u) greater than page size (%lu) falling back to device logical block size (%u)\n",
 964			idx, lim.logical_block_size,
 965			PAGE_SIZE, bdev_logical_block_size(cached_bdev));
 966
 967		/* This also adjusts physical block size/min io size if needed */
 968		lim.logical_block_size = bdev_logical_block_size(cached_bdev);
 969	}
 970
 971	d->disk = blk_alloc_disk(&lim, NUMA_NO_NODE);
 972	if (IS_ERR(d->disk))
 973		goto out_bioset_detach_exit;
 974
 975	set_capacity(d->disk, sectors);
 976	snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", idx);
 977
 978	d->disk->major		= bcache_major;
 979	d->disk->first_minor	= idx_to_first_minor(idx);
 980	d->disk->minors		= BCACHE_MINORS;
 981	d->disk->fops		= ops;
 982	d->disk->private_data	= d;
 983	return 0;
 984
 985out_bioset_detach_exit:
 986	bioset_exit(&d->bio_detached);
 987out_bioset_split_exit:
 988	bioset_exit(&d->bio_split);
 989out_ida_remove:
 990	ida_free(&bcache_device_idx, idx);
 991out_free_full_dirty_stripes:
 992	kvfree(d->full_dirty_stripes);
 993out_free_stripe_sectors_dirty:
 994	kvfree(d->stripe_sectors_dirty);
 995	return -ENOMEM;
 996
 997}
 998
 999/* Cached device */
1000
1001static void calc_cached_dev_sectors(struct cache_set *c)
1002{
1003	uint64_t sectors = 0;
1004	struct cached_dev *dc;
1005
1006	list_for_each_entry(dc, &c->cached_devs, list)
1007		sectors += bdev_nr_sectors(dc->bdev);
1008
1009	c->cached_dev_sectors = sectors;
1010}
1011
1012#define BACKING_DEV_OFFLINE_TIMEOUT 5
1013static int cached_dev_status_update(void *arg)
1014{
1015	struct cached_dev *dc = arg;
1016	struct request_queue *q;
1017
1018	/*
1019	 * If this delayed worker is stopping outside, directly quit here.
1020	 * dc->io_disable might be set via sysfs interface, so check it
1021	 * here too.
1022	 */
1023	while (!kthread_should_stop() && !dc->io_disable) {
1024		q = bdev_get_queue(dc->bdev);
1025		if (blk_queue_dying(q))
1026			dc->offline_seconds++;
1027		else
1028			dc->offline_seconds = 0;
1029
1030		if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) {
1031			pr_err("%pg: device offline for %d seconds\n",
1032			       dc->bdev,
1033			       BACKING_DEV_OFFLINE_TIMEOUT);
1034			pr_err("%s: disable I/O request due to backing device offline\n",
1035			       dc->disk.name);
1036			dc->io_disable = true;
1037			/* let others know earlier that io_disable is true */
1038			smp_mb();
1039			bcache_device_stop(&dc->disk);
1040			break;
1041		}
1042		schedule_timeout_interruptible(HZ);
1043	}
1044
1045	wait_for_kthread_stop();
1046	return 0;
1047}
1048
1049
1050int bch_cached_dev_run(struct cached_dev *dc)
1051{
1052	int ret = 0;
1053	struct bcache_device *d = &dc->disk;
1054	char *buf = kmemdup_nul(dc->sb.label, SB_LABEL_SIZE, GFP_KERNEL);
1055	char *env[] = {
1056		"DRIVER=bcache",
1057		kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid),
1058		kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf ? : ""),
1059		NULL,
1060	};
1061
1062	if (dc->io_disable) {
1063		pr_err("I/O disabled on cached dev %pg\n", dc->bdev);
1064		ret = -EIO;
1065		goto out;
1066	}
1067
1068	if (atomic_xchg(&dc->running, 1)) {
1069		pr_info("cached dev %pg is running already\n", dc->bdev);
1070		ret = -EBUSY;
1071		goto out;
1072	}
1073
1074	if (!d->c &&
1075	    BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {
1076		struct closure cl;
1077
1078		closure_init_stack(&cl);
1079
1080		SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE);
1081		bch_write_bdev_super(dc, &cl);
1082		closure_sync(&cl);
1083	}
1084
1085	ret = add_disk(d->disk);
1086	if (ret)
1087		goto out;
1088	bd_link_disk_holder(dc->bdev, dc->disk.disk);
1089	/*
1090	 * won't show up in the uevent file, use udevadm monitor -e instead
1091	 * only class / kset properties are persistent
1092	 */
1093	kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);
1094
1095	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
1096	    sysfs_create_link(&disk_to_dev(d->disk)->kobj,
1097			      &d->kobj, "bcache")) {
1098		pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");
1099		ret = -ENOMEM;
1100		goto out;
1101	}
1102
1103	dc->status_update_thread = kthread_run(cached_dev_status_update,
1104					       dc, "bcache_status_update");
1105	if (IS_ERR(dc->status_update_thread)) {
1106		pr_warn("failed to create bcache_status_update kthread, continue to run without monitoring backing device status\n");
1107	}
1108
1109out:
1110	kfree(env[1]);
1111	kfree(env[2]);
1112	kfree(buf);
1113	return ret;
1114}
1115
1116/*
1117 * If BCACHE_DEV_RATE_DW_RUNNING is set, it means routine of the delayed
1118 * work dc->writeback_rate_update is running. Wait until the routine
1119 * quits (BCACHE_DEV_RATE_DW_RUNNING is clear), then continue to
1120 * cancel it. If BCACHE_DEV_RATE_DW_RUNNING is not clear after time_out
1121 * seconds, give up waiting here and continue to cancel it too.
1122 */
1123static void cancel_writeback_rate_update_dwork(struct cached_dev *dc)
1124{
1125	int time_out = WRITEBACK_RATE_UPDATE_SECS_MAX * HZ;
1126
1127	do {
1128		if (!test_bit(BCACHE_DEV_RATE_DW_RUNNING,
1129			      &dc->disk.flags))
1130			break;
1131		time_out--;
1132		schedule_timeout_interruptible(1);
1133	} while (time_out > 0);
1134
1135	if (time_out == 0)
1136		pr_warn("give up waiting for dc->writeback_write_update to quit\n");
1137
1138	cancel_delayed_work_sync(&dc->writeback_rate_update);
1139}
1140
1141static void cached_dev_detach_finish(struct work_struct *w)
1142{
1143	struct cached_dev *dc = container_of(w, struct cached_dev, detach);
1144	struct cache_set *c = dc->disk.c;
1145
1146	BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
1147	BUG_ON(refcount_read(&dc->count));
1148
1149
1150	if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))
1151		cancel_writeback_rate_update_dwork(dc);
1152
1153	if (!IS_ERR_OR_NULL(dc->writeback_thread)) {
1154		kthread_stop(dc->writeback_thread);
1155		dc->writeback_thread = NULL;
1156	}
1157
1158	mutex_lock(&bch_register_lock);
1159
1160	bcache_device_detach(&dc->disk);
1161	list_move(&dc->list, &uncached_devices);
1162	calc_cached_dev_sectors(c);
1163
1164	clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);
1165	clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags);
1166
1167	mutex_unlock(&bch_register_lock);
1168
1169	pr_info("Caching disabled for %pg\n", dc->bdev);
1170
1171	/* Drop ref we took in cached_dev_detach() */
1172	closure_put(&dc->disk.cl);
1173}
1174
1175void bch_cached_dev_detach(struct cached_dev *dc)
1176{
1177	lockdep_assert_held(&bch_register_lock);
1178
1179	if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
1180		return;
1181
1182	if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
1183		return;
1184
1185	/*
1186	 * Block the device from being closed and freed until we're finished
1187	 * detaching
1188	 */
1189	closure_get(&dc->disk.cl);
1190
1191	bch_writeback_queue(dc);
1192
1193	cached_dev_put(dc);
1194}
1195
1196int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c,
1197			  uint8_t *set_uuid)
1198{
1199	uint32_t rtime = cpu_to_le32((u32)ktime_get_real_seconds());
1200	struct uuid_entry *u;
1201	struct cached_dev *exist_dc, *t;
1202	int ret = 0;
1203
1204	if ((set_uuid && memcmp(set_uuid, c->set_uuid, 16)) ||
1205	    (!set_uuid && memcmp(dc->sb.set_uuid, c->set_uuid, 16)))
1206		return -ENOENT;
1207
1208	if (dc->disk.c) {
1209		pr_err("Can't attach %pg: already attached\n", dc->bdev);
1210		return -EINVAL;
1211	}
1212
1213	if (test_bit(CACHE_SET_STOPPING, &c->flags)) {
1214		pr_err("Can't attach %pg: shutting down\n", dc->bdev);
1215		return -EINVAL;
1216	}
1217
1218	if (dc->sb.block_size < c->cache->sb.block_size) {
1219		/* Will die */
1220		pr_err("Couldn't attach %pg: block size less than set's block size\n",
1221		       dc->bdev);
1222		return -EINVAL;
1223	}
1224
1225	/* Check whether already attached */
1226	list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) {
1227		if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) {
1228			pr_err("Tried to attach %pg but duplicate UUID already attached\n",
1229				dc->bdev);
1230
1231			return -EINVAL;
1232		}
1233	}
1234
1235	u = uuid_find(c, dc->sb.uuid);
1236
1237	if (u &&
1238	    (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE ||
1239	     BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) {
1240		memcpy(u->uuid, invalid_uuid, 16);
1241		u->invalidated = cpu_to_le32((u32)ktime_get_real_seconds());
1242		u = NULL;
1243	}
1244
1245	if (!u) {
1246		if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
1247			pr_err("Couldn't find uuid for %pg in set\n", dc->bdev);
1248			return -ENOENT;
1249		}
1250
1251		u = uuid_find_empty(c);
1252		if (!u) {
1253			pr_err("Not caching %pg, no room for UUID\n", dc->bdev);
1254			return -EINVAL;
1255		}
1256	}
1257
1258	/*
1259	 * Deadlocks since we're called via sysfs...
1260	 * sysfs_remove_file(&dc->kobj, &sysfs_attach);
1261	 */
1262
1263	if (bch_is_zero(u->uuid, 16)) {
1264		struct closure cl;
1265
1266		closure_init_stack(&cl);
1267
1268		memcpy(u->uuid, dc->sb.uuid, 16);
1269		memcpy(u->label, dc->sb.label, SB_LABEL_SIZE);
1270		u->first_reg = u->last_reg = rtime;
1271		bch_uuid_write(c);
1272
1273		memcpy(dc->sb.set_uuid, c->set_uuid, 16);
1274		SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
1275
1276		bch_write_bdev_super(dc, &cl);
1277		closure_sync(&cl);
1278	} else {
1279		u->last_reg = rtime;
1280		bch_uuid_write(c);
1281	}
1282
1283	bcache_device_attach(&dc->disk, c, u - c->uuids);
1284	list_move(&dc->list, &c->cached_devs);
1285	calc_cached_dev_sectors(c);
1286
1287	/*
1288	 * dc->c must be set before dc->count != 0 - paired with the mb in
1289	 * cached_dev_get()
1290	 */
1291	smp_wmb();
1292	refcount_set(&dc->count, 1);
1293
1294	/* Block writeback thread, but spawn it */
1295	down_write(&dc->writeback_lock);
1296	if (bch_cached_dev_writeback_start(dc)) {
1297		up_write(&dc->writeback_lock);
1298		pr_err("Couldn't start writeback facilities for %s\n",
1299		       dc->disk.disk->disk_name);
1300		return -ENOMEM;
1301	}
1302
1303	if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
1304		atomic_set(&dc->has_dirty, 1);
1305		bch_writeback_queue(dc);
1306	}
1307
1308	bch_sectors_dirty_init(&dc->disk);
1309
1310	ret = bch_cached_dev_run(dc);
1311	if (ret && (ret != -EBUSY)) {
1312		up_write(&dc->writeback_lock);
1313		/*
1314		 * bch_register_lock is held, bcache_device_stop() is not
1315		 * able to be directly called. The kthread and kworker
1316		 * created previously in bch_cached_dev_writeback_start()
1317		 * have to be stopped manually here.
1318		 */
1319		kthread_stop(dc->writeback_thread);
1320		cancel_writeback_rate_update_dwork(dc);
1321		pr_err("Couldn't run cached device %pg\n", dc->bdev);
1322		return ret;
1323	}
1324
1325	bcache_device_link(&dc->disk, c, "bdev");
1326	atomic_inc(&c->attached_dev_nr);
1327
1328	if (bch_has_feature_obso_large_bucket(&(c->cache->sb))) {
1329		pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");
1330		pr_err("Please update to the latest bcache-tools to create the cache device\n");
1331		set_disk_ro(dc->disk.disk, 1);
1332	}
1333
1334	/* Allow the writeback thread to proceed */
1335	up_write(&dc->writeback_lock);
1336
1337	pr_info("Caching %pg as %s on set %pU\n",
1338		dc->bdev,
1339		dc->disk.disk->disk_name,
1340		dc->disk.c->set_uuid);
1341	return 0;
1342}
1343
1344/* when dc->disk.kobj released */
1345void bch_cached_dev_release(struct kobject *kobj)
1346{
1347	struct cached_dev *dc = container_of(kobj, struct cached_dev,
1348					     disk.kobj);
1349	kfree(dc);
1350	module_put(THIS_MODULE);
1351}
1352
1353static CLOSURE_CALLBACK(cached_dev_free)
1354{
1355	closure_type(dc, struct cached_dev, disk.cl);
1356
1357	if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))
1358		cancel_writeback_rate_update_dwork(dc);
1359
1360	if (!IS_ERR_OR_NULL(dc->writeback_thread))
1361		kthread_stop(dc->writeback_thread);
1362	if (!IS_ERR_OR_NULL(dc->status_update_thread))
1363		kthread_stop(dc->status_update_thread);
1364
1365	mutex_lock(&bch_register_lock);
1366
1367	if (atomic_read(&dc->running)) {
1368		bd_unlink_disk_holder(dc->bdev, dc->disk.disk);
1369		del_gendisk(dc->disk.disk);
1370	}
1371	bcache_device_free(&dc->disk);
1372	list_del(&dc->list);
1373
1374	mutex_unlock(&bch_register_lock);
1375
1376	/*
1377	 * Wait for any pending sb_write to complete before free.
1378	 * The sb_bio is embedded in struct cached_dev, so we must
1379	 * ensure no I/O is in progress.
1380	 */
1381	closure_sync(&dc->sb_write);
1382
1383	if (dc->sb_disk)
1384		folio_put(virt_to_folio(dc->sb_disk));
1385
1386	if (dc->bdev_file)
1387		fput(dc->bdev_file);
1388
1389	wake_up(&unregister_wait);
1390
1391	kobject_put(&dc->disk.kobj);
1392}
1393
1394static CLOSURE_CALLBACK(cached_dev_flush)
1395{
1396	closure_type(dc, struct cached_dev, disk.cl);
1397	struct bcache_device *d = &dc->disk;
1398
1399	mutex_lock(&bch_register_lock);
1400	bcache_device_unlink(d);
1401	mutex_unlock(&bch_register_lock);
1402
1403	bch_cache_accounting_destroy(&dc->accounting);
1404	kobject_del(&d->kobj);
1405
1406	continue_at(cl, cached_dev_free, system_percpu_wq);
1407}
1408
1409static int cached_dev_init(struct cached_dev *dc, unsigned int block_size)
1410{
1411	int ret;
1412	struct io *io;
1413	struct request_queue *q = bdev_get_queue(dc->bdev);
1414
1415	__module_get(THIS_MODULE);
1416	INIT_LIST_HEAD(&dc->list);
1417	closure_init(&dc->disk.cl, NULL);
1418	set_closure_fn(&dc->disk.cl, cached_dev_flush, system_percpu_wq);
1419	kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype);
1420	INIT_WORK(&dc->detach, cached_dev_detach_finish);
1421	sema_init(&dc->sb_write_mutex, 1);
1422	INIT_LIST_HEAD(&dc->io_lru);
1423	spin_lock_init(&dc->io_lock);
1424	bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);
1425
1426	dc->sequential_cutoff		= 4 << 20;
1427
1428	for (io = dc->io; io < dc->io + RECENT_IO; io++) {
1429		list_add(&io->lru, &dc->io_lru);
1430		hlist_add_head(&io->hash, dc->io_hash + RECENT_IO);
1431	}
1432
1433	if (bdev_io_opt(dc->bdev))
1434		dc->partial_stripes_expensive = !!(q->limits.features &
1435			BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE);
1436
1437	ret = bcache_device_init(&dc->disk, block_size,
1438			 bdev_nr_sectors(dc->bdev) - dc->sb.data_offset,
1439			 dc->bdev, &bcache_cached_ops);
1440	if (ret)
1441		return ret;
1442
1443	atomic_set(&dc->io_errors, 0);
1444	dc->io_disable = false;
1445	dc->error_limit = DEFAULT_CACHED_DEV_ERROR_LIMIT;
1446	/* default to auto */
1447	dc->stop_when_cache_set_failed = BCH_CACHED_DEV_STOP_AUTO;
1448
1449	bch_cached_dev_request_init(dc);
1450	bch_cached_dev_writeback_init(dc);
1451	return 0;
1452}
1453
1454/* Cached device - bcache superblock */
1455
1456static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
1457				 struct file *bdev_file,
1458				 struct cached_dev *dc)
1459{
1460	const char *err = "cannot allocate memory";
1461	struct cache_set *c;
1462	int ret = -ENOMEM;
1463
1464	memcpy(&dc->sb, sb, sizeof(struct cache_sb));
1465	dc->bdev_file = bdev_file;
1466	dc->bdev = file_bdev(bdev_file);
1467	dc->sb_disk = sb_disk;
1468
1469	if (cached_dev_init(dc, sb->block_size << 9))
1470		goto err;
1471
1472	err = "error creating kobject";
1473	if (kobject_add(&dc->disk.kobj, bdev_kobj(dc->bdev), "bcache"))
1474		goto err;
1475	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
1476		goto err;
1477
1478	pr_info("registered backing device %pg\n", dc->bdev);
1479
1480	list_add(&dc->list, &uncached_devices);
1481	/* attach to a matched cache set if it exists */
1482	list_for_each_entry(c, &bch_cache_sets, list)
1483		bch_cached_dev_attach(dc, c, NULL);
1484
1485	if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
1486	    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) {
1487		err = "failed to run cached device";
1488		ret = bch_cached_dev_run(dc);
1489		if (ret)
1490			goto err;
1491	}
1492
1493	return 0;
1494err:
1495	pr_notice("error %pg: %s\n", dc->bdev, err);
1496	bcache_device_stop(&dc->disk);
1497	return ret;
1498}
1499
1500/* Flash only volumes */
1501
1502/* When d->kobj released */
1503void bch_flash_dev_release(struct kobject *kobj)
1504{
1505	struct bcache_device *d = container_of(kobj, struct bcache_device,
1506					       kobj);
1507	kfree(d);
1508}
1509
1510static CLOSURE_CALLBACK(flash_dev_free)
1511{
1512	closure_type(d, struct bcache_device, cl);
1513
1514	mutex_lock(&bch_register_lock);
1515	atomic_long_sub(bcache_dev_sectors_dirty(d),
1516			&d->c->flash_dev_dirty_sectors);
1517	del_gendisk(d->disk);
1518	bcache_device_free(d);
1519	mutex_unlock(&bch_register_lock);
1520	kobject_put(&d->kobj);
1521}
1522
1523static CLOSURE_CALLBACK(flash_dev_flush)
1524{
1525	closure_type(d, struct bcache_device, cl);
1526
1527	mutex_lock(&bch_register_lock);
1528	bcache_device_unlink(d);
1529	mutex_unlock(&bch_register_lock);
1530	kobject_del(&d->kobj);
1531	continue_at(cl, flash_dev_free, system_percpu_wq);
1532}
1533
1534static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
1535{
1536	int err = -ENOMEM;
1537	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
1538					  GFP_KERNEL);
1539	if (!d)
1540		goto err_ret;
1541
1542	closure_init(&d->cl, NULL);
1543	set_closure_fn(&d->cl, flash_dev_flush, system_percpu_wq);
1544
1545	kobject_init(&d->kobj, &bch_flash_dev_ktype);
1546
1547	if (bcache_device_init(d, block_bytes(c->cache), u->sectors,
1548			NULL, &bcache_flash_ops))
1549		goto err;
1550
1551	bcache_device_attach(d, c, u - c->uuids);
1552	bch_sectors_dirty_init(d);
1553	bch_flash_dev_request_init(d);
1554	err = add_disk(d->disk);
1555	if (err)
1556		goto err;
1557
1558	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");
1559	if (err)
1560		goto err;
1561
1562	bcache_device_link(d, c, "volume");
1563
1564	if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
1565		pr_err("The obsoleted large bucket layout is unsupported, set the bcache device into read-only\n");
1566		pr_err("Please update to the latest bcache-tools to create the cache device\n");
1567		set_disk_ro(d->disk, 1);
1568	}
1569
1570	return 0;
1571err:
1572	kobject_put(&d->kobj);
1573err_ret:
1574	return err;
1575}
1576
1577static int flash_devs_run(struct cache_set *c)
1578{
1579	int ret = 0;
1580	struct uuid_entry *u;
1581
1582	for (u = c->uuids;
1583	     u < c->uuids + c->nr_uuids && !ret;
1584	     u++)
1585		if (UUID_FLASH_ONLY(u))
1586			ret = flash_dev_run(c, u);
1587
1588	return ret;
1589}
1590
1591int bch_flash_dev_create(struct cache_set *c, uint64_t size)
1592{
1593	struct uuid_entry *u;
1594
1595	if (test_bit(CACHE_SET_STOPPING, &c->flags))
1596		return -EINTR;
1597
1598	if (!test_bit(CACHE_SET_RUNNING, &c->flags))
1599		return -EPERM;
1600
1601	u = uuid_find_empty(c);
1602	if (!u) {
1603		pr_err("Can't create volume, no room for UUID\n");
1604		return -EINVAL;
1605	}
1606
1607	get_random_bytes(u->uuid, 16);
1608	memset(u->label, 0, 32);
1609	u->first_reg = u->last_reg = cpu_to_le32((u32)ktime_get_real_seconds());
1610
1611	SET_UUID_FLASH_ONLY(u, 1);
1612	u->sectors = size >> 9;
1613
1614	bch_uuid_write(c);
1615
1616	return flash_dev_run(c, u);
1617}
1618
1619bool bch_cached_dev_error(struct cached_dev *dc)
1620{
1621	if (!dc || test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
1622		return false;
1623
1624	dc->io_disable = true;
1625	/* make others know io_disable is true earlier */
1626	smp_mb();
1627
1628	pr_err("stop %s: too many IO errors on backing device %pg\n",
1629	       dc->disk.disk->disk_name, dc->bdev);
1630
1631	bcache_device_stop(&dc->disk);
1632	return true;
1633}
1634
1635/* Cache set */
1636
1637__printf(2, 3)
1638bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
1639{
1640	struct va_format vaf;
1641	va_list args;
1642
1643	if (c->on_error != ON_ERROR_PANIC &&
1644	    test_bit(CACHE_SET_STOPPING, &c->flags))
1645		return false;
1646
1647	if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags))
1648		pr_info("CACHE_SET_IO_DISABLE already set\n");
1649
1650	/*
1651	 * XXX: we can be called from atomic context
1652	 * acquire_console_sem();
1653	 */
1654
1655	va_start(args, fmt);
1656
1657	vaf.fmt = fmt;
1658	vaf.va = &args;
1659
1660	pr_err("error on %pU: %pV, disabling caching\n",
1661	       c->set_uuid, &vaf);
1662
1663	va_end(args);
1664
1665	if (c->on_error == ON_ERROR_PANIC)
1666		panic("panic forced after error\n");
1667
1668	bch_cache_set_unregister(c);
1669	return true;
1670}
1671
1672/* When c->kobj released */
1673void bch_cache_set_release(struct kobject *kobj)
1674{
1675	struct cache_set *c = container_of(kobj, struct cache_set, kobj);
1676
1677	kfree(c);
1678	module_put(THIS_MODULE);
1679}
1680
1681static CLOSURE_CALLBACK(cache_set_free)
1682{
1683	closure_type(c, struct cache_set, cl);
1684	struct cache *ca;
1685
1686	debugfs_remove(c->debug);
1687
1688	bch_open_buckets_free(c);
1689	bch_btree_cache_free(c);
1690	bch_journal_free(c);
1691
1692	mutex_lock(&bch_register_lock);
1693	bch_bset_sort_state_free(&c->sort);
1694	free_pages((unsigned long) c->uuids, ilog2(meta_bucket_pages(&c->cache->sb)));
1695
1696	ca = c->cache;
1697	if (ca) {
1698		ca->set = NULL;
1699		c->cache = NULL;
1700		kobject_put(&ca->kobj);
1701	}
1702
1703
1704	if (c->moving_gc_wq)
1705		destroy_workqueue(c->moving_gc_wq);
1706	bioset_exit(&c->bio_split);
1707	mempool_exit(&c->fill_iter);
1708	mempool_exit(&c->bio_meta);
1709	mempool_exit(&c->search);
1710	kfree(c->devices);
1711
1712	list_del(&c->list);
1713	mutex_unlock(&bch_register_lock);
1714
1715	pr_info("Cache set %pU unregistered\n", c->set_uuid);
1716	wake_up(&unregister_wait);
1717
1718	closure_debug_destroy(&c->cl);
1719	kobject_put(&c->kobj);
1720}
1721
1722static CLOSURE_CALLBACK(cache_set_flush)
1723{
1724	closure_type(c, struct cache_set, caching);
1725	struct cache *ca = c->cache;
1726	struct btree *b;
1727
1728	bch_cache_accounting_destroy(&c->accounting);
1729
1730	kobject_put(&c->internal);
1731	kobject_del(&c->kobj);
1732
1733	if (!IS_ERR_OR_NULL(c->gc_thread))
1734		kthread_stop(c->gc_thread);
1735
1736	if (!IS_ERR_OR_NULL(c->root))
1737		list_add(&c->root->list, &c->btree_cache);
1738
1739	/*
1740	 * Avoid flushing cached nodes if cache set is retiring
1741	 * due to too many I/O errors detected.
1742	 */
1743	if (!test_bit(CACHE_SET_IO_DISABLE, &c->flags))
1744		list_for_each_entry(b, &c->btree_cache, list) {
1745			mutex_lock(&b->write_lock);
1746			if (btree_node_dirty(b))
1747				__bch_btree_node_write(b, NULL);
1748			mutex_unlock(&b->write_lock);
1749		}
1750
1751	/*
1752	 * If the register_cache_set() call to bch_cache_set_alloc() failed,
1753	 * ca has not been assigned a value and return error.
1754	 * So we need check ca is not NULL during bch_cache_set_unregister().
1755	 */
1756	if (ca && ca->alloc_thread)
1757		kthread_stop(ca->alloc_thread);
1758
1759	if (c->journal.cur) {
1760		cancel_delayed_work_sync(&c->journal.work);
1761		/* flush last journal entry if needed */
1762		c->journal.work.work.func(&c->journal.work.work);
1763	}
1764
1765	closure_return(cl);
1766}
1767
1768/*
1769 * This function is only called when CACHE_SET_IO_DISABLE is set, which means
1770 * cache set is unregistering due to too many I/O errors. In this condition,
1771 * the bcache device might be stopped, it depends on stop_when_cache_set_failed
1772 * value and whether the broken cache has dirty data:
1773 *
1774 * dc->stop_when_cache_set_failed    dc->has_dirty   stop bcache device
1775 *  BCH_CACHED_STOP_AUTO               0               NO
1776 *  BCH_CACHED_STOP_AUTO               1               YES
1777 *  BCH_CACHED_DEV_STOP_ALWAYS         0               YES
1778 *  BCH_CACHED_DEV_STOP_ALWAYS         1               YES
1779 *
1780 * The expected behavior is, if stop_when_cache_set_failed is configured to
1781 * "auto" via sysfs interface, the bcache device will not be stopped if the
1782 * backing device is clean on the broken cache device.
1783 */
1784static void conditional_stop_bcache_device(struct cache_set *c,
1785					   struct bcache_device *d,
1786					   struct cached_dev *dc)
1787{
1788	if (dc->stop_when_cache_set_failed == BCH_CACHED_DEV_STOP_ALWAYS) {
1789		pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.\n",
1790			d->disk->disk_name, c->set_uuid);
1791		bcache_device_stop(d);
1792	} else if (atomic_read(&dc->has_dirty)) {
1793		/*
1794		 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO
1795		 * and dc->has_dirty == 1
1796		 */
1797		pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.\n",
1798			d->disk->disk_name);
1799		/*
1800		 * There might be a small time gap that cache set is
1801		 * released but bcache device is not. Inside this time
1802		 * gap, regular I/O requests will directly go into
1803		 * backing device as no cache set attached to. This
1804		 * behavior may also introduce potential inconsistence
1805		 * data in writeback mode while cache is dirty.
1806		 * Therefore before calling bcache_device_stop() due
1807		 * to a broken cache device, dc->io_disable should be
1808		 * explicitly set to true.
1809		 */
1810		dc->io_disable = true;
1811		/* make others know io_disable is true earlier */
1812		smp_mb();
1813		bcache_device_stop(d);
1814	} else {
1815		/*
1816		 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO
1817		 * and dc->has_dirty == 0
1818		 */
1819		pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.\n",
1820			d->disk->disk_name);
1821	}
1822}
1823
1824static CLOSURE_CALLBACK(__cache_set_unregister)
1825{
1826	closure_type(c, struct cache_set, caching);
1827	struct cached_dev *dc;
1828	struct bcache_device *d;
1829	size_t i;
1830
1831	mutex_lock(&bch_register_lock);
1832
1833	for (i = 0; i < c->devices_max_used; i++) {
1834		d = c->devices[i];
1835		if (!d)
1836			continue;
1837
1838		if (!UUID_FLASH_ONLY(&c->uuids[i]) &&
1839		    test_bit(CACHE_SET_UNREGISTERING, &c->flags)) {
1840			dc = container_of(d, struct cached_dev, disk);
1841			bch_cached_dev_detach(dc);
1842			if (test_bit(CACHE_SET_IO_DISABLE, &c->flags))
1843				conditional_stop_bcache_device(c, d, dc);
1844		} else {
1845			bcache_device_stop(d);
1846		}
1847	}
1848
1849	mutex_unlock(&bch_register_lock);
1850
1851	continue_at(cl, cache_set_flush, system_percpu_wq);
1852}
1853
1854void bch_cache_set_stop(struct cache_set *c)
1855{
1856	if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags))
1857		/* closure_fn set to __cache_set_unregister() */
1858		closure_queue(&c->caching);
1859}
1860
1861void bch_cache_set_unregister(struct cache_set *c)
1862{
1863	set_bit(CACHE_SET_UNREGISTERING, &c->flags);
1864	bch_cache_set_stop(c);
1865}
1866
1867#define alloc_meta_bucket_pages(gfp, sb)		\
1868	((void *) __get_free_pages(__GFP_ZERO|__GFP_COMP|gfp, ilog2(meta_bucket_pages(sb))))
1869
1870struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
1871{
1872	int iter_size;
1873	struct cache *ca = container_of(sb, struct cache, sb);
1874	struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);
1875
1876	if (!c)
1877		return NULL;
1878
1879	__module_get(THIS_MODULE);
1880	closure_init(&c->cl, NULL);
1881	set_closure_fn(&c->cl, cache_set_free, system_percpu_wq);
1882
1883	closure_init(&c->caching, &c->cl);
1884	set_closure_fn(&c->caching, __cache_set_unregister, system_percpu_wq);
1885
1886	/* Maybe create continue_at_noreturn() and use it here? */
1887	closure_set_stopped(&c->cl);
1888	closure_put(&c->cl);
1889
1890	kobject_init(&c->kobj, &bch_cache_set_ktype);
1891	kobject_init(&c->internal, &bch_cache_set_internal_ktype);
1892
1893	bch_cache_accounting_init(&c->accounting, &c->cl);
1894
1895	memcpy(c->set_uuid, sb->set_uuid, 16);
1896
1897	c->cache		= ca;
1898	c->cache->set		= c;
1899	c->bucket_bits		= ilog2(sb->bucket_size);
1900	c->block_bits		= ilog2(sb->block_size);
1901	c->nr_uuids		= meta_bucket_bytes(sb) / sizeof(struct uuid_entry);
1902	c->devices_max_used	= 0;
1903	atomic_set(&c->attached_dev_nr, 0);
1904	c->btree_pages		= meta_bucket_pages(sb);
1905	if (c->btree_pages > BTREE_MAX_PAGES)
1906		c->btree_pages = max_t(int, c->btree_pages / 4,
1907				       BTREE_MAX_PAGES);
1908
1909	sema_init(&c->sb_write_mutex, 1);
1910	mutex_init(&c->bucket_lock);
1911	init_waitqueue_head(&c->btree_cache_wait);
1912	spin_lock_init(&c->btree_cannibalize_lock);
1913	init_waitqueue_head(&c->bucket_wait);
1914	init_waitqueue_head(&c->gc_wait);
1915	sema_init(&c->uuid_write_mutex, 1);
1916
1917	spin_lock_init(&c->btree_gc_time.lock);
1918	spin_lock_init(&c->btree_split_time.lock);
1919	spin_lock_init(&c->btree_read_time.lock);
1920
1921	bch_moving_init_cache_set(c);
1922
1923	INIT_LIST_HEAD(&c->list);
1924	INIT_LIST_HEAD(&c->cached_devs);
1925	INIT_LIST_HEAD(&c->btree_cache);
1926	INIT_LIST_HEAD(&c->btree_cache_freeable);
1927	INIT_LIST_HEAD(&c->btree_cache_freed);
1928	INIT_LIST_HEAD(&c->data_buckets);
1929
1930	iter_size = sizeof(struct btree_iter) +
1931		    ((meta_bucket_pages(sb) * PAGE_SECTORS) / sb->block_size) *
1932			    sizeof(struct btree_iter_set);
1933
1934	c->devices = kcalloc(c->nr_uuids, sizeof(void *), GFP_KERNEL);
1935	if (!c->devices)
1936		goto err;
1937
1938	if (mempool_init_slab_pool(&c->search, 32, bch_search_cache))
1939		goto err;
1940
1941	if (mempool_init_kmalloc_pool(&c->bio_meta, 2,
1942			sizeof(struct bbio) +
1943			sizeof(struct bio_vec) * meta_bucket_pages(sb)))
1944		goto err;
1945
1946	if (mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size))
1947		goto err;
1948
1949	if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),
1950			BIOSET_NEED_RESCUER))
1951		goto err;
1952
1953	c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb);
1954	if (!c->uuids)
1955		goto err;
1956
1957	c->moving_gc_wq = alloc_workqueue("bcache_gc",
1958					  WQ_MEM_RECLAIM | WQ_PERCPU, 0);
1959	if (!c->moving_gc_wq)
1960		goto err;
1961
1962	if (bch_journal_alloc(c))
1963		goto err;
1964
1965	if (bch_btree_cache_alloc(c))
1966		goto err;
1967
1968	if (bch_open_buckets_alloc(c))
1969		goto err;
1970
1971	if (bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages)))
1972		goto err;
1973
1974	c->congested_read_threshold_us	= 2000;
1975	c->congested_write_threshold_us	= 20000;
1976	c->error_limit	= DEFAULT_IO_ERROR_LIMIT;
1977	c->idle_max_writeback_rate_enabled = 1;
1978	WARN_ON(test_and_clear_bit(CACHE_SET_IO_DISABLE, &c->flags));
1979
1980	return c;
1981err:
1982	bch_cache_set_unregister(c);
1983	return NULL;
1984}
1985
1986static int run_cache_set(struct cache_set *c)
1987{
1988	const char *err = "cannot allocate memory";
1989	struct cached_dev *dc, *t;
1990	struct cache *ca = c->cache;
1991	struct closure cl;
1992	LIST_HEAD(journal);
1993	struct journal_replay *l;
1994
1995	closure_init_stack(&cl);
1996
1997	c->nbuckets = ca->sb.nbuckets;
1998	set_gc_sectors(c);
1999
2000	if (CACHE_SYNC(&c->cache->sb)) {
2001		struct bkey *k;
2002		struct jset *j;
2003
2004		err = "cannot allocate memory for journal";
2005		if (bch_journal_read(c, &journal))
2006			goto err;
2007
2008		pr_debug("btree_journal_read() done\n");
2009
2010		err = "no journal entries found";
2011		if (list_empty(&journal))
2012			goto err;
2013
2014		j = &list_entry(journal.prev, struct journal_replay, list)->j;
2015
2016		err = "IO error reading priorities";
2017		if (prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]))
2018			goto err;
2019
2020		/*
2021		 * If prio_read() fails it'll call cache_set_error and we'll
2022		 * tear everything down right away, but if we perhaps checked
2023		 * sooner we could avoid journal replay.
2024		 */
2025
2026		k = &j->btree_root;
2027
2028		err = "bad btree root";
2029		if (__bch_btree_ptr_invalid(c, k))
2030			goto err;
2031
2032		err = "error reading btree root";
2033		c->root = bch_btree_node_get(c, NULL, k,
2034					     j->btree_level,
2035					     true, NULL);
2036		if (IS_ERR(c->root))
2037			goto err;
2038
2039		list_del_init(&c->root->list);
2040		rw_unlock(true, c->root);
2041
2042		err = uuid_read(c, j, &cl);
2043		if (err)
2044			goto err;
2045
2046		err = "error in recovery";
2047		if (bch_btree_check(c))
2048			goto err;
2049
2050		bch_journal_mark(c, &journal);
2051		bch_initial_gc_finish(c);
2052		pr_debug("btree_check() done\n");
2053
2054		/*
2055		 * bcache_journal_next() can't happen sooner, or
2056		 * btree_gc_finish() will give spurious errors about last_gc >
2057		 * gc_gen - this is a hack but oh well.
2058		 */
2059		bch_journal_next(&c->journal);
2060
2061		err = "error starting allocator thread";
2062		if (bch_cache_allocator_start(ca))
2063			goto err;
2064
2065		/*
2066		 * First place it's safe to allocate: btree_check() and
2067		 * btree_gc_finish() have to run before we have buckets to
2068		 * allocate, and bch_bucket_alloc_set() might cause a journal
2069		 * entry to be written so bcache_journal_next() has to be called
2070		 * first.
2071		 *
2072		 * If the uuids were in the old format we have to rewrite them
2073		 * before the next journal entry is written:
2074		 */
2075		if (j->version < BCACHE_JSET_VERSION_UUID)
2076			__uuid_write(c);
2077
2078		err = "bcache: replay journal failed";
2079		if (bch_journal_replay(c, &journal))
2080			goto err;
2081	} else {
2082		unsigned int j;
2083
2084		pr_notice("invalidating existing data\n");
2085		ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,
2086					2, SB_JOURNAL_BUCKETS);
2087
2088		for (j = 0; j < ca->sb.keys; j++)
2089			ca->sb.d[j] = ca->sb.first_bucket + j;
2090
2091		bch_initial_gc_finish(c);
2092
2093		err = "error starting allocator thread";
2094		if (bch_cache_allocator_start(ca))
2095			goto err;
2096
2097		mutex_lock(&c->bucket_lock);
2098		bch_prio_write(ca, true);
2099		mutex_unlock(&c->bucket_lock);
2100
2101		err = "cannot allocate new UUID bucket";
2102		if (__uuid_write(c))
2103			goto err;
2104
2105		err = "cannot allocate new btree root";
2106		c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL);
2107		if (IS_ERR(c->root))
2108			goto err;
2109
2110		mutex_lock(&c->root->write_lock);
2111		bkey_copy_key(&c->root->key, &MAX_KEY);
2112		bch_btree_node_write(c->root, &cl);
2113		mutex_unlock(&c->root->write_lock);
2114
2115		bch_btree_set_root(c->root);
2116		rw_unlock(true, c->root);
2117
2118		/*
2119		 * We don't want to write the first journal entry until
2120		 * everything is set up - fortunately journal entries won't be
2121		 * written until the SET_CACHE_SYNC() here:
2122		 */
2123		SET_CACHE_SYNC(&c->cache->sb, true);
2124
2125		bch_journal_next(&c->journal);
2126		bch_journal_meta(c, &cl);
2127	}
2128
2129	err = "error starting gc thread";
2130	if (bch_gc_thread_start(c))
2131		goto err;
2132
2133	closure_sync(&cl);
2134	c->cache->sb.last_mount = (u32)ktime_get_real_seconds();
2135	bcache_write_super(c);
2136
2137	if (bch_has_feature_obso_large_bucket(&c->cache->sb))
2138		pr_err("Detect obsoleted large bucket layout, all attached bcache device will be read-only\n");
2139
2140	list_for_each_entry_safe(dc, t, &uncached_devices, list)
2141		bch_cached_dev_attach(dc, c, NULL);
2142
2143	flash_devs_run(c);
2144
2145	bch_journal_space_reserve(&c->journal);
2146	set_bit(CACHE_SET_RUNNING, &c->flags);
2147	return 0;
2148err:
2149	while (!list_empty(&journal)) {
2150		l = list_first_entry(&journal, struct journal_replay, list);
2151		list_del(&l->list);
2152		kfree(l);
2153	}
2154
2155	closure_sync(&cl);
2156
2157	bch_cache_set_error(c, "%s", err);
2158
2159	return -EIO;
2160}
2161
2162static const char *register_cache_set(struct cache *ca)
2163{
2164	char buf[12];
2165	const char *err = "cannot allocate memory";
2166	struct cache_set *c;
2167
2168	list_for_each_entry(c, &bch_cache_sets, list)
2169		if (!memcmp(c->set_uuid, ca->sb.set_uuid, 16)) {
2170			if (c->cache)
2171				return "duplicate cache set member";
2172
2173			goto found;
2174		}
2175
2176	c = bch_cache_set_alloc(&ca->sb);
2177	if (!c)
2178		return err;
2179
2180	err = "error creating kobject";
2181	if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->set_uuid) ||
2182	    kobject_add(&c->internal, &c->kobj, "internal"))
2183		goto err;
2184
2185	if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj))
2186		goto err;
2187
2188	bch_debug_init_cache_set(c);
2189
2190	list_add(&c->list, &bch_cache_sets);
2191found:
2192	sprintf(buf, "cache%i", ca->sb.nr_this_dev);
2193	if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||
2194	    sysfs_create_link(&c->kobj, &ca->kobj, buf))
2195		goto err;
2196
2197	kobject_get(&ca->kobj);
2198	ca->set = c;
2199	ca->set->cache = ca;
2200
2201	err = "failed to run cache set";
2202	if (run_cache_set(c) < 0)
2203		goto err;
2204
2205	return NULL;
2206err:
2207	bch_cache_set_unregister(c);
2208	return err;
2209}
2210
2211/* Cache device */
2212
2213/* When ca->kobj released */
2214void bch_cache_release(struct kobject *kobj)
2215{
2216	struct cache *ca = container_of(kobj, struct cache, kobj);
2217	unsigned int i;
2218
2219	if (ca->set) {
2220		BUG_ON(ca->set->cache != ca);
2221		ca->set->cache = NULL;
2222	}
2223
2224	free_pages((unsigned long) ca->disk_buckets, ilog2(meta_bucket_pages(&ca->sb)));
2225	kfree(ca->prio_buckets);
2226	vfree(ca->buckets);
2227
2228	free_heap(&ca->heap);
2229	free_fifo(&ca->free_inc);
2230
2231	for (i = 0; i < RESERVE_NR; i++)
2232		free_fifo(&ca->free[i]);
2233
2234	if (ca->sb_disk)
2235		folio_put(virt_to_folio(ca->sb_disk));
2236
2237	if (ca->bdev_file)
2238		fput(ca->bdev_file);
2239
2240	kfree(ca);
2241	module_put(THIS_MODULE);
2242}
2243
2244static int cache_alloc(struct cache *ca)
2245{
2246	size_t free;
2247	size_t btree_buckets;
2248	struct bucket *b;
2249	int ret = -ENOMEM;
2250	const char *err = NULL;
2251
2252	__module_get(THIS_MODULE);
2253	kobject_init(&ca->kobj, &bch_cache_ktype);
2254
2255	bio_init_inline(&ca->journal.bio, NULL, 8, 0);
2256
2257	/*
2258	 * When the cache disk is first registered, ca->sb.njournal_buckets
2259	 * is zero, and it is assigned in run_cache_set().
2260	 *
2261	 * When ca->sb.njournal_buckets is not zero, journal exists,
2262	 * and in bch_journal_replay(), tree node may split.
2263	 * The worst situation is all journal buckets are valid journal,
2264	 * and all the keys need to replay, so the number of RESERVE_BTREE
2265	 * type buckets should be as much as journal buckets.
2266	 *
2267	 * If the number of RESERVE_BTREE type buckets is too few, the
2268	 * bch_allocator_thread() may hang up and unable to allocate
2269	 * bucket. The situation is roughly as follows:
2270	 *
2271	 * 1. In bch_data_insert_keys(), if the operation is not op->replace,
2272	 *    it will call the bch_journal(), which increments the journal_ref
2273	 *    counter. This counter is only decremented after bch_btree_insert
2274	 *    completes.
2275	 *
2276	 * 2. When calling bch_btree_insert, if the btree needs to split,
2277	 *    it will call btree_split() and btree_check_reserve() to check
2278	 *    whether there are enough reserved buckets in the RESERVE_BTREE
2279	 *    slot. If not enough, bcache_btree_root() will repeatedly retry.
2280	 *
2281	 * 3. Normally, the bch_allocator_thread is responsible for filling
2282	 *    the reservation slots from the free_inc bucket list. When the
2283	 *    free_inc bucket list is exhausted, the bch_allocator_thread
2284	 *    will call invalidate_buckets() until free_inc is refilled.
2285	 *    Then bch_allocator_thread calls bch_prio_write() once. and
2286	 *    bch_prio_write() will call bch_journal_meta() and waits for
2287	 *    the journal write to complete.
2288	 *
2289	 * 4. During journal_write, journal_write_unlocked() is be called.
2290	 *    If journal full occurs, journal_reclaim() and btree_flush_write()
2291	 *    will be called sequentially, then retry journal_write.
2292	 *
2293	 * 5. When 2 and 4 occur together, IO will hung up and cannot recover.
2294	 *
2295	 * Therefore, reserve more RESERVE_BTREE type buckets.
2296	 */
2297	btree_buckets = clamp_t(size_t, ca->sb.nbuckets >> 7,
2298				32, SB_JOURNAL_BUCKETS);
2299	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
2300	if (!free) {
2301		ret = -EPERM;
2302		err = "ca->sb.nbuckets is too small";
2303		goto err_free;
2304	}
2305
2306	if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets,
2307						GFP_KERNEL)) {
2308		err = "ca->free[RESERVE_BTREE] alloc failed";
2309		goto err_btree_alloc;
2310	}
2311
2312	if (!init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca),
2313							GFP_KERNEL)) {
2314		err = "ca->free[RESERVE_PRIO] alloc failed";
2315		goto err_prio_alloc;
2316	}
2317
2318	if (!init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL)) {
2319		err = "ca->free[RESERVE_MOVINGGC] alloc failed";
2320		goto err_movinggc_alloc;
2321	}
2322
2323	if (!init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL)) {
2324		err = "ca->free[RESERVE_NONE] alloc failed";
2325		goto err_none_alloc;
2326	}
2327
2328	if (!init_fifo(&ca->free_inc, free << 2, GFP_KERNEL)) {
2329		err = "ca->free_inc alloc failed";
2330		goto err_free_inc_alloc;
2331	}
2332
2333	if (!init_heap(&ca->heap, free << 3, GFP_KERNEL)) {
2334		err = "ca->heap alloc failed";
2335		goto err_heap_alloc;
2336	}
2337
2338	ca->buckets = vzalloc(array_size(sizeof(struct bucket),
2339			      ca->sb.nbuckets));
2340	if (!ca->buckets) {
2341		err = "ca->buckets alloc failed";
2342		goto err_buckets_alloc;
2343	}
2344
2345	ca->prio_buckets = kzalloc(array3_size(sizeof(uint64_t),
2346				   prio_buckets(ca), 2),
2347				   GFP_KERNEL);
2348	if (!ca->prio_buckets) {
2349		err = "ca->prio_buckets alloc failed";
2350		goto err_prio_buckets_alloc;
2351	}
2352
2353	ca->disk_buckets = alloc_meta_bucket_pages(GFP_KERNEL, &ca->sb);
2354	if (!ca->disk_buckets) {
2355		err = "ca->disk_buckets alloc failed";
2356		goto err_disk_buckets_alloc;
2357	}
2358
2359	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
2360
2361	for_each_bucket(b, ca)
2362		atomic_set(&b->pin, 0);
2363	return 0;
2364
2365err_disk_buckets_alloc:
2366	kfree(ca->prio_buckets);
2367err_prio_buckets_alloc:
2368	vfree(ca->buckets);
2369err_buckets_alloc:
2370	free_heap(&ca->heap);
2371err_heap_alloc:
2372	free_fifo(&ca->free_inc);
2373err_free_inc_alloc:
2374	free_fifo(&ca->free[RESERVE_NONE]);
2375err_none_alloc:
2376	free_fifo(&ca->free[RESERVE_MOVINGGC]);
2377err_movinggc_alloc:
2378	free_fifo(&ca->free[RESERVE_PRIO]);
2379err_prio_alloc:
2380	free_fifo(&ca->free[RESERVE_BTREE]);
2381err_btree_alloc:
2382err_free:
2383	module_put(THIS_MODULE);
2384	if (err)
2385		pr_notice("error %pg: %s\n", ca->bdev, err);
2386	return ret;
2387}
2388
2389static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
2390				struct file *bdev_file,
2391				struct cache *ca)
2392{
2393	const char *err = NULL; /* must be set for any error case */
2394	int ret = 0;
2395
2396	memcpy(&ca->sb, sb, sizeof(struct cache_sb));
2397	ca->bdev_file = bdev_file;
2398	ca->bdev = file_bdev(bdev_file);
2399	ca->sb_disk = sb_disk;
2400
2401	ret = cache_alloc(ca);
2402	if (ret != 0) {
2403		if (ret == -ENOMEM)
2404			err = "cache_alloc(): -ENOMEM";
2405		else if (ret == -EPERM)
2406			err = "cache_alloc(): cache device is too small";
2407		else
2408			err = "cache_alloc(): unknown error";
2409		pr_notice("error %pg: %s\n", file_bdev(bdev_file), err);
2410		/*
2411		 * If we failed here, it means ca->kobj is not initialized yet,
2412		 * kobject_put() won't be called and there is no chance to
2413		 * call fput() to bdev in bch_cache_release(). So
2414		 * we explicitly call fput() on the block device here.
2415		 */
2416		fput(bdev_file);
2417		return ret;
2418	}
2419
2420	if (kobject_add(&ca->kobj, bdev_kobj(file_bdev(bdev_file)), "bcache")) {
2421		pr_notice("error %pg: error calling kobject_add\n",
2422			  file_bdev(bdev_file));
2423		ret = -ENOMEM;
2424		goto out;
2425	}
2426
2427	mutex_lock(&bch_register_lock);
2428	err = register_cache_set(ca);
2429	mutex_unlock(&bch_register_lock);
2430
2431	if (err) {
2432		ret = -ENODEV;
2433		goto out;
2434	}
2435
2436	pr_info("registered cache device %pg\n", file_bdev(ca->bdev_file));
2437
2438out:
2439	kobject_put(&ca->kobj);
2440	return ret;
2441}
2442
2443/* Global interfaces/init */
2444
2445static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
2446			       const char *buffer, size_t size);
2447static ssize_t bch_pending_bdevs_cleanup(struct kobject *k,
2448					 struct kobj_attribute *attr,
2449					 const char *buffer, size_t size);
2450
2451kobj_attribute_write(register,		register_bcache);
2452kobj_attribute_write(register_quiet,	register_bcache);
2453kobj_attribute_write(pendings_cleanup,	bch_pending_bdevs_cleanup);
2454
2455static bool bch_is_open_backing(dev_t dev)
2456{
2457	struct cache_set *c, *tc;
2458	struct cached_dev *dc, *t;
2459
2460	list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
2461		list_for_each_entry_safe(dc, t, &c->cached_devs, list)
2462			if (dc->bdev->bd_dev == dev)
2463				return true;
2464	list_for_each_entry_safe(dc, t, &uncached_devices, list)
2465		if (dc->bdev->bd_dev == dev)
2466			return true;
2467	return false;
2468}
2469
2470static bool bch_is_open_cache(dev_t dev)
2471{
2472	struct cache_set *c, *tc;
2473
2474	list_for_each_entry_safe(c, tc, &bch_cache_sets, list) {
2475		struct cache *ca = c->cache;
2476
2477		if (ca->bdev->bd_dev == dev)
2478			return true;
2479	}
2480
2481	return false;
2482}
2483
2484static bool bch_is_open(dev_t dev)
2485{
2486	return bch_is_open_cache(dev) || bch_is_open_backing(dev);
2487}
2488
2489struct async_reg_args {
2490	struct delayed_work reg_work;
2491	char *path;
2492	struct cache_sb *sb;
2493	struct cache_sb_disk *sb_disk;
2494	struct file *bdev_file;
2495	void *holder;
2496};
2497
2498static void register_bdev_worker(struct work_struct *work)
2499{
2500	int fail = false;
2501	struct async_reg_args *args =
2502		container_of(work, struct async_reg_args, reg_work.work);
2503
2504	mutex_lock(&bch_register_lock);
2505	if (register_bdev(args->sb, args->sb_disk, args->bdev_file,
2506			  args->holder) < 0)
2507		fail = true;
2508	mutex_unlock(&bch_register_lock);
2509
2510	if (fail)
2511		pr_info("error %s: fail to register backing device\n",
2512			args->path);
2513	kfree(args->sb);
2514	kfree(args->path);
2515	kfree(args);
2516	module_put(THIS_MODULE);
2517}
2518
2519static void register_cache_worker(struct work_struct *work)
2520{
2521	int fail = false;
2522	struct async_reg_args *args =
2523		container_of(work, struct async_reg_args, reg_work.work);
2524
2525	/* blkdev_put() will be called in bch_cache_release() */
2526	if (register_cache(args->sb, args->sb_disk, args->bdev_file,
2527			   args->holder))
2528		fail = true;
2529
2530	if (fail)
2531		pr_info("error %s: fail to register cache device\n",
2532			args->path);
2533	kfree(args->sb);
2534	kfree(args->path);
2535	kfree(args);
2536	module_put(THIS_MODULE);
2537}
2538
2539static void register_device_async(struct async_reg_args *args)
2540{
2541	if (SB_IS_BDEV(args->sb))
2542		INIT_DELAYED_WORK(&args->reg_work, register_bdev_worker);
2543	else
2544		INIT_DELAYED_WORK(&args->reg_work, register_cache_worker);
2545
2546	/* 10 jiffies is enough for a delay */
2547	queue_delayed_work(system_percpu_wq, &args->reg_work, 10);
2548}
2549
2550static void *alloc_holder_object(struct cache_sb *sb)
2551{
2552	if (SB_IS_BDEV(sb))
2553		return kzalloc(sizeof(struct cached_dev), GFP_KERNEL);
2554	return kzalloc(sizeof(struct cache), GFP_KERNEL);
2555}
2556
2557static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
2558			       const char *buffer, size_t size)
2559{
2560	const char *err;
2561	char *path = NULL;
2562	struct cache_sb *sb;
2563	struct cache_sb_disk *sb_disk;
2564	struct file *bdev_file, *bdev_file2;
2565	void *holder = NULL;
2566	ssize_t ret;
2567	bool async_registration = false;
2568	bool quiet = false;
2569
2570#ifdef CONFIG_BCACHE_ASYNC_REGISTRATION
2571	async_registration = true;
2572#endif
2573
2574	ret = -EBUSY;
2575	err = "failed to reference bcache module";
2576	if (!try_module_get(THIS_MODULE))
2577		goto out;
2578
2579	/* For latest state of bcache_is_reboot */
2580	smp_mb();
2581	err = "bcache is in reboot";
2582	if (bcache_is_reboot)
2583		goto out_module_put;
2584
2585	ret = -ENOMEM;
2586	err = "cannot allocate memory";
2587	path = kstrndup(buffer, size, GFP_KERNEL);
2588	if (!path)
2589		goto out_module_put;
2590
2591	sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL);
2592	if (!sb)
2593		goto out_free_path;
2594
2595	ret = -EINVAL;
2596	err = "failed to open device";
2597	bdev_file = bdev_file_open_by_path(strim(path), BLK_OPEN_READ, NULL, NULL);
2598	if (IS_ERR(bdev_file))
2599		goto out_free_sb;
2600
2601	err = read_super(sb, file_bdev(bdev_file), &sb_disk);
2602	if (err)
2603		goto out_blkdev_put;
2604
2605	holder = alloc_holder_object(sb);
2606	if (!holder) {
2607		ret = -ENOMEM;
2608		err = "cannot allocate memory";
2609		goto out_put_sb_folio;
2610	}
2611
2612	/* Now reopen in exclusive mode with proper holder */
2613	bdev_file2 = bdev_file_open_by_dev(file_bdev(bdev_file)->bd_dev,
2614			BLK_OPEN_READ | BLK_OPEN_WRITE, holder, NULL);
2615	fput(bdev_file);
2616	bdev_file = bdev_file2;
2617	if (IS_ERR(bdev_file)) {
2618		ret = PTR_ERR(bdev_file);
2619		bdev_file = NULL;
2620		if (ret == -EBUSY) {
2621			dev_t dev;
2622
2623			mutex_lock(&bch_register_lock);
2624			if (lookup_bdev(strim(path), &dev) == 0 &&
2625			    bch_is_open(dev))
2626				err = "device already registered";
2627			else
2628				err = "device busy";
2629			mutex_unlock(&bch_register_lock);
2630			if (attr == &ksysfs_register_quiet) {
2631				quiet = true;
2632				ret = size;
2633			}
2634		}
2635		goto out_free_holder;
2636	}
2637
2638	err = "failed to register device";
2639
2640	if (async_registration) {
2641		/* register in asynchronous way */
2642		struct async_reg_args *args =
2643			kzalloc(sizeof(struct async_reg_args), GFP_KERNEL);
2644
2645		if (!args) {
2646			ret = -ENOMEM;
2647			err = "cannot allocate memory";
2648			goto out_free_holder;
2649		}
2650
2651		args->path	= path;
2652		args->sb	= sb;
2653		args->sb_disk	= sb_disk;
2654		args->bdev_file	= bdev_file;
2655		args->holder	= holder;
2656		register_device_async(args);
2657		/* No wait and returns to user space */
2658		goto async_done;
2659	}
2660
2661	if (SB_IS_BDEV(sb)) {
2662		mutex_lock(&bch_register_lock);
2663		ret = register_bdev(sb, sb_disk, bdev_file, holder);
2664		mutex_unlock(&bch_register_lock);
2665		/* blkdev_put() will be called in cached_dev_free() */
2666		if (ret < 0)
2667			goto out_free_sb;
2668	} else {
2669		/* blkdev_put() will be called in bch_cache_release() */
2670		ret = register_cache(sb, sb_disk, bdev_file, holder);
2671		if (ret)
2672			goto out_free_sb;
2673	}
2674
2675	kfree(sb);
2676	kfree(path);
2677	module_put(THIS_MODULE);
2678async_done:
2679	return size;
2680
2681out_free_holder:
2682	kfree(holder);
2683out_put_sb_folio:
2684	folio_put(virt_to_folio(sb_disk));
2685out_blkdev_put:
2686	if (bdev_file)
2687		fput(bdev_file);
2688out_free_sb:
2689	kfree(sb);
2690out_free_path:
2691	kfree(path);
2692	path = NULL;
2693out_module_put:
2694	module_put(THIS_MODULE);
2695out:
2696	if (!quiet)
2697		pr_info("error %s: %s\n", path?path:"", err);
2698	return ret;
2699}
2700
2701
2702struct pdev {
2703	struct list_head list;
2704	struct cached_dev *dc;
2705};
2706
2707static ssize_t bch_pending_bdevs_cleanup(struct kobject *k,
2708					 struct kobj_attribute *attr,
2709					 const char *buffer,
2710					 size_t size)
2711{
2712	LIST_HEAD(pending_devs);
2713	ssize_t ret = size;
2714	struct cached_dev *dc, *tdc;
2715	struct pdev *pdev, *tpdev;
2716	struct cache_set *c, *tc;
2717
2718	mutex_lock(&bch_register_lock);
2719	list_for_each_entry_safe(dc, tdc, &uncached_devices, list) {
2720		pdev = kmalloc(sizeof(struct pdev), GFP_KERNEL);
2721		if (!pdev)
2722			break;
2723		pdev->dc = dc;
2724		list_add(&pdev->list, &pending_devs);
2725	}
2726
2727	list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) {
2728		char *pdev_set_uuid = pdev->dc->sb.set_uuid;
2729		list_for_each_entry_safe(c, tc, &bch_cache_sets, list) {
2730			char *set_uuid = c->set_uuid;
2731
2732			if (!memcmp(pdev_set_uuid, set_uuid, 16)) {
2733				list_del(&pdev->list);
2734				kfree(pdev);
2735				break;
2736			}
2737		}
2738	}
2739	mutex_unlock(&bch_register_lock);
2740
2741	list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) {
2742		pr_info("delete pdev %p\n", pdev);
2743		list_del(&pdev->list);
2744		bcache_device_stop(&pdev->dc->disk);
2745		kfree(pdev);
2746	}
2747
2748	return ret;
2749}
2750
2751static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x)
2752{
2753	if (bcache_is_reboot)
2754		return NOTIFY_DONE;
2755
2756	if (code == SYS_DOWN ||
2757	    code == SYS_HALT ||
2758	    code == SYS_POWER_OFF) {
2759		DEFINE_WAIT(wait);
2760		unsigned long start = jiffies;
2761		bool stopped = false;
2762
2763		struct cache_set *c, *tc;
2764		struct cached_dev *dc, *tdc;
2765
2766		mutex_lock(&bch_register_lock);
2767
2768		if (bcache_is_reboot)
2769			goto out;
2770
2771		/* New registration is rejected since now */
2772		bcache_is_reboot = true;
2773		/*
2774		 * Make registering caller (if there is) on other CPU
2775		 * core know bcache_is_reboot set to true earlier
2776		 */
2777		smp_mb();
2778
2779		if (list_empty(&bch_cache_sets) &&
2780		    list_empty(&uncached_devices))
2781			goto out;
2782
2783		mutex_unlock(&bch_register_lock);
2784
2785		pr_info("Stopping all devices:\n");
2786
2787		/*
2788		 * The reason bch_register_lock is not held to call
2789		 * bch_cache_set_stop() and bcache_device_stop() is to
2790		 * avoid potential deadlock during reboot, because cache
2791		 * set or bcache device stopping process will acquire
2792		 * bch_register_lock too.
2793		 *
2794		 * We are safe here because bcache_is_reboot sets to
2795		 * true already, register_bcache() will reject new
2796		 * registration now. bcache_is_reboot also makes sure
2797		 * bcache_reboot() won't be re-entered on by other thread,
2798		 * so there is no race in following list iteration by
2799		 * list_for_each_entry_safe().
2800		 */
2801		list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
2802			bch_cache_set_stop(c);
2803
2804		list_for_each_entry_safe(dc, tdc, &uncached_devices, list)
2805			bcache_device_stop(&dc->disk);
2806
2807
2808		/*
2809		 * Give an early chance for other kthreads and
2810		 * kworkers to stop themselves
2811		 */
2812		schedule();
2813
2814		/* What's a condition variable? */
2815		while (1) {
2816			long timeout = start + 10 * HZ - jiffies;
2817
2818			mutex_lock(&bch_register_lock);
2819			stopped = list_empty(&bch_cache_sets) &&
2820				list_empty(&uncached_devices);
2821
2822			if (timeout < 0 || stopped)
2823				break;
2824
2825			prepare_to_wait(&unregister_wait, &wait,
2826					TASK_UNINTERRUPTIBLE);
2827
2828			mutex_unlock(&bch_register_lock);
2829			schedule_timeout(timeout);
2830		}
2831
2832		finish_wait(&unregister_wait, &wait);
2833
2834		if (stopped)
2835			pr_info("All devices stopped\n");
2836		else
2837			pr_notice("Timeout waiting for devices to be closed\n");
2838out:
2839		mutex_unlock(&bch_register_lock);
2840	}
2841
2842	return NOTIFY_DONE;
2843}
2844
2845static struct notifier_block reboot = {
2846	.notifier_call	= bcache_reboot,
2847	.priority	= INT_MAX, /* before any real devices */
2848};
2849
2850static void bcache_exit(void)
2851{
2852	bch_debug_exit();
2853	bch_request_exit();
2854	if (bcache_kobj)
2855		kobject_put(bcache_kobj);
2856	if (bcache_wq)
2857		destroy_workqueue(bcache_wq);
2858	if (bch_journal_wq)
2859		destroy_workqueue(bch_journal_wq);
2860	if (bch_flush_wq)
2861		destroy_workqueue(bch_flush_wq);
2862	bch_btree_exit();
2863
2864	if (bcache_major)
2865		unregister_blkdev(bcache_major, "bcache");
2866	unregister_reboot_notifier(&reboot);
2867	mutex_destroy(&bch_register_lock);
2868}
2869
2870/* Check and fixup module parameters */
2871static void check_module_parameters(void)
2872{
2873	if (bch_cutoff_writeback_sync == 0)
2874		bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC;
2875	else if (bch_cutoff_writeback_sync > CUTOFF_WRITEBACK_SYNC_MAX) {
2876		pr_warn("set bch_cutoff_writeback_sync (%u) to max value %u\n",
2877			bch_cutoff_writeback_sync, CUTOFF_WRITEBACK_SYNC_MAX);
2878		bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC_MAX;
2879	}
2880
2881	if (bch_cutoff_writeback == 0)
2882		bch_cutoff_writeback = CUTOFF_WRITEBACK;
2883	else if (bch_cutoff_writeback > CUTOFF_WRITEBACK_MAX) {
2884		pr_warn("set bch_cutoff_writeback (%u) to max value %u\n",
2885			bch_cutoff_writeback, CUTOFF_WRITEBACK_MAX);
2886		bch_cutoff_writeback = CUTOFF_WRITEBACK_MAX;
2887	}
2888
2889	if (bch_cutoff_writeback > bch_cutoff_writeback_sync) {
2890		pr_warn("set bch_cutoff_writeback (%u) to %u\n",
2891			bch_cutoff_writeback, bch_cutoff_writeback_sync);
2892		bch_cutoff_writeback = bch_cutoff_writeback_sync;
2893	}
2894}
2895
2896static int __init bcache_init(void)
2897{
2898	static const struct attribute *files[] = {
2899		&ksysfs_register.attr,
2900		&ksysfs_register_quiet.attr,
2901		&ksysfs_pendings_cleanup.attr,
2902		NULL
2903	};
2904
2905	check_module_parameters();
2906
2907	mutex_init(&bch_register_lock);
2908	init_waitqueue_head(&unregister_wait);
2909	register_reboot_notifier(&reboot);
2910
2911	bcache_major = register_blkdev(0, "bcache");
2912	if (bcache_major < 0) {
2913		unregister_reboot_notifier(&reboot);
2914		mutex_destroy(&bch_register_lock);
2915		return bcache_major;
2916	}
2917
2918	if (bch_btree_init())
2919		goto err;
2920
2921	bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2922	if (!bcache_wq)
2923		goto err;
2924
2925	/*
2926	 * Let's not make this `WQ_MEM_RECLAIM` for the following reasons:
2927	 *
2928	 * 1. It used `system_percpu_wq` before which also does no memory reclaim.
2929	 * 2. With `WQ_MEM_RECLAIM` desktop stalls, increased boot times, and
2930	 *    reduced throughput can be observed.
2931	 *
2932	 * We still want to user our own queue to not congest the `system_percpu_wq`.
2933	 */
2934	bch_flush_wq = alloc_workqueue("bch_flush", WQ_PERCPU, 0);
2935	if (!bch_flush_wq)
2936		goto err;
2937
2938	bch_journal_wq = alloc_workqueue("bch_journal",
2939					 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2940	if (!bch_journal_wq)
2941		goto err;
2942
2943	bcache_kobj = kobject_create_and_add("bcache", fs_kobj);
2944	if (!bcache_kobj)
2945		goto err;
2946
2947	if (bch_request_init() ||
2948	    sysfs_create_files(bcache_kobj, files))
2949		goto err;
2950
2951	bch_debug_init();
2952
2953	bcache_is_reboot = false;
2954
2955	return 0;
2956err:
2957	bcache_exit();
2958	return -ENOMEM;
2959}
2960
2961/*
2962 * Module hooks
2963 */
2964module_exit(bcache_exit);
2965module_init(bcache_init);
2966
2967module_param(bch_cutoff_writeback, uint, 0);
2968MODULE_PARM_DESC(bch_cutoff_writeback, "threshold to cutoff writeback");
2969
2970module_param(bch_cutoff_writeback_sync, uint, 0);
2971MODULE_PARM_DESC(bch_cutoff_writeback_sync, "hard threshold to cutoff writeback");
2972
2973MODULE_DESCRIPTION("Bcache: a Linux block layer cache");
2974MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
2975MODULE_LICENSE("GPL");