dm-thin.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2011-2012 Red Hat UK.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm-thin-metadata.h"
   9#include "dm-bio-prison-v1.h"
  10#include "dm.h"
  11
  12#include <linux/device-mapper.h>
  13#include <linux/dm-io.h>
  14#include <linux/dm-kcopyd.h>
  15#include <linux/jiffies.h>
  16#include <linux/log2.h>
  17#include <linux/list.h>
  18#include <linux/rculist.h>
  19#include <linux/init.h>
  20#include <linux/module.h>
  21#include <linux/slab.h>
  22#include <linux/vmalloc.h>
  23#include <linux/sort.h>
  24#include <linux/rbtree.h>
  25
  26#define	DM_MSG_PREFIX	"thin"
  27
  28/*
  29 * Tunable constants
  30 */
  31#define ENDIO_HOOK_POOL_SIZE 1024
  32#define MAPPING_POOL_SIZE 1024
  33#define COMMIT_PERIOD HZ
  34#define NO_SPACE_TIMEOUT_SECS 60
  35
  36static unsigned int no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
  37
  38DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
  39		"A percentage of time allocated for copy on write");
  40
  41/*
  42 * The block size of the device holding pool data must be
  43 * between 64KB and 1GB.
  44 */
  45#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
  46#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
  47
  48/*
  49 * Device id is restricted to 24 bits.
  50 */
  51#define MAX_DEV_ID ((1 << 24) - 1)
  52
  53/*
  54 * How do we handle breaking sharing of data blocks?
  55 * =================================================
  56 *
  57 * We use a standard copy-on-write btree to store the mappings for the
  58 * devices (note I'm talking about copy-on-write of the metadata here, not
  59 * the data).  When you take an internal snapshot you clone the root node
  60 * of the origin btree.  After this there is no concept of an origin or a
  61 * snapshot.  They are just two device trees that happen to point to the
  62 * same data blocks.
  63 *
  64 * When we get a write in we decide if it's to a shared data block using
  65 * some timestamp magic.  If it is, we have to break sharing.
  66 *
  67 * Let's say we write to a shared block in what was the origin.  The
  68 * steps are:
  69 *
  70 * i) plug io further to this physical block. (see bio_prison code).
  71 *
  72 * ii) quiesce any read io to that shared data block.  Obviously
  73 * including all devices that share this block.  (see dm_deferred_set code)
  74 *
  75 * iii) copy the data block to a newly allocate block.  This step can be
  76 * missed out if the io covers the block. (schedule_copy).
  77 *
  78 * iv) insert the new mapping into the origin's btree
  79 * (process_prepared_mapping).  This act of inserting breaks some
  80 * sharing of btree nodes between the two devices.  Breaking sharing only
  81 * effects the btree of that specific device.  Btrees for the other
  82 * devices that share the block never change.  The btree for the origin
  83 * device as it was after the last commit is untouched, ie. we're using
  84 * persistent data structures in the functional programming sense.
  85 *
  86 * v) unplug io to this physical block, including the io that triggered
  87 * the breaking of sharing.
  88 *
  89 * Steps (ii) and (iii) occur in parallel.
  90 *
  91 * The metadata _doesn't_ need to be committed before the io continues.  We
  92 * get away with this because the io is always written to a _new_ block.
  93 * If there's a crash, then:
  94 *
  95 * - The origin mapping will point to the old origin block (the shared
  96 * one).  This will contain the data as it was before the io that triggered
  97 * the breaking of sharing came in.
  98 *
  99 * - The snap mapping still points to the old block.  As it would after
 100 * the commit.
 101 *
 102 * The downside of this scheme is the timestamp magic isn't perfect, and
 103 * will continue to think that data block in the snapshot device is shared
 104 * even after the write to the origin has broken sharing.  I suspect data
 105 * blocks will typically be shared by many different devices, so we're
 106 * breaking sharing n + 1 times, rather than n, where n is the number of
 107 * devices that reference this data block.  At the moment I think the
 108 * benefits far, far outweigh the disadvantages.
 109 */
 110
 111/*----------------------------------------------------------------*/
 112
 113/*
 114 * Key building.
 115 */
 116enum lock_space {
 117	VIRTUAL,
 118	PHYSICAL
 119};
 120
 121static bool build_key(struct dm_thin_device *td, enum lock_space ls,
 122		      dm_block_t b, dm_block_t e, struct dm_cell_key *key)
 123{
 124	key->virtual = (ls == VIRTUAL);
 125	key->dev = dm_thin_dev_id(td);
 126	key->block_begin = b;
 127	key->block_end = e;
 128
 129	return dm_cell_key_has_valid_range(key);
 130}
 131
 132static void build_data_key(struct dm_thin_device *td, dm_block_t b,
 133			   struct dm_cell_key *key)
 134{
 135	(void) build_key(td, PHYSICAL, b, b + 1llu, key);
 136}
 137
 138static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
 139			      struct dm_cell_key *key)
 140{
 141	(void) build_key(td, VIRTUAL, b, b + 1llu, key);
 142}
 143
 144/*----------------------------------------------------------------*/
 145
 146#define THROTTLE_THRESHOLD (1 * HZ)
 147
 148struct throttle {
 149	struct rw_semaphore lock;
 150	unsigned long threshold;
 151	bool throttle_applied;
 152};
 153
 154static void throttle_init(struct throttle *t)
 155{
 156	init_rwsem(&t->lock);
 157	t->throttle_applied = false;
 158}
 159
 160static void throttle_work_start(struct throttle *t)
 161{
 162	t->threshold = jiffies + THROTTLE_THRESHOLD;
 163}
 164
 165static void throttle_work_update(struct throttle *t)
 166{
 167	if (!t->throttle_applied && time_is_before_jiffies(t->threshold)) {
 168		down_write(&t->lock);
 169		t->throttle_applied = true;
 170	}
 171}
 172
 173static void throttle_work_complete(struct throttle *t)
 174{
 175	if (t->throttle_applied) {
 176		t->throttle_applied = false;
 177		up_write(&t->lock);
 178	}
 179}
 180
 181static void throttle_lock(struct throttle *t)
 182{
 183	down_read(&t->lock);
 184}
 185
 186static void throttle_unlock(struct throttle *t)
 187{
 188	up_read(&t->lock);
 189}
 190
 191/*----------------------------------------------------------------*/
 192
 193/*
 194 * A pool device ties together a metadata device and a data device.  It
 195 * also provides the interface for creating and destroying internal
 196 * devices.
 197 */
 198struct dm_thin_new_mapping;
 199
 200/*
 201 * The pool runs in various modes.  Ordered in degraded order for comparisons.
 202 */
 203enum pool_mode {
 204	PM_WRITE,		/* metadata may be changed */
 205	PM_OUT_OF_DATA_SPACE,	/* metadata may be changed, though data may not be allocated */
 206
 207	/*
 208	 * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.
 209	 */
 210	PM_OUT_OF_METADATA_SPACE,
 211	PM_READ_ONLY,		/* metadata may not be changed */
 212
 213	PM_FAIL,		/* all I/O fails */
 214};
 215
 216struct pool_features {
 217	enum pool_mode mode;
 218
 219	bool zero_new_blocks:1;
 220	bool discard_enabled:1;
 221	bool discard_passdown:1;
 222	bool error_if_no_space:1;
 223};
 224
 225struct thin_c;
 226typedef void (*process_bio_fn)(struct thin_c *tc, struct bio *bio);
 227typedef void (*process_cell_fn)(struct thin_c *tc, struct dm_bio_prison_cell *cell);
 228typedef void (*process_mapping_fn)(struct dm_thin_new_mapping *m);
 229
 230#define CELL_SORT_ARRAY_SIZE 8192
 231
 232struct pool {
 233	struct list_head list;
 234	struct dm_target *ti;	/* Only set if a pool target is bound */
 235
 236	struct mapped_device *pool_md;
 237	struct block_device *data_dev;
 238	struct block_device *md_dev;
 239	struct dm_pool_metadata *pmd;
 240
 241	dm_block_t low_water_blocks;
 242	uint32_t sectors_per_block;
 243	int sectors_per_block_shift;
 244
 245	struct pool_features pf;
 246	bool low_water_triggered:1;	/* A dm event has been sent */
 247	bool suspended:1;
 248	bool out_of_data_space:1;
 249
 250	struct dm_bio_prison *prison;
 251	struct dm_kcopyd_client *copier;
 252
 253	struct work_struct worker;
 254	struct workqueue_struct *wq;
 255	struct throttle throttle;
 256	struct delayed_work waker;
 257	struct delayed_work no_space_timeout;
 258
 259	unsigned long last_commit_jiffies;
 260	unsigned int ref_count;
 261
 262	spinlock_t lock;
 263	struct bio_list deferred_flush_bios;
 264	struct bio_list deferred_flush_completions;
 265	struct list_head prepared_mappings;
 266	struct list_head prepared_discards;
 267	struct list_head prepared_discards_pt2;
 268	struct list_head active_thins;
 269
 270	struct dm_deferred_set *shared_read_ds;
 271	struct dm_deferred_set *all_io_ds;
 272
 273	struct dm_thin_new_mapping *next_mapping;
 274
 275	process_bio_fn process_bio;
 276	process_bio_fn process_discard;
 277
 278	process_cell_fn process_cell;
 279	process_cell_fn process_discard_cell;
 280
 281	process_mapping_fn process_prepared_mapping;
 282	process_mapping_fn process_prepared_discard;
 283	process_mapping_fn process_prepared_discard_pt2;
 284
 285	struct dm_bio_prison_cell **cell_sort_array;
 286
 287	mempool_t mapping_pool;
 288};
 289
 290static void metadata_operation_failed(struct pool *pool, const char *op, int r);
 291
 292static enum pool_mode get_pool_mode(struct pool *pool)
 293{
 294	return pool->pf.mode;
 295}
 296
 297static void notify_of_pool_mode_change(struct pool *pool)
 298{
 299	static const char *descs[] = {
 300		"write",
 301		"out-of-data-space",
 302		"read-only",
 303		"read-only",
 304		"fail"
 305	};
 306	const char *extra_desc = NULL;
 307	enum pool_mode mode = get_pool_mode(pool);
 308
 309	if (mode == PM_OUT_OF_DATA_SPACE) {
 310		if (!pool->pf.error_if_no_space)
 311			extra_desc = " (queue IO)";
 312		else
 313			extra_desc = " (error IO)";
 314	}
 315
 316	dm_table_event(pool->ti->table);
 317	DMINFO("%s: switching pool to %s%s mode",
 318	       dm_device_name(pool->pool_md),
 319	       descs[(int)mode], extra_desc ? : "");
 320}
 321
 322/*
 323 * Target context for a pool.
 324 */
 325struct pool_c {
 326	struct dm_target *ti;
 327	struct pool *pool;
 328	struct dm_dev *data_dev;
 329	struct dm_dev *metadata_dev;
 330
 331	dm_block_t low_water_blocks;
 332	struct pool_features requested_pf; /* Features requested during table load */
 333	struct pool_features adjusted_pf;  /* Features used after adjusting for constituent devices */
 334};
 335
 336/*
 337 * Target context for a thin.
 338 */
 339struct thin_c {
 340	struct list_head list;
 341	struct dm_dev *pool_dev;
 342	struct dm_dev *origin_dev;
 343	sector_t origin_size;
 344	dm_thin_id dev_id;
 345
 346	struct pool *pool;
 347	struct dm_thin_device *td;
 348	struct mapped_device *thin_md;
 349
 350	bool requeue_mode:1;
 351	spinlock_t lock;
 352	struct list_head deferred_cells;
 353	struct bio_list deferred_bio_list;
 354	struct bio_list retry_on_resume_list;
 355	struct rb_root sort_bio_list; /* sorted list of deferred bios */
 356
 357	/*
 358	 * Ensures the thin is not destroyed until the worker has finished
 359	 * iterating the active_thins list.
 360	 */
 361	refcount_t refcount;
 362	struct completion can_destroy;
 363};
 364
 365/*----------------------------------------------------------------*/
 366
 367static bool block_size_is_power_of_two(struct pool *pool)
 368{
 369	return pool->sectors_per_block_shift >= 0;
 370}
 371
 372static sector_t block_to_sectors(struct pool *pool, dm_block_t b)
 373{
 374	return block_size_is_power_of_two(pool) ?
 375		(b << pool->sectors_per_block_shift) :
 376		(b * pool->sectors_per_block);
 377}
 378
 379/*----------------------------------------------------------------*/
 380
 381struct discard_op {
 382	struct thin_c *tc;
 383	struct blk_plug plug;
 384	struct bio *parent_bio;
 385	struct bio *bio;
 386};
 387
 388static void begin_discard(struct discard_op *op, struct thin_c *tc, struct bio *parent)
 389{
 390	BUG_ON(!parent);
 391
 392	op->tc = tc;
 393	blk_start_plug(&op->plug);
 394	op->parent_bio = parent;
 395	op->bio = NULL;
 396}
 397
 398static void issue_discard(struct discard_op *op, dm_block_t data_b, dm_block_t data_e)
 399{
 400	struct thin_c *tc = op->tc;
 401	sector_t s = block_to_sectors(tc->pool, data_b);
 402	sector_t len = block_to_sectors(tc->pool, data_e - data_b);
 403
 404	__blkdev_issue_discard(tc->pool_dev->bdev, s, len, GFP_NOIO, &op->bio);
 405}
 406
 407static void end_discard(struct discard_op *op, int r)
 408{
 409	if (op->bio) {
 410		/*
 411		 * Even if one of the calls to issue_discard failed, we
 412		 * need to wait for the chain to complete.
 413		 */
 414		bio_chain(op->bio, op->parent_bio);
 415		op->bio->bi_opf = REQ_OP_DISCARD;
 416		submit_bio(op->bio);
 417	}
 418
 419	blk_finish_plug(&op->plug);
 420
 421	/*
 422	 * Even if r is set, there could be sub discards in flight that we
 423	 * need to wait for.
 424	 */
 425	if (r && !op->parent_bio->bi_status)
 426		op->parent_bio->bi_status = errno_to_blk_status(r);
 427	bio_endio(op->parent_bio);
 428}
 429
 430/*----------------------------------------------------------------*/
 431
 432/*
 433 * wake_worker() is used when new work is queued and when pool_resume is
 434 * ready to continue deferred IO processing.
 435 */
 436static void wake_worker(struct pool *pool)
 437{
 438	queue_work(pool->wq, &pool->worker);
 439}
 440
 441/*----------------------------------------------------------------*/
 442
 443static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bio,
 444		      struct dm_bio_prison_cell **cell_result)
 445{
 446	int r;
 447	struct dm_bio_prison_cell *cell_prealloc;
 448
 449	/*
 450	 * Allocate a cell from the prison's mempool.
 451	 * This might block but it can't fail.
 452	 */
 453	cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO);
 454
 455	r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result);
 456	if (r) {
 457		/*
 458		 * We reused an old cell; we can get rid of
 459		 * the new one.
 460		 */
 461		dm_bio_prison_free_cell(pool->prison, cell_prealloc);
 462	}
 463
 464	return r;
 465}
 466
 467static void cell_release(struct pool *pool,
 468			 struct dm_bio_prison_cell *cell,
 469			 struct bio_list *bios)
 470{
 471	dm_cell_release(pool->prison, cell, bios);
 472	dm_bio_prison_free_cell(pool->prison, cell);
 473}
 474
 475static void cell_visit_release(struct pool *pool,
 476			       void (*fn)(void *, struct dm_bio_prison_cell *),
 477			       void *context,
 478			       struct dm_bio_prison_cell *cell)
 479{
 480	dm_cell_visit_release(pool->prison, fn, context, cell);
 481	dm_bio_prison_free_cell(pool->prison, cell);
 482}
 483
 484static void cell_release_no_holder(struct pool *pool,
 485				   struct dm_bio_prison_cell *cell,
 486				   struct bio_list *bios)
 487{
 488	dm_cell_release_no_holder(pool->prison, cell, bios);
 489	dm_bio_prison_free_cell(pool->prison, cell);
 490}
 491
 492static void cell_error_with_code(struct pool *pool,
 493		struct dm_bio_prison_cell *cell, blk_status_t error_code)
 494{
 495	dm_cell_error(pool->prison, cell, error_code);
 496	dm_bio_prison_free_cell(pool->prison, cell);
 497}
 498
 499static blk_status_t get_pool_io_error_code(struct pool *pool)
 500{
 501	return pool->out_of_data_space ? BLK_STS_NOSPC : BLK_STS_IOERR;
 502}
 503
 504static void cell_error(struct pool *pool, struct dm_bio_prison_cell *cell)
 505{
 506	cell_error_with_code(pool, cell, get_pool_io_error_code(pool));
 507}
 508
 509static void cell_success(struct pool *pool, struct dm_bio_prison_cell *cell)
 510{
 511	cell_error_with_code(pool, cell, 0);
 512}
 513
 514static void cell_requeue(struct pool *pool, struct dm_bio_prison_cell *cell)
 515{
 516	cell_error_with_code(pool, cell, BLK_STS_DM_REQUEUE);
 517}
 518
 519/*----------------------------------------------------------------*/
 520
 521/*
 522 * A global list of pools that uses a struct mapped_device as a key.
 523 */
 524static struct dm_thin_pool_table {
 525	struct mutex mutex;
 526	struct list_head pools;
 527} dm_thin_pool_table;
 528
 529static void pool_table_init(void)
 530{
 531	mutex_init(&dm_thin_pool_table.mutex);
 532	INIT_LIST_HEAD(&dm_thin_pool_table.pools);
 533}
 534
 535static void pool_table_exit(void)
 536{
 537	mutex_destroy(&dm_thin_pool_table.mutex);
 538}
 539
 540static void __pool_table_insert(struct pool *pool)
 541{
 542	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 543	list_add(&pool->list, &dm_thin_pool_table.pools);
 544}
 545
 546static void __pool_table_remove(struct pool *pool)
 547{
 548	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 549	list_del(&pool->list);
 550}
 551
 552static struct pool *__pool_table_lookup(struct mapped_device *md)
 553{
 554	struct pool *pool = NULL, *tmp;
 555
 556	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 557
 558	list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
 559		if (tmp->pool_md == md) {
 560			pool = tmp;
 561			break;
 562		}
 563	}
 564
 565	return pool;
 566}
 567
 568static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
 569{
 570	struct pool *pool = NULL, *tmp;
 571
 572	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 573
 574	list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
 575		if (tmp->md_dev == md_dev) {
 576			pool = tmp;
 577			break;
 578		}
 579	}
 580
 581	return pool;
 582}
 583
 584/*----------------------------------------------------------------*/
 585
 586struct dm_thin_endio_hook {
 587	struct thin_c *tc;
 588	struct dm_deferred_entry *shared_read_entry;
 589	struct dm_deferred_entry *all_io_entry;
 590	struct dm_thin_new_mapping *overwrite_mapping;
 591	struct rb_node rb_node;
 592	struct dm_bio_prison_cell *cell;
 593};
 594
 595static void error_bio_list(struct bio_list *bios, blk_status_t error)
 596{
 597	struct bio *bio;
 598
 599	while ((bio = bio_list_pop(bios))) {
 600		bio->bi_status = error;
 601		bio_endio(bio);
 602	}
 603}
 604
 605static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master,
 606		blk_status_t error)
 607{
 608	struct bio_list bios;
 609
 610	bio_list_init(&bios);
 611
 612	spin_lock_irq(&tc->lock);
 613	bio_list_merge_init(&bios, master);
 614	spin_unlock_irq(&tc->lock);
 615
 616	error_bio_list(&bios, error);
 617}
 618
 619static void requeue_deferred_cells(struct thin_c *tc)
 620{
 621	struct pool *pool = tc->pool;
 622	struct list_head cells;
 623	struct dm_bio_prison_cell *cell, *tmp;
 624
 625	INIT_LIST_HEAD(&cells);
 626
 627	spin_lock_irq(&tc->lock);
 628	list_splice_init(&tc->deferred_cells, &cells);
 629	spin_unlock_irq(&tc->lock);
 630
 631	list_for_each_entry_safe(cell, tmp, &cells, user_list)
 632		cell_requeue(pool, cell);
 633}
 634
 635static void requeue_io(struct thin_c *tc)
 636{
 637	struct bio_list bios;
 638
 639	bio_list_init(&bios);
 640
 641	spin_lock_irq(&tc->lock);
 642	bio_list_merge_init(&bios, &tc->deferred_bio_list);
 643	bio_list_merge_init(&bios, &tc->retry_on_resume_list);
 644	spin_unlock_irq(&tc->lock);
 645
 646	error_bio_list(&bios, BLK_STS_DM_REQUEUE);
 647	requeue_deferred_cells(tc);
 648}
 649
 650static void error_retry_list_with_code(struct pool *pool, blk_status_t error)
 651{
 652	struct thin_c *tc;
 653
 654	rcu_read_lock();
 655	list_for_each_entry_rcu(tc, &pool->active_thins, list)
 656		error_thin_bio_list(tc, &tc->retry_on_resume_list, error);
 657	rcu_read_unlock();
 658}
 659
 660static void error_retry_list(struct pool *pool)
 661{
 662	error_retry_list_with_code(pool, get_pool_io_error_code(pool));
 663}
 664
 665/*
 666 * This section of code contains the logic for processing a thin device's IO.
 667 * Much of the code depends on pool object resources (lists, workqueues, etc)
 668 * but most is exclusively called from the thin target rather than the thin-pool
 669 * target.
 670 */
 671
 672static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
 673{
 674	struct pool *pool = tc->pool;
 675	sector_t block_nr = bio->bi_iter.bi_sector;
 676
 677	if (block_size_is_power_of_two(pool))
 678		block_nr >>= pool->sectors_per_block_shift;
 679	else
 680		(void) sector_div(block_nr, pool->sectors_per_block);
 681
 682	return block_nr;
 683}
 684
 685/*
 686 * Returns the _complete_ blocks that this bio covers.
 687 */
 688static void get_bio_block_range(struct thin_c *tc, struct bio *bio,
 689				dm_block_t *begin, dm_block_t *end)
 690{
 691	struct pool *pool = tc->pool;
 692	sector_t b = bio->bi_iter.bi_sector;
 693	sector_t e = b + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
 694
 695	b += pool->sectors_per_block - 1ull; /* so we round up */
 696
 697	if (block_size_is_power_of_two(pool)) {
 698		b >>= pool->sectors_per_block_shift;
 699		e >>= pool->sectors_per_block_shift;
 700	} else {
 701		(void) sector_div(b, pool->sectors_per_block);
 702		(void) sector_div(e, pool->sectors_per_block);
 703	}
 704
 705	if (e < b) {
 706		/* Can happen if the bio is within a single block. */
 707		e = b;
 708	}
 709
 710	*begin = b;
 711	*end = e;
 712}
 713
 714static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
 715{
 716	struct pool *pool = tc->pool;
 717	sector_t bi_sector = bio->bi_iter.bi_sector;
 718
 719	bio_set_dev(bio, tc->pool_dev->bdev);
 720	if (block_size_is_power_of_two(pool)) {
 721		bio->bi_iter.bi_sector =
 722			(block << pool->sectors_per_block_shift) |
 723			(bi_sector & (pool->sectors_per_block - 1));
 724	} else {
 725		bio->bi_iter.bi_sector = (block * pool->sectors_per_block) +
 726				 sector_div(bi_sector, pool->sectors_per_block);
 727	}
 728}
 729
 730static void remap_to_origin(struct thin_c *tc, struct bio *bio)
 731{
 732	bio_set_dev(bio, tc->origin_dev->bdev);
 733}
 734
 735static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
 736{
 737	return op_is_flush(bio->bi_opf) &&
 738		dm_thin_changed_this_transaction(tc->td);
 739}
 740
 741static void inc_all_io_entry(struct pool *pool, struct bio *bio)
 742{
 743	struct dm_thin_endio_hook *h;
 744
 745	if (bio_op(bio) == REQ_OP_DISCARD)
 746		return;
 747
 748	h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
 749	h->all_io_entry = dm_deferred_entry_inc(pool->all_io_ds);
 750}
 751
 752static void issue(struct thin_c *tc, struct bio *bio)
 753{
 754	struct pool *pool = tc->pool;
 755
 756	if (!bio_triggers_commit(tc, bio)) {
 757		dm_submit_bio_remap(bio, NULL);
 758		return;
 759	}
 760
 761	/*
 762	 * Complete bio with an error if earlier I/O caused changes to
 763	 * the metadata that can't be committed e.g, due to I/O errors
 764	 * on the metadata device.
 765	 */
 766	if (dm_thin_aborted_changes(tc->td)) {
 767		bio_io_error(bio);
 768		return;
 769	}
 770
 771	/*
 772	 * Batch together any bios that trigger commits and then issue a
 773	 * single commit for them in process_deferred_bios().
 774	 */
 775	spin_lock_irq(&pool->lock);
 776	bio_list_add(&pool->deferred_flush_bios, bio);
 777	spin_unlock_irq(&pool->lock);
 778}
 779
 780static void remap_to_origin_and_issue(struct thin_c *tc, struct bio *bio)
 781{
 782	remap_to_origin(tc, bio);
 783	issue(tc, bio);
 784}
 785
 786static void remap_and_issue(struct thin_c *tc, struct bio *bio,
 787			    dm_block_t block)
 788{
 789	remap(tc, bio, block);
 790	issue(tc, bio);
 791}
 792
 793/*----------------------------------------------------------------*/
 794
 795/*
 796 * Bio endio functions.
 797 */
 798struct dm_thin_new_mapping {
 799	struct list_head list;
 800
 801	bool pass_discard:1;
 802	bool maybe_shared:1;
 803
 804	/*
 805	 * Track quiescing, copying and zeroing preparation actions.  When this
 806	 * counter hits zero the block is prepared and can be inserted into the
 807	 * btree.
 808	 */
 809	atomic_t prepare_actions;
 810
 811	blk_status_t status;
 812	struct thin_c *tc;
 813	dm_block_t virt_begin, virt_end;
 814	dm_block_t data_block;
 815	struct dm_bio_prison_cell *cell;
 816
 817	/*
 818	 * If the bio covers the whole area of a block then we can avoid
 819	 * zeroing or copying.  Instead this bio is hooked.  The bio will
 820	 * still be in the cell, so care has to be taken to avoid issuing
 821	 * the bio twice.
 822	 */
 823	struct bio *bio;
 824	bio_end_io_t *saved_bi_end_io;
 825};
 826
 827static void __complete_mapping_preparation(struct dm_thin_new_mapping *m)
 828{
 829	struct pool *pool = m->tc->pool;
 830
 831	if (atomic_dec_and_test(&m->prepare_actions)) {
 832		list_add_tail(&m->list, &pool->prepared_mappings);
 833		wake_worker(pool);
 834	}
 835}
 836
 837static void complete_mapping_preparation(struct dm_thin_new_mapping *m)
 838{
 839	unsigned long flags;
 840	struct pool *pool = m->tc->pool;
 841
 842	spin_lock_irqsave(&pool->lock, flags);
 843	__complete_mapping_preparation(m);
 844	spin_unlock_irqrestore(&pool->lock, flags);
 845}
 846
 847static void copy_complete(int read_err, unsigned long write_err, void *context)
 848{
 849	struct dm_thin_new_mapping *m = context;
 850
 851	m->status = read_err || write_err ? BLK_STS_IOERR : 0;
 852	complete_mapping_preparation(m);
 853}
 854
 855static void overwrite_endio(struct bio *bio)
 856{
 857	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
 858	struct dm_thin_new_mapping *m = h->overwrite_mapping;
 859
 860	bio->bi_end_io = m->saved_bi_end_io;
 861
 862	m->status = bio->bi_status;
 863	complete_mapping_preparation(m);
 864}
 865
 866/*----------------------------------------------------------------*/
 867
 868/*
 869 * Workqueue.
 870 */
 871
 872/*
 873 * Prepared mapping jobs.
 874 */
 875
 876/*
 877 * This sends the bios in the cell, except the original holder, back
 878 * to the deferred_bios list.
 879 */
 880static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *cell)
 881{
 882	struct pool *pool = tc->pool;
 883	unsigned long flags;
 884	struct bio_list bios;
 885
 886	bio_list_init(&bios);
 887	cell_release_no_holder(pool, cell, &bios);
 888
 889	if (!bio_list_empty(&bios)) {
 890		spin_lock_irqsave(&tc->lock, flags);
 891		bio_list_merge(&tc->deferred_bio_list, &bios);
 892		spin_unlock_irqrestore(&tc->lock, flags);
 893		wake_worker(pool);
 894	}
 895}
 896
 897static void thin_defer_bio(struct thin_c *tc, struct bio *bio);
 898
 899struct remap_info {
 900	struct thin_c *tc;
 901	struct bio_list defer_bios;
 902	struct bio_list issue_bios;
 903};
 904
 905static void __inc_remap_and_issue_cell(void *context,
 906				       struct dm_bio_prison_cell *cell)
 907{
 908	struct remap_info *info = context;
 909	struct bio *bio;
 910
 911	while ((bio = bio_list_pop(&cell->bios))) {
 912		if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD)
 913			bio_list_add(&info->defer_bios, bio);
 914		else {
 915			inc_all_io_entry(info->tc->pool, bio);
 916
 917			/*
 918			 * We can't issue the bios with the bio prison lock
 919			 * held, so we add them to a list to issue on
 920			 * return from this function.
 921			 */
 922			bio_list_add(&info->issue_bios, bio);
 923		}
 924	}
 925}
 926
 927static void inc_remap_and_issue_cell(struct thin_c *tc,
 928				     struct dm_bio_prison_cell *cell,
 929				     dm_block_t block)
 930{
 931	struct bio *bio;
 932	struct remap_info info;
 933
 934	info.tc = tc;
 935	bio_list_init(&info.defer_bios);
 936	bio_list_init(&info.issue_bios);
 937
 938	/*
 939	 * We have to be careful to inc any bios we're about to issue
 940	 * before the cell is released, and avoid a race with new bios
 941	 * being added to the cell.
 942	 */
 943	cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
 944			   &info, cell);
 945
 946	while ((bio = bio_list_pop(&info.defer_bios)))
 947		thin_defer_bio(tc, bio);
 948
 949	while ((bio = bio_list_pop(&info.issue_bios)))
 950		remap_and_issue(info.tc, bio, block);
 951}
 952
 953static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
 954{
 955	cell_error(m->tc->pool, m->cell);
 956	list_del(&m->list);
 957	mempool_free(m, &m->tc->pool->mapping_pool);
 958}
 959
 960static void complete_overwrite_bio(struct thin_c *tc, struct bio *bio)
 961{
 962	struct pool *pool = tc->pool;
 963
 964	/*
 965	 * If the bio has the REQ_FUA flag set we must commit the metadata
 966	 * before signaling its completion.
 967	 */
 968	if (!bio_triggers_commit(tc, bio)) {
 969		bio_endio(bio);
 970		return;
 971	}
 972
 973	/*
 974	 * Complete bio with an error if earlier I/O caused changes to the
 975	 * metadata that can't be committed, e.g, due to I/O errors on the
 976	 * metadata device.
 977	 */
 978	if (dm_thin_aborted_changes(tc->td)) {
 979		bio_io_error(bio);
 980		return;
 981	}
 982
 983	/*
 984	 * Batch together any bios that trigger commits and then issue a
 985	 * single commit for them in process_deferred_bios().
 986	 */
 987	spin_lock_irq(&pool->lock);
 988	bio_list_add(&pool->deferred_flush_completions, bio);
 989	spin_unlock_irq(&pool->lock);
 990}
 991
 992static void process_prepared_mapping(struct dm_thin_new_mapping *m)
 993{
 994	struct thin_c *tc = m->tc;
 995	struct pool *pool = tc->pool;
 996	struct bio *bio = m->bio;
 997	int r;
 998
 999	if (m->status) {
1000		cell_error(pool, m->cell);
1001		goto out;
1002	}
1003
1004	/*
1005	 * Commit the prepared block into the mapping btree.
1006	 * Any I/O for this block arriving after this point will get
1007	 * remapped to it directly.
1008	 */
1009	r = dm_thin_insert_block(tc->td, m->virt_begin, m->data_block);
1010	if (r) {
1011		metadata_operation_failed(pool, "dm_thin_insert_block", r);
1012		cell_error(pool, m->cell);
1013		goto out;
1014	}
1015
1016	/*
1017	 * Release any bios held while the block was being provisioned.
1018	 * If we are processing a write bio that completely covers the block,
1019	 * we already processed it so can ignore it now when processing
1020	 * the bios in the cell.
1021	 */
1022	if (bio) {
1023		inc_remap_and_issue_cell(tc, m->cell, m->data_block);
1024		complete_overwrite_bio(tc, bio);
1025	} else {
1026		inc_all_io_entry(tc->pool, m->cell->holder);
1027		remap_and_issue(tc, m->cell->holder, m->data_block);
1028		inc_remap_and_issue_cell(tc, m->cell, m->data_block);
1029	}
1030
1031out:
1032	list_del(&m->list);
1033	mempool_free(m, &pool->mapping_pool);
1034}
1035
1036/*----------------------------------------------------------------*/
1037
1038static void free_discard_mapping(struct dm_thin_new_mapping *m)
1039{
1040	struct thin_c *tc = m->tc;
1041
1042	if (m->cell)
1043		cell_defer_no_holder(tc, m->cell);
1044	mempool_free(m, &tc->pool->mapping_pool);
1045}
1046
1047static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
1048{
1049	bio_io_error(m->bio);
1050	free_discard_mapping(m);
1051}
1052
1053static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
1054{
1055	bio_endio(m->bio);
1056	free_discard_mapping(m);
1057}
1058
1059static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
1060{
1061	int r;
1062	struct thin_c *tc = m->tc;
1063
1064	r = dm_thin_remove_range(tc->td, m->cell->key.block_begin, m->cell->key.block_end);
1065	if (r) {
1066		metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
1067		bio_io_error(m->bio);
1068	} else
1069		bio_endio(m->bio);
1070
1071	cell_defer_no_holder(tc, m->cell);
1072	mempool_free(m, &tc->pool->mapping_pool);
1073}
1074
1075/*----------------------------------------------------------------*/
1076
1077static void passdown_double_checking_shared_status(struct dm_thin_new_mapping *m,
1078						   struct bio *discard_parent)
1079{
1080	/*
1081	 * We've already unmapped this range of blocks, but before we
1082	 * passdown we have to check that these blocks are now unused.
1083	 */
1084	int r = 0;
1085	bool shared = true;
1086	struct thin_c *tc = m->tc;
1087	struct pool *pool = tc->pool;
1088	dm_block_t b = m->data_block, e, end = m->data_block + m->virt_end - m->virt_begin;
1089	struct discard_op op;
1090
1091	begin_discard(&op, tc, discard_parent);
1092	while (b != end) {
1093		/* find start of unmapped run */
1094		for (; b < end; b++) {
1095			r = dm_pool_block_is_shared(pool->pmd, b, &shared);
1096			if (r)
1097				goto out;
1098
1099			if (!shared)
1100				break;
1101		}
1102
1103		if (b == end)
1104			break;
1105
1106		/* find end of run */
1107		for (e = b + 1; e != end; e++) {
1108			r = dm_pool_block_is_shared(pool->pmd, e, &shared);
1109			if (r)
1110				goto out;
1111
1112			if (shared)
1113				break;
1114		}
1115
1116		issue_discard(&op, b, e);
1117
1118		b = e;
1119	}
1120out:
1121	end_discard(&op, r);
1122}
1123
1124static void queue_passdown_pt2(struct dm_thin_new_mapping *m)
1125{
1126	unsigned long flags;
1127	struct pool *pool = m->tc->pool;
1128
1129	spin_lock_irqsave(&pool->lock, flags);
1130	list_add_tail(&m->list, &pool->prepared_discards_pt2);
1131	spin_unlock_irqrestore(&pool->lock, flags);
1132	wake_worker(pool);
1133}
1134
1135static void passdown_endio(struct bio *bio)
1136{
1137	/*
1138	 * It doesn't matter if the passdown discard failed, we still want
1139	 * to unmap (we ignore err).
1140	 */
1141	queue_passdown_pt2(bio->bi_private);
1142	bio_put(bio);
1143}
1144
1145static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping *m)
1146{
1147	int r;
1148	struct thin_c *tc = m->tc;
1149	struct pool *pool = tc->pool;
1150	struct bio *discard_parent;
1151	dm_block_t data_end = m->data_block + (m->virt_end - m->virt_begin);
1152
1153	/*
1154	 * Only this thread allocates blocks, so we can be sure that the
1155	 * newly unmapped blocks will not be allocated before the end of
1156	 * the function.
1157	 */
1158	r = dm_thin_remove_range(tc->td, m->virt_begin, m->virt_end);
1159	if (r) {
1160		metadata_operation_failed(pool, "dm_thin_remove_range", r);
1161		bio_io_error(m->bio);
1162		cell_defer_no_holder(tc, m->cell);
1163		mempool_free(m, &pool->mapping_pool);
1164		return;
1165	}
1166
1167	/*
1168	 * Increment the unmapped blocks.  This prevents a race between the
1169	 * passdown io and reallocation of freed blocks.
1170	 */
1171	r = dm_pool_inc_data_range(pool->pmd, m->data_block, data_end);
1172	if (r) {
1173		metadata_operation_failed(pool, "dm_pool_inc_data_range", r);
1174		bio_io_error(m->bio);
1175		cell_defer_no_holder(tc, m->cell);
1176		mempool_free(m, &pool->mapping_pool);
1177		return;
1178	}
1179
1180	discard_parent = bio_alloc(NULL, 1, 0, GFP_NOIO);
1181	discard_parent->bi_end_io = passdown_endio;
1182	discard_parent->bi_private = m;
1183	if (m->maybe_shared)
1184		passdown_double_checking_shared_status(m, discard_parent);
1185	else {
1186		struct discard_op op;
1187
1188		begin_discard(&op, tc, discard_parent);
1189		issue_discard(&op, m->data_block, data_end);
1190		end_discard(&op, 0);
1191	}
1192}
1193
1194static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping *m)
1195{
1196	int r;
1197	struct thin_c *tc = m->tc;
1198	struct pool *pool = tc->pool;
1199
1200	/*
1201	 * The passdown has completed, so now we can decrement all those
1202	 * unmapped blocks.
1203	 */
1204	r = dm_pool_dec_data_range(pool->pmd, m->data_block,
1205				   m->data_block + (m->virt_end - m->virt_begin));
1206	if (r) {
1207		metadata_operation_failed(pool, "dm_pool_dec_data_range", r);
1208		bio_io_error(m->bio);
1209	} else
1210		bio_endio(m->bio);
1211
1212	cell_defer_no_holder(tc, m->cell);
1213	mempool_free(m, &pool->mapping_pool);
1214}
1215
1216static void process_prepared(struct pool *pool, struct list_head *head,
1217			     process_mapping_fn *fn)
1218{
1219	struct list_head maps;
1220	struct dm_thin_new_mapping *m, *tmp;
1221
1222	INIT_LIST_HEAD(&maps);
1223	spin_lock_irq(&pool->lock);
1224	list_splice_init(head, &maps);
1225	spin_unlock_irq(&pool->lock);
1226
1227	list_for_each_entry_safe(m, tmp, &maps, list)
1228		(*fn)(m);
1229}
1230
1231/*
1232 * Deferred bio jobs.
1233 */
1234static int io_overlaps_block(struct pool *pool, struct bio *bio)
1235{
1236	return bio->bi_iter.bi_size ==
1237		(pool->sectors_per_block << SECTOR_SHIFT);
1238}
1239
1240static int io_overwrites_block(struct pool *pool, struct bio *bio)
1241{
1242	return (bio_data_dir(bio) == WRITE) &&
1243		io_overlaps_block(pool, bio);
1244}
1245
1246static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
1247			       bio_end_io_t *fn)
1248{
1249	*save = bio->bi_end_io;
1250	bio->bi_end_io = fn;
1251}
1252
1253static int ensure_next_mapping(struct pool *pool)
1254{
1255	if (pool->next_mapping)
1256		return 0;
1257
1258	pool->next_mapping = mempool_alloc(&pool->mapping_pool, GFP_ATOMIC);
1259
1260	return pool->next_mapping ? 0 : -ENOMEM;
1261}
1262
1263static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool)
1264{
1265	struct dm_thin_new_mapping *m = pool->next_mapping;
1266
1267	BUG_ON(!pool->next_mapping);
1268
1269	memset(m, 0, sizeof(struct dm_thin_new_mapping));
1270	INIT_LIST_HEAD(&m->list);
1271	m->bio = NULL;
1272
1273	pool->next_mapping = NULL;
1274
1275	return m;
1276}
1277
1278static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m,
1279		    sector_t begin, sector_t end)
1280{
1281	struct dm_io_region to;
1282
1283	to.bdev = tc->pool_dev->bdev;
1284	to.sector = begin;
1285	to.count = end - begin;
1286
1287	dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m);
1288}
1289
1290static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio,
1291				      dm_block_t data_begin,
1292				      struct dm_thin_new_mapping *m)
1293{
1294	struct pool *pool = tc->pool;
1295	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1296
1297	h->overwrite_mapping = m;
1298	m->bio = bio;
1299	save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1300	inc_all_io_entry(pool, bio);
1301	remap_and_issue(tc, bio, data_begin);
1302}
1303
1304/*
1305 * A partial copy also needs to zero the uncopied region.
1306 */
1307static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
1308			  struct dm_dev *origin, dm_block_t data_origin,
1309			  dm_block_t data_dest,
1310			  struct dm_bio_prison_cell *cell, struct bio *bio,
1311			  sector_t len)
1312{
1313	struct pool *pool = tc->pool;
1314	struct dm_thin_new_mapping *m = get_next_mapping(pool);
1315
1316	m->tc = tc;
1317	m->virt_begin = virt_block;
1318	m->virt_end = virt_block + 1u;
1319	m->data_block = data_dest;
1320	m->cell = cell;
1321
1322	/*
1323	 * quiesce action + copy action + an extra reference held for the
1324	 * duration of this function (we may need to inc later for a
1325	 * partial zero).
1326	 */
1327	atomic_set(&m->prepare_actions, 3);
1328
1329	if (!dm_deferred_set_add_work(pool->shared_read_ds, &m->list))
1330		complete_mapping_preparation(m); /* already quiesced */
1331
1332	/*
1333	 * IO to pool_dev remaps to the pool target's data_dev.
1334	 *
1335	 * If the whole block of data is being overwritten, we can issue the
1336	 * bio immediately. Otherwise we use kcopyd to clone the data first.
1337	 */
1338	if (io_overwrites_block(pool, bio))
1339		remap_and_issue_overwrite(tc, bio, data_dest, m);
1340	else {
1341		struct dm_io_region from, to;
1342
1343		from.bdev = origin->bdev;
1344		from.sector = data_origin * pool->sectors_per_block;
1345		from.count = len;
1346
1347		to.bdev = tc->pool_dev->bdev;
1348		to.sector = data_dest * pool->sectors_per_block;
1349		to.count = len;
1350
1351		dm_kcopyd_copy(pool->copier, &from, 1, &to,
1352			       0, copy_complete, m);
1353
1354		/*
1355		 * Do we need to zero a tail region?
1356		 */
1357		if (len < pool->sectors_per_block && pool->pf.zero_new_blocks) {
1358			atomic_inc(&m->prepare_actions);
1359			ll_zero(tc, m,
1360				data_dest * pool->sectors_per_block + len,
1361				(data_dest + 1) * pool->sectors_per_block);
1362		}
1363	}
1364
1365	complete_mapping_preparation(m); /* drop our ref */
1366}
1367
1368static void schedule_internal_copy(struct thin_c *tc, dm_block_t virt_block,
1369				   dm_block_t data_origin, dm_block_t data_dest,
1370				   struct dm_bio_prison_cell *cell, struct bio *bio)
1371{
1372	schedule_copy(tc, virt_block, tc->pool_dev,
1373		      data_origin, data_dest, cell, bio,
1374		      tc->pool->sectors_per_block);
1375}
1376
1377static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
1378			  dm_block_t data_block, struct dm_bio_prison_cell *cell,
1379			  struct bio *bio)
1380{
1381	struct pool *pool = tc->pool;
1382	struct dm_thin_new_mapping *m = get_next_mapping(pool);
1383
1384	atomic_set(&m->prepare_actions, 1); /* no need to quiesce */
1385	m->tc = tc;
1386	m->virt_begin = virt_block;
1387	m->virt_end = virt_block + 1u;
1388	m->data_block = data_block;
1389	m->cell = cell;
1390
1391	/*
1392	 * If the whole block of data is being overwritten or we are not
1393	 * zeroing pre-existing data, we can issue the bio immediately.
1394	 * Otherwise we use kcopyd to zero the data first.
1395	 */
1396	if (pool->pf.zero_new_blocks) {
1397		if (io_overwrites_block(pool, bio))
1398			remap_and_issue_overwrite(tc, bio, data_block, m);
1399		else {
1400			ll_zero(tc, m, data_block * pool->sectors_per_block,
1401				(data_block + 1) * pool->sectors_per_block);
1402		}
1403	} else
1404		process_prepared_mapping(m);
1405}
1406
1407static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block,
1408				   dm_block_t data_dest,
1409				   struct dm_bio_prison_cell *cell, struct bio *bio)
1410{
1411	struct pool *pool = tc->pool;
1412	sector_t virt_block_begin = virt_block * pool->sectors_per_block;
1413	sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block;
1414
1415	if (virt_block_end <= tc->origin_size) {
1416		schedule_copy(tc, virt_block, tc->origin_dev,
1417			      virt_block, data_dest, cell, bio,
1418			      pool->sectors_per_block);
1419
1420	} else if (virt_block_begin < tc->origin_size) {
1421		schedule_copy(tc, virt_block, tc->origin_dev,
1422			      virt_block, data_dest, cell, bio,
1423			      tc->origin_size - virt_block_begin);
1424
1425	} else
1426		schedule_zero(tc, virt_block, data_dest, cell, bio);
1427}
1428
1429static void set_pool_mode(struct pool *pool, enum pool_mode new_mode);
1430
1431static void requeue_bios(struct pool *pool);
1432
1433static bool is_read_only_pool_mode(enum pool_mode mode)
1434{
1435	return (mode == PM_OUT_OF_METADATA_SPACE || mode == PM_READ_ONLY);
1436}
1437
1438static bool is_read_only(struct pool *pool)
1439{
1440	return is_read_only_pool_mode(get_pool_mode(pool));
1441}
1442
1443static void check_for_metadata_space(struct pool *pool)
1444{
1445	int r;
1446	const char *ooms_reason = NULL;
1447	dm_block_t nr_free;
1448
1449	r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free);
1450	if (r)
1451		ooms_reason = "Could not get free metadata blocks";
1452	else if (!nr_free)
1453		ooms_reason = "No free metadata blocks";
1454
1455	if (ooms_reason && !is_read_only(pool)) {
1456		DMERR("%s", ooms_reason);
1457		set_pool_mode(pool, PM_OUT_OF_METADATA_SPACE);
1458	}
1459}
1460
1461static void check_for_data_space(struct pool *pool)
1462{
1463	int r;
1464	dm_block_t nr_free;
1465
1466	if (get_pool_mode(pool) != PM_OUT_OF_DATA_SPACE)
1467		return;
1468
1469	r = dm_pool_get_free_block_count(pool->pmd, &nr_free);
1470	if (r)
1471		return;
1472
1473	if (nr_free) {
1474		set_pool_mode(pool, PM_WRITE);
1475		requeue_bios(pool);
1476	}
1477}
1478
1479/*
1480 * A non-zero return indicates read_only or fail_io mode.
1481 * Many callers don't care about the return value.
1482 */
1483static int commit(struct pool *pool)
1484{
1485	int r;
1486
1487	if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE)
1488		return -EINVAL;
1489
1490	r = dm_pool_commit_metadata(pool->pmd);
1491	if (r)
1492		metadata_operation_failed(pool, "dm_pool_commit_metadata", r);
1493	else {
1494		check_for_metadata_space(pool);
1495		check_for_data_space(pool);
1496	}
1497
1498	return r;
1499}
1500
1501static void check_low_water_mark(struct pool *pool, dm_block_t free_blocks)
1502{
1503	if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1504		DMWARN("%s: reached low water mark for data device: sending event.",
1505		       dm_device_name(pool->pool_md));
1506		spin_lock_irq(&pool->lock);
1507		pool->low_water_triggered = true;
1508		spin_unlock_irq(&pool->lock);
1509		dm_table_event(pool->ti->table);
1510	}
1511}
1512
1513static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1514{
1515	int r;
1516	dm_block_t free_blocks;
1517	struct pool *pool = tc->pool;
1518
1519	if (WARN_ON(get_pool_mode(pool) != PM_WRITE))
1520		return -EINVAL;
1521
1522	r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1523	if (r) {
1524		metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
1525		return r;
1526	}
1527
1528	check_low_water_mark(pool, free_blocks);
1529
1530	if (!free_blocks) {
1531		/*
1532		 * Try to commit to see if that will free up some
1533		 * more space.
1534		 */
1535		r = commit(pool);
1536		if (r)
1537			return r;
1538
1539		r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1540		if (r) {
1541			metadata_operation_failed(pool, "dm_pool_get_free_block_count", r);
1542			return r;
1543		}
1544
1545		if (!free_blocks) {
1546			set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
1547			return -ENOSPC;
1548		}
1549	}
1550
1551	r = dm_pool_alloc_data_block(pool->pmd, result);
1552	if (r) {
1553		if (r == -ENOSPC)
1554			set_pool_mode(pool, PM_OUT_OF_DATA_SPACE);
1555		else
1556			metadata_operation_failed(pool, "dm_pool_alloc_data_block", r);
1557		return r;
1558	}
1559
1560	r = dm_pool_get_free_metadata_block_count(pool->pmd, &free_blocks);
1561	if (r) {
1562		metadata_operation_failed(pool, "dm_pool_get_free_metadata_block_count", r);
1563		return r;
1564	}
1565
1566	if (!free_blocks) {
1567		/* Let's commit before we use up the metadata reserve. */
1568		r = commit(pool);
1569		if (r)
1570			return r;
1571	}
1572
1573	return 0;
1574}
1575
1576/*
1577 * If we have run out of space, queue bios until the device is
1578 * resumed, presumably after having been reloaded with more space.
1579 */
1580static void retry_on_resume(struct bio *bio)
1581{
1582	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1583	struct thin_c *tc = h->tc;
1584
1585	spin_lock_irq(&tc->lock);
1586	bio_list_add(&tc->retry_on_resume_list, bio);
1587	spin_unlock_irq(&tc->lock);
1588}
1589
1590static blk_status_t should_error_unserviceable_bio(struct pool *pool)
1591{
1592	enum pool_mode m = get_pool_mode(pool);
1593
1594	switch (m) {
1595	case PM_WRITE:
1596		/* Shouldn't get here */
1597		DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
1598		return BLK_STS_IOERR;
1599
1600	case PM_OUT_OF_DATA_SPACE:
1601		return pool->pf.error_if_no_space ? BLK_STS_NOSPC : 0;
1602
1603	case PM_OUT_OF_METADATA_SPACE:
1604	case PM_READ_ONLY:
1605	case PM_FAIL:
1606		return BLK_STS_IOERR;
1607	default:
1608		/* Shouldn't get here */
1609		DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
1610		return BLK_STS_IOERR;
1611	}
1612}
1613
1614static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1615{
1616	blk_status_t error = should_error_unserviceable_bio(pool);
1617
1618	if (error) {
1619		bio->bi_status = error;
1620		bio_endio(bio);
1621	} else
1622		retry_on_resume(bio);
1623}
1624
1625static void retry_bios_on_resume(struct pool *pool, struct dm_bio_prison_cell *cell)
1626{
1627	struct bio *bio;
1628	struct bio_list bios;
1629	blk_status_t error;
1630
1631	error = should_error_unserviceable_bio(pool);
1632	if (error) {
1633		cell_error_with_code(pool, cell, error);
1634		return;
1635	}
1636
1637	bio_list_init(&bios);
1638	cell_release(pool, cell, &bios);
1639
1640	while ((bio = bio_list_pop(&bios)))
1641		retry_on_resume(bio);
1642}
1643
1644static void process_discard_cell_no_passdown(struct thin_c *tc,
1645					     struct dm_bio_prison_cell *virt_cell)
1646{
1647	struct pool *pool = tc->pool;
1648	struct dm_thin_new_mapping *m = get_next_mapping(pool);
1649
1650	/*
1651	 * We don't need to lock the data blocks, since there's no
1652	 * passdown.  We only lock data blocks for allocation and breaking sharing.
1653	 */
1654	m->tc = tc;
1655	m->virt_begin = virt_cell->key.block_begin;
1656	m->virt_end = virt_cell->key.block_end;
1657	m->cell = virt_cell;
1658	m->bio = virt_cell->holder;
1659
1660	if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1661		pool->process_prepared_discard(m);
1662}
1663
1664static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t end,
1665				 struct bio *bio)
1666{
1667	struct pool *pool = tc->pool;
1668
1669	int r;
1670	bool maybe_shared;
1671	struct dm_cell_key data_key;
1672	struct dm_bio_prison_cell *data_cell;
1673	struct dm_thin_new_mapping *m;
1674	dm_block_t virt_begin, virt_end, data_begin, data_end;
1675	dm_block_t len, next_boundary;
1676
1677	while (begin != end) {
1678		r = dm_thin_find_mapped_range(tc->td, begin, end, &virt_begin, &virt_end,
1679					      &data_begin, &maybe_shared);
1680		if (r) {
1681			/*
1682			 * Silently fail, letting any mappings we've
1683			 * created complete.
1684			 */
1685			break;
1686		}
1687
1688		data_end = data_begin + (virt_end - virt_begin);
1689
1690		/*
1691		 * Make sure the data region obeys the bio prison restrictions.
1692		 */
1693		while (data_begin < data_end) {
1694			r = ensure_next_mapping(pool);
1695			if (r)
1696				return; /* we did our best */
1697
1698			next_boundary = ((data_begin >> BIO_PRISON_MAX_RANGE_SHIFT) + 1)
1699				<< BIO_PRISON_MAX_RANGE_SHIFT;
1700			len = min_t(sector_t, data_end - data_begin, next_boundary - data_begin);
1701
1702			/* This key is certainly within range given the above splitting */
1703			(void) build_key(tc->td, PHYSICAL, data_begin, data_begin + len, &data_key);
1704			if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) {
1705				/* contention, we'll give up with this range */
1706				data_begin += len;
1707				continue;
1708			}
1709
1710			/*
1711			 * IO may still be going to the destination block.  We must
1712			 * quiesce before we can do the removal.
1713			 */
1714			m = get_next_mapping(pool);
1715			m->tc = tc;
1716			m->maybe_shared = maybe_shared;
1717			m->virt_begin = virt_begin;
1718			m->virt_end = virt_begin + len;
1719			m->data_block = data_begin;
1720			m->cell = data_cell;
1721			m->bio = bio;
1722
1723			/*
1724			 * The parent bio must not complete before sub discard bios are
1725			 * chained to it (see end_discard's bio_chain)!
1726			 *
1727			 * This per-mapping bi_remaining increment is paired with
1728			 * the implicit decrement that occurs via bio_endio() in
1729			 * end_discard().
1730			 */
1731			bio_inc_remaining(bio);
1732			if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
1733				pool->process_prepared_discard(m);
1734
1735			virt_begin += len;
1736			data_begin += len;
1737		}
1738
1739		begin = virt_end;
1740	}
1741}
1742
1743static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_prison_cell *virt_cell)
1744{
1745	struct bio *bio = virt_cell->holder;
1746	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1747
1748	/*
1749	 * The virt_cell will only get freed once the origin bio completes.
1750	 * This means it will remain locked while all the individual
1751	 * passdown bios are in flight.
1752	 */
1753	h->cell = virt_cell;
1754	break_up_discard_bio(tc, virt_cell->key.block_begin, virt_cell->key.block_end, bio);
1755
1756	/*
1757	 * We complete the bio now, knowing that the bi_remaining field
1758	 * will prevent completion until the sub range discards have
1759	 * completed.
1760	 */
1761	bio_endio(bio);
1762}
1763
1764static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1765{
1766	dm_block_t begin, end;
1767	struct dm_cell_key virt_key;
1768	struct dm_bio_prison_cell *virt_cell;
1769
1770	get_bio_block_range(tc, bio, &begin, &end);
1771	if (begin == end) {
1772		/*
1773		 * The discard covers less than a block.
1774		 */
1775		bio_endio(bio);
1776		return;
1777	}
1778
1779	if (unlikely(!build_key(tc->td, VIRTUAL, begin, end, &virt_key))) {
1780		DMERR_LIMIT("Discard doesn't respect bio prison limits");
1781		bio_endio(bio);
1782		return;
1783	}
1784
1785	if (bio_detain(tc->pool, &virt_key, bio, &virt_cell)) {
1786		/*
1787		 * Potential starvation issue: We're relying on the
1788		 * fs/application being well behaved, and not trying to
1789		 * send IO to a region at the same time as discarding it.
1790		 * If they do this persistently then it's possible this
1791		 * cell will never be granted.
1792		 */
1793		return;
1794	}
1795
1796	tc->pool->process_discard_cell(tc, virt_cell);
1797}
1798
1799static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1800			  struct dm_cell_key *key,
1801			  struct dm_thin_lookup_result *lookup_result,
1802			  struct dm_bio_prison_cell *cell)
1803{
1804	int r;
1805	dm_block_t data_block;
1806	struct pool *pool = tc->pool;
1807
1808	r = alloc_data_block(tc, &data_block);
1809	switch (r) {
1810	case 0:
1811		schedule_internal_copy(tc, block, lookup_result->block,
1812				       data_block, cell, bio);
1813		break;
1814
1815	case -ENOSPC:
1816		retry_bios_on_resume(pool, cell);
1817		break;
1818
1819	default:
1820		DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1821			    __func__, r);
1822		cell_error(pool, cell);
1823		break;
1824	}
1825}
1826
1827static void __remap_and_issue_shared_cell(void *context,
1828					  struct dm_bio_prison_cell *cell)
1829{
1830	struct remap_info *info = context;
1831	struct bio *bio;
1832
1833	while ((bio = bio_list_pop(&cell->bios))) {
1834		if (bio_data_dir(bio) == WRITE || op_is_flush(bio->bi_opf) ||
1835		    bio_op(bio) == REQ_OP_DISCARD)
1836			bio_list_add(&info->defer_bios, bio);
1837		else {
1838			struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1839
1840			h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
1841			inc_all_io_entry(info->tc->pool, bio);
1842			bio_list_add(&info->issue_bios, bio);
1843		}
1844	}
1845}
1846
1847static void remap_and_issue_shared_cell(struct thin_c *tc,
1848					struct dm_bio_prison_cell *cell,
1849					dm_block_t block)
1850{
1851	struct bio *bio;
1852	struct remap_info info;
1853
1854	info.tc = tc;
1855	bio_list_init(&info.defer_bios);
1856	bio_list_init(&info.issue_bios);
1857
1858	cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
1859			   &info, cell);
1860
1861	while ((bio = bio_list_pop(&info.defer_bios)))
1862		thin_defer_bio(tc, bio);
1863
1864	while ((bio = bio_list_pop(&info.issue_bios)))
1865		remap_and_issue(tc, bio, block);
1866}
1867
1868static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1869			       dm_block_t block,
1870			       struct dm_thin_lookup_result *lookup_result,
1871			       struct dm_bio_prison_cell *virt_cell)
1872{
1873	struct dm_bio_prison_cell *data_cell;
1874	struct pool *pool = tc->pool;
1875	struct dm_cell_key key;
1876
1877	/*
1878	 * If cell is already occupied, then sharing is already in the process
1879	 * of being broken so we have nothing further to do here.
1880	 */
1881	build_data_key(tc->td, lookup_result->block, &key);
1882	if (bio_detain(pool, &key, bio, &data_cell)) {
1883		cell_defer_no_holder(tc, virt_cell);
1884		return;
1885	}
1886
1887	if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
1888		break_sharing(tc, bio, block, &key, lookup_result, data_cell);
1889		cell_defer_no_holder(tc, virt_cell);
1890	} else {
1891		struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1892
1893		h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
1894		inc_all_io_entry(pool, bio);
1895		remap_and_issue(tc, bio, lookup_result->block);
1896
1897		remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
1898		remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
1899	}
1900}
1901
1902static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1903			    struct dm_bio_prison_cell *cell)
1904{
1905	int r;
1906	dm_block_t data_block;
1907	struct pool *pool = tc->pool;
1908
1909	/*
1910	 * Remap empty bios (flushes) immediately, without provisioning.
1911	 */
1912	if (!bio->bi_iter.bi_size) {
1913		inc_all_io_entry(pool, bio);
1914		cell_defer_no_holder(tc, cell);
1915
1916		remap_and_issue(tc, bio, 0);
1917		return;
1918	}
1919
1920	/*
1921	 * Fill read bios with zeroes and complete them immediately.
1922	 */
1923	if (bio_data_dir(bio) == READ) {
1924		zero_fill_bio(bio);
1925		cell_defer_no_holder(tc, cell);
1926		bio_endio(bio);
1927		return;
1928	}
1929
1930	r = alloc_data_block(tc, &data_block);
1931	switch (r) {
1932	case 0:
1933		if (tc->origin_dev)
1934			schedule_external_copy(tc, block, data_block, cell, bio);
1935		else
1936			schedule_zero(tc, block, data_block, cell, bio);
1937		break;
1938
1939	case -ENOSPC:
1940		retry_bios_on_resume(pool, cell);
1941		break;
1942
1943	default:
1944		DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1945			    __func__, r);
1946		cell_error(pool, cell);
1947		break;
1948	}
1949}
1950
1951static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1952{
1953	int r;
1954	struct pool *pool = tc->pool;
1955	struct bio *bio = cell->holder;
1956	dm_block_t block = get_bio_block(tc, bio);
1957	struct dm_thin_lookup_result lookup_result;
1958
1959	if (tc->requeue_mode) {
1960		cell_requeue(pool, cell);
1961		return;
1962	}
1963
1964	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1965	switch (r) {
1966	case 0:
1967		if (lookup_result.shared)
1968			process_shared_bio(tc, bio, block, &lookup_result, cell);
1969		else {
1970			inc_all_io_entry(pool, bio);
1971			remap_and_issue(tc, bio, lookup_result.block);
1972			inc_remap_and_issue_cell(tc, cell, lookup_result.block);
1973		}
1974		break;
1975
1976	case -ENODATA:
1977		if (bio_data_dir(bio) == READ && tc->origin_dev) {
1978			inc_all_io_entry(pool, bio);
1979			cell_defer_no_holder(tc, cell);
1980
1981			if (bio_end_sector(bio) <= tc->origin_size)
1982				remap_to_origin_and_issue(tc, bio);
1983
1984			else if (bio->bi_iter.bi_sector < tc->origin_size) {
1985				zero_fill_bio(bio);
1986				bio->bi_iter.bi_size = (tc->origin_size - bio->bi_iter.bi_sector) << SECTOR_SHIFT;
1987				remap_to_origin_and_issue(tc, bio);
1988
1989			} else {
1990				zero_fill_bio(bio);
1991				bio_endio(bio);
1992			}
1993		} else
1994			provision_block(tc, bio, block, cell);
1995		break;
1996
1997	default:
1998		DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1999			    __func__, r);
2000		cell_defer_no_holder(tc, cell);
2001		bio_io_error(bio);
2002		break;
2003	}
2004}
2005
2006static void process_bio(struct thin_c *tc, struct bio *bio)
2007{
2008	struct pool *pool = tc->pool;
2009	dm_block_t block = get_bio_block(tc, bio);
2010	struct dm_bio_prison_cell *cell;
2011	struct dm_cell_key key;
2012
2013	/*
2014	 * If cell is already occupied, then the block is already
2015	 * being provisioned so we have nothing further to do here.
2016	 */
2017	build_virtual_key(tc->td, block, &key);
2018	if (bio_detain(pool, &key, bio, &cell))
2019		return;
2020
2021	process_cell(tc, cell);
2022}
2023
2024static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
2025				    struct dm_bio_prison_cell *cell)
2026{
2027	int r;
2028	int rw = bio_data_dir(bio);
2029	dm_block_t block = get_bio_block(tc, bio);
2030	struct dm_thin_lookup_result lookup_result;
2031
2032	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
2033	switch (r) {
2034	case 0:
2035		if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
2036			handle_unserviceable_bio(tc->pool, bio);
2037			if (cell)
2038				cell_defer_no_holder(tc, cell);
2039		} else {
2040			inc_all_io_entry(tc->pool, bio);
2041			remap_and_issue(tc, bio, lookup_result.block);
2042			if (cell)
2043				inc_remap_and_issue_cell(tc, cell, lookup_result.block);
2044		}
2045		break;
2046
2047	case -ENODATA:
2048		if (cell)
2049			cell_defer_no_holder(tc, cell);
2050		if (rw != READ) {
2051			handle_unserviceable_bio(tc->pool, bio);
2052			break;
2053		}
2054
2055		if (tc->origin_dev) {
2056			inc_all_io_entry(tc->pool, bio);
2057			remap_to_origin_and_issue(tc, bio);
2058			break;
2059		}
2060
2061		zero_fill_bio(bio);
2062		bio_endio(bio);
2063		break;
2064
2065	default:
2066		DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
2067			    __func__, r);
2068		if (cell)
2069			cell_defer_no_holder(tc, cell);
2070		bio_io_error(bio);
2071		break;
2072	}
2073}
2074
2075static void process_bio_read_only(struct thin_c *tc, struct bio *bio)
2076{
2077	__process_bio_read_only(tc, bio, NULL);
2078}
2079
2080static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2081{
2082	__process_bio_read_only(tc, cell->holder, cell);
2083}
2084
2085static void process_bio_success(struct thin_c *tc, struct bio *bio)
2086{
2087	bio_endio(bio);
2088}
2089
2090static void process_bio_fail(struct thin_c *tc, struct bio *bio)
2091{
2092	bio_io_error(bio);
2093}
2094
2095static void process_cell_success(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2096{
2097	cell_success(tc->pool, cell);
2098}
2099
2100static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2101{
2102	cell_error(tc->pool, cell);
2103}
2104
2105/*
2106 * FIXME: should we also commit due to size of transaction, measured in
2107 * metadata blocks?
2108 */
2109static int need_commit_due_to_time(struct pool *pool)
2110{
2111	return !time_in_range(jiffies, pool->last_commit_jiffies,
2112			      pool->last_commit_jiffies + COMMIT_PERIOD);
2113}
2114
2115#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
2116#define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
2117
2118static void __thin_bio_rb_add(struct thin_c *tc, struct bio *bio)
2119{
2120	struct rb_node **rbp, *parent;
2121	struct dm_thin_endio_hook *pbd;
2122	sector_t bi_sector = bio->bi_iter.bi_sector;
2123
2124	rbp = &tc->sort_bio_list.rb_node;
2125	parent = NULL;
2126	while (*rbp) {
2127		parent = *rbp;
2128		pbd = thin_pbd(parent);
2129
2130		if (bi_sector < thin_bio(pbd)->bi_iter.bi_sector)
2131			rbp = &(*rbp)->rb_left;
2132		else
2133			rbp = &(*rbp)->rb_right;
2134	}
2135
2136	pbd = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2137	rb_link_node(&pbd->rb_node, parent, rbp);
2138	rb_insert_color(&pbd->rb_node, &tc->sort_bio_list);
2139}
2140
2141static void __extract_sorted_bios(struct thin_c *tc)
2142{
2143	struct rb_node *node;
2144	struct dm_thin_endio_hook *pbd;
2145	struct bio *bio;
2146
2147	for (node = rb_first(&tc->sort_bio_list); node; node = rb_next(node)) {
2148		pbd = thin_pbd(node);
2149		bio = thin_bio(pbd);
2150
2151		bio_list_add(&tc->deferred_bio_list, bio);
2152		rb_erase(&pbd->rb_node, &tc->sort_bio_list);
2153	}
2154
2155	WARN_ON(!RB_EMPTY_ROOT(&tc->sort_bio_list));
2156}
2157
2158static void __sort_thin_deferred_bios(struct thin_c *tc)
2159{
2160	struct bio *bio;
2161	struct bio_list bios;
2162
2163	bio_list_init(&bios);
2164	bio_list_merge(&bios, &tc->deferred_bio_list);
2165	bio_list_init(&tc->deferred_bio_list);
2166
2167	/* Sort deferred_bio_list using rb-tree */
2168	while ((bio = bio_list_pop(&bios)))
2169		__thin_bio_rb_add(tc, bio);
2170
2171	/*
2172	 * Transfer the sorted bios in sort_bio_list back to
2173	 * deferred_bio_list to allow lockless submission of
2174	 * all bios.
2175	 */
2176	__extract_sorted_bios(tc);
2177}
2178
2179static void process_thin_deferred_bios(struct thin_c *tc)
2180{
2181	struct pool *pool = tc->pool;
2182	struct bio *bio;
2183	struct bio_list bios;
2184	struct blk_plug plug;
2185	unsigned int count = 0;
2186
2187	if (tc->requeue_mode) {
2188		error_thin_bio_list(tc, &tc->deferred_bio_list,
2189				BLK_STS_DM_REQUEUE);
2190		return;
2191	}
2192
2193	bio_list_init(&bios);
2194
2195	spin_lock_irq(&tc->lock);
2196
2197	if (bio_list_empty(&tc->deferred_bio_list)) {
2198		spin_unlock_irq(&tc->lock);
2199		return;
2200	}
2201
2202	__sort_thin_deferred_bios(tc);
2203
2204	bio_list_merge(&bios, &tc->deferred_bio_list);
2205	bio_list_init(&tc->deferred_bio_list);
2206
2207	spin_unlock_irq(&tc->lock);
2208
2209	blk_start_plug(&plug);
2210	while ((bio = bio_list_pop(&bios))) {
2211		/*
2212		 * If we've got no free new_mapping structs, and processing
2213		 * this bio might require one, we pause until there are some
2214		 * prepared mappings to process.
2215		 */
2216		if (ensure_next_mapping(pool)) {
2217			spin_lock_irq(&tc->lock);
2218			bio_list_add(&tc->deferred_bio_list, bio);
2219			bio_list_merge(&tc->deferred_bio_list, &bios);
2220			spin_unlock_irq(&tc->lock);
2221			break;
2222		}
2223
2224		if (bio_op(bio) == REQ_OP_DISCARD)
2225			pool->process_discard(tc, bio);
2226		else
2227			pool->process_bio(tc, bio);
2228
2229		if ((count++ & 127) == 0) {
2230			throttle_work_update(&pool->throttle);
2231			dm_pool_issue_prefetches(pool->pmd);
2232		}
2233		cond_resched();
2234	}
2235	blk_finish_plug(&plug);
2236}
2237
2238static int cmp_cells(const void *lhs, const void *rhs)
2239{
2240	struct dm_bio_prison_cell *lhs_cell = *((struct dm_bio_prison_cell **) lhs);
2241	struct dm_bio_prison_cell *rhs_cell = *((struct dm_bio_prison_cell **) rhs);
2242
2243	BUG_ON(!lhs_cell->holder);
2244	BUG_ON(!rhs_cell->holder);
2245
2246	if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
2247		return -1;
2248
2249	if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
2250		return 1;
2251
2252	return 0;
2253}
2254
2255static unsigned int sort_cells(struct pool *pool, struct list_head *cells)
2256{
2257	unsigned int count = 0;
2258	struct dm_bio_prison_cell *cell, *tmp;
2259
2260	list_for_each_entry_safe(cell, tmp, cells, user_list) {
2261		if (count >= CELL_SORT_ARRAY_SIZE)
2262			break;
2263
2264		pool->cell_sort_array[count++] = cell;
2265		list_del(&cell->user_list);
2266	}
2267
2268	sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
2269
2270	return count;
2271}
2272
2273static void process_thin_deferred_cells(struct thin_c *tc)
2274{
2275	struct pool *pool = tc->pool;
2276	struct list_head cells;
2277	struct dm_bio_prison_cell *cell;
2278	unsigned int i, j, count;
2279
2280	INIT_LIST_HEAD(&cells);
2281
2282	spin_lock_irq(&tc->lock);
2283	list_splice_init(&tc->deferred_cells, &cells);
2284	spin_unlock_irq(&tc->lock);
2285
2286	if (list_empty(&cells))
2287		return;
2288
2289	do {
2290		count = sort_cells(tc->pool, &cells);
2291
2292		for (i = 0; i < count; i++) {
2293			cell = pool->cell_sort_array[i];
2294			BUG_ON(!cell->holder);
2295
2296			/*
2297			 * If we've got no free new_mapping structs, and processing
2298			 * this bio might require one, we pause until there are some
2299			 * prepared mappings to process.
2300			 */
2301			if (ensure_next_mapping(pool)) {
2302				for (j = i; j < count; j++)
2303					list_add(&pool->cell_sort_array[j]->user_list, &cells);
2304
2305				spin_lock_irq(&tc->lock);
2306				list_splice(&cells, &tc->deferred_cells);
2307				spin_unlock_irq(&tc->lock);
2308				return;
2309			}
2310
2311			if (bio_op(cell->holder) == REQ_OP_DISCARD)
2312				pool->process_discard_cell(tc, cell);
2313			else
2314				pool->process_cell(tc, cell);
2315		}
2316		cond_resched();
2317	} while (!list_empty(&cells));
2318}
2319
2320static void thin_get(struct thin_c *tc);
2321static void thin_put(struct thin_c *tc);
2322
2323/*
2324 * We can't hold rcu_read_lock() around code that can block.  So we
2325 * find a thin with the rcu lock held; bump a refcount; then drop
2326 * the lock.
2327 */
2328static struct thin_c *get_first_thin(struct pool *pool)
2329{
2330	struct thin_c *tc = NULL;
2331
2332	rcu_read_lock();
2333	tc = list_first_or_null_rcu(&pool->active_thins, struct thin_c, list);
2334	if (tc)
2335		thin_get(tc);
2336	rcu_read_unlock();
2337
2338	return tc;
2339}
2340
2341static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
2342{
2343	struct thin_c *old_tc = tc;
2344
2345	rcu_read_lock();
2346	list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
2347		thin_get(tc);
2348		thin_put(old_tc);
2349		rcu_read_unlock();
2350		return tc;
2351	}
2352	thin_put(old_tc);
2353	rcu_read_unlock();
2354
2355	return NULL;
2356}
2357
2358static void process_deferred_bios(struct pool *pool)
2359{
2360	struct bio *bio;
2361	struct bio_list bios, bio_completions;
2362	struct thin_c *tc;
2363
2364	tc = get_first_thin(pool);
2365	while (tc) {
2366		process_thin_deferred_cells(tc);
2367		process_thin_deferred_bios(tc);
2368		tc = get_next_thin(pool, tc);
2369	}
2370
2371	/*
2372	 * If there are any deferred flush bios, we must commit the metadata
2373	 * before issuing them or signaling their completion.
2374	 */
2375	bio_list_init(&bios);
2376	bio_list_init(&bio_completions);
2377
2378	spin_lock_irq(&pool->lock);
2379	bio_list_merge(&bios, &pool->deferred_flush_bios);
2380	bio_list_init(&pool->deferred_flush_bios);
2381
2382	bio_list_merge(&bio_completions, &pool->deferred_flush_completions);
2383	bio_list_init(&pool->deferred_flush_completions);
2384	spin_unlock_irq(&pool->lock);
2385
2386	if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
2387	    !(dm_pool_changed_this_transaction(pool->pmd) && need_commit_due_to_time(pool)))
2388		return;
2389
2390	if (commit(pool)) {
2391		bio_list_merge(&bios, &bio_completions);
2392
2393		while ((bio = bio_list_pop(&bios)))
2394			bio_io_error(bio);
2395		return;
2396	}
2397	pool->last_commit_jiffies = jiffies;
2398
2399	while ((bio = bio_list_pop(&bio_completions)))
2400		bio_endio(bio);
2401
2402	while ((bio = bio_list_pop(&bios))) {
2403		/*
2404		 * The data device was flushed as part of metadata commit,
2405		 * so complete redundant flushes immediately.
2406		 */
2407		if (bio->bi_opf & REQ_PREFLUSH)
2408			bio_endio(bio);
2409		else
2410			dm_submit_bio_remap(bio, NULL);
2411	}
2412}
2413
2414static void do_worker(struct work_struct *ws)
2415{
2416	struct pool *pool = container_of(ws, struct pool, worker);
2417
2418	throttle_work_start(&pool->throttle);
2419	dm_pool_issue_prefetches(pool->pmd);
2420	throttle_work_update(&pool->throttle);
2421	process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
2422	throttle_work_update(&pool->throttle);
2423	process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
2424	throttle_work_update(&pool->throttle);
2425	process_prepared(pool, &pool->prepared_discards_pt2, &pool->process_prepared_discard_pt2);
2426	throttle_work_update(&pool->throttle);
2427	process_deferred_bios(pool);
2428	throttle_work_complete(&pool->throttle);
2429}
2430
2431/*
2432 * We want to commit periodically so that not too much
2433 * unwritten data builds up.
2434 */
2435static void do_waker(struct work_struct *ws)
2436{
2437	struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
2438
2439	wake_worker(pool);
2440	queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
2441}
2442
2443/*
2444 * We're holding onto IO to allow userland time to react.  After the
2445 * timeout either the pool will have been resized (and thus back in
2446 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
2447 */
2448static void do_no_space_timeout(struct work_struct *ws)
2449{
2450	struct pool *pool = container_of(to_delayed_work(ws), struct pool,
2451					 no_space_timeout);
2452
2453	if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space) {
2454		pool->pf.error_if_no_space = true;
2455		notify_of_pool_mode_change(pool);
2456		error_retry_list_with_code(pool, BLK_STS_NOSPC);
2457	}
2458}
2459
2460/*----------------------------------------------------------------*/
2461
2462struct pool_work {
2463	struct work_struct worker;
2464	struct completion complete;
2465};
2466
2467static struct pool_work *to_pool_work(struct work_struct *ws)
2468{
2469	return container_of(ws, struct pool_work, worker);
2470}
2471
2472static void pool_work_complete(struct pool_work *pw)
2473{
2474	complete(&pw->complete);
2475}
2476
2477static void pool_work_wait(struct pool_work *pw, struct pool *pool,
2478			   void (*fn)(struct work_struct *))
2479{
2480	INIT_WORK_ONSTACK(&pw->worker, fn);
2481	init_completion(&pw->complete);
2482	queue_work(pool->wq, &pw->worker);
2483	wait_for_completion(&pw->complete);
2484	destroy_work_on_stack(&pw->worker);
2485}
2486
2487/*----------------------------------------------------------------*/
2488
2489struct noflush_work {
2490	struct pool_work pw;
2491	struct thin_c *tc;
2492};
2493
2494static struct noflush_work *to_noflush(struct work_struct *ws)
2495{
2496	return container_of(to_pool_work(ws), struct noflush_work, pw);
2497}
2498
2499static void do_noflush_start(struct work_struct *ws)
2500{
2501	struct noflush_work *w = to_noflush(ws);
2502
2503	w->tc->requeue_mode = true;
2504	requeue_io(w->tc);
2505	pool_work_complete(&w->pw);
2506}
2507
2508static void do_noflush_stop(struct work_struct *ws)
2509{
2510	struct noflush_work *w = to_noflush(ws);
2511
2512	w->tc->requeue_mode = false;
2513	pool_work_complete(&w->pw);
2514}
2515
2516static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
2517{
2518	struct noflush_work w;
2519
2520	w.tc = tc;
2521	pool_work_wait(&w.pw, tc->pool, fn);
2522}
2523
2524/*----------------------------------------------------------------*/
2525
2526static void set_discard_callbacks(struct pool *pool)
2527{
2528	struct pool_c *pt = pool->ti->private;
2529
2530	if (pt->adjusted_pf.discard_passdown) {
2531		pool->process_discard_cell = process_discard_cell_passdown;
2532		pool->process_prepared_discard = process_prepared_discard_passdown_pt1;
2533		pool->process_prepared_discard_pt2 = process_prepared_discard_passdown_pt2;
2534	} else {
2535		pool->process_discard_cell = process_discard_cell_no_passdown;
2536		pool->process_prepared_discard = process_prepared_discard_no_passdown;
2537	}
2538}
2539
2540static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
2541{
2542	struct pool_c *pt = pool->ti->private;
2543	bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
2544	enum pool_mode old_mode = get_pool_mode(pool);
2545	unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ;
2546
2547	/*
2548	 * Never allow the pool to transition to PM_WRITE mode if user
2549	 * intervention is required to verify metadata and data consistency.
2550	 */
2551	if (new_mode == PM_WRITE && needs_check) {
2552		DMERR("%s: unable to switch pool to write mode until repaired.",
2553		      dm_device_name(pool->pool_md));
2554		if (old_mode != new_mode)
2555			new_mode = old_mode;
2556		else
2557			new_mode = PM_READ_ONLY;
2558	}
2559	/*
2560	 * If we were in PM_FAIL mode, rollback of metadata failed.  We're
2561	 * not going to recover without a thin_repair.	So we never let the
2562	 * pool move out of the old mode.
2563	 */
2564	if (old_mode == PM_FAIL)
2565		new_mode = old_mode;
2566
2567	switch (new_mode) {
2568	case PM_FAIL:
2569		dm_pool_metadata_read_only(pool->pmd);
2570		pool->process_bio = process_bio_fail;
2571		pool->process_discard = process_bio_fail;
2572		pool->process_cell = process_cell_fail;
2573		pool->process_discard_cell = process_cell_fail;
2574		pool->process_prepared_mapping = process_prepared_mapping_fail;
2575		pool->process_prepared_discard = process_prepared_discard_fail;
2576
2577		error_retry_list(pool);
2578		break;
2579
2580	case PM_OUT_OF_METADATA_SPACE:
2581	case PM_READ_ONLY:
2582		dm_pool_metadata_read_only(pool->pmd);
2583		pool->process_bio = process_bio_read_only;
2584		pool->process_discard = process_bio_success;
2585		pool->process_cell = process_cell_read_only;
2586		pool->process_discard_cell = process_cell_success;
2587		pool->process_prepared_mapping = process_prepared_mapping_fail;
2588		pool->process_prepared_discard = process_prepared_discard_success;
2589
2590		error_retry_list(pool);
2591		break;
2592
2593	case PM_OUT_OF_DATA_SPACE:
2594		/*
2595		 * Ideally we'd never hit this state; the low water mark
2596		 * would trigger userland to extend the pool before we
2597		 * completely run out of data space.  However, many small
2598		 * IOs to unprovisioned space can consume data space at an
2599		 * alarming rate.  Adjust your low water mark if you're
2600		 * frequently seeing this mode.
2601		 */
2602		pool->out_of_data_space = true;
2603		pool->process_bio = process_bio_read_only;
2604		pool->process_discard = process_discard_bio;
2605		pool->process_cell = process_cell_read_only;
2606		pool->process_prepared_mapping = process_prepared_mapping;
2607		set_discard_callbacks(pool);
2608
2609		if (!pool->pf.error_if_no_space && no_space_timeout)
2610			queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
2611		break;
2612
2613	case PM_WRITE:
2614		if (old_mode == PM_OUT_OF_DATA_SPACE)
2615			cancel_delayed_work_sync(&pool->no_space_timeout);
2616		pool->out_of_data_space = false;
2617		pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space;
2618		dm_pool_metadata_read_write(pool->pmd);
2619		pool->process_bio = process_bio;
2620		pool->process_discard = process_discard_bio;
2621		pool->process_cell = process_cell;
2622		pool->process_prepared_mapping = process_prepared_mapping;
2623		set_discard_callbacks(pool);
2624		break;
2625	}
2626
2627	pool->pf.mode = new_mode;
2628	/*
2629	 * The pool mode may have changed, sync it so bind_control_target()
2630	 * doesn't cause an unexpected mode transition on resume.
2631	 */
2632	pt->adjusted_pf.mode = new_mode;
2633
2634	if (old_mode != new_mode)
2635		notify_of_pool_mode_change(pool);
2636}
2637
2638static void abort_transaction(struct pool *pool)
2639{
2640	const char *dev_name = dm_device_name(pool->pool_md);
2641
2642	DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
2643	if (dm_pool_abort_metadata(pool->pmd)) {
2644		DMERR("%s: failed to abort metadata transaction", dev_name);
2645		set_pool_mode(pool, PM_FAIL);
2646	}
2647
2648	if (dm_pool_metadata_set_needs_check(pool->pmd)) {
2649		DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
2650		set_pool_mode(pool, PM_FAIL);
2651	}
2652}
2653
2654static void metadata_operation_failed(struct pool *pool, const char *op, int r)
2655{
2656	DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2657		    dm_device_name(pool->pool_md), op, r);
2658
2659	abort_transaction(pool);
2660	set_pool_mode(pool, PM_READ_ONLY);
2661}
2662
2663/*----------------------------------------------------------------*/
2664
2665/*
2666 * Mapping functions.
2667 */
2668
2669/*
2670 * Called only while mapping a thin bio to hand it over to the workqueue.
2671 */
2672static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
2673{
2674	struct pool *pool = tc->pool;
2675
2676	spin_lock_irq(&tc->lock);
2677	bio_list_add(&tc->deferred_bio_list, bio);
2678	spin_unlock_irq(&tc->lock);
2679
2680	wake_worker(pool);
2681}
2682
2683static void thin_defer_bio_with_throttle(struct thin_c *tc, struct bio *bio)
2684{
2685	struct pool *pool = tc->pool;
2686
2687	throttle_lock(&pool->throttle);
2688	thin_defer_bio(tc, bio);
2689	throttle_unlock(&pool->throttle);
2690}
2691
2692static void thin_defer_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
2693{
2694	struct pool *pool = tc->pool;
2695
2696	throttle_lock(&pool->throttle);
2697	spin_lock_irq(&tc->lock);
2698	list_add_tail(&cell->user_list, &tc->deferred_cells);
2699	spin_unlock_irq(&tc->lock);
2700	throttle_unlock(&pool->throttle);
2701
2702	wake_worker(pool);
2703}
2704
2705static void thin_hook_bio(struct thin_c *tc, struct bio *bio)
2706{
2707	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
2708
2709	h->tc = tc;
2710	h->shared_read_entry = NULL;
2711	h->all_io_entry = NULL;
2712	h->overwrite_mapping = NULL;
2713	h->cell = NULL;
2714}
2715
2716/*
2717 * Non-blocking function called from the thin target's map function.
2718 */
2719static int thin_bio_map(struct dm_target *ti, struct bio *bio)
2720{
2721	int r;
2722	struct thin_c *tc = ti->private;
2723	dm_block_t block = get_bio_block(tc, bio);
2724	struct dm_thin_device *td = tc->td;
2725	struct dm_thin_lookup_result result;
2726	struct dm_bio_prison_cell *virt_cell, *data_cell;
2727	struct dm_cell_key key;
2728
2729	thin_hook_bio(tc, bio);
2730
2731	if (tc->requeue_mode) {
2732		bio->bi_status = BLK_STS_DM_REQUEUE;
2733		bio_endio(bio);
2734		return DM_MAPIO_SUBMITTED;
2735	}
2736
2737	if (get_pool_mode(tc->pool) == PM_FAIL) {
2738		bio_io_error(bio);
2739		return DM_MAPIO_SUBMITTED;
2740	}
2741
2742	if (op_is_flush(bio->bi_opf) || bio_op(bio) == REQ_OP_DISCARD) {
2743		thin_defer_bio_with_throttle(tc, bio);
2744		return DM_MAPIO_SUBMITTED;
2745	}
2746
2747	/*
2748	 * We must hold the virtual cell before doing the lookup, otherwise
2749	 * there's a race with discard.
2750	 */
2751	build_virtual_key(tc->td, block, &key);
2752	if (bio_detain(tc->pool, &key, bio, &virt_cell))
2753		return DM_MAPIO_SUBMITTED;
2754
2755	r = dm_thin_find_block(td, block, 0, &result);
2756
2757	/*
2758	 * Note that we defer readahead too.
2759	 */
2760	switch (r) {
2761	case 0:
2762		if (unlikely(result.shared)) {
2763			/*
2764			 * We have a race condition here between the
2765			 * result.shared value returned by the lookup and
2766			 * snapshot creation, which may cause new
2767			 * sharing.
2768			 *
2769			 * To avoid this always quiesce the origin before
2770			 * taking the snap.  You want to do this anyway to
2771			 * ensure a consistent application view
2772			 * (i.e. lockfs).
2773			 *
2774			 * More distant ancestors are irrelevant. The
2775			 * shared flag will be set in their case.
2776			 */
2777			thin_defer_cell(tc, virt_cell);
2778			return DM_MAPIO_SUBMITTED;
2779		}
2780
2781		build_data_key(tc->td, result.block, &key);
2782		if (bio_detain(tc->pool, &key, bio, &data_cell)) {
2783			cell_defer_no_holder(tc, virt_cell);
2784			return DM_MAPIO_SUBMITTED;
2785		}
2786
2787		inc_all_io_entry(tc->pool, bio);
2788		cell_defer_no_holder(tc, data_cell);
2789		cell_defer_no_holder(tc, virt_cell);
2790
2791		remap(tc, bio, result.block);
2792		return DM_MAPIO_REMAPPED;
2793
2794	case -ENODATA:
2795	case -EWOULDBLOCK:
2796		thin_defer_cell(tc, virt_cell);
2797		return DM_MAPIO_SUBMITTED;
2798
2799	default:
2800		/*
2801		 * Must always call bio_io_error on failure.
2802		 * dm_thin_find_block can fail with -EINVAL if the
2803		 * pool is switched to fail-io mode.
2804		 */
2805		bio_io_error(bio);
2806		cell_defer_no_holder(tc, virt_cell);
2807		return DM_MAPIO_SUBMITTED;
2808	}
2809}
2810
2811static void requeue_bios(struct pool *pool)
2812{
2813	struct thin_c *tc;
2814
2815	rcu_read_lock();
2816	list_for_each_entry_rcu(tc, &pool->active_thins, list) {
2817		spin_lock_irq(&tc->lock);
2818		bio_list_merge(&tc->deferred_bio_list, &tc->retry_on_resume_list);
2819		bio_list_init(&tc->retry_on_resume_list);
2820		spin_unlock_irq(&tc->lock);
2821	}
2822	rcu_read_unlock();
2823}
2824
2825/*
2826 *--------------------------------------------------------------
2827 * Binding of control targets to a pool object
2828 *--------------------------------------------------------------
2829 */
2830static bool is_factor(sector_t block_size, uint32_t n)
2831{
2832	return !sector_div(block_size, n);
2833}
2834
2835/*
2836 * If discard_passdown was enabled verify that the data device
2837 * supports discards.  Disable discard_passdown if not.
2838 */
2839static void disable_discard_passdown_if_not_supported(struct pool_c *pt)
2840{
2841	struct pool *pool = pt->pool;
2842	struct block_device *data_bdev = pt->data_dev->bdev;
2843	struct queue_limits *data_limits = bdev_limits(data_bdev);
2844	const char *reason = NULL;
2845
2846	if (!pt->adjusted_pf.discard_passdown)
2847		return;
2848
2849	if (!bdev_max_discard_sectors(pt->data_dev->bdev))
2850		reason = "discard unsupported";
2851
2852	else if (data_limits->max_discard_sectors < pool->sectors_per_block)
2853		reason = "max discard sectors smaller than a block";
2854
2855	if (reason) {
2856		DMWARN("Data device (%pg) %s: Disabling discard passdown.", data_bdev, reason);
2857		pt->adjusted_pf.discard_passdown = false;
2858	}
2859}
2860
2861static int bind_control_target(struct pool *pool, struct dm_target *ti)
2862{
2863	struct pool_c *pt = ti->private;
2864
2865	/*
2866	 * We want to make sure that a pool in PM_FAIL mode is never upgraded.
2867	 */
2868	enum pool_mode old_mode = get_pool_mode(pool);
2869	enum pool_mode new_mode = pt->adjusted_pf.mode;
2870
2871	/*
2872	 * Don't change the pool's mode until set_pool_mode() below.
2873	 * Otherwise the pool's process_* function pointers may
2874	 * not match the desired pool mode.
2875	 */
2876	pt->adjusted_pf.mode = old_mode;
2877
2878	pool->ti = ti;
2879	pool->pf = pt->adjusted_pf;
2880	pool->low_water_blocks = pt->low_water_blocks;
2881
2882	set_pool_mode(pool, new_mode);
2883
2884	return 0;
2885}
2886
2887static void unbind_control_target(struct pool *pool, struct dm_target *ti)
2888{
2889	if (pool->ti == ti)
2890		pool->ti = NULL;
2891}
2892
2893/*
2894 *--------------------------------------------------------------
2895 * Pool creation
2896 *--------------------------------------------------------------
2897 */
2898/* Initialize pool features. */
2899static void pool_features_init(struct pool_features *pf)
2900{
2901	pf->mode = PM_WRITE;
2902	pf->zero_new_blocks = true;
2903	pf->discard_enabled = true;
2904	pf->discard_passdown = true;
2905	pf->error_if_no_space = false;
2906}
2907
2908static void __pool_destroy(struct pool *pool)
2909{
2910	__pool_table_remove(pool);
2911
2912	vfree(pool->cell_sort_array);
2913	if (dm_pool_metadata_close(pool->pmd) < 0)
2914		DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
2915
2916	dm_bio_prison_destroy(pool->prison);
2917	dm_kcopyd_client_destroy(pool->copier);
2918
2919	cancel_delayed_work_sync(&pool->waker);
2920	cancel_delayed_work_sync(&pool->no_space_timeout);
2921	if (pool->wq)
2922		destroy_workqueue(pool->wq);
2923
2924	if (pool->next_mapping)
2925		mempool_free(pool->next_mapping, &pool->mapping_pool);
2926	mempool_exit(&pool->mapping_pool);
2927	dm_deferred_set_destroy(pool->shared_read_ds);
2928	dm_deferred_set_destroy(pool->all_io_ds);
2929	kfree(pool);
2930}
2931
2932static struct kmem_cache *_new_mapping_cache;
2933
2934static struct pool *pool_create(struct mapped_device *pool_md,
2935				struct block_device *metadata_dev,
2936				struct block_device *data_dev,
2937				unsigned long block_size,
2938				int read_only, char **error)
2939{
2940	int r;
2941	void *err_p;
2942	struct pool *pool;
2943	struct dm_pool_metadata *pmd;
2944	bool format_device = read_only ? false : true;
2945
2946	pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device);
2947	if (IS_ERR(pmd)) {
2948		*error = "Error creating metadata object";
2949		return ERR_CAST(pmd);
2950	}
2951
2952	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
2953	if (!pool) {
2954		*error = "Error allocating memory for pool";
2955		err_p = ERR_PTR(-ENOMEM);
2956		goto bad_pool;
2957	}
2958
2959	pool->pmd = pmd;
2960	pool->sectors_per_block = block_size;
2961	if (block_size & (block_size - 1))
2962		pool->sectors_per_block_shift = -1;
2963	else
2964		pool->sectors_per_block_shift = __ffs(block_size);
2965	pool->low_water_blocks = 0;
2966	pool_features_init(&pool->pf);
2967	pool->prison = dm_bio_prison_create();
2968	if (!pool->prison) {
2969		*error = "Error creating pool's bio prison";
2970		err_p = ERR_PTR(-ENOMEM);
2971		goto bad_prison;
2972	}
2973
2974	pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2975	if (IS_ERR(pool->copier)) {
2976		r = PTR_ERR(pool->copier);
2977		*error = "Error creating pool's kcopyd client";
2978		err_p = ERR_PTR(r);
2979		goto bad_kcopyd_client;
2980	}
2981
2982	/*
2983	 * Create singlethreaded workqueue that will service all devices
2984	 * that use this metadata.
2985	 */
2986	pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2987	if (!pool->wq) {
2988		*error = "Error creating pool's workqueue";
2989		err_p = ERR_PTR(-ENOMEM);
2990		goto bad_wq;
2991	}
2992
2993	throttle_init(&pool->throttle);
2994	INIT_WORK(&pool->worker, do_worker);
2995	INIT_DELAYED_WORK(&pool->waker, do_waker);
2996	INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
2997	spin_lock_init(&pool->lock);
2998	bio_list_init(&pool->deferred_flush_bios);
2999	bio_list_init(&pool->deferred_flush_completions);
3000	INIT_LIST_HEAD(&pool->prepared_mappings);
3001	INIT_LIST_HEAD(&pool->prepared_discards);
3002	INIT_LIST_HEAD(&pool->prepared_discards_pt2);
3003	INIT_LIST_HEAD(&pool->active_thins);
3004	pool->low_water_triggered = false;
3005	pool->suspended = true;
3006	pool->out_of_data_space = false;
3007
3008	pool->shared_read_ds = dm_deferred_set_create();
3009	if (!pool->shared_read_ds) {
3010		*error = "Error creating pool's shared read deferred set";
3011		err_p = ERR_PTR(-ENOMEM);
3012		goto bad_shared_read_ds;
3013	}
3014
3015	pool->all_io_ds = dm_deferred_set_create();
3016	if (!pool->all_io_ds) {
3017		*error = "Error creating pool's all io deferred set";
3018		err_p = ERR_PTR(-ENOMEM);
3019		goto bad_all_io_ds;
3020	}
3021
3022	pool->next_mapping = NULL;
3023	r = mempool_init_slab_pool(&pool->mapping_pool, MAPPING_POOL_SIZE,
3024				   _new_mapping_cache);
3025	if (r) {
3026		*error = "Error creating pool's mapping mempool";
3027		err_p = ERR_PTR(r);
3028		goto bad_mapping_pool;
3029	}
3030
3031	pool->cell_sort_array =
3032		vmalloc_array(CELL_SORT_ARRAY_SIZE,
3033			      sizeof(*pool->cell_sort_array));
3034	if (!pool->cell_sort_array) {
3035		*error = "Error allocating cell sort array";
3036		err_p = ERR_PTR(-ENOMEM);
3037		goto bad_sort_array;
3038	}
3039
3040	pool->ref_count = 1;
3041	pool->last_commit_jiffies = jiffies;
3042	pool->pool_md = pool_md;
3043	pool->md_dev = metadata_dev;
3044	pool->data_dev = data_dev;
3045	__pool_table_insert(pool);
3046
3047	return pool;
3048
3049bad_sort_array:
3050	mempool_exit(&pool->mapping_pool);
3051bad_mapping_pool:
3052	dm_deferred_set_destroy(pool->all_io_ds);
3053bad_all_io_ds:
3054	dm_deferred_set_destroy(pool->shared_read_ds);
3055bad_shared_read_ds:
3056	destroy_workqueue(pool->wq);
3057bad_wq:
3058	dm_kcopyd_client_destroy(pool->copier);
3059bad_kcopyd_client:
3060	dm_bio_prison_destroy(pool->prison);
3061bad_prison:
3062	kfree(pool);
3063bad_pool:
3064	if (dm_pool_metadata_close(pmd))
3065		DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
3066
3067	return err_p;
3068}
3069
3070static void __pool_inc(struct pool *pool)
3071{
3072	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
3073	pool->ref_count++;
3074}
3075
3076static void __pool_dec(struct pool *pool)
3077{
3078	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
3079	BUG_ON(!pool->ref_count);
3080	if (!--pool->ref_count)
3081		__pool_destroy(pool);
3082}
3083
3084static struct pool *__pool_find(struct mapped_device *pool_md,
3085				struct block_device *metadata_dev,
3086				struct block_device *data_dev,
3087				unsigned long block_size, int read_only,
3088				char **error, int *created)
3089{
3090	struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
3091
3092	if (pool) {
3093		if (pool->pool_md != pool_md) {
3094			*error = "metadata device already in use by a pool";
3095			return ERR_PTR(-EBUSY);
3096		}
3097		if (pool->data_dev != data_dev) {
3098			*error = "data device already in use by a pool";
3099			return ERR_PTR(-EBUSY);
3100		}
3101		__pool_inc(pool);
3102
3103	} else {
3104		pool = __pool_table_lookup(pool_md);
3105		if (pool) {
3106			if (pool->md_dev != metadata_dev || pool->data_dev != data_dev) {
3107				*error = "different pool cannot replace a pool";
3108				return ERR_PTR(-EINVAL);
3109			}
3110			__pool_inc(pool);
3111
3112		} else {
3113			pool = pool_create(pool_md, metadata_dev, data_dev, block_size, read_only, error);
3114			*created = 1;
3115		}
3116	}
3117
3118	return pool;
3119}
3120
3121/*
3122 *--------------------------------------------------------------
3123 * Pool target methods
3124 *--------------------------------------------------------------
3125 */
3126static void pool_dtr(struct dm_target *ti)
3127{
3128	struct pool_c *pt = ti->private;
3129
3130	mutex_lock(&dm_thin_pool_table.mutex);
3131
3132	unbind_control_target(pt->pool, ti);
3133	__pool_dec(pt->pool);
3134	dm_put_device(ti, pt->metadata_dev);
3135	dm_put_device(ti, pt->data_dev);
3136	kfree(pt);
3137
3138	mutex_unlock(&dm_thin_pool_table.mutex);
3139}
3140
3141static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
3142			       struct dm_target *ti)
3143{
3144	int r;
3145	unsigned int argc;
3146	const char *arg_name;
3147
3148	static const struct dm_arg _args[] = {
3149		{0, 4, "Invalid number of pool feature arguments"},
3150	};
3151
3152	/*
3153	 * No feature arguments supplied.
3154	 */
3155	if (!as->argc)
3156		return 0;
3157
3158	r = dm_read_arg_group(_args, as, &argc, &ti->error);
3159	if (r)
3160		return -EINVAL;
3161
3162	while (argc && !r) {
3163		arg_name = dm_shift_arg(as);
3164		argc--;
3165
3166		if (!strcasecmp(arg_name, "skip_block_zeroing"))
3167			pf->zero_new_blocks = false;
3168
3169		else if (!strcasecmp(arg_name, "ignore_discard"))
3170			pf->discard_enabled = false;
3171
3172		else if (!strcasecmp(arg_name, "no_discard_passdown"))
3173			pf->discard_passdown = false;
3174
3175		else if (!strcasecmp(arg_name, "read_only"))
3176			pf->mode = PM_READ_ONLY;
3177
3178		else if (!strcasecmp(arg_name, "error_if_no_space"))
3179			pf->error_if_no_space = true;
3180
3181		else {
3182			ti->error = "Unrecognised pool feature requested";
3183			r = -EINVAL;
3184			break;
3185		}
3186	}
3187
3188	return r;
3189}
3190
3191static void metadata_low_callback(void *context)
3192{
3193	struct pool *pool = context;
3194
3195	DMWARN("%s: reached low water mark for metadata device: sending event.",
3196	       dm_device_name(pool->pool_md));
3197
3198	dm_table_event(pool->ti->table);
3199}
3200
3201/*
3202 * We need to flush the data device **before** committing the metadata.
3203 *
3204 * This ensures that the data blocks of any newly inserted mappings are
3205 * properly written to non-volatile storage and won't be lost in case of a
3206 * crash.
3207 *
3208 * Failure to do so can result in data corruption in the case of internal or
3209 * external snapshots and in the case of newly provisioned blocks, when block
3210 * zeroing is enabled.
3211 */
3212static int metadata_pre_commit_callback(void *context)
3213{
3214	struct pool *pool = context;
3215
3216	return blkdev_issue_flush(pool->data_dev);
3217}
3218
3219static sector_t get_dev_size(struct block_device *bdev)
3220{
3221	return bdev_nr_sectors(bdev);
3222}
3223
3224static void warn_if_metadata_device_too_big(struct block_device *bdev)
3225{
3226	sector_t metadata_dev_size = get_dev_size(bdev);
3227
3228	if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
3229		DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
3230		       bdev, THIN_METADATA_MAX_SECTORS);
3231}
3232
3233static sector_t get_metadata_dev_size(struct block_device *bdev)
3234{
3235	sector_t metadata_dev_size = get_dev_size(bdev);
3236
3237	if (metadata_dev_size > THIN_METADATA_MAX_SECTORS)
3238		metadata_dev_size = THIN_METADATA_MAX_SECTORS;
3239
3240	return metadata_dev_size;
3241}
3242
3243static dm_block_t get_metadata_dev_size_in_blocks(struct block_device *bdev)
3244{
3245	sector_t metadata_dev_size = get_metadata_dev_size(bdev);
3246
3247	sector_div(metadata_dev_size, THIN_METADATA_BLOCK_SIZE);
3248
3249	return metadata_dev_size;
3250}
3251
3252/*
3253 * When a metadata threshold is crossed a dm event is triggered, and
3254 * userland should respond by growing the metadata device.  We could let
3255 * userland set the threshold, like we do with the data threshold, but I'm
3256 * not sure they know enough to do this well.
3257 */
3258static dm_block_t calc_metadata_threshold(struct pool_c *pt)
3259{
3260	/*
3261	 * 4M is ample for all ops with the possible exception of thin
3262	 * device deletion which is harmless if it fails (just retry the
3263	 * delete after you've grown the device).
3264	 */
3265	dm_block_t quarter = get_metadata_dev_size_in_blocks(pt->metadata_dev->bdev) / 4;
3266
3267	return min((dm_block_t)1024ULL /* 4M */, quarter);
3268}
3269
3270/*
3271 * thin-pool <metadata dev> <data dev>
3272 *	     <data block size (sectors)>
3273 *	     <low water mark (blocks)>
3274 *	     [<#feature args> [<arg>]*]
3275 *
3276 * Optional feature arguments are:
3277 *	     skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
3278 *	     ignore_discard: disable discard
3279 *	     no_discard_passdown: don't pass discards down to the data device
3280 *	     read_only: Don't allow any changes to be made to the pool metadata.
3281 *	     error_if_no_space: error IOs, instead of queueing, if no space.
3282 */
3283static int pool_ctr(struct dm_target *ti, unsigned int argc, char **argv)
3284{
3285	int r, pool_created = 0;
3286	struct pool_c *pt;
3287	struct pool *pool;
3288	struct pool_features pf;
3289	struct dm_arg_set as;
3290	struct dm_dev *data_dev;
3291	unsigned long block_size;
3292	dm_block_t low_water_blocks;
3293	struct dm_dev *metadata_dev;
3294	blk_mode_t metadata_mode;
3295
3296	/*
3297	 * FIXME Remove validation from scope of lock.
3298	 */
3299	mutex_lock(&dm_thin_pool_table.mutex);
3300
3301	if (argc < 4) {
3302		ti->error = "Invalid argument count";
3303		r = -EINVAL;
3304		goto out_unlock;
3305	}
3306
3307	as.argc = argc;
3308	as.argv = argv;
3309
3310	/* make sure metadata and data are different devices */
3311	if (!strcmp(argv[0], argv[1])) {
3312		ti->error = "Error setting metadata or data device";
3313		r = -EINVAL;
3314		goto out_unlock;
3315	}
3316
3317	/*
3318	 * Set default pool features.
3319	 */
3320	pool_features_init(&pf);
3321
3322	dm_consume_args(&as, 4);
3323	r = parse_pool_features(&as, &pf, ti);
3324	if (r)
3325		goto out_unlock;
3326
3327	metadata_mode = BLK_OPEN_READ |
3328		((pf.mode == PM_READ_ONLY) ? 0 : BLK_OPEN_WRITE);
3329	r = dm_get_device(ti, argv[0], metadata_mode, &metadata_dev);
3330	if (r) {
3331		ti->error = "Error opening metadata block device";
3332		goto out_unlock;
3333	}
3334	warn_if_metadata_device_too_big(metadata_dev->bdev);
3335
3336	r = dm_get_device(ti, argv[1], BLK_OPEN_READ | BLK_OPEN_WRITE, &data_dev);
3337	if (r) {
3338		ti->error = "Error getting data device";
3339		goto out_metadata;
3340	}
3341
3342	if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
3343	    block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
3344	    block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
3345	    block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
3346		ti->error = "Invalid block size";
3347		r = -EINVAL;
3348		goto out;
3349	}
3350
3351	if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
3352		ti->error = "Invalid low water mark";
3353		r = -EINVAL;
3354		goto out;
3355	}
3356
3357	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
3358	if (!pt) {
3359		r = -ENOMEM;
3360		goto out;
3361	}
3362
3363	pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev, data_dev->bdev,
3364			   block_size, pf.mode == PM_READ_ONLY, &ti->error, &pool_created);
3365	if (IS_ERR(pool)) {
3366		r = PTR_ERR(pool);
3367		goto out_free_pt;
3368	}
3369
3370	/*
3371	 * 'pool_created' reflects whether this is the first table load.
3372	 * Top level discard support is not allowed to be changed after
3373	 * initial load.  This would require a pool reload to trigger thin
3374	 * device changes.
3375	 */
3376	if (!pool_created && pf.discard_enabled != pool->pf.discard_enabled) {
3377		ti->error = "Discard support cannot be disabled once enabled";
3378		r = -EINVAL;
3379		goto out_flags_changed;
3380	}
3381
3382	pt->pool = pool;
3383	pt->ti = ti;
3384	pt->metadata_dev = metadata_dev;
3385	pt->data_dev = data_dev;
3386	pt->low_water_blocks = low_water_blocks;
3387	pt->adjusted_pf = pt->requested_pf = pf;
3388	ti->num_flush_bios = 1;
3389	ti->limit_swap_bios = true;
3390
3391	/*
3392	 * Only need to enable discards if the pool should pass
3393	 * them down to the data device.  The thin device's discard
3394	 * processing will cause mappings to be removed from the btree.
3395	 */
3396	if (pf.discard_enabled && pf.discard_passdown) {
3397		ti->num_discard_bios = 1;
3398		/*
3399		 * Setting 'discards_supported' circumvents the normal
3400		 * stacking of discard limits (this keeps the pool and
3401		 * thin devices' discard limits consistent).
3402		 */
3403		ti->discards_supported = true;
3404		ti->max_discard_granularity = true;
3405	}
3406	ti->private = pt;
3407
3408	r = dm_pool_register_metadata_threshold(pt->pool->pmd,
3409						calc_metadata_threshold(pt),
3410						metadata_low_callback,
3411						pool);
3412	if (r) {
3413		ti->error = "Error registering metadata threshold";
3414		goto out_flags_changed;
3415	}
3416
3417	dm_pool_register_pre_commit_callback(pool->pmd,
3418					     metadata_pre_commit_callback, pool);
3419
3420	mutex_unlock(&dm_thin_pool_table.mutex);
3421
3422	return 0;
3423
3424out_flags_changed:
3425	__pool_dec(pool);
3426out_free_pt:
3427	kfree(pt);
3428out:
3429	dm_put_device(ti, data_dev);
3430out_metadata:
3431	dm_put_device(ti, metadata_dev);
3432out_unlock:
3433	mutex_unlock(&dm_thin_pool_table.mutex);
3434
3435	return r;
3436}
3437
3438static int pool_map(struct dm_target *ti, struct bio *bio)
3439{
3440	struct pool_c *pt = ti->private;
3441	struct pool *pool = pt->pool;
3442
3443	/*
3444	 * As this is a singleton target, ti->begin is always zero.
3445	 */
3446	spin_lock_irq(&pool->lock);
3447	bio_set_dev(bio, pt->data_dev->bdev);
3448	spin_unlock_irq(&pool->lock);
3449
3450	return DM_MAPIO_REMAPPED;
3451}
3452
3453static int maybe_resize_data_dev(struct dm_target *ti, bool *need_commit)
3454{
3455	int r;
3456	struct pool_c *pt = ti->private;
3457	struct pool *pool = pt->pool;
3458	sector_t data_size = ti->len;
3459	dm_block_t sb_data_size;
3460
3461	*need_commit = false;
3462
3463	(void) sector_div(data_size, pool->sectors_per_block);
3464
3465	r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
3466	if (r) {
3467		DMERR("%s: failed to retrieve data device size",
3468		      dm_device_name(pool->pool_md));
3469		return r;
3470	}
3471
3472	if (data_size < sb_data_size) {
3473		DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3474		      dm_device_name(pool->pool_md),
3475		      (unsigned long long)data_size, sb_data_size);
3476		return -EINVAL;
3477
3478	} else if (data_size > sb_data_size) {
3479		if (dm_pool_metadata_needs_check(pool->pmd)) {
3480			DMERR("%s: unable to grow the data device until repaired.",
3481			      dm_device_name(pool->pool_md));
3482			return 0;
3483		}
3484
3485		if (sb_data_size)
3486			DMINFO("%s: growing the data device from %llu to %llu blocks",
3487			       dm_device_name(pool->pool_md),
3488			       sb_data_size, (unsigned long long)data_size);
3489		r = dm_pool_resize_data_dev(pool->pmd, data_size);
3490		if (r) {
3491			metadata_operation_failed(pool, "dm_pool_resize_data_dev", r);
3492			return r;
3493		}
3494
3495		*need_commit = true;
3496	}
3497
3498	return 0;
3499}
3500
3501static int maybe_resize_metadata_dev(struct dm_target *ti, bool *need_commit)
3502{
3503	int r;
3504	struct pool_c *pt = ti->private;
3505	struct pool *pool = pt->pool;
3506	dm_block_t metadata_dev_size, sb_metadata_dev_size;
3507
3508	*need_commit = false;
3509
3510	metadata_dev_size = get_metadata_dev_size_in_blocks(pool->md_dev);
3511
3512	r = dm_pool_get_metadata_dev_size(pool->pmd, &sb_metadata_dev_size);
3513	if (r) {
3514		DMERR("%s: failed to retrieve metadata device size",
3515		      dm_device_name(pool->pool_md));
3516		return r;
3517	}
3518
3519	if (metadata_dev_size < sb_metadata_dev_size) {
3520		DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3521		      dm_device_name(pool->pool_md),
3522		      metadata_dev_size, sb_metadata_dev_size);
3523		return -EINVAL;
3524
3525	} else if (metadata_dev_size > sb_metadata_dev_size) {
3526		if (dm_pool_metadata_needs_check(pool->pmd)) {
3527			DMERR("%s: unable to grow the metadata device until repaired.",
3528			      dm_device_name(pool->pool_md));
3529			return 0;
3530		}
3531
3532		warn_if_metadata_device_too_big(pool->md_dev);
3533		DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3534		       dm_device_name(pool->pool_md),
3535		       sb_metadata_dev_size, metadata_dev_size);
3536
3537		if (get_pool_mode(pool) == PM_OUT_OF_METADATA_SPACE)
3538			set_pool_mode(pool, PM_WRITE);
3539
3540		r = dm_pool_resize_metadata_dev(pool->pmd, metadata_dev_size);
3541		if (r) {
3542			metadata_operation_failed(pool, "dm_pool_resize_metadata_dev", r);
3543			return r;
3544		}
3545
3546		*need_commit = true;
3547	}
3548
3549	return 0;
3550}
3551
3552/*
3553 * Retrieves the number of blocks of the data device from
3554 * the superblock and compares it to the actual device size,
3555 * thus resizing the data device in case it has grown.
3556 *
3557 * This both copes with opening preallocated data devices in the ctr
3558 * being followed by a resume
3559 * -and-
3560 * calling the resume method individually after userspace has
3561 * grown the data device in reaction to a table event.
3562 */
3563static int pool_preresume(struct dm_target *ti)
3564{
3565	int r;
3566	bool need_commit1, need_commit2;
3567	struct pool_c *pt = ti->private;
3568	struct pool *pool = pt->pool;
3569
3570	/*
3571	 * Take control of the pool object.
3572	 */
3573	r = bind_control_target(pool, ti);
3574	if (r)
3575		goto out;
3576
3577	r = maybe_resize_data_dev(ti, &need_commit1);
3578	if (r)
3579		goto out;
3580
3581	r = maybe_resize_metadata_dev(ti, &need_commit2);
3582	if (r)
3583		goto out;
3584
3585	if (need_commit1 || need_commit2)
3586		(void) commit(pool);
3587out:
3588	/*
3589	 * When a thin-pool is PM_FAIL, it cannot be rebuilt if
3590	 * bio is in deferred list. Therefore need to return 0
3591	 * to allow pool_resume() to flush IO.
3592	 */
3593	if (r && get_pool_mode(pool) == PM_FAIL)
3594		r = 0;
3595
3596	return r;
3597}
3598
3599static void pool_suspend_active_thins(struct pool *pool)
3600{
3601	struct thin_c *tc;
3602
3603	/* Suspend all active thin devices */
3604	tc = get_first_thin(pool);
3605	while (tc) {
3606		dm_internal_suspend_noflush(tc->thin_md);
3607		tc = get_next_thin(pool, tc);
3608	}
3609}
3610
3611static void pool_resume_active_thins(struct pool *pool)
3612{
3613	struct thin_c *tc;
3614
3615	/* Resume all active thin devices */
3616	tc = get_first_thin(pool);
3617	while (tc) {
3618		dm_internal_resume(tc->thin_md);
3619		tc = get_next_thin(pool, tc);
3620	}
3621}
3622
3623static void pool_resume(struct dm_target *ti)
3624{
3625	struct pool_c *pt = ti->private;
3626	struct pool *pool = pt->pool;
3627
3628	/*
3629	 * Must requeue active_thins' bios and then resume
3630	 * active_thins _before_ clearing 'suspend' flag.
3631	 */
3632	requeue_bios(pool);
3633	pool_resume_active_thins(pool);
3634
3635	spin_lock_irq(&pool->lock);
3636	pool->low_water_triggered = false;
3637	pool->suspended = false;
3638	spin_unlock_irq(&pool->lock);
3639
3640	do_waker(&pool->waker.work);
3641}
3642
3643static void pool_presuspend(struct dm_target *ti)
3644{
3645	struct pool_c *pt = ti->private;
3646	struct pool *pool = pt->pool;
3647
3648	spin_lock_irq(&pool->lock);
3649	pool->suspended = true;
3650	spin_unlock_irq(&pool->lock);
3651
3652	pool_suspend_active_thins(pool);
3653}
3654
3655static void pool_presuspend_undo(struct dm_target *ti)
3656{
3657	struct pool_c *pt = ti->private;
3658	struct pool *pool = pt->pool;
3659
3660	pool_resume_active_thins(pool);
3661
3662	spin_lock_irq(&pool->lock);
3663	pool->suspended = false;
3664	spin_unlock_irq(&pool->lock);
3665}
3666
3667static void pool_postsuspend(struct dm_target *ti)
3668{
3669	struct pool_c *pt = ti->private;
3670	struct pool *pool = pt->pool;
3671
3672	cancel_delayed_work_sync(&pool->waker);
3673	cancel_delayed_work_sync(&pool->no_space_timeout);
3674	flush_workqueue(pool->wq);
3675	(void) commit(pool);
3676}
3677
3678static int check_arg_count(unsigned int argc, unsigned int args_required)
3679{
3680	if (argc != args_required) {
3681		DMWARN("Message received with %u arguments instead of %u.",
3682		       argc, args_required);
3683		return -EINVAL;
3684	}
3685
3686	return 0;
3687}
3688
3689static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
3690{
3691	if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
3692	    *dev_id <= MAX_DEV_ID)
3693		return 0;
3694
3695	if (warning)
3696		DMWARN("Message received with invalid device id: %s", arg);
3697
3698	return -EINVAL;
3699}
3700
3701static int process_create_thin_mesg(unsigned int argc, char **argv, struct pool *pool)
3702{
3703	dm_thin_id dev_id;
3704	int r;
3705
3706	r = check_arg_count(argc, 2);
3707	if (r)
3708		return r;
3709
3710	r = read_dev_id(argv[1], &dev_id, 1);
3711	if (r)
3712		return r;
3713
3714	r = dm_pool_create_thin(pool->pmd, dev_id);
3715	if (r) {
3716		DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3717		       argv[1]);
3718		return r;
3719	}
3720
3721	return 0;
3722}
3723
3724static int process_create_snap_mesg(unsigned int argc, char **argv, struct pool *pool)
3725{
3726	dm_thin_id dev_id;
3727	dm_thin_id origin_dev_id;
3728	int r;
3729
3730	r = check_arg_count(argc, 3);
3731	if (r)
3732		return r;
3733
3734	r = read_dev_id(argv[1], &dev_id, 1);
3735	if (r)
3736		return r;
3737
3738	r = read_dev_id(argv[2], &origin_dev_id, 1);
3739	if (r)
3740		return r;
3741
3742	r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
3743	if (r) {
3744		DMWARN("Creation of new snapshot %s of device %s failed.",
3745		       argv[1], argv[2]);
3746		return r;
3747	}
3748
3749	return 0;
3750}
3751
3752static int process_delete_mesg(unsigned int argc, char **argv, struct pool *pool)
3753{
3754	dm_thin_id dev_id;
3755	int r;
3756
3757	r = check_arg_count(argc, 2);
3758	if (r)
3759		return r;
3760
3761	r = read_dev_id(argv[1], &dev_id, 1);
3762	if (r)
3763		return r;
3764
3765	r = dm_pool_delete_thin_device(pool->pmd, dev_id);
3766	if (r)
3767		DMWARN("Deletion of thin device %s failed.", argv[1]);
3768
3769	return r;
3770}
3771
3772static int process_set_transaction_id_mesg(unsigned int argc, char **argv, struct pool *pool)
3773{
3774	dm_thin_id old_id, new_id;
3775	int r;
3776
3777	r = check_arg_count(argc, 3);
3778	if (r)
3779		return r;
3780
3781	if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
3782		DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
3783		return -EINVAL;
3784	}
3785
3786	if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
3787		DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
3788		return -EINVAL;
3789	}
3790
3791	r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
3792	if (r) {
3793		DMWARN("Failed to change transaction id from %s to %s.",
3794		       argv[1], argv[2]);
3795		return r;
3796	}
3797
3798	return 0;
3799}
3800
3801static int process_reserve_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool)
3802{
3803	int r;
3804
3805	r = check_arg_count(argc, 1);
3806	if (r)
3807		return r;
3808
3809	(void) commit(pool);
3810
3811	r = dm_pool_reserve_metadata_snap(pool->pmd);
3812	if (r)
3813		DMWARN("reserve_metadata_snap message failed.");
3814
3815	return r;
3816}
3817
3818static int process_release_metadata_snap_mesg(unsigned int argc, char **argv, struct pool *pool)
3819{
3820	int r;
3821
3822	r = check_arg_count(argc, 1);
3823	if (r)
3824		return r;
3825
3826	r = dm_pool_release_metadata_snap(pool->pmd);
3827	if (r)
3828		DMWARN("release_metadata_snap message failed.");
3829
3830	return r;
3831}
3832
3833/*
3834 * Messages supported:
3835 *   create_thin	<dev_id>
3836 *   create_snap	<dev_id> <origin_id>
3837 *   delete		<dev_id>
3838 *   set_transaction_id <current_trans_id> <new_trans_id>
3839 *   reserve_metadata_snap
3840 *   release_metadata_snap
3841 */
3842static int pool_message(struct dm_target *ti, unsigned int argc, char **argv,
3843			char *result, unsigned int maxlen)
3844{
3845	int r = -EINVAL;
3846	struct pool_c *pt = ti->private;
3847	struct pool *pool = pt->pool;
3848
3849	if (get_pool_mode(pool) >= PM_OUT_OF_METADATA_SPACE) {
3850		DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3851		      dm_device_name(pool->pool_md));
3852		return -EOPNOTSUPP;
3853	}
3854
3855	if (!strcasecmp(argv[0], "create_thin"))
3856		r = process_create_thin_mesg(argc, argv, pool);
3857
3858	else if (!strcasecmp(argv[0], "create_snap"))
3859		r = process_create_snap_mesg(argc, argv, pool);
3860
3861	else if (!strcasecmp(argv[0], "delete"))
3862		r = process_delete_mesg(argc, argv, pool);
3863
3864	else if (!strcasecmp(argv[0], "set_transaction_id"))
3865		r = process_set_transaction_id_mesg(argc, argv, pool);
3866
3867	else if (!strcasecmp(argv[0], "reserve_metadata_snap"))
3868		r = process_reserve_metadata_snap_mesg(argc, argv, pool);
3869
3870	else if (!strcasecmp(argv[0], "release_metadata_snap"))
3871		r = process_release_metadata_snap_mesg(argc, argv, pool);
3872
3873	else
3874		DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
3875
3876	if (!r)
3877		(void) commit(pool);
3878
3879	return r;
3880}
3881
3882static void emit_flags(struct pool_features *pf, char *result,
3883		       unsigned int sz, unsigned int maxlen)
3884{
3885	unsigned int count = !pf->zero_new_blocks + !pf->discard_enabled +
3886		!pf->discard_passdown + (pf->mode == PM_READ_ONLY) +
3887		pf->error_if_no_space;
3888	DMEMIT("%u ", count);
3889
3890	if (!pf->zero_new_blocks)
3891		DMEMIT("skip_block_zeroing ");
3892
3893	if (!pf->discard_enabled)
3894		DMEMIT("ignore_discard ");
3895
3896	if (!pf->discard_passdown)
3897		DMEMIT("no_discard_passdown ");
3898
3899	if (pf->mode == PM_READ_ONLY)
3900		DMEMIT("read_only ");
3901
3902	if (pf->error_if_no_space)
3903		DMEMIT("error_if_no_space ");
3904}
3905
3906/*
3907 * Status line is:
3908 *    <transaction id> <used metadata sectors>/<total metadata sectors>
3909 *    <used data sectors>/<total data sectors> <held metadata root>
3910 *    <pool mode> <discard config> <no space config> <needs_check>
3911 */
3912static void pool_status(struct dm_target *ti, status_type_t type,
3913			unsigned int status_flags, char *result, unsigned int maxlen)
3914{
3915	int r;
3916	unsigned int sz = 0;
3917	uint64_t transaction_id;
3918	dm_block_t nr_free_blocks_data;
3919	dm_block_t nr_free_blocks_metadata;
3920	dm_block_t nr_blocks_data;
3921	dm_block_t nr_blocks_metadata;
3922	dm_block_t held_root;
3923	enum pool_mode mode;
3924	char buf[BDEVNAME_SIZE];
3925	char buf2[BDEVNAME_SIZE];
3926	struct pool_c *pt = ti->private;
3927	struct pool *pool = pt->pool;
3928
3929	switch (type) {
3930	case STATUSTYPE_INFO:
3931		if (get_pool_mode(pool) == PM_FAIL) {
3932			DMEMIT("Fail");
3933			break;
3934		}
3935
3936		/* Commit to ensure statistics aren't out-of-date */
3937		if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3938			(void) commit(pool);
3939
3940		r = dm_pool_get_metadata_transaction_id(pool->pmd, &transaction_id);
3941		if (r) {
3942			DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3943			      dm_device_name(pool->pool_md), r);
3944			goto err;
3945		}
3946
3947		r = dm_pool_get_free_metadata_block_count(pool->pmd, &nr_free_blocks_metadata);
3948		if (r) {
3949			DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3950			      dm_device_name(pool->pool_md), r);
3951			goto err;
3952		}
3953
3954		r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
3955		if (r) {
3956			DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3957			      dm_device_name(pool->pool_md), r);
3958			goto err;
3959		}
3960
3961		r = dm_pool_get_free_block_count(pool->pmd, &nr_free_blocks_data);
3962		if (r) {
3963			DMERR("%s: dm_pool_get_free_block_count returned %d",
3964			      dm_device_name(pool->pool_md), r);
3965			goto err;
3966		}
3967
3968		r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
3969		if (r) {
3970			DMERR("%s: dm_pool_get_data_dev_size returned %d",
3971			      dm_device_name(pool->pool_md), r);
3972			goto err;
3973		}
3974
3975		r = dm_pool_get_metadata_snap(pool->pmd, &held_root);
3976		if (r) {
3977			DMERR("%s: dm_pool_get_metadata_snap returned %d",
3978			      dm_device_name(pool->pool_md), r);
3979			goto err;
3980		}
3981
3982		DMEMIT("%llu %llu/%llu %llu/%llu ",
3983		       (unsigned long long)transaction_id,
3984		       (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3985		       (unsigned long long)nr_blocks_metadata,
3986		       (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
3987		       (unsigned long long)nr_blocks_data);
3988
3989		if (held_root)
3990			DMEMIT("%llu ", held_root);
3991		else
3992			DMEMIT("- ");
3993
3994		mode = get_pool_mode(pool);
3995		if (mode == PM_OUT_OF_DATA_SPACE)
3996			DMEMIT("out_of_data_space ");
3997		else if (is_read_only_pool_mode(mode))
3998			DMEMIT("ro ");
3999		else
4000			DMEMIT("rw ");
4001
4002		if (!pool->pf.discard_enabled)
4003			DMEMIT("ignore_discard ");
4004		else if (pool->pf.discard_passdown)
4005			DMEMIT("discard_passdown ");
4006		else
4007			DMEMIT("no_discard_passdown ");
4008
4009		if (pool->pf.error_if_no_space)
4010			DMEMIT("error_if_no_space ");
4011		else
4012			DMEMIT("queue_if_no_space ");
4013
4014		if (dm_pool_metadata_needs_check(pool->pmd))
4015			DMEMIT("needs_check ");
4016		else
4017			DMEMIT("- ");
4018
4019		DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt));
4020
4021		break;
4022
4023	case STATUSTYPE_TABLE:
4024		DMEMIT("%s %s %lu %llu ",
4025		       format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
4026		       format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
4027		       (unsigned long)pool->sectors_per_block,
4028		       (unsigned long long)pt->low_water_blocks);
4029		emit_flags(&pt->requested_pf, result, sz, maxlen);
4030		break;
4031
4032	case STATUSTYPE_IMA:
4033		*result = '\0';
4034		break;
4035	}
4036	return;
4037
4038err:
4039	DMEMIT("Error");
4040}
4041
4042static int pool_iterate_devices(struct dm_target *ti,
4043				iterate_devices_callout_fn fn, void *data)
4044{
4045	struct pool_c *pt = ti->private;
4046
4047	return fn(ti, pt->data_dev, 0, ti->len, data);
4048}
4049
4050static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
4051{
4052	struct pool_c *pt = ti->private;
4053	struct pool *pool = pt->pool;
4054	sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
4055
4056	/*
4057	 * If max_sectors is smaller than pool->sectors_per_block adjust it
4058	 * to the highest possible power-of-2 factor of pool->sectors_per_block.
4059	 * This is especially beneficial when the pool's data device is a RAID
4060	 * device that has a full stripe width that matches pool->sectors_per_block
4061	 * -- because even though partial RAID stripe-sized IOs will be issued to a
4062	 *    single RAID stripe; when aggregated they will end on a full RAID stripe
4063	 *    boundary.. which avoids additional partial RAID stripe writes cascading
4064	 */
4065	if (limits->max_sectors < pool->sectors_per_block) {
4066		while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
4067			if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
4068				limits->max_sectors--;
4069			limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
4070		}
4071	}
4072
4073	/*
4074	 * If the system-determined stacked limits are compatible with the
4075	 * pool's blocksize (io_opt is a factor) do not override them.
4076	 */
4077	if (io_opt_sectors < pool->sectors_per_block ||
4078	    !is_factor(io_opt_sectors, pool->sectors_per_block)) {
4079		if (is_factor(pool->sectors_per_block, limits->max_sectors))
4080			limits->io_min = limits->max_sectors << SECTOR_SHIFT;
4081		else
4082			limits->io_min = pool->sectors_per_block << SECTOR_SHIFT;
4083		limits->io_opt = pool->sectors_per_block << SECTOR_SHIFT;
4084	}
4085
4086	/*
4087	 * pt->adjusted_pf is a staging area for the actual features to use.
4088	 * They get transferred to the live pool in bind_control_target()
4089	 * called from pool_preresume().
4090	 */
4091
4092	if (pt->adjusted_pf.discard_enabled) {
4093		disable_discard_passdown_if_not_supported(pt);
4094		if (!pt->adjusted_pf.discard_passdown)
4095			limits->max_hw_discard_sectors = 0;
4096		/*
4097		 * The pool uses the same discard limits as the underlying data
4098		 * device.  DM core has already set this up.
4099		 */
4100	} else {
4101		/*
4102		 * Must explicitly disallow stacking discard limits otherwise the
4103		 * block layer will stack them if pool's data device has support.
4104		 */
4105		limits->discard_granularity = 0;
4106	}
4107}
4108
4109static struct target_type pool_target = {
4110	.name = "thin-pool",
4111	.features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
4112		    DM_TARGET_IMMUTABLE | DM_TARGET_PASSES_CRYPTO,
4113	.version = {1, 24, 0},
4114	.module = THIS_MODULE,
4115	.ctr = pool_ctr,
4116	.dtr = pool_dtr,
4117	.map = pool_map,
4118	.presuspend = pool_presuspend,
4119	.presuspend_undo = pool_presuspend_undo,
4120	.postsuspend = pool_postsuspend,
4121	.preresume = pool_preresume,
4122	.resume = pool_resume,
4123	.message = pool_message,
4124	.status = pool_status,
4125	.iterate_devices = pool_iterate_devices,
4126	.io_hints = pool_io_hints,
4127};
4128
4129/*
4130 *--------------------------------------------------------------
4131 * Thin target methods
4132 *--------------------------------------------------------------
4133 */
4134static void thin_get(struct thin_c *tc)
4135{
4136	refcount_inc(&tc->refcount);
4137}
4138
4139static void thin_put(struct thin_c *tc)
4140{
4141	if (refcount_dec_and_test(&tc->refcount))
4142		complete(&tc->can_destroy);
4143}
4144
4145static void thin_dtr(struct dm_target *ti)
4146{
4147	struct thin_c *tc = ti->private;
4148
4149	spin_lock_irq(&tc->pool->lock);
4150	list_del_rcu(&tc->list);
4151	spin_unlock_irq(&tc->pool->lock);
4152	synchronize_rcu();
4153
4154	thin_put(tc);
4155	wait_for_completion(&tc->can_destroy);
4156
4157	mutex_lock(&dm_thin_pool_table.mutex);
4158
4159	__pool_dec(tc->pool);
4160	dm_pool_close_thin_device(tc->td);
4161	dm_put_device(ti, tc->pool_dev);
4162	if (tc->origin_dev)
4163		dm_put_device(ti, tc->origin_dev);
4164	kfree(tc);
4165
4166	mutex_unlock(&dm_thin_pool_table.mutex);
4167}
4168
4169/*
4170 * Thin target parameters:
4171 *
4172 * <pool_dev> <dev_id> [origin_dev]
4173 *
4174 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
4175 * dev_id: the internal device identifier
4176 * origin_dev: a device external to the pool that should act as the origin
4177 *
4178 * If the pool device has discards disabled, they get disabled for the thin
4179 * device as well.
4180 */
4181static int thin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
4182{
4183	int r;
4184	struct thin_c *tc;
4185	struct dm_dev *pool_dev, *origin_dev;
4186	struct mapped_device *pool_md;
4187
4188	mutex_lock(&dm_thin_pool_table.mutex);
4189
4190	if (argc != 2 && argc != 3) {
4191		ti->error = "Invalid argument count";
4192		r = -EINVAL;
4193		goto out_unlock;
4194	}
4195
4196	tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
4197	if (!tc) {
4198		ti->error = "Out of memory";
4199		r = -ENOMEM;
4200		goto out_unlock;
4201	}
4202	tc->thin_md = dm_table_get_md(ti->table);
4203	spin_lock_init(&tc->lock);
4204	INIT_LIST_HEAD(&tc->deferred_cells);
4205	bio_list_init(&tc->deferred_bio_list);
4206	bio_list_init(&tc->retry_on_resume_list);
4207	tc->sort_bio_list = RB_ROOT;
4208
4209	if (argc == 3) {
4210		if (!strcmp(argv[0], argv[2])) {
4211			ti->error = "Error setting origin device";
4212			r = -EINVAL;
4213			goto bad_origin_dev;
4214		}
4215
4216		r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &origin_dev);
4217		if (r) {
4218			ti->error = "Error opening origin device";
4219			goto bad_origin_dev;
4220		}
4221		tc->origin_dev = origin_dev;
4222	}
4223
4224	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
4225	if (r) {
4226		ti->error = "Error opening pool device";
4227		goto bad_pool_dev;
4228	}
4229	tc->pool_dev = pool_dev;
4230
4231	if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
4232		ti->error = "Invalid device id";
4233		r = -EINVAL;
4234		goto bad_common;
4235	}
4236
4237	pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
4238	if (!pool_md) {
4239		ti->error = "Couldn't get pool mapped device";
4240		r = -EINVAL;
4241		goto bad_common;
4242	}
4243
4244	tc->pool = __pool_table_lookup(pool_md);
4245	if (!tc->pool) {
4246		ti->error = "Couldn't find pool object";
4247		r = -EINVAL;
4248		goto bad_pool_lookup;
4249	}
4250	__pool_inc(tc->pool);
4251
4252	if (get_pool_mode(tc->pool) == PM_FAIL) {
4253		ti->error = "Couldn't open thin device, Pool is in fail mode";
4254		r = -EINVAL;
4255		goto bad_pool;
4256	}
4257
4258	r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
4259	if (r) {
4260		ti->error = "Couldn't open thin internal device";
4261		goto bad_pool;
4262	}
4263
4264	r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
4265	if (r)
4266		goto bad;
4267
4268	ti->num_flush_bios = 1;
4269	ti->limit_swap_bios = true;
4270	ti->flush_supported = true;
4271	ti->accounts_remapped_io = true;
4272	ti->per_io_data_size = sizeof(struct dm_thin_endio_hook);
4273
4274	/* In case the pool supports discards, pass them on. */
4275	if (tc->pool->pf.discard_enabled) {
4276		ti->discards_supported = true;
4277		ti->num_discard_bios = 1;
4278		ti->max_discard_granularity = true;
4279	}
4280
4281	mutex_unlock(&dm_thin_pool_table.mutex);
4282
4283	spin_lock_irq(&tc->pool->lock);
4284	if (tc->pool->suspended) {
4285		spin_unlock_irq(&tc->pool->lock);
4286		mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
4287		ti->error = "Unable to activate thin device while pool is suspended";
4288		r = -EINVAL;
4289		goto bad;
4290	}
4291	refcount_set(&tc->refcount, 1);
4292	init_completion(&tc->can_destroy);
4293	list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
4294	spin_unlock_irq(&tc->pool->lock);
4295	/*
4296	 * This synchronize_rcu() call is needed here otherwise we risk a
4297	 * wake_worker() call finding no bios to process (because the newly
4298	 * added tc isn't yet visible).  So this reduces latency since we
4299	 * aren't then dependent on the periodic commit to wake_worker().
4300	 */
4301	synchronize_rcu();
4302
4303	dm_put(pool_md);
4304
4305	return 0;
4306
4307bad:
4308	dm_pool_close_thin_device(tc->td);
4309bad_pool:
4310	__pool_dec(tc->pool);
4311bad_pool_lookup:
4312	dm_put(pool_md);
4313bad_common:
4314	dm_put_device(ti, tc->pool_dev);
4315bad_pool_dev:
4316	if (tc->origin_dev)
4317		dm_put_device(ti, tc->origin_dev);
4318bad_origin_dev:
4319	kfree(tc);
4320out_unlock:
4321	mutex_unlock(&dm_thin_pool_table.mutex);
4322
4323	return r;
4324}
4325
4326static int thin_map(struct dm_target *ti, struct bio *bio)
4327{
4328	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
4329
4330	return thin_bio_map(ti, bio);
4331}
4332
4333static int thin_endio(struct dm_target *ti, struct bio *bio,
4334		blk_status_t *err)
4335{
4336	unsigned long flags;
4337	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
4338	struct list_head work;
4339	struct dm_thin_new_mapping *m, *tmp;
4340	struct pool *pool = h->tc->pool;
4341
4342	if (h->shared_read_entry) {
4343		INIT_LIST_HEAD(&work);
4344		dm_deferred_entry_dec(h->shared_read_entry, &work);
4345
4346		spin_lock_irqsave(&pool->lock, flags);
4347		list_for_each_entry_safe(m, tmp, &work, list) {
4348			list_del(&m->list);
4349			__complete_mapping_preparation(m);
4350		}
4351		spin_unlock_irqrestore(&pool->lock, flags);
4352	}
4353
4354	if (h->all_io_entry) {
4355		INIT_LIST_HEAD(&work);
4356		dm_deferred_entry_dec(h->all_io_entry, &work);
4357		if (!list_empty(&work)) {
4358			spin_lock_irqsave(&pool->lock, flags);
4359			list_for_each_entry_safe(m, tmp, &work, list)
4360				list_add_tail(&m->list, &pool->prepared_discards);
4361			spin_unlock_irqrestore(&pool->lock, flags);
4362			wake_worker(pool);
4363		}
4364	}
4365
4366	if (h->cell)
4367		cell_defer_no_holder(h->tc, h->cell);
4368
4369	return DM_ENDIO_DONE;
4370}
4371
4372static void thin_presuspend(struct dm_target *ti)
4373{
4374	struct thin_c *tc = ti->private;
4375
4376	if (dm_noflush_suspending(ti))
4377		noflush_work(tc, do_noflush_start);
4378}
4379
4380static void thin_postsuspend(struct dm_target *ti)
4381{
4382	struct thin_c *tc = ti->private;
4383
4384	if (dm_noflush_suspending(ti))
4385		noflush_work(tc, do_noflush_stop);
4386}
4387
4388static int thin_preresume(struct dm_target *ti)
4389{
4390	struct thin_c *tc = ti->private;
4391
4392	if (tc->origin_dev)
4393		tc->origin_size = get_dev_size(tc->origin_dev->bdev);
4394
4395	return 0;
4396}
4397
4398/*
4399 * <nr mapped sectors> <highest mapped sector>
4400 */
4401static void thin_status(struct dm_target *ti, status_type_t type,
4402			unsigned int status_flags, char *result, unsigned int maxlen)
4403{
4404	int r;
4405	ssize_t sz = 0;
4406	dm_block_t mapped, highest;
4407	char buf[BDEVNAME_SIZE];
4408	struct thin_c *tc = ti->private;
4409
4410	if (get_pool_mode(tc->pool) == PM_FAIL) {
4411		DMEMIT("Fail");
4412		return;
4413	}
4414
4415	if (!tc->td)
4416		DMEMIT("-");
4417	else {
4418		switch (type) {
4419		case STATUSTYPE_INFO:
4420			r = dm_thin_get_mapped_count(tc->td, &mapped);
4421			if (r) {
4422				DMERR("dm_thin_get_mapped_count returned %d", r);
4423				goto err;
4424			}
4425
4426			r = dm_thin_get_highest_mapped_block(tc->td, &highest);
4427			if (r < 0) {
4428				DMERR("dm_thin_get_highest_mapped_block returned %d", r);
4429				goto err;
4430			}
4431
4432			DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
4433			if (r)
4434				DMEMIT("%llu", ((highest + 1) *
4435						tc->pool->sectors_per_block) - 1);
4436			else
4437				DMEMIT("-");
4438			break;
4439
4440		case STATUSTYPE_TABLE:
4441			DMEMIT("%s %lu",
4442			       format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
4443			       (unsigned long) tc->dev_id);
4444			if (tc->origin_dev)
4445				DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
4446			break;
4447
4448		case STATUSTYPE_IMA:
4449			*result = '\0';
4450			break;
4451		}
4452	}
4453
4454	return;
4455
4456err:
4457	DMEMIT("Error");
4458}
4459
4460static int thin_iterate_devices(struct dm_target *ti,
4461				iterate_devices_callout_fn fn, void *data)
4462{
4463	sector_t blocks;
4464	struct thin_c *tc = ti->private;
4465	struct pool *pool = tc->pool;
4466
4467	/*
4468	 * We can't call dm_pool_get_data_dev_size() since that blocks.  So
4469	 * we follow a more convoluted path through to the pool's target.
4470	 */
4471	if (!pool->ti)
4472		return 0;	/* nothing is bound */
4473
4474	blocks = pool->ti->len;
4475	(void) sector_div(blocks, pool->sectors_per_block);
4476	if (blocks)
4477		return fn(ti, tc->pool_dev, 0, pool->sectors_per_block * blocks, data);
4478
4479	return 0;
4480}
4481
4482static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
4483{
4484	struct thin_c *tc = ti->private;
4485	struct pool *pool = tc->pool;
4486
4487	if (pool->pf.discard_enabled) {
4488		limits->discard_granularity = pool->sectors_per_block << SECTOR_SHIFT;
4489		limits->max_hw_discard_sectors = pool->sectors_per_block * BIO_PRISON_MAX_RANGE;
4490	}
4491}
4492
4493static struct target_type thin_target = {
4494	.name = "thin",
4495	.features = DM_TARGET_PASSES_CRYPTO,
4496	.version = {1, 24, 0},
4497	.module	= THIS_MODULE,
4498	.ctr = thin_ctr,
4499	.dtr = thin_dtr,
4500	.map = thin_map,
4501	.end_io = thin_endio,
4502	.preresume = thin_preresume,
4503	.presuspend = thin_presuspend,
4504	.postsuspend = thin_postsuspend,
4505	.status = thin_status,
4506	.iterate_devices = thin_iterate_devices,
4507	.io_hints = thin_io_hints,
4508};
4509
4510/*----------------------------------------------------------------*/
4511
4512static int __init dm_thin_init(void)
4513{
4514	int r = -ENOMEM;
4515
4516	pool_table_init();
4517
4518	_new_mapping_cache = KMEM_CACHE(dm_thin_new_mapping, 0);
4519	if (!_new_mapping_cache)
4520		return r;
4521
4522	r = dm_register_target(&thin_target);
4523	if (r)
4524		goto bad_new_mapping_cache;
4525
4526	r = dm_register_target(&pool_target);
4527	if (r)
4528		goto bad_thin_target;
4529
4530	return 0;
4531
4532bad_thin_target:
4533	dm_unregister_target(&thin_target);
4534bad_new_mapping_cache:
4535	kmem_cache_destroy(_new_mapping_cache);
4536
4537	return r;
4538}
4539
4540static void dm_thin_exit(void)
4541{
4542	dm_unregister_target(&thin_target);
4543	dm_unregister_target(&pool_target);
4544
4545	kmem_cache_destroy(_new_mapping_cache);
4546
4547	pool_table_exit();
4548}
4549
4550module_init(dm_thin_init);
4551module_exit(dm_thin_exit);
4552
4553module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644);
4554MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
4555
4556MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
4557MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>");
4558MODULE_LICENSE("GPL");