dm-table.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2001 Sistina Software (UK) Limited.
   4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
   5 *
   6 * This file is released under the GPL.
   7 */
   8
   9#include "dm-core.h"
  10#include "dm-rq.h"
  11
  12#include <linux/module.h>
  13#include <linux/vmalloc.h>
  14#include <linux/blkdev.h>
  15#include <linux/blk-integrity.h>
  16#include <linux/namei.h>
  17#include <linux/ctype.h>
  18#include <linux/string.h>
  19#include <linux/slab.h>
  20#include <linux/interrupt.h>
  21#include <linux/mutex.h>
  22#include <linux/delay.h>
  23#include <linux/atomic.h>
  24#include <linux/blk-mq.h>
  25#include <linux/mount.h>
  26#include <linux/dax.h>
  27
  28#define DM_MSG_PREFIX "table"
  29
  30#define NODE_SIZE L1_CACHE_BYTES
  31#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
  32#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
  33
  34/*
  35 * Similar to ceiling(log_size(n))
  36 */
  37static unsigned int int_log(unsigned int n, unsigned int base)
  38{
  39	int result = 0;
  40
  41	while (n > 1) {
  42		n = dm_div_up(n, base);
  43		result++;
  44	}
  45
  46	return result;
  47}
  48
  49/*
  50 * Calculate the index of the child node of the n'th node k'th key.
  51 */
  52static inline unsigned int get_child(unsigned int n, unsigned int k)
  53{
  54	return (n * CHILDREN_PER_NODE) + k;
  55}
  56
  57/*
  58 * Return the n'th node of level l from table t.
  59 */
  60static inline sector_t *get_node(struct dm_table *t,
  61				 unsigned int l, unsigned int n)
  62{
  63	return t->index[l] + (n * KEYS_PER_NODE);
  64}
  65
  66/*
  67 * Return the highest key that you could lookup from the n'th
  68 * node on level l of the btree.
  69 */
  70static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
  71{
  72	for (; l < t->depth - 1; l++)
  73		n = get_child(n, CHILDREN_PER_NODE - 1);
  74
  75	if (n >= t->counts[l])
  76		return (sector_t) -1;
  77
  78	return get_node(t, l, n)[KEYS_PER_NODE - 1];
  79}
  80
  81/*
  82 * Fills in a level of the btree based on the highs of the level
  83 * below it.
  84 */
  85static int setup_btree_index(unsigned int l, struct dm_table *t)
  86{
  87	unsigned int n, k;
  88	sector_t *node;
  89
  90	for (n = 0U; n < t->counts[l]; n++) {
  91		node = get_node(t, l, n);
  92
  93		for (k = 0U; k < KEYS_PER_NODE; k++)
  94			node[k] = high(t, l + 1, get_child(n, k));
  95	}
  96
  97	return 0;
  98}
  99
 100/*
 101 * highs, and targets are managed as dynamic arrays during a
 102 * table load.
 103 */
 104static int alloc_targets(struct dm_table *t, unsigned int num)
 105{
 106	sector_t *n_highs;
 107	struct dm_target *n_targets;
 108
 109	/*
 110	 * Allocate both the target array and offset array at once.
 111	 */
 112	n_highs = kvcalloc(num, sizeof(struct dm_target) + sizeof(sector_t),
 113			   GFP_KERNEL);
 114	if (!n_highs)
 115		return -ENOMEM;
 116
 117	n_targets = (struct dm_target *) (n_highs + num);
 118
 119	memset(n_highs, -1, sizeof(*n_highs) * num);
 120
 121	t->num_allocated = num;
 122	t->highs = n_highs;
 123	t->targets = n_targets;
 124
 125	return 0;
 126}
 127
 128int dm_table_create(struct dm_table **result, blk_mode_t mode,
 129		    unsigned int num_targets, struct mapped_device *md)
 130{
 131	struct dm_table *t;
 132
 133	if (num_targets > DM_MAX_TARGETS)
 134		return -EOVERFLOW;
 135
 136	t = kzalloc(sizeof(*t), GFP_KERNEL);
 137
 138	if (!t)
 139		return -ENOMEM;
 140
 141	INIT_LIST_HEAD(&t->devices);
 142	init_rwsem(&t->devices_lock);
 143
 144	if (!num_targets)
 145		num_targets = KEYS_PER_NODE;
 146
 147	num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
 148
 149	if (!num_targets) {
 150		kfree(t);
 151		return -EOVERFLOW;
 152	}
 153
 154	if (alloc_targets(t, num_targets)) {
 155		kfree(t);
 156		return -ENOMEM;
 157	}
 158
 159	t->type = DM_TYPE_NONE;
 160	t->mode = mode;
 161	t->md = md;
 162	t->flush_bypasses_map = true;
 163	*result = t;
 164	return 0;
 165}
 166
 167static void free_devices(struct list_head *devices, struct mapped_device *md)
 168{
 169	struct list_head *tmp, *next;
 170
 171	list_for_each_safe(tmp, next, devices) {
 172		struct dm_dev_internal *dd =
 173		    list_entry(tmp, struct dm_dev_internal, list);
 174		DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
 175		       dm_device_name(md), dd->dm_dev->name);
 176		dm_put_table_device(md, dd->dm_dev);
 177		kfree(dd);
 178	}
 179}
 180
 181static void dm_table_destroy_crypto_profile(struct dm_table *t);
 182
 183void dm_table_destroy(struct dm_table *t)
 184{
 185	if (!t)
 186		return;
 187
 188	/* free the indexes */
 189	if (t->depth >= 2)
 190		kvfree(t->index[t->depth - 2]);
 191
 192	/* free the targets */
 193	for (unsigned int i = 0; i < t->num_targets; i++) {
 194		struct dm_target *ti = dm_table_get_target(t, i);
 195
 196		if (ti->type->dtr)
 197			ti->type->dtr(ti);
 198
 199		dm_put_target_type(ti->type);
 200	}
 201
 202	kvfree(t->highs);
 203
 204	/* free the device list */
 205	free_devices(&t->devices, t->md);
 206
 207	dm_free_md_mempools(t->mempools);
 208
 209	dm_table_destroy_crypto_profile(t);
 210
 211	kfree(t);
 212}
 213
 214/*
 215 * See if we've already got a device in the list.
 216 */
 217static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
 218{
 219	struct dm_dev_internal *dd;
 220
 221	list_for_each_entry(dd, l, list)
 222		if (dd->dm_dev->bdev->bd_dev == dev)
 223			return dd;
 224
 225	return NULL;
 226}
 227
 228/*
 229 * If possible, this checks an area of a destination device is invalid.
 230 */
 231static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
 232				  sector_t start, sector_t len, void *data)
 233{
 234	struct queue_limits *limits = data;
 235	struct block_device *bdev = dev->bdev;
 236	sector_t dev_size = bdev_nr_sectors(bdev);
 237	unsigned short logical_block_size_sectors =
 238		limits->logical_block_size >> SECTOR_SHIFT;
 239
 240	if (!dev_size)
 241		return 0;
 242
 243	if ((start >= dev_size) || (start + len > dev_size)) {
 244		DMERR("%s: %pg too small for target: start=%llu, len=%llu, dev_size=%llu",
 245		      dm_device_name(ti->table->md), bdev,
 246		      (unsigned long long)start,
 247		      (unsigned long long)len,
 248		      (unsigned long long)dev_size);
 249		return 1;
 250	}
 251
 252	/*
 253	 * If the target is mapped to zoned block device(s), check
 254	 * that the zones are not partially mapped.
 255	 */
 256	if (bdev_is_zoned(bdev)) {
 257		unsigned int zone_sectors = bdev_zone_sectors(bdev);
 258
 259		if (!bdev_is_zone_aligned(bdev, start)) {
 260			DMERR("%s: start=%llu not aligned to h/w zone size %u of %pg",
 261			      dm_device_name(ti->table->md),
 262			      (unsigned long long)start,
 263			      zone_sectors, bdev);
 264			return 1;
 265		}
 266
 267		/*
 268		 * Note: The last zone of a zoned block device may be smaller
 269		 * than other zones. So for a target mapping the end of a
 270		 * zoned block device with such a zone, len would not be zone
 271		 * aligned. We do not allow such last smaller zone to be part
 272		 * of the mapping here to ensure that mappings with multiple
 273		 * devices do not end up with a smaller zone in the middle of
 274		 * the sector range.
 275		 */
 276		if (!bdev_is_zone_aligned(bdev, len)) {
 277			DMERR("%s: len=%llu not aligned to h/w zone size %u of %pg",
 278			      dm_device_name(ti->table->md),
 279			      (unsigned long long)len,
 280			      zone_sectors, bdev);
 281			return 1;
 282		}
 283	}
 284
 285	if (logical_block_size_sectors <= 1)
 286		return 0;
 287
 288	if (start & (logical_block_size_sectors - 1)) {
 289		DMERR("%s: start=%llu not aligned to h/w logical block size %u of %pg",
 290		      dm_device_name(ti->table->md),
 291		      (unsigned long long)start,
 292		      limits->logical_block_size, bdev);
 293		return 1;
 294	}
 295
 296	if (len & (logical_block_size_sectors - 1)) {
 297		DMERR("%s: len=%llu not aligned to h/w logical block size %u of %pg",
 298		      dm_device_name(ti->table->md),
 299		      (unsigned long long)len,
 300		      limits->logical_block_size, bdev);
 301		return 1;
 302	}
 303
 304	return 0;
 305}
 306
 307/*
 308 * This upgrades the mode on an already open dm_dev, being
 309 * careful to leave things as they were if we fail to reopen the
 310 * device and not to touch the existing bdev field in case
 311 * it is accessed concurrently.
 312 */
 313static int upgrade_mode(struct dm_dev_internal *dd, blk_mode_t new_mode,
 314			struct mapped_device *md)
 315{
 316	int r;
 317	struct dm_dev *old_dev, *new_dev;
 318
 319	old_dev = dd->dm_dev;
 320
 321	r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
 322				dd->dm_dev->mode | new_mode, &new_dev);
 323	if (r)
 324		return r;
 325
 326	dd->dm_dev = new_dev;
 327	dm_put_table_device(md, old_dev);
 328
 329	return 0;
 330}
 331
 332/*
 333 * Note: the __ref annotation is because this function can call the __init
 334 * marked early_lookup_bdev when called during early boot code from dm-init.c.
 335 */
 336int __ref dm_devt_from_path(const char *path, dev_t *dev_p)
 337{
 338	int r;
 339	dev_t dev;
 340	unsigned int major, minor;
 341	char dummy;
 342
 343	if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) {
 344		/* Extract the major/minor numbers */
 345		dev = MKDEV(major, minor);
 346		if (MAJOR(dev) != major || MINOR(dev) != minor)
 347			return -EOVERFLOW;
 348	} else {
 349		r = lookup_bdev(path, &dev);
 350#ifndef MODULE
 351		if (r && system_state < SYSTEM_RUNNING)
 352			r = early_lookup_bdev(path, &dev);
 353#endif
 354		if (r)
 355			return r;
 356	}
 357	*dev_p = dev;
 358	return 0;
 359}
 360EXPORT_SYMBOL(dm_devt_from_path);
 361
 362/*
 363 * Add a device to the list, or just increment the usage count if
 364 * it's already present.
 365 */
 366int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode,
 367		  struct dm_dev **result)
 368{
 369	int r;
 370	dev_t dev;
 371	struct dm_dev_internal *dd;
 372	struct dm_table *t = ti->table;
 373
 374	BUG_ON(!t);
 375
 376	r = dm_devt_from_path(path, &dev);
 377	if (r)
 378		return r;
 379
 380	if (dev == disk_devt(t->md->disk))
 381		return -EINVAL;
 382
 383	down_write(&t->devices_lock);
 384
 385	dd = find_device(&t->devices, dev);
 386	if (!dd) {
 387		dd = kmalloc(sizeof(*dd), GFP_KERNEL);
 388		if (!dd) {
 389			r = -ENOMEM;
 390			goto unlock_ret_r;
 391		}
 392
 393		r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev);
 394		if (r) {
 395			kfree(dd);
 396			goto unlock_ret_r;
 397		}
 398
 399		refcount_set(&dd->count, 1);
 400		list_add(&dd->list, &t->devices);
 401		goto out;
 402
 403	} else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
 404		r = upgrade_mode(dd, mode, t->md);
 405		if (r)
 406			goto unlock_ret_r;
 407	}
 408	refcount_inc(&dd->count);
 409out:
 410	up_write(&t->devices_lock);
 411	*result = dd->dm_dev;
 412	return 0;
 413
 414unlock_ret_r:
 415	up_write(&t->devices_lock);
 416	return r;
 417}
 418EXPORT_SYMBOL(dm_get_device);
 419
 420static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
 421				sector_t start, sector_t len, void *data)
 422{
 423	struct queue_limits *limits = data;
 424	struct block_device *bdev = dev->bdev;
 425	struct request_queue *q = bdev_get_queue(bdev);
 426
 427	if (unlikely(!q)) {
 428		DMWARN("%s: Cannot set limits for nonexistent device %pg",
 429		       dm_device_name(ti->table->md), bdev);
 430		return 0;
 431	}
 432
 433	mutex_lock(&q->limits_lock);
 434	/*
 435	 * BLK_FEAT_ATOMIC_WRITES is not inherited from the bottom device in
 436	 * blk_stack_limits(), so do it manually.
 437	 */
 438	limits->features |= (q->limits.features & BLK_FEAT_ATOMIC_WRITES);
 439
 440	if (blk_stack_limits(limits, &q->limits,
 441			get_start_sect(bdev) + start) < 0)
 442		DMWARN("%s: adding target device %pg caused an alignment inconsistency: "
 443		       "physical_block_size=%u, logical_block_size=%u, "
 444		       "alignment_offset=%u, start=%llu",
 445		       dm_device_name(ti->table->md), bdev,
 446		       q->limits.physical_block_size,
 447		       q->limits.logical_block_size,
 448		       q->limits.alignment_offset,
 449		       (unsigned long long) start << SECTOR_SHIFT);
 450
 451	/*
 452	 * Only stack the integrity profile if the target doesn't have native
 453	 * integrity support.
 454	 */
 455	if (!dm_target_has_integrity(ti->type))
 456		queue_limits_stack_integrity_bdev(limits, bdev);
 457	mutex_unlock(&q->limits_lock);
 458	return 0;
 459}
 460
 461/*
 462 * Decrement a device's use count and remove it if necessary.
 463 */
 464void dm_put_device(struct dm_target *ti, struct dm_dev *d)
 465{
 466	int found = 0;
 467	struct dm_table *t = ti->table;
 468	struct list_head *devices = &t->devices;
 469	struct dm_dev_internal *dd;
 470
 471	down_write(&t->devices_lock);
 472
 473	list_for_each_entry(dd, devices, list) {
 474		if (dd->dm_dev == d) {
 475			found = 1;
 476			break;
 477		}
 478	}
 479	if (!found) {
 480		DMERR("%s: device %s not in table devices list",
 481		      dm_device_name(t->md), d->name);
 482		goto unlock_ret;
 483	}
 484	if (refcount_dec_and_test(&dd->count)) {
 485		dm_put_table_device(t->md, d);
 486		list_del(&dd->list);
 487		kfree(dd);
 488	}
 489
 490unlock_ret:
 491	up_write(&t->devices_lock);
 492}
 493EXPORT_SYMBOL(dm_put_device);
 494
 495/*
 496 * Checks to see if the target joins onto the end of the table.
 497 */
 498static int adjoin(struct dm_table *t, struct dm_target *ti)
 499{
 500	struct dm_target *prev;
 501
 502	if (!t->num_targets)
 503		return !ti->begin;
 504
 505	prev = &t->targets[t->num_targets - 1];
 506	return (ti->begin == (prev->begin + prev->len));
 507}
 508
 509/*
 510 * Used to dynamically allocate the arg array.
 511 *
 512 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
 513 * process messages even if some device is suspended. These messages have a
 514 * small fixed number of arguments.
 515 *
 516 * On the other hand, dm-switch needs to process bulk data using messages and
 517 * excessive use of GFP_NOIO could cause trouble.
 518 */
 519static char **realloc_argv(unsigned int *size, char **old_argv)
 520{
 521	char **argv;
 522	unsigned int new_size;
 523	gfp_t gfp;
 524
 525	if (*size) {
 526		new_size = *size * 2;
 527		gfp = GFP_KERNEL;
 528	} else {
 529		new_size = 8;
 530		gfp = GFP_NOIO;
 531	}
 532	argv = kmalloc_array(new_size, sizeof(*argv), gfp);
 533	if (argv) {
 534		if (old_argv)
 535			memcpy(argv, old_argv, *size * sizeof(*argv));
 536		*size = new_size;
 537	}
 538
 539	kfree(old_argv);
 540	return argv;
 541}
 542
 543/*
 544 * Destructively splits up the argument list to pass to ctr.
 545 */
 546int dm_split_args(int *argc, char ***argvp, char *input)
 547{
 548	char *start, *end = input, *out, **argv = NULL;
 549	unsigned int array_size = 0;
 550
 551	*argc = 0;
 552
 553	if (!input) {
 554		*argvp = NULL;
 555		return 0;
 556	}
 557
 558	argv = realloc_argv(&array_size, argv);
 559	if (!argv)
 560		return -ENOMEM;
 561
 562	while (1) {
 563		/* Skip whitespace */
 564		start = skip_spaces(end);
 565
 566		if (!*start)
 567			break;	/* success, we hit the end */
 568
 569		/* 'out' is used to remove any back-quotes */
 570		end = out = start;
 571		while (*end) {
 572			/* Everything apart from '\0' can be quoted */
 573			if (*end == '\\' && *(end + 1)) {
 574				*out++ = *(end + 1);
 575				end += 2;
 576				continue;
 577			}
 578
 579			if (isspace(*end))
 580				break;	/* end of token */
 581
 582			*out++ = *end++;
 583		}
 584
 585		/* have we already filled the array ? */
 586		if ((*argc + 1) > array_size) {
 587			argv = realloc_argv(&array_size, argv);
 588			if (!argv)
 589				return -ENOMEM;
 590		}
 591
 592		/* we know this is whitespace */
 593		if (*end)
 594			end++;
 595
 596		/* terminate the string and put it in the array */
 597		*out = '\0';
 598		argv[*argc] = start;
 599		(*argc)++;
 600	}
 601
 602	*argvp = argv;
 603	return 0;
 604}
 605
 606static void dm_set_stacking_limits(struct queue_limits *limits)
 607{
 608	blk_set_stacking_limits(limits);
 609	limits->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL;
 610}
 611
 612/*
 613 * Impose necessary and sufficient conditions on a devices's table such
 614 * that any incoming bio which respects its logical_block_size can be
 615 * processed successfully.  If it falls across the boundary between
 616 * two or more targets, the size of each piece it gets split into must
 617 * be compatible with the logical_block_size of the target processing it.
 618 */
 619static int validate_hardware_logical_block_alignment(struct dm_table *t,
 620						     struct queue_limits *limits)
 621{
 622	/*
 623	 * This function uses arithmetic modulo the logical_block_size
 624	 * (in units of 512-byte sectors).
 625	 */
 626	unsigned short device_logical_block_size_sects =
 627		limits->logical_block_size >> SECTOR_SHIFT;
 628
 629	/*
 630	 * Offset of the start of the next table entry, mod logical_block_size.
 631	 */
 632	unsigned short next_target_start = 0;
 633
 634	/*
 635	 * Given an aligned bio that extends beyond the end of a
 636	 * target, how many sectors must the next target handle?
 637	 */
 638	unsigned short remaining = 0;
 639
 640	struct dm_target *ti;
 641	struct queue_limits ti_limits;
 642	unsigned int i;
 643
 644	/*
 645	 * Check each entry in the table in turn.
 646	 */
 647	for (i = 0; i < t->num_targets; i++) {
 648		ti = dm_table_get_target(t, i);
 649
 650		dm_set_stacking_limits(&ti_limits);
 651
 652		/* combine all target devices' limits */
 653		if (ti->type->iterate_devices)
 654			ti->type->iterate_devices(ti, dm_set_device_limits,
 655						  &ti_limits);
 656
 657		/*
 658		 * If the remaining sectors fall entirely within this
 659		 * table entry are they compatible with its logical_block_size?
 660		 */
 661		if (remaining < ti->len &&
 662		    remaining & ((ti_limits.logical_block_size >>
 663				  SECTOR_SHIFT) - 1))
 664			break;	/* Error */
 665
 666		next_target_start =
 667		    (unsigned short) ((next_target_start + ti->len) &
 668				      (device_logical_block_size_sects - 1));
 669		remaining = next_target_start ?
 670		    device_logical_block_size_sects - next_target_start : 0;
 671	}
 672
 673	if (remaining) {
 674		DMERR("%s: table line %u (start sect %llu len %llu) "
 675		      "not aligned to h/w logical block size %u",
 676		      dm_device_name(t->md), i,
 677		      (unsigned long long) ti->begin,
 678		      (unsigned long long) ti->len,
 679		      limits->logical_block_size);
 680		return -EINVAL;
 681	}
 682
 683	return 0;
 684}
 685
 686int dm_table_add_target(struct dm_table *t, const char *type,
 687			sector_t start, sector_t len, char *params)
 688{
 689	int r = -EINVAL, argc;
 690	char **argv;
 691	struct dm_target *ti;
 692
 693	if (t->singleton) {
 694		DMERR("%s: target type %s must appear alone in table",
 695		      dm_device_name(t->md), t->targets->type->name);
 696		return -EINVAL;
 697	}
 698
 699	BUG_ON(t->num_targets >= t->num_allocated);
 700
 701	ti = t->targets + t->num_targets;
 702	memset(ti, 0, sizeof(*ti));
 703
 704	if (!len) {
 705		DMERR("%s: zero-length target", dm_device_name(t->md));
 706		return -EINVAL;
 707	}
 708	if (start + len < start || start + len > LLONG_MAX >> SECTOR_SHIFT) {
 709		DMERR("%s: too large device", dm_device_name(t->md));
 710		return -EINVAL;
 711	}
 712
 713	ti->type = dm_get_target_type(type);
 714	if (!ti->type) {
 715		DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
 716		return -EINVAL;
 717	}
 718
 719	if (dm_target_needs_singleton(ti->type)) {
 720		if (t->num_targets) {
 721			ti->error = "singleton target type must appear alone in table";
 722			goto bad;
 723		}
 724		t->singleton = true;
 725	}
 726
 727	if (dm_target_always_writeable(ti->type) &&
 728	    !(t->mode & BLK_OPEN_WRITE)) {
 729		ti->error = "target type may not be included in a read-only table";
 730		goto bad;
 731	}
 732
 733	if (t->immutable_target_type) {
 734		if (t->immutable_target_type != ti->type) {
 735			ti->error = "immutable target type cannot be mixed with other target types";
 736			goto bad;
 737		}
 738	} else if (dm_target_is_immutable(ti->type)) {
 739		if (t->num_targets) {
 740			ti->error = "immutable target type cannot be mixed with other target types";
 741			goto bad;
 742		}
 743		t->immutable_target_type = ti->type;
 744	}
 745
 746	ti->table = t;
 747	ti->begin = start;
 748	ti->len = len;
 749	ti->error = "Unknown error";
 750
 751	/*
 752	 * Does this target adjoin the previous one ?
 753	 */
 754	if (!adjoin(t, ti)) {
 755		ti->error = "Gap in table";
 756		goto bad;
 757	}
 758
 759	r = dm_split_args(&argc, &argv, params);
 760	if (r) {
 761		ti->error = "couldn't split parameters";
 762		goto bad;
 763	}
 764
 765	r = ti->type->ctr(ti, argc, argv);
 766	kfree(argv);
 767	if (r)
 768		goto bad;
 769
 770	t->highs[t->num_targets++] = ti->begin + ti->len - 1;
 771
 772	if (!ti->num_discard_bios && ti->discards_supported)
 773		DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
 774		       dm_device_name(t->md), type);
 775
 776	if (ti->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))
 777		static_branch_enable(&swap_bios_enabled);
 778
 779	if (!ti->flush_bypasses_map)
 780		t->flush_bypasses_map = false;
 781
 782	return 0;
 783
 784 bad:
 785	DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r));
 786	dm_put_target_type(ti->type);
 787	return r;
 788}
 789
 790/*
 791 * Target argument parsing helpers.
 792 */
 793static int validate_next_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 794			     unsigned int *value, char **error, unsigned int grouped)
 795{
 796	const char *arg_str = dm_shift_arg(arg_set);
 797	char dummy;
 798
 799	if (!arg_str ||
 800	    (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
 801	    (*value < arg->min) ||
 802	    (*value > arg->max) ||
 803	    (grouped && arg_set->argc < *value)) {
 804		*error = arg->error;
 805		return -EINVAL;
 806	}
 807
 808	return 0;
 809}
 810
 811int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 812		unsigned int *value, char **error)
 813{
 814	return validate_next_arg(arg, arg_set, value, error, 0);
 815}
 816EXPORT_SYMBOL(dm_read_arg);
 817
 818int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 819		      unsigned int *value, char **error)
 820{
 821	return validate_next_arg(arg, arg_set, value, error, 1);
 822}
 823EXPORT_SYMBOL(dm_read_arg_group);
 824
 825const char *dm_shift_arg(struct dm_arg_set *as)
 826{
 827	char *r;
 828
 829	if (as->argc) {
 830		as->argc--;
 831		r = *as->argv;
 832		as->argv++;
 833		return r;
 834	}
 835
 836	return NULL;
 837}
 838EXPORT_SYMBOL(dm_shift_arg);
 839
 840void dm_consume_args(struct dm_arg_set *as, unsigned int num_args)
 841{
 842	BUG_ON(as->argc < num_args);
 843	as->argc -= num_args;
 844	as->argv += num_args;
 845}
 846EXPORT_SYMBOL(dm_consume_args);
 847
 848static bool __table_type_bio_based(enum dm_queue_mode table_type)
 849{
 850	return (table_type == DM_TYPE_BIO_BASED ||
 851		table_type == DM_TYPE_DAX_BIO_BASED);
 852}
 853
 854static bool __table_type_request_based(enum dm_queue_mode table_type)
 855{
 856	return table_type == DM_TYPE_REQUEST_BASED;
 857}
 858
 859void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)
 860{
 861	t->type = type;
 862}
 863EXPORT_SYMBOL_GPL(dm_table_set_type);
 864
 865/* validate the dax capability of the target device span */
 866static int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev,
 867			sector_t start, sector_t len, void *data)
 868{
 869	if (dev->dax_dev)
 870		return false;
 871
 872	DMDEBUG("%pg: error: dax unsupported by block device", dev->bdev);
 873	return true;
 874}
 875
 876/* Check devices support synchronous DAX */
 877static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_dev *dev,
 878					      sector_t start, sector_t len, void *data)
 879{
 880	return !dev->dax_dev || !dax_synchronous(dev->dax_dev);
 881}
 882
 883static bool dm_table_supports_dax(struct dm_table *t,
 884				  iterate_devices_callout_fn iterate_fn)
 885{
 886	/* Ensure that all targets support DAX. */
 887	for (unsigned int i = 0; i < t->num_targets; i++) {
 888		struct dm_target *ti = dm_table_get_target(t, i);
 889
 890		if (!ti->type->direct_access)
 891			return false;
 892
 893		if (dm_target_is_wildcard(ti->type) ||
 894		    !ti->type->iterate_devices ||
 895		    ti->type->iterate_devices(ti, iterate_fn, NULL))
 896			return false;
 897	}
 898
 899	return true;
 900}
 901
 902static int device_is_not_rq_stackable(struct dm_target *ti, struct dm_dev *dev,
 903				      sector_t start, sector_t len, void *data)
 904{
 905	struct block_device *bdev = dev->bdev;
 906	struct request_queue *q = bdev_get_queue(bdev);
 907
 908	/* request-based cannot stack on partitions! */
 909	if (bdev_is_partition(bdev))
 910		return true;
 911
 912	return !queue_is_mq(q);
 913}
 914
 915static int dm_table_determine_type(struct dm_table *t)
 916{
 917	unsigned int bio_based = 0, request_based = 0, hybrid = 0;
 918	struct dm_target *ti;
 919	struct list_head *devices = dm_table_get_devices(t);
 920	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
 921
 922	if (t->type != DM_TYPE_NONE) {
 923		/* target already set the table's type */
 924		if (t->type == DM_TYPE_BIO_BASED) {
 925			/* possibly upgrade to a variant of bio-based */
 926			goto verify_bio_based;
 927		}
 928		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
 929		goto verify_rq_based;
 930	}
 931
 932	for (unsigned int i = 0; i < t->num_targets; i++) {
 933		ti = dm_table_get_target(t, i);
 934		if (dm_target_hybrid(ti))
 935			hybrid = 1;
 936		else if (dm_target_request_based(ti))
 937			request_based = 1;
 938		else
 939			bio_based = 1;
 940
 941		if (bio_based && request_based) {
 942			DMERR("Inconsistent table: different target types can't be mixed up");
 943			return -EINVAL;
 944		}
 945	}
 946
 947	if (hybrid && !bio_based && !request_based) {
 948		/*
 949		 * The targets can work either way.
 950		 * Determine the type from the live device.
 951		 * Default to bio-based if device is new.
 952		 */
 953		if (__table_type_request_based(live_md_type))
 954			request_based = 1;
 955		else
 956			bio_based = 1;
 957	}
 958
 959	if (bio_based) {
 960verify_bio_based:
 961		/* We must use this table as bio-based */
 962		t->type = DM_TYPE_BIO_BASED;
 963		if (dm_table_supports_dax(t, device_not_dax_capable) ||
 964		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
 965			t->type = DM_TYPE_DAX_BIO_BASED;
 966		}
 967		return 0;
 968	}
 969
 970	BUG_ON(!request_based); /* No targets in this table */
 971
 972	t->type = DM_TYPE_REQUEST_BASED;
 973
 974verify_rq_based:
 975	/*
 976	 * Request-based dm supports only tables that have a single target now.
 977	 * To support multiple targets, request splitting support is needed,
 978	 * and that needs lots of changes in the block-layer.
 979	 * (e.g. request completion process for partial completion.)
 980	 */
 981	if (t->num_targets > 1) {
 982		DMERR("request-based DM doesn't support multiple targets");
 983		return -EINVAL;
 984	}
 985
 986	if (list_empty(devices)) {
 987		int srcu_idx;
 988		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);
 989
 990		/* inherit live table's type */
 991		if (live_table)
 992			t->type = live_table->type;
 993		dm_put_live_table(t->md, srcu_idx);
 994		return 0;
 995	}
 996
 997	ti = dm_table_get_immutable_target(t);
 998	if (!ti) {
 999		DMERR("table load rejected: immutable target is required");
1000		return -EINVAL;
1001	} else if (ti->max_io_len) {
1002		DMERR("table load rejected: immutable target that splits IO is not supported");
1003		return -EINVAL;
1004	}
1005
1006	/* Non-request-stackable devices can't be used for request-based dm */
1007	if (!ti->type->iterate_devices ||
1008	    ti->type->iterate_devices(ti, device_is_not_rq_stackable, NULL)) {
1009		DMERR("table load rejected: including non-request-stackable devices");
1010		return -EINVAL;
1011	}
1012
1013	return 0;
1014}
1015
1016enum dm_queue_mode dm_table_get_type(struct dm_table *t)
1017{
1018	return t->type;
1019}
1020
1021struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
1022{
1023	return t->immutable_target_type;
1024}
1025
1026struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
1027{
1028	/* Immutable target is implicitly a singleton */
1029	if (t->num_targets > 1 ||
1030	    !dm_target_is_immutable(t->targets[0].type))
1031		return NULL;
1032
1033	return t->targets;
1034}
1035
1036struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
1037{
1038	for (unsigned int i = 0; i < t->num_targets; i++) {
1039		struct dm_target *ti = dm_table_get_target(t, i);
1040
1041		if (dm_target_is_wildcard(ti->type))
1042			return ti;
1043	}
1044
1045	return NULL;
1046}
1047
1048bool dm_table_request_based(struct dm_table *t)
1049{
1050	return __table_type_request_based(dm_table_get_type(t));
1051}
1052
1053static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
1054{
1055	enum dm_queue_mode type = dm_table_get_type(t);
1056	unsigned int per_io_data_size = 0, front_pad, io_front_pad;
1057	unsigned int min_pool_size = 0, pool_size;
1058	struct dm_md_mempools *pools;
1059	unsigned int bioset_flags = 0;
1060
1061	if (unlikely(type == DM_TYPE_NONE)) {
1062		DMERR("no table type is set, can't allocate mempools");
1063		return -EINVAL;
1064	}
1065
1066	pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
1067	if (!pools)
1068		return -ENOMEM;
1069
1070	if (type == DM_TYPE_REQUEST_BASED) {
1071		pool_size = dm_get_reserved_rq_based_ios();
1072		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
1073		goto init_bs;
1074	}
1075
1076	if (md->queue->limits.features & BLK_FEAT_POLL)
1077		bioset_flags |= BIOSET_PERCPU_CACHE;
1078
1079	for (unsigned int i = 0; i < t->num_targets; i++) {
1080		struct dm_target *ti = dm_table_get_target(t, i);
1081
1082		per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
1083		min_pool_size = max(min_pool_size, ti->num_flush_bios);
1084	}
1085	pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
1086	front_pad = roundup(per_io_data_size,
1087		__alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
1088
1089	io_front_pad = roundup(per_io_data_size,
1090		__alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
1091	if (bioset_init(&pools->io_bs, pool_size, io_front_pad, bioset_flags))
1092		goto out_free_pools;
1093init_bs:
1094	if (bioset_init(&pools->bs, pool_size, front_pad, 0))
1095		goto out_free_pools;
1096
1097	t->mempools = pools;
1098	return 0;
1099
1100out_free_pools:
1101	dm_free_md_mempools(pools);
1102	return -ENOMEM;
1103}
1104
1105static int setup_indexes(struct dm_table *t)
1106{
1107	int i;
1108	unsigned int total = 0;
1109	sector_t *indexes;
1110
1111	/* allocate the space for *all* the indexes */
1112	for (i = t->depth - 2; i >= 0; i--) {
1113		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
1114		total += t->counts[i];
1115	}
1116
1117	indexes = kvcalloc(total, NODE_SIZE, GFP_KERNEL);
1118	if (!indexes)
1119		return -ENOMEM;
1120
1121	/* set up internal nodes, bottom-up */
1122	for (i = t->depth - 2; i >= 0; i--) {
1123		t->index[i] = indexes;
1124		indexes += (KEYS_PER_NODE * t->counts[i]);
1125		setup_btree_index(i, t);
1126	}
1127
1128	return 0;
1129}
1130
1131/*
1132 * Builds the btree to index the map.
1133 */
1134static int dm_table_build_index(struct dm_table *t)
1135{
1136	int r = 0;
1137	unsigned int leaf_nodes;
1138
1139	/* how many indexes will the btree have ? */
1140	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
1141	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
1142
1143	/* leaf layer has already been set up */
1144	t->counts[t->depth - 1] = leaf_nodes;
1145	t->index[t->depth - 1] = t->highs;
1146
1147	if (t->depth >= 2)
1148		r = setup_indexes(t);
1149
1150	return r;
1151}
1152
1153#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1154
1155struct dm_crypto_profile {
1156	struct blk_crypto_profile profile;
1157	struct mapped_device *md;
1158};
1159
1160static int dm_keyslot_evict_callback(struct dm_target *ti, struct dm_dev *dev,
1161				     sector_t start, sector_t len, void *data)
1162{
1163	const struct blk_crypto_key *key = data;
1164
1165	blk_crypto_evict_key(dev->bdev, key);
1166	return 0;
1167}
1168
1169/*
1170 * When an inline encryption key is evicted from a device-mapper device, evict
1171 * it from all the underlying devices.
1172 */
1173static int dm_keyslot_evict(struct blk_crypto_profile *profile,
1174			    const struct blk_crypto_key *key, unsigned int slot)
1175{
1176	struct mapped_device *md =
1177		container_of(profile, struct dm_crypto_profile, profile)->md;
1178	struct dm_table *t;
1179	int srcu_idx;
1180
1181	t = dm_get_live_table(md, &srcu_idx);
1182	if (!t)
1183		goto put_live_table;
1184
1185	for (unsigned int i = 0; i < t->num_targets; i++) {
1186		struct dm_target *ti = dm_table_get_target(t, i);
1187
1188		if (!ti->type->iterate_devices)
1189			continue;
1190		ti->type->iterate_devices(ti, dm_keyslot_evict_callback,
1191					  (void *)key);
1192	}
1193
1194put_live_table:
1195	dm_put_live_table(md, srcu_idx);
1196	return 0;
1197}
1198
1199enum dm_wrappedkey_op {
1200	DERIVE_SW_SECRET,
1201	IMPORT_KEY,
1202	GENERATE_KEY,
1203	PREPARE_KEY,
1204};
1205
1206struct dm_wrappedkey_op_args {
1207	enum dm_wrappedkey_op op;
1208	int err;
1209	union {
1210		struct {
1211			const u8 *eph_key;
1212			size_t eph_key_size;
1213			u8 *sw_secret;
1214		} derive_sw_secret;
1215		struct {
1216			const u8 *raw_key;
1217			size_t raw_key_size;
1218			u8 *lt_key;
1219		} import_key;
1220		struct {
1221			u8 *lt_key;
1222		} generate_key;
1223		struct {
1224			const u8 *lt_key;
1225			size_t lt_key_size;
1226			u8 *eph_key;
1227		} prepare_key;
1228	};
1229};
1230
1231static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev,
1232				     sector_t start, sector_t len, void *data)
1233{
1234	struct dm_wrappedkey_op_args *args = data;
1235	struct block_device *bdev = dev->bdev;
1236	struct blk_crypto_profile *profile =
1237		bdev_get_queue(bdev)->crypto_profile;
1238	int err = -EOPNOTSUPP;
1239
1240	switch (args->op) {
1241	case DERIVE_SW_SECRET:
1242		err = blk_crypto_derive_sw_secret(
1243					bdev,
1244					args->derive_sw_secret.eph_key,
1245					args->derive_sw_secret.eph_key_size,
1246					args->derive_sw_secret.sw_secret);
1247		break;
1248	case IMPORT_KEY:
1249		err = blk_crypto_import_key(profile,
1250					    args->import_key.raw_key,
1251					    args->import_key.raw_key_size,
1252					    args->import_key.lt_key);
1253		break;
1254	case GENERATE_KEY:
1255		err = blk_crypto_generate_key(profile,
1256					      args->generate_key.lt_key);
1257		break;
1258	case PREPARE_KEY:
1259		err = blk_crypto_prepare_key(profile,
1260					     args->prepare_key.lt_key,
1261					     args->prepare_key.lt_key_size,
1262					     args->prepare_key.eph_key);
1263		break;
1264	}
1265	args->err = err;
1266	return 1; /* No need to continue the iteration. */
1267}
1268
1269static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile,
1270				 struct dm_wrappedkey_op_args *args)
1271{
1272	struct mapped_device *md =
1273		container_of(profile, struct dm_crypto_profile, profile)->md;
1274	struct dm_target *ti;
1275	struct dm_table *t;
1276	int srcu_idx;
1277	int i;
1278
1279	args->err = -EOPNOTSUPP;
1280
1281	t = dm_get_live_table(md, &srcu_idx);
1282	if (!t)
1283		goto out;
1284
1285	/*
1286	 * blk-crypto currently has no support for multiple incompatible
1287	 * implementations of wrapped inline crypto keys on a single system.
1288	 * It was already checked earlier that support for wrapped keys was
1289	 * declared on all underlying devices.  Thus, all the underlying devices
1290	 * should support all wrapped key operations and they should behave
1291	 * identically, i.e. work with the same keys.  So, just executing the
1292	 * operation on the first device suffices for now.
1293	 */
1294	for (i = 0; i < t->num_targets; i++) {
1295		ti = dm_table_get_target(t, i);
1296		if (!ti->type->iterate_devices)
1297			continue;
1298		if (ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args) != 0)
1299			break;
1300	}
1301out:
1302	dm_put_live_table(md, srcu_idx);
1303	return args->err;
1304}
1305
1306static int dm_derive_sw_secret(struct blk_crypto_profile *profile,
1307			       const u8 *eph_key, size_t eph_key_size,
1308			       u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
1309{
1310	struct dm_wrappedkey_op_args args = {
1311		.op = DERIVE_SW_SECRET,
1312		.derive_sw_secret = {
1313			.eph_key = eph_key,
1314			.eph_key_size = eph_key_size,
1315			.sw_secret = sw_secret,
1316		},
1317	};
1318	return dm_exec_wrappedkey_op(profile, &args);
1319}
1320
1321static int dm_import_key(struct blk_crypto_profile *profile,
1322			 const u8 *raw_key, size_t raw_key_size,
1323			 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
1324{
1325	struct dm_wrappedkey_op_args args = {
1326		.op = IMPORT_KEY,
1327		.import_key = {
1328			.raw_key = raw_key,
1329			.raw_key_size = raw_key_size,
1330			.lt_key = lt_key,
1331		},
1332	};
1333	return dm_exec_wrappedkey_op(profile, &args);
1334}
1335
1336static int dm_generate_key(struct blk_crypto_profile *profile,
1337			   u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
1338{
1339	struct dm_wrappedkey_op_args args = {
1340		.op = GENERATE_KEY,
1341		.generate_key = {
1342			.lt_key = lt_key,
1343		},
1344	};
1345	return dm_exec_wrappedkey_op(profile, &args);
1346}
1347
1348static int dm_prepare_key(struct blk_crypto_profile *profile,
1349			  const u8 *lt_key, size_t lt_key_size,
1350			  u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
1351{
1352	struct dm_wrappedkey_op_args args = {
1353		.op = PREPARE_KEY,
1354		.prepare_key = {
1355			.lt_key = lt_key,
1356			.lt_key_size = lt_key_size,
1357			.eph_key = eph_key,
1358		},
1359	};
1360	return dm_exec_wrappedkey_op(profile, &args);
1361}
1362
1363static int
1364device_intersect_crypto_capabilities(struct dm_target *ti, struct dm_dev *dev,
1365				     sector_t start, sector_t len, void *data)
1366{
1367	struct blk_crypto_profile *parent = data;
1368	struct blk_crypto_profile *child =
1369		bdev_get_queue(dev->bdev)->crypto_profile;
1370
1371	blk_crypto_intersect_capabilities(parent, child);
1372	return 0;
1373}
1374
1375void dm_destroy_crypto_profile(struct blk_crypto_profile *profile)
1376{
1377	struct dm_crypto_profile *dmcp = container_of(profile,
1378						      struct dm_crypto_profile,
1379						      profile);
1380
1381	if (!profile)
1382		return;
1383
1384	blk_crypto_profile_destroy(profile);
1385	kfree(dmcp);
1386}
1387
1388static void dm_table_destroy_crypto_profile(struct dm_table *t)
1389{
1390	dm_destroy_crypto_profile(t->crypto_profile);
1391	t->crypto_profile = NULL;
1392}
1393
1394/*
1395 * Constructs and initializes t->crypto_profile with a crypto profile that
1396 * represents the common set of crypto capabilities of the devices described by
1397 * the dm_table.  However, if the constructed crypto profile doesn't support all
1398 * crypto capabilities that are supported by the current mapped_device, it
1399 * returns an error instead, since we don't support removing crypto capabilities
1400 * on table changes.  Finally, if the constructed crypto profile is "empty" (has
1401 * no crypto capabilities at all), it just sets t->crypto_profile to NULL.
1402 */
1403static int dm_table_construct_crypto_profile(struct dm_table *t)
1404{
1405	struct dm_crypto_profile *dmcp;
1406	struct blk_crypto_profile *profile;
1407	unsigned int i;
1408	bool empty_profile = true;
1409
1410	dmcp = kmalloc(sizeof(*dmcp), GFP_KERNEL);
1411	if (!dmcp)
1412		return -ENOMEM;
1413	dmcp->md = t->md;
1414
1415	profile = &dmcp->profile;
1416	blk_crypto_profile_init(profile, 0);
1417	profile->ll_ops.keyslot_evict = dm_keyslot_evict;
1418	profile->max_dun_bytes_supported = UINT_MAX;
1419	memset(profile->modes_supported, 0xFF,
1420	       sizeof(profile->modes_supported));
1421	profile->key_types_supported = ~0;
1422
1423	for (i = 0; i < t->num_targets; i++) {
1424		struct dm_target *ti = dm_table_get_target(t, i);
1425
1426		if (!dm_target_passes_crypto(ti->type)) {
1427			blk_crypto_intersect_capabilities(profile, NULL);
1428			break;
1429		}
1430		if (!ti->type->iterate_devices)
1431			continue;
1432		ti->type->iterate_devices(ti,
1433					  device_intersect_crypto_capabilities,
1434					  profile);
1435	}
1436
1437	if (profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED) {
1438		profile->ll_ops.derive_sw_secret = dm_derive_sw_secret;
1439		profile->ll_ops.import_key = dm_import_key;
1440		profile->ll_ops.generate_key = dm_generate_key;
1441		profile->ll_ops.prepare_key = dm_prepare_key;
1442	}
1443
1444	if (t->md->queue &&
1445	    !blk_crypto_has_capabilities(profile,
1446					 t->md->queue->crypto_profile)) {
1447		DMERR("Inline encryption capabilities of new DM table were more restrictive than the old table's. This is not supported!");
1448		dm_destroy_crypto_profile(profile);
1449		return -EINVAL;
1450	}
1451
1452	/*
1453	 * If the new profile doesn't actually support any crypto capabilities,
1454	 * we may as well represent it with a NULL profile.
1455	 */
1456	for (i = 0; i < ARRAY_SIZE(profile->modes_supported); i++) {
1457		if (profile->modes_supported[i]) {
1458			empty_profile = false;
1459			break;
1460		}
1461	}
1462
1463	if (empty_profile) {
1464		dm_destroy_crypto_profile(profile);
1465		profile = NULL;
1466	}
1467
1468	/*
1469	 * t->crypto_profile is only set temporarily while the table is being
1470	 * set up, and it gets set to NULL after the profile has been
1471	 * transferred to the request_queue.
1472	 */
1473	t->crypto_profile = profile;
1474
1475	return 0;
1476}
1477
1478static void dm_update_crypto_profile(struct request_queue *q,
1479				     struct dm_table *t)
1480{
1481	if (!t->crypto_profile)
1482		return;
1483
1484	/* Make the crypto profile less restrictive. */
1485	if (!q->crypto_profile) {
1486		blk_crypto_register(t->crypto_profile, q);
1487	} else {
1488		blk_crypto_update_capabilities(q->crypto_profile,
1489					       t->crypto_profile);
1490		dm_destroy_crypto_profile(t->crypto_profile);
1491	}
1492	t->crypto_profile = NULL;
1493}
1494
1495#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1496
1497static int dm_table_construct_crypto_profile(struct dm_table *t)
1498{
1499	return 0;
1500}
1501
1502void dm_destroy_crypto_profile(struct blk_crypto_profile *profile)
1503{
1504}
1505
1506static void dm_table_destroy_crypto_profile(struct dm_table *t)
1507{
1508}
1509
1510static void dm_update_crypto_profile(struct request_queue *q,
1511				     struct dm_table *t)
1512{
1513}
1514
1515#endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1516
1517/*
1518 * Prepares the table for use by building the indices,
1519 * setting the type, and allocating mempools.
1520 */
1521int dm_table_complete(struct dm_table *t)
1522{
1523	int r;
1524
1525	r = dm_table_determine_type(t);
1526	if (r) {
1527		DMERR("unable to determine table type");
1528		return r;
1529	}
1530
1531	r = dm_table_build_index(t);
1532	if (r) {
1533		DMERR("unable to build btrees");
1534		return r;
1535	}
1536
1537	r = dm_table_construct_crypto_profile(t);
1538	if (r) {
1539		DMERR("could not construct crypto profile.");
1540		return r;
1541	}
1542
1543	r = dm_table_alloc_md_mempools(t, t->md);
1544	if (r)
1545		DMERR("unable to allocate mempools");
1546
1547	return r;
1548}
1549
1550static DEFINE_MUTEX(_event_lock);
1551void dm_table_event_callback(struct dm_table *t,
1552			     void (*fn)(void *), void *context)
1553{
1554	mutex_lock(&_event_lock);
1555	t->event_fn = fn;
1556	t->event_context = context;
1557	mutex_unlock(&_event_lock);
1558}
1559
1560void dm_table_event(struct dm_table *t)
1561{
1562	mutex_lock(&_event_lock);
1563	if (t->event_fn)
1564		t->event_fn(t->event_context);
1565	mutex_unlock(&_event_lock);
1566}
1567EXPORT_SYMBOL(dm_table_event);
1568
1569inline sector_t dm_table_get_size(struct dm_table *t)
1570{
1571	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
1572}
1573EXPORT_SYMBOL(dm_table_get_size);
1574
1575/*
1576 * Search the btree for the correct target.
1577 *
1578 * Caller should check returned pointer for NULL
1579 * to trap I/O beyond end of device.
1580 */
1581struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
1582{
1583	unsigned int l, n = 0, k = 0;
1584	sector_t *node;
1585
1586	if (unlikely(sector >= dm_table_get_size(t)))
1587		return NULL;
1588
1589	for (l = 0; l < t->depth; l++) {
1590		n = get_child(n, k);
1591		node = get_node(t, l, n);
1592
1593		for (k = 0; k < KEYS_PER_NODE; k++)
1594			if (node[k] >= sector)
1595				break;
1596	}
1597
1598	return &t->targets[(KEYS_PER_NODE * n) + k];
1599}
1600
1601/*
1602 * type->iterate_devices() should be called when the sanity check needs to
1603 * iterate and check all underlying data devices. iterate_devices() will
1604 * iterate all underlying data devices until it encounters a non-zero return
1605 * code, returned by whether the input iterate_devices_callout_fn, or
1606 * iterate_devices() itself internally.
1607 *
1608 * For some target type (e.g. dm-stripe), one call of iterate_devices() may
1609 * iterate multiple underlying devices internally, in which case a non-zero
1610 * return code returned by iterate_devices_callout_fn will stop the iteration
1611 * in advance.
1612 *
1613 * Cases requiring _any_ underlying device supporting some kind of attribute,
1614 * should use the iteration structure like dm_table_any_dev_attr(), or call
1615 * it directly. @func should handle semantics of positive examples, e.g.
1616 * capable of something.
1617 *
1618 * Cases requiring _all_ underlying devices supporting some kind of attribute,
1619 * should use the iteration structure like dm_table_supports_nowait() or
1620 * dm_table_supports_discards(). Or introduce dm_table_all_devs_attr() that
1621 * uses an @anti_func that handle semantics of counter examples, e.g. not
1622 * capable of something. So: return !dm_table_any_dev_attr(t, anti_func, data);
1623 */
1624static bool dm_table_any_dev_attr(struct dm_table *t,
1625				  iterate_devices_callout_fn func, void *data)
1626{
1627	for (unsigned int i = 0; i < t->num_targets; i++) {
1628		struct dm_target *ti = dm_table_get_target(t, i);
1629
1630		if (ti->type->iterate_devices &&
1631		    ti->type->iterate_devices(ti, func, data))
1632			return true;
1633	}
1634
1635	return false;
1636}
1637
1638static int count_device(struct dm_target *ti, struct dm_dev *dev,
1639			sector_t start, sector_t len, void *data)
1640{
1641	unsigned int *num_devices = data;
1642
1643	(*num_devices)++;
1644
1645	return 0;
1646}
1647
1648/*
1649 * Check whether a table has no data devices attached using each
1650 * target's iterate_devices method.
1651 * Returns false if the result is unknown because a target doesn't
1652 * support iterate_devices.
1653 */
1654bool dm_table_has_no_data_devices(struct dm_table *t)
1655{
1656	for (unsigned int i = 0; i < t->num_targets; i++) {
1657		struct dm_target *ti = dm_table_get_target(t, i);
1658		unsigned int num_devices = 0;
1659
1660		if (!ti->type->iterate_devices)
1661			return false;
1662
1663		ti->type->iterate_devices(ti, count_device, &num_devices);
1664		if (num_devices)
1665			return false;
1666	}
1667
1668	return true;
1669}
1670
1671bool dm_table_is_wildcard(struct dm_table *t)
1672{
1673	for (unsigned int i = 0; i < t->num_targets; i++) {
1674		struct dm_target *ti = dm_table_get_target(t, i);
1675
1676		if (!dm_target_is_wildcard(ti->type))
1677			return false;
1678	}
1679
1680	return true;
1681}
1682
1683static int device_not_zoned(struct dm_target *ti, struct dm_dev *dev,
1684			    sector_t start, sector_t len, void *data)
1685{
1686	bool *zoned = data;
1687
1688	return bdev_is_zoned(dev->bdev) != *zoned;
1689}
1690
1691static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev,
1692				 sector_t start, sector_t len, void *data)
1693{
1694	return bdev_is_zoned(dev->bdev);
1695}
1696
1697/*
1698 * Check the device zoned model based on the target feature flag. If the target
1699 * has the DM_TARGET_ZONED_HM feature flag set, host-managed zoned devices are
1700 * also accepted but all devices must have the same zoned model. If the target
1701 * has the DM_TARGET_MIXED_ZONED_MODEL feature set, the devices can have any
1702 * zoned model with all zoned devices having the same zone size.
1703 */
1704static bool dm_table_supports_zoned(struct dm_table *t, bool zoned)
1705{
1706	for (unsigned int i = 0; i < t->num_targets; i++) {
1707		struct dm_target *ti = dm_table_get_target(t, i);
1708
1709		/*
1710		 * For the wildcard target (dm-error), if we do not have a
1711		 * backing device, we must always return false. If we have a
1712		 * backing device, the result must depend on checking zoned
1713		 * model, like for any other target. So for this, check directly
1714		 * if the target backing device is zoned as we get "false" when
1715		 * dm-error was set without a backing device.
1716		 */
1717		if (dm_target_is_wildcard(ti->type) &&
1718		    !ti->type->iterate_devices(ti, device_is_zoned_model, NULL))
1719			return false;
1720
1721		if (dm_target_supports_zoned_hm(ti->type)) {
1722			if (!ti->type->iterate_devices ||
1723			    ti->type->iterate_devices(ti, device_not_zoned,
1724						      &zoned))
1725				return false;
1726		} else if (!dm_target_supports_mixed_zoned_model(ti->type)) {
1727			if (zoned)
1728				return false;
1729		}
1730	}
1731
1732	return true;
1733}
1734
1735static int device_not_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
1736					   sector_t start, sector_t len, void *data)
1737{
1738	unsigned int *zone_sectors = data;
1739
1740	if (!bdev_is_zoned(dev->bdev))
1741		return 0;
1742	return bdev_zone_sectors(dev->bdev) != *zone_sectors;
1743}
1744
1745/*
1746 * Check consistency of zoned model and zone sectors across all targets. For
1747 * zone sectors, if the destination device is a zoned block device, it shall
1748 * have the specified zone_sectors.
1749 */
1750static int validate_hardware_zoned(struct dm_table *t, bool zoned,
1751				   unsigned int zone_sectors)
1752{
1753	if (!zoned)
1754		return 0;
1755
1756	if (!dm_table_supports_zoned(t, zoned)) {
1757		DMERR("%s: zoned model is not consistent across all devices",
1758		      dm_device_name(t->md));
1759		return -EINVAL;
1760	}
1761
1762	/* Check zone size validity and compatibility */
1763	if (!zone_sectors || !is_power_of_2(zone_sectors))
1764		return -EINVAL;
1765
1766	if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) {
1767		DMERR("%s: zone sectors is not consistent across all zoned devices",
1768		      dm_device_name(t->md));
1769		return -EINVAL;
1770	}
1771
1772	return 0;
1773}
1774
1775/*
1776 * Establish the new table's queue_limits and validate them.
1777 */
1778int dm_calculate_queue_limits(struct dm_table *t,
1779			      struct queue_limits *limits)
1780{
1781	struct queue_limits ti_limits;
1782	unsigned int zone_sectors = 0;
1783	bool zoned = false;
1784
1785	dm_set_stacking_limits(limits);
1786
1787	t->integrity_supported = true;
1788	for (unsigned int i = 0; i < t->num_targets; i++) {
1789		struct dm_target *ti = dm_table_get_target(t, i);
1790
1791		if (!dm_target_passes_integrity(ti->type))
1792			t->integrity_supported = false;
1793	}
1794
1795	for (unsigned int i = 0; i < t->num_targets; i++) {
1796		struct dm_target *ti = dm_table_get_target(t, i);
1797
1798		dm_set_stacking_limits(&ti_limits);
1799
1800		if (!ti->type->iterate_devices) {
1801			/* Set I/O hints portion of queue limits */
1802			if (ti->type->io_hints)
1803				ti->type->io_hints(ti, &ti_limits);
1804			goto combine_limits;
1805		}
1806
1807		/*
1808		 * Combine queue limits of all the devices this target uses.
1809		 */
1810		ti->type->iterate_devices(ti, dm_set_device_limits,
1811					  &ti_limits);
1812
1813		if (!zoned && (ti_limits.features & BLK_FEAT_ZONED)) {
1814			/*
1815			 * After stacking all limits, validate all devices
1816			 * in table support this zoned model and zone sectors.
1817			 */
1818			zoned = (ti_limits.features & BLK_FEAT_ZONED);
1819			zone_sectors = ti_limits.chunk_sectors;
1820		}
1821
1822		/* Set I/O hints portion of queue limits */
1823		if (ti->type->io_hints)
1824			ti->type->io_hints(ti, &ti_limits);
1825
1826		/*
1827		 * Check each device area is consistent with the target's
1828		 * overall queue limits.
1829		 */
1830		if (ti->type->iterate_devices(ti, device_area_is_invalid,
1831					      &ti_limits))
1832			return -EINVAL;
1833
1834combine_limits:
1835		/*
1836		 * Merge this target's queue limits into the overall limits
1837		 * for the table.
1838		 */
1839		if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1840			DMWARN("%s: adding target device (start sect %llu len %llu) "
1841			       "caused an alignment inconsistency",
1842			       dm_device_name(t->md),
1843			       (unsigned long long) ti->begin,
1844			       (unsigned long long) ti->len);
1845
1846		if (t->integrity_supported ||
1847		    dm_target_has_integrity(ti->type)) {
1848			if (!queue_limits_stack_integrity(limits, &ti_limits)) {
1849				DMWARN("%s: adding target device (start sect %llu len %llu) "
1850				       "disabled integrity support due to incompatibility",
1851				       dm_device_name(t->md),
1852				       (unsigned long long) ti->begin,
1853				       (unsigned long long) ti->len);
1854				t->integrity_supported = false;
1855			}
1856		}
1857	}
1858
1859	/*
1860	 * Verify that the zoned model and zone sectors, as determined before
1861	 * any .io_hints override, are the same across all devices in the table.
1862	 * - this is especially relevant if .io_hints is emulating a disk-managed
1863	 *   zoned model on host-managed zoned block devices.
1864	 * BUT...
1865	 */
1866	if (limits->features & BLK_FEAT_ZONED) {
1867		/*
1868		 * ...IF the above limits stacking determined a zoned model
1869		 * validate that all of the table's devices conform to it.
1870		 */
1871		zoned = limits->features & BLK_FEAT_ZONED;
1872		zone_sectors = limits->chunk_sectors;
1873	}
1874	if (validate_hardware_zoned(t, zoned, zone_sectors))
1875		return -EINVAL;
1876
1877	return validate_hardware_logical_block_alignment(t, limits);
1878}
1879
1880/*
1881 * Check if a target requires flush support even if none of the underlying
1882 * devices need it (e.g. to persist target-specific metadata).
1883 */
1884static bool dm_table_supports_flush(struct dm_table *t)
1885{
1886	for (unsigned int i = 0; i < t->num_targets; i++) {
1887		struct dm_target *ti = dm_table_get_target(t, i);
1888
1889		if (ti->num_flush_bios && ti->flush_supported)
1890			return true;
1891	}
1892
1893	return false;
1894}
1895
1896static int device_dax_write_cache_enabled(struct dm_target *ti,
1897					  struct dm_dev *dev, sector_t start,
1898					  sector_t len, void *data)
1899{
1900	struct dax_device *dax_dev = dev->dax_dev;
1901
1902	if (!dax_dev)
1903		return false;
1904
1905	if (dax_write_cache_enabled(dax_dev))
1906		return true;
1907	return false;
1908}
1909
1910static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,
1911					   sector_t start, sector_t len, void *data)
1912{
1913	struct request_queue *q = bdev_get_queue(dev->bdev);
1914	int b;
1915
1916	mutex_lock(&q->limits_lock);
1917	b = !q->limits.max_write_zeroes_sectors;
1918	mutex_unlock(&q->limits_lock);
1919	return b;
1920}
1921
1922static bool dm_table_supports_write_zeroes(struct dm_table *t)
1923{
1924	for (unsigned int i = 0; i < t->num_targets; i++) {
1925		struct dm_target *ti = dm_table_get_target(t, i);
1926
1927		if (!ti->num_write_zeroes_bios)
1928			return false;
1929
1930		if (!ti->type->iterate_devices ||
1931		    ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))
1932			return false;
1933	}
1934
1935	return true;
1936}
1937
1938static bool dm_table_supports_nowait(struct dm_table *t)
1939{
1940	for (unsigned int i = 0; i < t->num_targets; i++) {
1941		struct dm_target *ti = dm_table_get_target(t, i);
1942
1943		if (!dm_target_supports_nowait(ti->type))
1944			return false;
1945	}
1946
1947	return true;
1948}
1949
1950static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
1951				      sector_t start, sector_t len, void *data)
1952{
1953	return !bdev_max_discard_sectors(dev->bdev);
1954}
1955
1956static bool dm_table_supports_discards(struct dm_table *t)
1957{
1958	for (unsigned int i = 0; i < t->num_targets; i++) {
1959		struct dm_target *ti = dm_table_get_target(t, i);
1960
1961		if (!ti->num_discard_bios)
1962			return false;
1963
1964		/*
1965		 * Either the target provides discard support (as implied by setting
1966		 * 'discards_supported') or it relies on _all_ data devices having
1967		 * discard support.
1968		 */
1969		if (!ti->discards_supported &&
1970		    (!ti->type->iterate_devices ||
1971		     ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))
1972			return false;
1973	}
1974
1975	return true;
1976}
1977
1978static int device_not_secure_erase_capable(struct dm_target *ti,
1979					   struct dm_dev *dev, sector_t start,
1980					   sector_t len, void *data)
1981{
1982	return !bdev_max_secure_erase_sectors(dev->bdev);
1983}
1984
1985static bool dm_table_supports_secure_erase(struct dm_table *t)
1986{
1987	for (unsigned int i = 0; i < t->num_targets; i++) {
1988		struct dm_target *ti = dm_table_get_target(t, i);
1989
1990		if (!ti->num_secure_erase_bios)
1991			return false;
1992
1993		if (!ti->type->iterate_devices ||
1994		    ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))
1995			return false;
1996	}
1997
1998	return true;
1999}
2000
2001static int device_not_atomic_write_capable(struct dm_target *ti,
2002			struct dm_dev *dev, sector_t start,
2003			sector_t len, void *data)
2004{
2005	return !bdev_can_atomic_write(dev->bdev);
2006}
2007
2008static bool dm_table_supports_atomic_writes(struct dm_table *t)
2009{
2010	for (unsigned int i = 0; i < t->num_targets; i++) {
2011		struct dm_target *ti = dm_table_get_target(t, i);
2012
2013		if (!dm_target_supports_atomic_writes(ti->type))
2014			return false;
2015
2016		if (!ti->type->iterate_devices)
2017			return false;
2018
2019		if (ti->type->iterate_devices(ti,
2020			device_not_atomic_write_capable, NULL)) {
2021			return false;
2022		}
2023	}
2024	return true;
2025}
2026
2027bool dm_table_supports_size_change(struct dm_table *t, sector_t old_size,
2028				   sector_t new_size)
2029{
2030	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && dm_has_zone_plugs(t->md) &&
2031	    old_size != new_size) {
2032		DMWARN("%s: device has zone write plug resources. "
2033		       "Cannot change size",
2034		       dm_device_name(t->md));
2035		return false;
2036	}
2037	return true;
2038}
2039
2040/*
2041 * This function will be skipped by noflush reloads of immutable request
2042 * based devices (dm-mpath).
2043 */
2044int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
2045			      struct queue_limits *limits)
2046{
2047	int r;
2048	struct queue_limits old_limits;
2049
2050	if (!dm_table_supports_nowait(t))
2051		limits->features &= ~BLK_FEAT_NOWAIT;
2052
2053	/*
2054	 * The current polling impementation does not support request based
2055	 * stacking.
2056	 */
2057	if (!__table_type_bio_based(t->type))
2058		limits->features &= ~BLK_FEAT_POLL;
2059
2060	if (!dm_table_supports_discards(t)) {
2061		limits->max_hw_discard_sectors = 0;
2062		limits->discard_granularity = 0;
2063		limits->discard_alignment = 0;
2064	}
2065
2066	if (!dm_table_supports_write_zeroes(t)) {
2067		limits->max_write_zeroes_sectors = 0;
2068		limits->max_hw_wzeroes_unmap_sectors = 0;
2069	}
2070
2071	if (!dm_table_supports_secure_erase(t))
2072		limits->max_secure_erase_sectors = 0;
2073
2074	if (dm_table_supports_flush(t))
2075		limits->features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;
2076
2077	if (dm_table_supports_dax(t, device_not_dax_capable))
2078		limits->features |= BLK_FEAT_DAX;
2079	else
2080		limits->features &= ~BLK_FEAT_DAX;
2081
2082	/* For a zoned table, setup the zone related queue attributes. */
2083	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2084		if (limits->features & BLK_FEAT_ZONED) {
2085			r = dm_set_zones_restrictions(t, q, limits);
2086			if (r)
2087				return r;
2088		} else if (dm_has_zone_plugs(t->md)) {
2089			DMWARN("%s: device has zone write plug resources. "
2090			       "Cannot switch to non-zoned table.",
2091			       dm_device_name(t->md));
2092			return -EINVAL;
2093		}
2094	}
2095
2096	if (dm_table_supports_atomic_writes(t))
2097		limits->features |= BLK_FEAT_ATOMIC_WRITES;
2098
2099	old_limits = queue_limits_start_update(q);
2100	r = queue_limits_commit_update(q, limits);
2101	if (r)
2102		return r;
2103
2104	/*
2105	 * Now that the limits are set, check the zones mapped by the table
2106	 * and setup the resources for zone append emulation if necessary.
2107	 */
2108	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
2109	    (limits->features & BLK_FEAT_ZONED)) {
2110		r = dm_revalidate_zones(t, q);
2111		if (r) {
2112			queue_limits_set(q, &old_limits);
2113			return r;
2114		}
2115	}
2116
2117	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
2118		dm_finalize_zone_settings(t, limits);
2119
2120	if (dm_table_supports_dax(t, device_not_dax_synchronous_capable))
2121		set_dax_synchronous(t->md->dax_dev);
2122
2123	if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL))
2124		dax_write_cache(t->md->dax_dev, true);
2125
2126	dm_update_crypto_profile(q, t);
2127	return 0;
2128}
2129
2130struct list_head *dm_table_get_devices(struct dm_table *t)
2131{
2132	return &t->devices;
2133}
2134
2135blk_mode_t dm_table_get_mode(struct dm_table *t)
2136{
2137	return t->mode;
2138}
2139EXPORT_SYMBOL(dm_table_get_mode);
2140
2141enum suspend_mode {
2142	PRESUSPEND,
2143	PRESUSPEND_UNDO,
2144	POSTSUSPEND,
2145};
2146
2147static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
2148{
2149	lockdep_assert_held(&t->md->suspend_lock);
2150
2151	for (unsigned int i = 0; i < t->num_targets; i++) {
2152		struct dm_target *ti = dm_table_get_target(t, i);
2153
2154		switch (mode) {
2155		case PRESUSPEND:
2156			if (ti->type->presuspend)
2157				ti->type->presuspend(ti);
2158			break;
2159		case PRESUSPEND_UNDO:
2160			if (ti->type->presuspend_undo)
2161				ti->type->presuspend_undo(ti);
2162			break;
2163		case POSTSUSPEND:
2164			if (ti->type->postsuspend)
2165				ti->type->postsuspend(ti);
2166			break;
2167		}
2168	}
2169}
2170
2171void dm_table_presuspend_targets(struct dm_table *t)
2172{
2173	if (!t)
2174		return;
2175
2176	suspend_targets(t, PRESUSPEND);
2177}
2178
2179void dm_table_presuspend_undo_targets(struct dm_table *t)
2180{
2181	if (!t)
2182		return;
2183
2184	suspend_targets(t, PRESUSPEND_UNDO);
2185}
2186
2187void dm_table_postsuspend_targets(struct dm_table *t)
2188{
2189	if (!t)
2190		return;
2191
2192	suspend_targets(t, POSTSUSPEND);
2193}
2194
2195int dm_table_resume_targets(struct dm_table *t)
2196{
2197	unsigned int i;
2198	int r = 0;
2199
2200	lockdep_assert_held(&t->md->suspend_lock);
2201
2202	for (i = 0; i < t->num_targets; i++) {
2203		struct dm_target *ti = dm_table_get_target(t, i);
2204
2205		if (!ti->type->preresume)
2206			continue;
2207
2208		r = ti->type->preresume(ti);
2209		if (r) {
2210			DMERR("%s: %s: preresume failed, error = %d",
2211			      dm_device_name(t->md), ti->type->name, r);
2212			return r;
2213		}
2214	}
2215
2216	for (i = 0; i < t->num_targets; i++) {
2217		struct dm_target *ti = dm_table_get_target(t, i);
2218
2219		if (ti->type->resume)
2220			ti->type->resume(ti);
2221	}
2222
2223	return 0;
2224}
2225
2226struct mapped_device *dm_table_get_md(struct dm_table *t)
2227{
2228	return t->md;
2229}
2230EXPORT_SYMBOL(dm_table_get_md);
2231
2232const char *dm_table_device_name(struct dm_table *t)
2233{
2234	return dm_device_name(t->md);
2235}
2236EXPORT_SYMBOL_GPL(dm_table_device_name);
2237
2238void dm_table_run_md_queue_async(struct dm_table *t)
2239{
2240	if (!dm_table_request_based(t))
2241		return;
2242
2243	if (t->md->queue)
2244		blk_mq_run_hw_queues(t->md->queue, true);
2245}
2246EXPORT_SYMBOL(dm_table_run_md_queue_async);
2247