dev-replace.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) STRATO AG 2012.  All rights reserved.
   4 */
   5
   6#include <linux/sched.h>
   7#include <linux/bio.h>
   8#include <linux/slab.h>
   9#include <linux/blkdev.h>
  10#include <linux/kthread.h>
  11#include <linux/math64.h>
  12#include "misc.h"
  13#include "ctree.h"
  14#include "disk-io.h"
  15#include "transaction.h"
  16#include "volumes.h"
  17#include "async-thread.h"
  18#include "dev-replace.h"
  19#include "sysfs.h"
  20#include "zoned.h"
  21#include "block-group.h"
  22#include "fs.h"
  23#include "accessors.h"
  24#include "scrub.h"
  25
  26/*
  27 * Device replace overview
  28 *
  29 * [Objective]
  30 * To copy all extents (both new and on-disk) from source device to target
  31 * device, while still keeping the filesystem read-write.
  32 *
  33 * [Method]
  34 * There are two main methods involved:
  35 *
  36 * - Write duplication
  37 *
  38 *   All new writes will be written to both target and source devices, so even
  39 *   if replace gets canceled, sources device still contains up-to-date data.
  40 *
  41 *   Location:		handle_ops_on_dev_replace() from btrfs_map_block()
  42 *   Start:		btrfs_dev_replace_start()
  43 *   End:		btrfs_dev_replace_finishing()
  44 *   Content:		Latest data/metadata
  45 *
  46 * - Copy existing extents
  47 *
  48 *   This happens by reusing scrub facility, as scrub also iterates through
  49 *   existing extents from commit root.
  50 *
  51 *   Location:		scrub_write_block_to_dev_replace() from
  52 *   			scrub_block_complete()
  53 *   Content:		Data/meta from commit root.
  54 *
  55 * Due to the content difference, we need to avoid nocow write when dev-replace
  56 * is happening.  This is done by marking the block group read-only and waiting
  57 * for NOCOW writes.
  58 *
  59 * After replace is done, the finishing part is done by swapping the target and
  60 * source devices.
  61 *
  62 *   Location:		btrfs_dev_replace_update_device_in_mapping_tree() from
  63 *   			btrfs_dev_replace_finishing()
  64 */
  65
  66static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
  67				       int scrub_ret);
  68static int btrfs_dev_replace_kthread(void *data);
  69
  70int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
  71{
  72	struct btrfs_dev_lookup_args args = { .devid = BTRFS_DEV_REPLACE_DEVID };
  73	struct btrfs_key key;
  74	struct btrfs_root *dev_root = fs_info->dev_root;
  75	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
  76	struct extent_buffer *eb;
  77	int slot;
  78	int ret = 0;
  79	BTRFS_PATH_AUTO_FREE(path);
  80	int item_size;
  81	struct btrfs_dev_replace_item *ptr;
  82	u64 src_devid;
  83
  84	if (!dev_root)
  85		return 0;
  86
  87	path = btrfs_alloc_path();
  88	if (!path)
  89		return -ENOMEM;
  90
  91	key.objectid = 0;
  92	key.type = BTRFS_DEV_REPLACE_KEY;
  93	key.offset = 0;
  94	ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
  95	if (ret) {
  96no_valid_dev_replace_entry_found:
  97		/*
  98		 * We don't have a replace item or it's corrupted.  If there is
  99		 * a replace target, fail the mount.
 100		 */
 101		if (unlikely(btrfs_find_device(fs_info->fs_devices, &args))) {
 102			btrfs_err(fs_info,
 103			"found replace target device without a valid replace item");
 104			return -EUCLEAN;
 105		}
 106		dev_replace->replace_state =
 107			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
 108		dev_replace->cont_reading_from_srcdev_mode =
 109		    BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS;
 110		dev_replace->time_started = 0;
 111		dev_replace->time_stopped = 0;
 112		atomic64_set(&dev_replace->num_write_errors, 0);
 113		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
 114		dev_replace->cursor_left = 0;
 115		dev_replace->committed_cursor_left = 0;
 116		dev_replace->cursor_left_last_write_of_item = 0;
 117		dev_replace->cursor_right = 0;
 118		dev_replace->srcdev = NULL;
 119		dev_replace->tgtdev = NULL;
 120		dev_replace->is_valid = 0;
 121		dev_replace->item_needs_writeback = 0;
 122		return 0;
 123	}
 124	slot = path->slots[0];
 125	eb = path->nodes[0];
 126	item_size = btrfs_item_size(eb, slot);
 127	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item);
 128
 129	if (item_size != sizeof(struct btrfs_dev_replace_item)) {
 130		btrfs_warn(fs_info,
 131			"dev_replace entry found has unexpected size, ignore entry");
 132		goto no_valid_dev_replace_entry_found;
 133	}
 134
 135	src_devid = btrfs_dev_replace_src_devid(eb, ptr);
 136	dev_replace->cont_reading_from_srcdev_mode =
 137		btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr);
 138	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);
 139	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);
 140	dev_replace->time_stopped =
 141		btrfs_dev_replace_time_stopped(eb, ptr);
 142	atomic64_set(&dev_replace->num_write_errors,
 143		     btrfs_dev_replace_num_write_errors(eb, ptr));
 144	atomic64_set(&dev_replace->num_uncorrectable_read_errors,
 145		     btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr));
 146	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);
 147	dev_replace->committed_cursor_left = dev_replace->cursor_left;
 148	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;
 149	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);
 150	dev_replace->is_valid = 1;
 151
 152	dev_replace->item_needs_writeback = 0;
 153	switch (dev_replace->replace_state) {
 154	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
 155	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
 156	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
 157		/*
 158		 * We don't have an active replace item but if there is a
 159		 * replace target, fail the mount.
 160		 */
 161		if (unlikely(btrfs_find_device(fs_info->fs_devices, &args))) {
 162			btrfs_err(fs_info,
 163"replace without active item, run 'device scan --forget' on the target device");
 164			ret = -EUCLEAN;
 165		} else {
 166			dev_replace->srcdev = NULL;
 167			dev_replace->tgtdev = NULL;
 168		}
 169		break;
 170	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
 171	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
 172		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices, &args);
 173		args.devid = src_devid;
 174		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices, &args);
 175
 176		/*
 177		 * allow 'btrfs dev replace_cancel' if src/tgt device is
 178		 * missing
 179		 */
 180		if (unlikely(!dev_replace->srcdev && !btrfs_test_opt(fs_info, DEGRADED))) {
 181			ret = -EIO;
 182			btrfs_warn(fs_info,
 183			   "cannot mount because device replace operation is ongoing and");
 184			btrfs_warn(fs_info,
 185			   "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
 186			   src_devid);
 187		}
 188		if (unlikely(!dev_replace->tgtdev && !btrfs_test_opt(fs_info, DEGRADED))) {
 189			ret = -EIO;
 190			btrfs_warn(fs_info,
 191			   "cannot mount because device replace operation is ongoing and");
 192			btrfs_warn(fs_info,
 193			   "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?",
 194				BTRFS_DEV_REPLACE_DEVID);
 195		}
 196		if (dev_replace->tgtdev) {
 197			if (dev_replace->srcdev) {
 198				dev_replace->tgtdev->total_bytes =
 199					dev_replace->srcdev->total_bytes;
 200				dev_replace->tgtdev->disk_total_bytes =
 201					dev_replace->srcdev->disk_total_bytes;
 202				dev_replace->tgtdev->commit_total_bytes =
 203					dev_replace->srcdev->commit_total_bytes;
 204				dev_replace->tgtdev->bytes_used =
 205					dev_replace->srcdev->bytes_used;
 206				dev_replace->tgtdev->commit_bytes_used =
 207					dev_replace->srcdev->commit_bytes_used;
 208			}
 209			set_bit(BTRFS_DEV_STATE_REPLACE_TGT,
 210				&dev_replace->tgtdev->dev_state);
 211
 212			WARN_ON(fs_info->fs_devices->rw_devices == 0);
 213			dev_replace->tgtdev->io_width = fs_info->sectorsize;
 214			dev_replace->tgtdev->io_align = fs_info->sectorsize;
 215			dev_replace->tgtdev->sector_size = fs_info->sectorsize;
 216			dev_replace->tgtdev->fs_info = fs_info;
 217			set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
 218				&dev_replace->tgtdev->dev_state);
 219		}
 220		break;
 221	}
 222
 223	return ret;
 224}
 225
 226/*
 227 * Initialize a new device for device replace target from a given source dev
 228 * and path.
 229 *
 230 * Return 0 and new device in @device_out, otherwise return < 0
 231 */
 232static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 233				  const char *device_path,
 234				  struct btrfs_device *srcdev,
 235				  struct btrfs_device **device_out)
 236{
 237	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 238	struct btrfs_device *device;
 239	struct file *bdev_file;
 240	struct block_device *bdev;
 241	u64 devid = BTRFS_DEV_REPLACE_DEVID;
 242	int ret = 0;
 243
 244	*device_out = NULL;
 245	if (srcdev->fs_devices->seeding) {
 246		btrfs_err(fs_info, "the filesystem is a seed filesystem!");
 247		return -EINVAL;
 248	}
 249
 250	bdev_file = bdev_file_open_by_path(device_path, BLK_OPEN_WRITE,
 251					   fs_info->sb, &fs_holder_ops);
 252	if (IS_ERR(bdev_file)) {
 253		btrfs_err(fs_info, "target device %s is invalid!", device_path);
 254		return PTR_ERR(bdev_file);
 255	}
 256	bdev = file_bdev(bdev_file);
 257
 258	if (!btrfs_check_device_zone_type(fs_info, bdev)) {
 259		btrfs_err(fs_info,
 260		"dev-replace: zoned type of target device mismatch with filesystem");
 261		ret = -EINVAL;
 262		goto error;
 263	}
 264
 265	sync_blockdev(bdev);
 266
 267	list_for_each_entry(device, &fs_devices->devices, dev_list) {
 268		if (device->bdev == bdev) {
 269			btrfs_err(fs_info,
 270				  "target device is in the filesystem!");
 271			ret = -EEXIST;
 272			goto error;
 273		}
 274	}
 275
 276
 277	if (bdev_nr_bytes(bdev) < btrfs_device_get_total_bytes(srcdev)) {
 278		btrfs_err(fs_info,
 279			  "target device is smaller than source device!");
 280		ret = -EINVAL;
 281		goto error;
 282	}
 283
 284
 285	device = btrfs_alloc_device(NULL, &devid, NULL, device_path);
 286	if (IS_ERR(device)) {
 287		ret = PTR_ERR(device);
 288		goto error;
 289	}
 290
 291	ret = lookup_bdev(device_path, &device->devt);
 292	if (ret)
 293		goto error;
 294
 295	set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
 296	device->generation = 0;
 297	device->io_width = fs_info->sectorsize;
 298	device->io_align = fs_info->sectorsize;
 299	device->sector_size = fs_info->sectorsize;
 300	device->total_bytes = btrfs_device_get_total_bytes(srcdev);
 301	device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev);
 302	device->bytes_used = btrfs_device_get_bytes_used(srcdev);
 303	device->commit_total_bytes = srcdev->commit_total_bytes;
 304	device->commit_bytes_used = device->bytes_used;
 305	device->fs_info = fs_info;
 306	device->bdev = bdev;
 307	device->bdev_file = bdev_file;
 308	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
 309	set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
 310	device->dev_stats_valid = 1;
 311	set_blocksize(bdev_file, BTRFS_BDEV_BLOCKSIZE);
 312	device->fs_devices = fs_devices;
 313
 314	ret = btrfs_get_dev_zone_info(device, false);
 315	if (ret)
 316		goto error;
 317
 318	mutex_lock(&fs_devices->device_list_mutex);
 319	list_add(&device->dev_list, &fs_devices->devices);
 320	fs_devices->num_devices++;
 321	fs_devices->open_devices++;
 322	mutex_unlock(&fs_devices->device_list_mutex);
 323
 324	*device_out = device;
 325	return 0;
 326
 327error:
 328	bdev_fput(bdev_file);
 329	return ret;
 330}
 331
 332/*
 333 * called from commit_transaction. Writes changed device replace state to
 334 * disk.
 335 */
 336int btrfs_run_dev_replace(struct btrfs_trans_handle *trans)
 337{
 338	struct btrfs_fs_info *fs_info = trans->fs_info;
 339	int ret;
 340	struct btrfs_root *dev_root = fs_info->dev_root;
 341	BTRFS_PATH_AUTO_FREE(path);
 342	struct btrfs_key key;
 343	struct extent_buffer *eb;
 344	struct btrfs_dev_replace_item *ptr;
 345	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
 346
 347	down_read(&dev_replace->rwsem);
 348	if (!dev_replace->is_valid ||
 349	    !dev_replace->item_needs_writeback) {
 350		up_read(&dev_replace->rwsem);
 351		return 0;
 352	}
 353	up_read(&dev_replace->rwsem);
 354
 355	key.objectid = 0;
 356	key.type = BTRFS_DEV_REPLACE_KEY;
 357	key.offset = 0;
 358
 359	path = btrfs_alloc_path();
 360	if (!path)
 361		return -ENOMEM;
 362
 363	ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
 364	if (ret < 0) {
 365		btrfs_warn(fs_info,
 366			   "error %d while searching for dev_replace item!",
 367			   ret);
 368		return ret;
 369	}
 370
 371	if (ret == 0 &&
 372	    btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
 373		/*
 374		 * need to delete old one and insert a new one.
 375		 * Since no attempt is made to recover any old state, if the
 376		 * dev_replace state is 'running', the data on the target
 377		 * drive is lost.
 378		 * It would be possible to recover the state: just make sure
 379		 * that the beginning of the item is never changed and always
 380		 * contains all the essential information. Then read this
 381		 * minimal set of information and use it as a base for the
 382		 * new state.
 383		 */
 384		ret = btrfs_del_item(trans, dev_root, path);
 385		if (ret != 0) {
 386			btrfs_warn(fs_info,
 387				   "delete too small dev_replace item failed %d!",
 388				   ret);
 389			return ret;
 390		}
 391		ret = 1;
 392	}
 393
 394	if (ret == 1) {
 395		/* need to insert a new item */
 396		btrfs_release_path(path);
 397		ret = btrfs_insert_empty_item(trans, dev_root, path,
 398					      &key, sizeof(*ptr));
 399		if (ret < 0) {
 400			btrfs_warn(fs_info,
 401				   "insert dev_replace item failed %d!", ret);
 402			return ret;
 403		}
 404	}
 405
 406	eb = path->nodes[0];
 407	ptr = btrfs_item_ptr(eb, path->slots[0],
 408			     struct btrfs_dev_replace_item);
 409
 410	down_write(&dev_replace->rwsem);
 411	if (dev_replace->srcdev)
 412		btrfs_set_dev_replace_src_devid(eb, ptr,
 413			dev_replace->srcdev->devid);
 414	else
 415		btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1);
 416	btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr,
 417		dev_replace->cont_reading_from_srcdev_mode);
 418	btrfs_set_dev_replace_replace_state(eb, ptr,
 419		dev_replace->replace_state);
 420	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);
 421	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);
 422	btrfs_set_dev_replace_num_write_errors(eb, ptr,
 423		atomic64_read(&dev_replace->num_write_errors));
 424	btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr,
 425		atomic64_read(&dev_replace->num_uncorrectable_read_errors));
 426	dev_replace->cursor_left_last_write_of_item =
 427		dev_replace->cursor_left;
 428	btrfs_set_dev_replace_cursor_left(eb, ptr,
 429		dev_replace->cursor_left_last_write_of_item);
 430	btrfs_set_dev_replace_cursor_right(eb, ptr,
 431		dev_replace->cursor_right);
 432	dev_replace->item_needs_writeback = 0;
 433	up_write(&dev_replace->rwsem);
 434
 435	return ret;
 436}
 437
 438static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info,
 439				    struct btrfs_device *src_dev)
 440{
 441	struct btrfs_path *path;
 442	struct btrfs_key key;
 443	struct btrfs_key found_key;
 444	struct btrfs_root *root = fs_info->dev_root;
 445	struct btrfs_dev_extent *dev_extent = NULL;
 446	struct btrfs_block_group *cache;
 447	struct btrfs_trans_handle *trans;
 448	int iter_ret = 0;
 449	int ret = 0;
 450	u64 chunk_offset;
 451
 452	/* Do not use "to_copy" on non zoned filesystem for now */
 453	if (!btrfs_is_zoned(fs_info))
 454		return 0;
 455
 456	mutex_lock(&fs_info->chunk_mutex);
 457
 458	/* Ensure we don't have pending new block group */
 459	spin_lock(&fs_info->trans_lock);
 460	while (fs_info->running_transaction &&
 461	       !list_empty(&fs_info->running_transaction->dev_update_list)) {
 462		spin_unlock(&fs_info->trans_lock);
 463		mutex_unlock(&fs_info->chunk_mutex);
 464		trans = btrfs_attach_transaction(root);
 465		if (IS_ERR(trans)) {
 466			ret = PTR_ERR(trans);
 467			mutex_lock(&fs_info->chunk_mutex);
 468			if (ret == -ENOENT) {
 469				spin_lock(&fs_info->trans_lock);
 470				continue;
 471			} else {
 472				goto unlock;
 473			}
 474		}
 475
 476		ret = btrfs_commit_transaction(trans);
 477		mutex_lock(&fs_info->chunk_mutex);
 478		if (ret)
 479			goto unlock;
 480
 481		spin_lock(&fs_info->trans_lock);
 482	}
 483	spin_unlock(&fs_info->trans_lock);
 484
 485	path = btrfs_alloc_path();
 486	if (!path) {
 487		ret = -ENOMEM;
 488		goto unlock;
 489	}
 490
 491	path->reada = READA_FORWARD;
 492	path->search_commit_root = true;
 493	path->skip_locking = true;
 494
 495	key.objectid = src_dev->devid;
 496	key.type = BTRFS_DEV_EXTENT_KEY;
 497	key.offset = 0;
 498
 499	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
 500		struct extent_buffer *leaf = path->nodes[0];
 501
 502		if (found_key.objectid != src_dev->devid)
 503			break;
 504
 505		if (found_key.type != BTRFS_DEV_EXTENT_KEY)
 506			break;
 507
 508		if (found_key.offset < key.offset)
 509			break;
 510
 511		dev_extent = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent);
 512
 513		chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dev_extent);
 514
 515		cache = btrfs_lookup_block_group(fs_info, chunk_offset);
 516		if (!cache)
 517			continue;
 518
 519		set_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
 520		btrfs_put_block_group(cache);
 521	}
 522	if (iter_ret < 0)
 523		ret = iter_ret;
 524
 525	btrfs_free_path(path);
 526unlock:
 527	mutex_unlock(&fs_info->chunk_mutex);
 528
 529	return ret;
 530}
 531
 532bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev,
 533				      struct btrfs_block_group *cache,
 534				      u64 physical)
 535{
 536	struct btrfs_fs_info *fs_info = cache->fs_info;
 537	struct btrfs_chunk_map *map;
 538	u64 chunk_offset = cache->start;
 539	int num_extents, cur_extent;
 540	int i;
 541
 542	/* Do not use "to_copy" on non zoned filesystem for now */
 543	if (!btrfs_is_zoned(fs_info))
 544		return true;
 545
 546	spin_lock(&cache->lock);
 547	if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) {
 548		spin_unlock(&cache->lock);
 549		return true;
 550	}
 551	spin_unlock(&cache->lock);
 552
 553	map = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
 554	ASSERT(!IS_ERR(map));
 555
 556	num_extents = 0;
 557	cur_extent = 0;
 558	for (i = 0; i < map->num_stripes; i++) {
 559		/* We have more device extent to copy */
 560		if (srcdev != map->stripes[i].dev)
 561			continue;
 562
 563		num_extents++;
 564		if (physical == map->stripes[i].physical)
 565			cur_extent = i;
 566	}
 567
 568	btrfs_free_chunk_map(map);
 569
 570	if (num_extents > 1 && cur_extent < num_extents - 1) {
 571		/*
 572		 * Has more stripes on this device. Keep this block group
 573		 * readonly until we finish all the stripes.
 574		 */
 575		return false;
 576	}
 577
 578	/* Last stripe on this device */
 579	clear_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
 580
 581	return true;
 582}
 583
 584static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
 585		const char *tgtdev_name, u64 srcdevid, const char *srcdev_name,
 586		int read_src)
 587{
 588	struct btrfs_root *root = fs_info->dev_root;
 589	struct btrfs_trans_handle *trans;
 590	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
 591	int ret;
 592	struct btrfs_device *tgt_device = NULL;
 593	struct btrfs_device *src_device = NULL;
 594
 595	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,
 596						  srcdev_name);
 597	if (IS_ERR(src_device))
 598		return PTR_ERR(src_device);
 599
 600	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {
 601		btrfs_warn(fs_info,
 602	  "cannot replace device %s (devid %llu) due to active swapfile",
 603			btrfs_dev_name(src_device), src_device->devid);
 604		return -ETXTBSY;
 605	}
 606
 607	/*
 608	 * Here we commit the transaction to make sure commit_total_bytes
 609	 * of all the devices are updated.
 610	 */
 611	trans = btrfs_attach_transaction(root);
 612	if (!IS_ERR(trans)) {
 613		ret = btrfs_commit_transaction(trans);
 614		if (ret)
 615			return ret;
 616	} else if (PTR_ERR(trans) != -ENOENT) {
 617		return PTR_ERR(trans);
 618	}
 619
 620	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,
 621					    src_device, &tgt_device);
 622	if (ret)
 623		return ret;
 624
 625	ret = mark_block_group_to_copy(fs_info, src_device);
 626	if (ret)
 627		return ret;
 628
 629	down_write(&dev_replace->rwsem);
 630	dev_replace->replace_task = current;
 631	switch (dev_replace->replace_state) {
 632	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
 633	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
 634	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
 635		break;
 636	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
 637	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
 638		DEBUG_WARN("unexpected STARTED or SUSPENDED dev-replace state");
 639		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED;
 640		up_write(&dev_replace->rwsem);
 641		goto leave;
 642	}
 643
 644	dev_replace->cont_reading_from_srcdev_mode = read_src;
 645	dev_replace->srcdev = src_device;
 646	dev_replace->tgtdev = tgt_device;
 647
 648	btrfs_info(fs_info,
 649		      "dev_replace from %s (devid %llu) to %s started",
 650		      btrfs_dev_name(src_device),
 651		      src_device->devid,
 652		      btrfs_dev_name(tgt_device));
 653
 654	/*
 655	 * from now on, the writes to the srcdev are all duplicated to
 656	 * go to the tgtdev as well (refer to btrfs_map_block()).
 657	 */
 658	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
 659	dev_replace->time_started = ktime_get_real_seconds();
 660	dev_replace->cursor_left = 0;
 661	dev_replace->committed_cursor_left = 0;
 662	dev_replace->cursor_left_last_write_of_item = 0;
 663	dev_replace->cursor_right = 0;
 664	dev_replace->is_valid = 1;
 665	dev_replace->item_needs_writeback = 1;
 666	atomic64_set(&dev_replace->num_write_errors, 0);
 667	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
 668	up_write(&dev_replace->rwsem);
 669
 670	ret = btrfs_sysfs_add_device(tgt_device);
 671	if (ret)
 672		btrfs_err(fs_info, "kobj add dev failed %d", ret);
 673
 674	btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);
 675
 676	/*
 677	 * Commit dev_replace state and reserve 1 item for it.
 678	 * This is crucial to ensure we won't miss copying extents for new block
 679	 * groups that are allocated after we started the device replace, and
 680	 * must be done after setting up the device replace state.
 681	 */
 682	trans = btrfs_start_transaction(root, 1);
 683	if (IS_ERR(trans)) {
 684		ret = PTR_ERR(trans);
 685		down_write(&dev_replace->rwsem);
 686		dev_replace->replace_state =
 687			BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED;
 688		dev_replace->srcdev = NULL;
 689		dev_replace->tgtdev = NULL;
 690		up_write(&dev_replace->rwsem);
 691		goto leave;
 692	}
 693
 694	ret = btrfs_commit_transaction(trans);
 695	WARN_ON(ret);
 696
 697	/* the disk copy procedure reuses the scrub code */
 698	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,
 699			      btrfs_device_get_total_bytes(src_device),
 700			      &dev_replace->scrub_progress, 0, 1);
 701
 702	ret = btrfs_dev_replace_finishing(fs_info, ret);
 703	if (ret == -EINPROGRESS)
 704		ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS;
 705
 706	return ret;
 707
 708leave:
 709	btrfs_destroy_dev_replace_tgtdev(tgt_device);
 710	return ret;
 711}
 712
 713static int btrfs_check_replace_dev_names(struct btrfs_ioctl_dev_replace_args *args)
 714{
 715	if (args->start.srcdevid == 0) {
 716		if (memchr(args->start.srcdev_name, 0,
 717			   sizeof(args->start.srcdev_name)) == NULL)
 718			return -ENAMETOOLONG;
 719	} else {
 720		args->start.srcdev_name[0] = 0;
 721	}
 722
 723	if (memchr(args->start.tgtdev_name, 0,
 724		   sizeof(args->start.tgtdev_name)) == NULL)
 725	    return -ENAMETOOLONG;
 726
 727	return 0;
 728}
 729
 730int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,
 731			    struct btrfs_ioctl_dev_replace_args *args)
 732{
 733	int ret;
 734
 735	switch (args->start.cont_reading_from_srcdev_mode) {
 736	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS:
 737	case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID:
 738		break;
 739	default:
 740		return -EINVAL;
 741	}
 742	ret = btrfs_check_replace_dev_names(args);
 743	if (ret < 0)
 744		return ret;
 745
 746	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,
 747					args->start.srcdevid,
 748					args->start.srcdev_name,
 749					args->start.cont_reading_from_srcdev_mode);
 750	args->result = ret;
 751	/* don't warn if EINPROGRESS, someone else might be running scrub */
 752	if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS ||
 753	    ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR)
 754		return 0;
 755
 756	return ret;
 757}
 758
 759/*
 760 * blocked until all in-flight bios operations are finished.
 761 */
 762static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)
 763{
 764	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
 765	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(
 766		   &fs_info->dev_replace.bio_counter));
 767}
 768
 769/*
 770 * we have removed target device, it is safe to allow new bios request.
 771 */
 772static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)
 773{
 774	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);
 775	wake_up(&fs_info->dev_replace.replace_wait);
 776}
 777
 778/*
 779 * When finishing the device replace, before swapping the source device with the
 780 * target device we must update the chunk allocation state in the target device,
 781 * as it is empty because replace works by directly copying the chunks and not
 782 * through the normal chunk allocation path.
 783 */
 784static int btrfs_set_target_alloc_state(struct btrfs_device *srcdev,
 785					struct btrfs_device *tgtdev)
 786{
 787	struct extent_state *cached_state = NULL;
 788	u64 start = 0;
 789	u64 found_start;
 790	u64 found_end;
 791	int ret = 0;
 792
 793	lockdep_assert_held(&srcdev->fs_info->chunk_mutex);
 794
 795	while (btrfs_find_first_extent_bit(&srcdev->alloc_state, start,
 796					   &found_start, &found_end,
 797					   CHUNK_ALLOCATED, &cached_state)) {
 798		ret = btrfs_set_extent_bit(&tgtdev->alloc_state, found_start,
 799					   found_end, CHUNK_ALLOCATED, NULL);
 800		if (ret)
 801			break;
 802		start = found_end + 1;
 803	}
 804
 805	btrfs_free_extent_state(cached_state);
 806	return ret;
 807}
 808
 809static void btrfs_dev_replace_update_device_in_mapping_tree(
 810						struct btrfs_fs_info *fs_info,
 811						struct btrfs_device *srcdev,
 812						struct btrfs_device *tgtdev)
 813{
 814	struct rb_node *node;
 815
 816	/*
 817	 * The chunk mutex must be held so that no new chunks can be created
 818	 * while we are updating existing chunks. This guarantees we don't miss
 819	 * any new chunk that gets created for a range that falls before the
 820	 * range of the last chunk we processed.
 821	 */
 822	lockdep_assert_held(&fs_info->chunk_mutex);
 823
 824	write_lock(&fs_info->mapping_tree_lock);
 825	node = rb_first_cached(&fs_info->mapping_tree);
 826	while (node) {
 827		struct rb_node *next = rb_next(node);
 828		struct btrfs_chunk_map *map;
 829		u64 next_start;
 830
 831		map = rb_entry(node, struct btrfs_chunk_map, rb_node);
 832		next_start = map->start + map->chunk_len;
 833
 834		for (int i = 0; i < map->num_stripes; i++)
 835			if (srcdev == map->stripes[i].dev)
 836				map->stripes[i].dev = tgtdev;
 837
 838		if (cond_resched_rwlock_write(&fs_info->mapping_tree_lock)) {
 839			map = btrfs_find_chunk_map_nolock(fs_info, next_start, U64_MAX);
 840			if (!map)
 841				break;
 842			node = &map->rb_node;
 843			/*
 844			 * Drop the lookup reference since we are holding the
 845			 * lock in write mode and no one can remove the chunk
 846			 * map from the tree and drop its tree reference.
 847			 */
 848			btrfs_free_chunk_map(map);
 849		} else {
 850			node = next;
 851		}
 852	}
 853	write_unlock(&fs_info->mapping_tree_lock);
 854}
 855
 856static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
 857				       int scrub_ret)
 858{
 859	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
 860	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 861	struct btrfs_device *tgt_device;
 862	struct btrfs_device *src_device;
 863	struct btrfs_root *root = fs_info->tree_root;
 864	u8 uuid_tmp[BTRFS_UUID_SIZE];
 865	struct btrfs_trans_handle *trans;
 866	int ret = 0;
 867
 868	/* don't allow cancel or unmount to disturb the finishing procedure */
 869	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
 870
 871	down_read(&dev_replace->rwsem);
 872	/* was the operation canceled, or is it finished? */
 873	if (dev_replace->replace_state !=
 874	    BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) {
 875		up_read(&dev_replace->rwsem);
 876		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 877		return 0;
 878	}
 879
 880	tgt_device = dev_replace->tgtdev;
 881	src_device = dev_replace->srcdev;
 882	up_read(&dev_replace->rwsem);
 883
 884	/*
 885	 * flush all outstanding I/O and inode extent mappings before the
 886	 * copy operation is declared as being finished
 887	 */
 888	ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
 889	if (ret) {
 890		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 891		return ret;
 892	}
 893	btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL);
 894
 895	/*
 896	 * We have to use this loop approach because at this point src_device
 897	 * has to be available for transaction commit to complete, yet new
 898	 * chunks shouldn't be allocated on the device.
 899	 */
 900	while (1) {
 901		trans = btrfs_start_transaction(root, 0);
 902		if (IS_ERR(trans)) {
 903			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 904			return PTR_ERR(trans);
 905		}
 906		ret = btrfs_commit_transaction(trans);
 907		WARN_ON(ret);
 908
 909		/* Prevent write_all_supers() during the finishing procedure */
 910		mutex_lock(&fs_devices->device_list_mutex);
 911		/* Prevent new chunks being allocated on the source device */
 912		mutex_lock(&fs_info->chunk_mutex);
 913
 914		if (!list_empty(&src_device->post_commit_list)) {
 915			mutex_unlock(&fs_devices->device_list_mutex);
 916			mutex_unlock(&fs_info->chunk_mutex);
 917		} else {
 918			break;
 919		}
 920	}
 921
 922	down_write(&dev_replace->rwsem);
 923	dev_replace->replace_state =
 924		scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED
 925			  : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED;
 926	dev_replace->tgtdev = NULL;
 927	dev_replace->srcdev = NULL;
 928	dev_replace->time_stopped = ktime_get_real_seconds();
 929	dev_replace->item_needs_writeback = 1;
 930
 931	/*
 932	 * Update allocation state in the new device and replace the old device
 933	 * with the new one in the mapping tree.
 934	 */
 935	if (!scrub_ret) {
 936		scrub_ret = btrfs_set_target_alloc_state(src_device, tgt_device);
 937		if (scrub_ret)
 938			goto error;
 939		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
 940								src_device,
 941								tgt_device);
 942	} else {
 943		if (scrub_ret != -ECANCELED)
 944			btrfs_err(fs_info,
 945				 "btrfs_scrub_dev(%s, %llu, %s) failed %d",
 946				 btrfs_dev_name(src_device),
 947				 src_device->devid,
 948				 btrfs_dev_name(tgt_device), scrub_ret);
 949error:
 950		up_write(&dev_replace->rwsem);
 951		mutex_unlock(&fs_info->chunk_mutex);
 952		mutex_unlock(&fs_devices->device_list_mutex);
 953		btrfs_rm_dev_replace_blocked(fs_info);
 954		if (tgt_device)
 955			btrfs_destroy_dev_replace_tgtdev(tgt_device);
 956		btrfs_rm_dev_replace_unblocked(fs_info);
 957		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 958
 959		return scrub_ret;
 960	}
 961
 962	btrfs_info(fs_info,
 963			  "dev_replace from %s (devid %llu) to %s finished",
 964			  btrfs_dev_name(src_device),
 965			  src_device->devid,
 966			  btrfs_dev_name(tgt_device));
 967	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state);
 968	tgt_device->devid = src_device->devid;
 969	src_device->devid = BTRFS_DEV_REPLACE_DEVID;
 970	memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp));
 971	memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid));
 972	memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid));
 973	btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes);
 974	btrfs_device_set_disk_total_bytes(tgt_device,
 975					  src_device->disk_total_bytes);
 976	btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used);
 977	tgt_device->commit_bytes_used = src_device->bytes_used;
 978
 979	btrfs_assign_next_active_device(src_device, tgt_device);
 980
 981	list_add(&tgt_device->dev_alloc_list, &fs_devices->alloc_list);
 982	fs_devices->rw_devices++;
 983
 984	dev_replace->replace_task = NULL;
 985	up_write(&dev_replace->rwsem);
 986	btrfs_rm_dev_replace_blocked(fs_info);
 987
 988	btrfs_rm_dev_replace_remove_srcdev(src_device);
 989
 990	btrfs_rm_dev_replace_unblocked(fs_info);
 991
 992	/*
 993	 * Increment dev_stats_ccnt so that btrfs_run_dev_stats() will
 994	 * update on-disk dev stats value during commit transaction
 995	 */
 996	atomic_inc(&tgt_device->dev_stats_ccnt);
 997
 998	/*
 999	 * this is again a consistent state where no dev_replace procedure
1000	 * is running, the target device is part of the filesystem, the
1001	 * source device is not part of the filesystem anymore and its 1st
1002	 * superblock is scratched out so that it is no longer marked to
1003	 * belong to this filesystem.
1004	 */
1005	mutex_unlock(&fs_info->chunk_mutex);
1006	mutex_unlock(&fs_devices->device_list_mutex);
1007
1008	/* replace the sysfs entry */
1009	btrfs_sysfs_remove_device(src_device);
1010	btrfs_sysfs_update_devid(tgt_device);
1011	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &src_device->dev_state))
1012		btrfs_scratch_superblocks(fs_info, src_device);
1013
1014	/* write back the superblocks */
1015	trans = btrfs_start_transaction(root, 0);
1016	if (!IS_ERR(trans))
1017		btrfs_commit_transaction(trans);
1018
1019	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1020
1021	btrfs_rm_dev_replace_free_srcdev(src_device);
1022
1023	return 0;
1024}
1025
1026/*
1027 * Read progress of device replace status according to the state and last
1028 * stored position. The value format is the same as for
1029 * btrfs_dev_replace::progress_1000
1030 */
1031static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)
1032{
1033	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1034	u64 ret = 0;
1035
1036	switch (dev_replace->replace_state) {
1037	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1038	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1039		ret = 0;
1040		break;
1041	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1042		ret = 1000;
1043		break;
1044	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1045	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1046		ret = div64_u64(dev_replace->cursor_left,
1047				div_u64(btrfs_device_get_total_bytes(
1048						dev_replace->srcdev), 1000));
1049		break;
1050	}
1051
1052	return ret;
1053}
1054
1055void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,
1056			      struct btrfs_ioctl_dev_replace_args *args)
1057{
1058	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1059
1060	down_read(&dev_replace->rwsem);
1061	/* even if !dev_replace_is_valid, the values are good enough for
1062	 * the replace_status ioctl */
1063	args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
1064	args->status.replace_state = dev_replace->replace_state;
1065	args->status.time_started = dev_replace->time_started;
1066	args->status.time_stopped = dev_replace->time_stopped;
1067	args->status.num_write_errors =
1068		atomic64_read(&dev_replace->num_write_errors);
1069	args->status.num_uncorrectable_read_errors =
1070		atomic64_read(&dev_replace->num_uncorrectable_read_errors);
1071	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);
1072	up_read(&dev_replace->rwsem);
1073}
1074
1075int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
1076{
1077	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1078	struct btrfs_device *tgt_device = NULL;
1079	struct btrfs_device *src_device = NULL;
1080	struct btrfs_trans_handle *trans;
1081	struct btrfs_root *root = fs_info->tree_root;
1082	int result;
1083	int ret;
1084
1085	if (sb_rdonly(fs_info->sb))
1086		return -EROFS;
1087
1088	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
1089	down_write(&dev_replace->rwsem);
1090	switch (dev_replace->replace_state) {
1091	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1092	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1093	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1094		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
1095		up_write(&dev_replace->rwsem);
1096		break;
1097	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1098		tgt_device = dev_replace->tgtdev;
1099		src_device = dev_replace->srcdev;
1100		up_write(&dev_replace->rwsem);
1101		ret = btrfs_scrub_cancel(fs_info);
1102		if (ret < 0) {
1103			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED;
1104		} else {
1105			result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
1106			/*
1107			 * btrfs_dev_replace_finishing() will handle the
1108			 * cleanup part
1109			 */
1110			btrfs_info(fs_info,
1111				"dev_replace from %s (devid %llu) to %s canceled",
1112				btrfs_dev_name(src_device), src_device->devid,
1113				btrfs_dev_name(tgt_device));
1114		}
1115		break;
1116	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1117		/*
1118		 * Scrub doing the replace isn't running so we need to do the
1119		 * cleanup step of btrfs_dev_replace_finishing() here
1120		 */
1121		result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR;
1122		tgt_device = dev_replace->tgtdev;
1123		src_device = dev_replace->srcdev;
1124		dev_replace->tgtdev = NULL;
1125		dev_replace->srcdev = NULL;
1126		dev_replace->replace_state =
1127				BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;
1128		dev_replace->time_stopped = ktime_get_real_seconds();
1129		dev_replace->item_needs_writeback = 1;
1130
1131		up_write(&dev_replace->rwsem);
1132
1133		/* Scrub for replace must not be running in suspended state */
1134		btrfs_scrub_cancel(fs_info);
1135
1136		trans = btrfs_start_transaction(root, 0);
1137		if (IS_ERR(trans)) {
1138			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1139			return PTR_ERR(trans);
1140		}
1141		ret = btrfs_commit_transaction(trans);
1142		WARN_ON(ret);
1143
1144		btrfs_info(fs_info,
1145		"suspended dev_replace from %s (devid %llu) to %s canceled",
1146			btrfs_dev_name(src_device), src_device->devid,
1147			btrfs_dev_name(tgt_device));
1148
1149		if (tgt_device)
1150			btrfs_destroy_dev_replace_tgtdev(tgt_device);
1151		break;
1152	default:
1153		up_write(&dev_replace->rwsem);
1154		result = -EINVAL;
1155	}
1156
1157	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1158	return result;
1159}
1160
1161void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
1162{
1163	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1164
1165	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);
1166	down_write(&dev_replace->rwsem);
1167
1168	switch (dev_replace->replace_state) {
1169	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1170	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1171	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1172	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1173		break;
1174	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1175		dev_replace->replace_state =
1176			BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1177		dev_replace->time_stopped = ktime_get_real_seconds();
1178		dev_replace->item_needs_writeback = 1;
1179		btrfs_info(fs_info, "suspending dev_replace for unmount");
1180		break;
1181	}
1182
1183	up_write(&dev_replace->rwsem);
1184	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
1185}
1186
1187/* resume dev_replace procedure that was interrupted by unmount */
1188int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
1189{
1190	struct task_struct *task;
1191	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1192
1193	down_write(&dev_replace->rwsem);
1194
1195	switch (dev_replace->replace_state) {
1196	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1197	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1198	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1199		up_write(&dev_replace->rwsem);
1200		return 0;
1201	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1202		break;
1203	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1204		dev_replace->replace_state =
1205			BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;
1206		break;
1207	}
1208	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {
1209		btrfs_info(fs_info,
1210			   "cannot continue dev_replace, tgtdev is missing");
1211		btrfs_info(fs_info,
1212			   "you may cancel the operation after 'mount -o degraded'");
1213		dev_replace->replace_state =
1214					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1215		up_write(&dev_replace->rwsem);
1216		return 0;
1217	}
1218	up_write(&dev_replace->rwsem);
1219
1220	/*
1221	 * This could collide with a paused balance, but the exclusive op logic
1222	 * should never allow both to start and pause. We don't want to allow
1223	 * dev-replace to start anyway.
1224	 */
1225	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {
1226		down_write(&dev_replace->rwsem);
1227		dev_replace->replace_state =
1228					BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
1229		up_write(&dev_replace->rwsem);
1230		btrfs_info(fs_info,
1231		"cannot resume dev-replace, other exclusive operation running");
1232		return 0;
1233	}
1234
1235	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
1236	return PTR_ERR_OR_ZERO(task);
1237}
1238
1239static int btrfs_dev_replace_kthread(void *data)
1240{
1241	struct btrfs_fs_info *fs_info = data;
1242	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
1243	u64 progress;
1244	int ret;
1245
1246	progress = btrfs_dev_replace_progress(fs_info);
1247	progress = div_u64(progress, 10);
1248	btrfs_info(fs_info,
1249		"continuing dev_replace from %s (devid %llu) to target %s @%u%%",
1250		btrfs_dev_name(dev_replace->srcdev),
1251		dev_replace->srcdev->devid,
1252		btrfs_dev_name(dev_replace->tgtdev),
1253		(unsigned int)progress);
1254
1255	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,
1256			      dev_replace->committed_cursor_left,
1257			      btrfs_device_get_total_bytes(dev_replace->srcdev),
1258			      &dev_replace->scrub_progress, 0, 1);
1259	ret = btrfs_dev_replace_finishing(fs_info, ret);
1260	WARN_ON(ret && ret != -ECANCELED);
1261
1262	btrfs_exclop_finish(fs_info);
1263	return 0;
1264}
1265
1266bool __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)
1267{
1268	if (!dev_replace->is_valid)
1269		return false;
1270
1271	switch (dev_replace->replace_state) {
1272	case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED:
1273	case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED:
1274	case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED:
1275		return false;
1276	case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED:
1277	case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED:
1278		/*
1279		 * return true even if tgtdev is missing (this is
1280		 * something that can happen if the dev_replace
1281		 * procedure is suspended by an umount and then
1282		 * the tgtdev is missing (or "btrfs dev scan") was
1283		 * not called and the filesystem is remounted
1284		 * in degraded state. This does not stop the
1285		 * dev_replace procedure. It needs to be canceled
1286		 * manually if the cancellation is wanted.
1287		 */
1288		break;
1289	}
1290	return true;
1291}
1292
1293void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)
1294{
1295	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);
1296	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);
1297}
1298
1299void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)
1300{
1301	while (1) {
1302		percpu_counter_inc(&fs_info->dev_replace.bio_counter);
1303		if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1304				     &fs_info->fs_state)))
1305			break;
1306
1307		btrfs_bio_counter_dec(fs_info);
1308		wait_event(fs_info->dev_replace.replace_wait,
1309			   !test_bit(BTRFS_FS_STATE_DEV_REPLACING,
1310				     &fs_info->fs_state));
1311	}
1312}