extent_io.c

   1// SPDX-License-Identifier: GPL-2.0
   2
   3#include <linux/bitops.h>
   4#include <linux/slab.h>
   5#include <linux/bio.h>
   6#include <linux/mm.h>
   7#include <linux/pagemap.h>
   8#include <linux/page-flags.h>
   9#include <linux/sched/mm.h>
  10#include <linux/spinlock.h>
  11#include <linux/blkdev.h>
  12#include <linux/swap.h>
  13#include <linux/writeback.h>
  14#include <linux/pagevec.h>
  15#include <linux/prefetch.h>
  16#include <linux/fsverity.h>
  17#include "extent_io.h"
  18#include "extent-io-tree.h"
  19#include "extent_map.h"
  20#include "ctree.h"
  21#include "btrfs_inode.h"
  22#include "bio.h"
  23#include "locking.h"
  24#include "backref.h"
  25#include "disk-io.h"
  26#include "subpage.h"
  27#include "zoned.h"
  28#include "block-group.h"
  29#include "compression.h"
  30#include "fs.h"
  31#include "accessors.h"
  32#include "file-item.h"
  33#include "file.h"
  34#include "dev-replace.h"
  35#include "super.h"
  36#include "transaction.h"
  37
  38static struct kmem_cache *extent_buffer_cache;
  39
  40#ifdef CONFIG_BTRFS_DEBUG
  41static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
  42{
  43	struct btrfs_fs_info *fs_info = eb->fs_info;
  44	unsigned long flags;
  45
  46	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
  47	list_add(&eb->leak_list, &fs_info->allocated_ebs);
  48	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
  49}
  50
  51static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
  52{
  53	struct btrfs_fs_info *fs_info = eb->fs_info;
  54	unsigned long flags;
  55
  56	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
  57	list_del(&eb->leak_list);
  58	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
  59}
  60
  61void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
  62{
  63	struct extent_buffer *eb;
  64	unsigned long flags;
  65
  66	/*
  67	 * If we didn't get into open_ctree our allocated_ebs will not be
  68	 * initialized, so just skip this.
  69	 */
  70	if (!fs_info->allocated_ebs.next)
  71		return;
  72
  73	WARN_ON(!list_empty(&fs_info->allocated_ebs));
  74	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
  75	while (!list_empty(&fs_info->allocated_ebs)) {
  76		eb = list_first_entry(&fs_info->allocated_ebs,
  77				      struct extent_buffer, leak_list);
  78		btrfs_err(fs_info,
  79		       "buffer leak start %llu len %u refs %d bflags %lu owner %llu",
  80		       eb->start, eb->len, refcount_read(&eb->refs), eb->bflags,
  81		       btrfs_header_owner(eb));
  82		list_del(&eb->leak_list);
  83		WARN_ON_ONCE(1);
  84		kmem_cache_free(extent_buffer_cache, eb);
  85	}
  86	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
  87}
  88#else
  89#define btrfs_leak_debug_add_eb(eb)			do {} while (0)
  90#define btrfs_leak_debug_del_eb(eb)			do {} while (0)
  91#endif
  92
  93/*
  94 * Structure to record info about the bio being assembled, and other info like
  95 * how many bytes are there before stripe/ordered extent boundary.
  96 */
  97struct btrfs_bio_ctrl {
  98	struct btrfs_bio *bbio;
  99	/* Last byte contained in bbio + 1 . */
 100	loff_t next_file_offset;
 101	enum btrfs_compression_type compress_type;
 102	u32 len_to_oe_boundary;
 103	blk_opf_t opf;
 104	/*
 105	 * For data read bios, we attempt to optimize csum lookups if the extent
 106	 * generation is older than the current one. To make this possible, we
 107	 * need to track the maximum generation of an extent in a bio_ctrl to
 108	 * make the decision when submitting the bio.
 109	 *
 110	 * The pattern between do_readpage(), submit_one_bio() and
 111	 * submit_extent_folio() is quite subtle, so tracking this is tricky.
 112	 *
 113	 * As we process extent E, we might submit a bio with existing built up
 114	 * extents before adding E to a new bio, or we might just add E to the
 115	 * bio. As a result, E's generation could apply to the current bio or
 116	 * to the next one, so we need to be careful to update the bio_ctrl's
 117	 * generation with E's only when we are sure E is added to bio_ctrl->bbio
 118	 * in submit_extent_folio().
 119	 *
 120	 * See the comment in btrfs_lookup_bio_sums() for more detail on the
 121	 * need for this optimization.
 122	 */
 123	u64 generation;
 124	btrfs_bio_end_io_t end_io_func;
 125	struct writeback_control *wbc;
 126
 127	/*
 128	 * The sectors of the page which are going to be submitted by
 129	 * extent_writepage_io().
 130	 * This is to avoid touching ranges covered by compression/inline.
 131	 */
 132	unsigned long submit_bitmap;
 133	struct readahead_control *ractl;
 134
 135	/*
 136	 * The start offset of the last used extent map by a read operation.
 137	 *
 138	 * This is for proper compressed read merge.
 139	 * U64_MAX means we are starting the read and have made no progress yet.
 140	 *
 141	 * The current btrfs_bio_is_contig() only uses disk_bytenr as
 142	 * the condition to check if the read can be merged with previous
 143	 * bio, which is not correct. E.g. two file extents pointing to the
 144	 * same extent but with different offset.
 145	 *
 146	 * So here we need to do extra checks to only merge reads that are
 147	 * covered by the same extent map.
 148	 * Just extent_map::start will be enough, as they are unique
 149	 * inside the same inode.
 150	 */
 151	u64 last_em_start;
 152};
 153
 154/*
 155 * Helper to set the csum search commit root option for a bio_ctrl's bbio
 156 * before submitting the bio.
 157 *
 158 * Only for use by submit_one_bio().
 159 */
 160static void bio_set_csum_search_commit_root(struct btrfs_bio_ctrl *bio_ctrl)
 161{
 162	struct btrfs_bio *bbio = bio_ctrl->bbio;
 163
 164	ASSERT(bbio);
 165
 166	if (!(btrfs_op(&bbio->bio) == BTRFS_MAP_READ && is_data_inode(bbio->inode)))
 167		return;
 168
 169	bio_ctrl->bbio->csum_search_commit_root =
 170		(bio_ctrl->generation &&
 171		 bio_ctrl->generation < btrfs_get_fs_generation(bbio->inode->root->fs_info));
 172}
 173
 174static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
 175{
 176	struct btrfs_bio *bbio = bio_ctrl->bbio;
 177
 178	if (!bbio)
 179		return;
 180
 181	/* Caller should ensure the bio has at least some range added */
 182	ASSERT(bbio->bio.bi_iter.bi_size);
 183
 184	bio_set_csum_search_commit_root(bio_ctrl);
 185
 186	if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
 187	    bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
 188		btrfs_submit_compressed_read(bbio);
 189	else
 190		btrfs_submit_bbio(bbio, 0);
 191
 192	/* The bbio is owned by the end_io handler now */
 193	bio_ctrl->bbio = NULL;
 194	/*
 195	 * We used the generation to decide whether to lookup csums in the
 196	 * commit_root or not when we called bio_set_csum_search_commit_root()
 197	 * above. Now, reset the generation for the next bio.
 198	 */
 199	bio_ctrl->generation = 0;
 200}
 201
 202/*
 203 * Submit or fail the current bio in the bio_ctrl structure.
 204 */
 205static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
 206{
 207	struct btrfs_bio *bbio = bio_ctrl->bbio;
 208
 209	if (!bbio)
 210		return;
 211
 212	if (ret) {
 213		ASSERT(ret < 0);
 214		btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
 215		/* The bio is owned by the end_io handler now */
 216		bio_ctrl->bbio = NULL;
 217	} else {
 218		submit_one_bio(bio_ctrl);
 219	}
 220}
 221
 222int __init extent_buffer_init_cachep(void)
 223{
 224	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
 225						sizeof(struct extent_buffer), 0, 0,
 226						NULL);
 227	if (!extent_buffer_cache)
 228		return -ENOMEM;
 229
 230	return 0;
 231}
 232
 233void __cold extent_buffer_free_cachep(void)
 234{
 235	/*
 236	 * Make sure all delayed rcu free are flushed before we
 237	 * destroy caches.
 238	 */
 239	rcu_barrier();
 240	kmem_cache_destroy(extent_buffer_cache);
 241}
 242
 243static void process_one_folio(struct btrfs_fs_info *fs_info,
 244			      struct folio *folio, const struct folio *locked_folio,
 245			      unsigned long page_ops, u64 start, u64 end)
 246{
 247	u32 len;
 248
 249	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
 250	len = end + 1 - start;
 251
 252	if (page_ops & PAGE_SET_ORDERED)
 253		btrfs_folio_clamp_set_ordered(fs_info, folio, start, len);
 254	if (page_ops & PAGE_START_WRITEBACK) {
 255		btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len);
 256		btrfs_folio_clamp_set_writeback(fs_info, folio, start, len);
 257	}
 258	if (page_ops & PAGE_END_WRITEBACK)
 259		btrfs_folio_clamp_clear_writeback(fs_info, folio, start, len);
 260
 261	if (folio != locked_folio && (page_ops & PAGE_UNLOCK))
 262		btrfs_folio_end_lock(fs_info, folio, start, len);
 263}
 264
 265static void __process_folios_contig(struct address_space *mapping,
 266				    const struct folio *locked_folio, u64 start,
 267				    u64 end, unsigned long page_ops)
 268{
 269	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
 270	pgoff_t index = start >> PAGE_SHIFT;
 271	pgoff_t end_index = end >> PAGE_SHIFT;
 272	struct folio_batch fbatch;
 273	int i;
 274
 275	folio_batch_init(&fbatch);
 276	while (index <= end_index) {
 277		int found_folios;
 278
 279		found_folios = filemap_get_folios_contig(mapping, &index,
 280				end_index, &fbatch);
 281		for (i = 0; i < found_folios; i++) {
 282			struct folio *folio = fbatch.folios[i];
 283
 284			process_one_folio(fs_info, folio, locked_folio,
 285					  page_ops, start, end);
 286		}
 287		folio_batch_release(&fbatch);
 288		cond_resched();
 289	}
 290}
 291
 292static noinline void unlock_delalloc_folio(const struct inode *inode,
 293					   struct folio *locked_folio,
 294					   u64 start, u64 end)
 295{
 296	ASSERT(locked_folio);
 297
 298	__process_folios_contig(inode->i_mapping, locked_folio, start, end,
 299				PAGE_UNLOCK);
 300}
 301
 302static noinline int lock_delalloc_folios(struct inode *inode,
 303					 struct folio *locked_folio,
 304					 u64 start, u64 end)
 305{
 306	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
 307	struct address_space *mapping = inode->i_mapping;
 308	pgoff_t index = start >> PAGE_SHIFT;
 309	pgoff_t end_index = end >> PAGE_SHIFT;
 310	u64 processed_end = start;
 311	struct folio_batch fbatch;
 312
 313	folio_batch_init(&fbatch);
 314	while (index <= end_index) {
 315		unsigned int found_folios, i;
 316
 317		found_folios = filemap_get_folios_contig(mapping, &index,
 318				end_index, &fbatch);
 319		if (found_folios == 0)
 320			goto out;
 321
 322		for (i = 0; i < found_folios; i++) {
 323			struct folio *folio = fbatch.folios[i];
 324			u64 range_start;
 325			u32 range_len;
 326
 327			if (folio == locked_folio)
 328				continue;
 329
 330			folio_lock(folio);
 331			if (!folio_test_dirty(folio) || folio->mapping != mapping) {
 332				folio_unlock(folio);
 333				goto out;
 334			}
 335			range_start = max_t(u64, folio_pos(folio), start);
 336			range_len = min_t(u64, folio_next_pos(folio), end + 1) - range_start;
 337			btrfs_folio_set_lock(fs_info, folio, range_start, range_len);
 338
 339			processed_end = range_start + range_len - 1;
 340		}
 341		folio_batch_release(&fbatch);
 342		cond_resched();
 343	}
 344
 345	return 0;
 346out:
 347	folio_batch_release(&fbatch);
 348	if (processed_end > start)
 349		unlock_delalloc_folio(inode, locked_folio, start, processed_end);
 350	return -EAGAIN;
 351}
 352
 353/*
 354 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
 355 * more than @max_bytes.
 356 *
 357 * @start:	The original start bytenr to search.
 358 *		Will store the extent range start bytenr.
 359 * @end:	The original end bytenr of the search range
 360 *		Will store the extent range end bytenr.
 361 *
 362 * Return true if we find a delalloc range which starts inside the original
 363 * range, and @start/@end will store the delalloc range start/end.
 364 *
 365 * Return false if we can't find any delalloc range which starts inside the
 366 * original range, and @start/@end will be the non-delalloc range start/end.
 367 */
 368EXPORT_FOR_TESTS
 369noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
 370						 struct folio *locked_folio,
 371						 u64 *start, u64 *end)
 372{
 373	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
 374	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
 375	const u64 orig_start = *start;
 376	const u64 orig_end = *end;
 377	u64 max_bytes = fs_info->max_extent_size;
 378	u64 delalloc_start;
 379	u64 delalloc_end;
 380	bool found;
 381	struct extent_state *cached_state = NULL;
 382	int ret;
 383	int loops = 0;
 384
 385	/* Caller should pass a valid @end to indicate the search range end */
 386	ASSERT(orig_end > orig_start);
 387
 388	/* The range should at least cover part of the folio */
 389	ASSERT(!(orig_start >= folio_next_pos(locked_folio) ||
 390		 orig_end <= folio_pos(locked_folio)));
 391again:
 392	/* step one, find a bunch of delalloc bytes starting at start */
 393	delalloc_start = *start;
 394	delalloc_end = 0;
 395
 396	/*
 397	 * If @max_bytes is smaller than a block, btrfs_find_delalloc_range() can
 398	 * return early without handling any dirty ranges.
 399	 */
 400	ASSERT(max_bytes >= fs_info->sectorsize);
 401
 402	found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
 403					  max_bytes, &cached_state);
 404	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
 405		*start = delalloc_start;
 406
 407		/* @delalloc_end can be -1, never go beyond @orig_end */
 408		*end = min(delalloc_end, orig_end);
 409		btrfs_free_extent_state(cached_state);
 410		return false;
 411	}
 412
 413	/*
 414	 * start comes from the offset of locked_folio.  We have to lock
 415	 * folios in order, so we can't process delalloc bytes before
 416	 * locked_folio
 417	 */
 418	if (delalloc_start < *start)
 419		delalloc_start = *start;
 420
 421	/*
 422	 * make sure to limit the number of folios we try to lock down
 423	 */
 424	if (delalloc_end + 1 - delalloc_start > max_bytes)
 425		delalloc_end = delalloc_start + max_bytes - 1;
 426
 427	/* step two, lock all the folios after the folios that has start */
 428	ret = lock_delalloc_folios(inode, locked_folio, delalloc_start,
 429				   delalloc_end);
 430	ASSERT(!ret || ret == -EAGAIN);
 431	if (ret == -EAGAIN) {
 432		/*
 433		 * Some of the folios are gone, lets avoid looping by
 434		 * shortening the size of the delalloc range we're searching.
 435		 */
 436		btrfs_free_extent_state(cached_state);
 437		cached_state = NULL;
 438		if (!loops) {
 439			max_bytes = fs_info->sectorsize;
 440			loops = 1;
 441			goto again;
 442		} else {
 443			found = false;
 444			goto out_failed;
 445		}
 446	}
 447
 448	/* step three, lock the state bits for the whole range */
 449	btrfs_lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
 450
 451	/* then test to make sure it is all still delalloc */
 452	ret = btrfs_test_range_bit(tree, delalloc_start, delalloc_end,
 453				   EXTENT_DELALLOC, cached_state);
 454
 455	btrfs_unlock_extent(tree, delalloc_start, delalloc_end, &cached_state);
 456	if (!ret) {
 457		unlock_delalloc_folio(inode, locked_folio, delalloc_start,
 458				      delalloc_end);
 459		cond_resched();
 460		goto again;
 461	}
 462	*start = delalloc_start;
 463	*end = delalloc_end;
 464out_failed:
 465	return found;
 466}
 467
 468void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
 469				  const struct folio *locked_folio,
 470				  struct extent_state **cached,
 471				  u32 clear_bits, unsigned long page_ops)
 472{
 473	btrfs_clear_extent_bit(&inode->io_tree, start, end, clear_bits, cached);
 474
 475	__process_folios_contig(inode->vfs_inode.i_mapping, locked_folio, start,
 476				end, page_ops);
 477}
 478
 479static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len)
 480{
 481	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
 482
 483	if (!fsverity_active(folio->mapping->host) ||
 484	    btrfs_folio_test_uptodate(fs_info, folio, start, len) ||
 485	    start >= i_size_read(folio->mapping->host))
 486		return true;
 487	return fsverity_verify_folio(folio);
 488}
 489
 490static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len)
 491{
 492	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
 493
 494	ASSERT(folio_pos(folio) <= start &&
 495	       start + len <= folio_next_pos(folio));
 496
 497	if (uptodate && btrfs_verify_folio(folio, start, len))
 498		btrfs_folio_set_uptodate(fs_info, folio, start, len);
 499	else
 500		btrfs_folio_clear_uptodate(fs_info, folio, start, len);
 501
 502	if (!btrfs_is_subpage(fs_info, folio))
 503		folio_unlock(folio);
 504	else
 505		btrfs_folio_end_lock(fs_info, folio, start, len);
 506}
 507
 508/*
 509 * After a write IO is done, we need to:
 510 *
 511 * - clear the uptodate bits on error
 512 * - clear the writeback bits in the extent tree for the range
 513 * - filio_end_writeback()  if there is no more pending io for the folio
 514 *
 515 * Scheduling is not allowed, so the extent state tree is expected
 516 * to have one and only one object corresponding to this IO.
 517 */
 518static void end_bbio_data_write(struct btrfs_bio *bbio)
 519{
 520	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
 521	struct bio *bio = &bbio->bio;
 522	int error = blk_status_to_errno(bio->bi_status);
 523	struct folio_iter fi;
 524	const u32 sectorsize = fs_info->sectorsize;
 525
 526	ASSERT(!bio_flagged(bio, BIO_CLONED));
 527	bio_for_each_folio_all(fi, bio) {
 528		struct folio *folio = fi.folio;
 529		u64 start = folio_pos(folio) + fi.offset;
 530		u32 len = fi.length;
 531
 532		/* Our read/write should always be sector aligned. */
 533		if (!IS_ALIGNED(fi.offset, sectorsize))
 534			btrfs_err(fs_info,
 535		"partial page write in btrfs with offset %zu and length %zu",
 536				  fi.offset, fi.length);
 537		else if (!IS_ALIGNED(fi.length, sectorsize))
 538			btrfs_info(fs_info,
 539		"incomplete page write with offset %zu and length %zu",
 540				   fi.offset, fi.length);
 541
 542		btrfs_finish_ordered_extent(bbio->ordered, folio, start, len,
 543					    !error);
 544		if (error)
 545			mapping_set_error(folio->mapping, error);
 546		btrfs_folio_clear_writeback(fs_info, folio, start, len);
 547	}
 548
 549	bio_put(bio);
 550}
 551
 552static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio)
 553{
 554	ASSERT(folio_test_locked(folio));
 555	if (!btrfs_is_subpage(fs_info, folio))
 556		return;
 557
 558	ASSERT(folio_test_private(folio));
 559	btrfs_folio_set_lock(fs_info, folio, folio_pos(folio), folio_size(folio));
 560}
 561
 562/*
 563 * After a data read IO is done, we need to:
 564 *
 565 * - clear the uptodate bits on error
 566 * - set the uptodate bits if things worked
 567 * - set the folio up to date if all extents in the tree are uptodate
 568 * - clear the lock bit in the extent tree
 569 * - unlock the folio if there are no other extents locked for it
 570 *
 571 * Scheduling is not allowed, so the extent state tree is expected
 572 * to have one and only one object corresponding to this IO.
 573 */
 574static void end_bbio_data_read(struct btrfs_bio *bbio)
 575{
 576	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
 577	struct bio *bio = &bbio->bio;
 578	struct folio_iter fi;
 579
 580	ASSERT(!bio_flagged(bio, BIO_CLONED));
 581	bio_for_each_folio_all(fi, &bbio->bio) {
 582		bool uptodate = !bio->bi_status;
 583		struct folio *folio = fi.folio;
 584		struct inode *inode = folio->mapping->host;
 585		u64 start = folio_pos(folio) + fi.offset;
 586
 587		btrfs_debug(fs_info,
 588			"%s: bi_sector=%llu, err=%d, mirror=%u",
 589			__func__, bio->bi_iter.bi_sector, bio->bi_status,
 590			bbio->mirror_num);
 591
 592
 593		if (likely(uptodate)) {
 594			u64 end = start + fi.length - 1;
 595			loff_t i_size = i_size_read(inode);
 596
 597			/*
 598			 * Zero out the remaining part if this range straddles
 599			 * i_size.
 600			 *
 601			 * Here we should only zero the range inside the folio,
 602			 * not touch anything else.
 603			 *
 604			 * NOTE: i_size is exclusive while end is inclusive and
 605			 * folio_contains() takes PAGE_SIZE units.
 606			 */
 607			if (folio_contains(folio, i_size >> PAGE_SHIFT) &&
 608			    i_size <= end) {
 609				u32 zero_start = max(offset_in_folio(folio, i_size),
 610						     offset_in_folio(folio, start));
 611				u32 zero_len = offset_in_folio(folio, end) + 1 -
 612					       zero_start;
 613
 614				folio_zero_range(folio, zero_start, zero_len);
 615			}
 616		}
 617
 618		/* Update page status and unlock. */
 619		end_folio_read(folio, uptodate, start, fi.length);
 620	}
 621	bio_put(bio);
 622}
 623
 624/*
 625 * Populate every free slot in a provided array with folios using GFP_NOFS.
 626 *
 627 * @nr_folios:   number of folios to allocate
 628 * @order:	 the order of the folios to be allocated
 629 * @folio_array: the array to fill with folios; any existing non-NULL entries in
 630 *		 the array will be skipped
 631 *
 632 * Return: 0        if all folios were able to be allocated;
 633 *         -ENOMEM  otherwise, the partially allocated folios would be freed and
 634 *                  the array slots zeroed
 635 */
 636int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order,
 637			    struct folio **folio_array)
 638{
 639	for (int i = 0; i < nr_folios; i++) {
 640		if (folio_array[i])
 641			continue;
 642		folio_array[i] = folio_alloc(GFP_NOFS, order);
 643		if (!folio_array[i])
 644			goto error;
 645	}
 646	return 0;
 647error:
 648	for (int i = 0; i < nr_folios; i++) {
 649		if (folio_array[i])
 650			folio_put(folio_array[i]);
 651		folio_array[i] = NULL;
 652	}
 653	return -ENOMEM;
 654}
 655
 656/*
 657 * Populate every free slot in a provided array with pages, using GFP_NOFS.
 658 *
 659 * @nr_pages:   number of pages to allocate
 660 * @page_array: the array to fill with pages; any existing non-null entries in
 661 *		the array will be skipped
 662 * @nofail:	whether using __GFP_NOFAIL flag
 663 *
 664 * Return: 0        if all pages were able to be allocated;
 665 *         -ENOMEM  otherwise, the partially allocated pages would be freed and
 666 *                  the array slots zeroed
 667 */
 668int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array,
 669			   bool nofail)
 670{
 671	const gfp_t gfp = nofail ? (GFP_NOFS | __GFP_NOFAIL) : GFP_NOFS;
 672	unsigned int allocated;
 673
 674	for (allocated = 0; allocated < nr_pages;) {
 675		unsigned int last = allocated;
 676
 677		allocated = alloc_pages_bulk(gfp, nr_pages, page_array);
 678		if (unlikely(allocated == last)) {
 679			/* No progress, fail and do cleanup. */
 680			for (int i = 0; i < allocated; i++) {
 681				__free_page(page_array[i]);
 682				page_array[i] = NULL;
 683			}
 684			return -ENOMEM;
 685		}
 686	}
 687	return 0;
 688}
 689
 690/*
 691 * Populate needed folios for the extent buffer.
 692 *
 693 * For now, the folios populated are always in order 0 (aka, single page).
 694 */
 695static int alloc_eb_folio_array(struct extent_buffer *eb, bool nofail)
 696{
 697	struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 };
 698	int num_pages = num_extent_pages(eb);
 699	int ret;
 700
 701	ret = btrfs_alloc_page_array(num_pages, page_array, nofail);
 702	if (ret < 0)
 703		return ret;
 704
 705	for (int i = 0; i < num_pages; i++)
 706		eb->folios[i] = page_folio(page_array[i]);
 707	eb->folio_size = PAGE_SIZE;
 708	eb->folio_shift = PAGE_SHIFT;
 709	return 0;
 710}
 711
 712static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
 713				u64 disk_bytenr, loff_t file_offset)
 714{
 715	struct bio *bio = &bio_ctrl->bbio->bio;
 716	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
 717
 718	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
 719		/*
 720		 * For compression, all IO should have its logical bytenr set
 721		 * to the starting bytenr of the compressed extent.
 722		 */
 723		return bio->bi_iter.bi_sector == sector;
 724	}
 725
 726	/*
 727	 * To merge into a bio both the disk sector and the logical offset in
 728	 * the file need to be contiguous.
 729	 */
 730	return bio_ctrl->next_file_offset == file_offset &&
 731		bio_end_sector(bio) == sector;
 732}
 733
 734static void alloc_new_bio(struct btrfs_inode *inode,
 735			  struct btrfs_bio_ctrl *bio_ctrl,
 736			  u64 disk_bytenr, u64 file_offset)
 737{
 738	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 739	struct btrfs_bio *bbio;
 740
 741	bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, inode,
 742			       file_offset, bio_ctrl->end_io_func, NULL);
 743	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
 744	bbio->bio.bi_write_hint = inode->vfs_inode.i_write_hint;
 745	bio_ctrl->bbio = bbio;
 746	bio_ctrl->len_to_oe_boundary = U32_MAX;
 747	bio_ctrl->next_file_offset = file_offset;
 748
 749	/* Limit data write bios to the ordered boundary. */
 750	if (bio_ctrl->wbc) {
 751		struct btrfs_ordered_extent *ordered;
 752
 753		ordered = btrfs_lookup_ordered_extent(inode, file_offset);
 754		if (ordered) {
 755			bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
 756					ordered->file_offset +
 757					ordered->disk_num_bytes - file_offset);
 758			bbio->ordered = ordered;
 759		}
 760
 761		/*
 762		 * Pick the last added device to support cgroup writeback.  For
 763		 * multi-device file systems this means blk-cgroup policies have
 764		 * to always be set on the last added/replaced device.
 765		 * This is a bit odd but has been like that for a long time.
 766		 */
 767		bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
 768		wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
 769	}
 770}
 771
 772/*
 773 * @disk_bytenr: logical bytenr where the write will be
 774 * @page:	page to add to the bio
 775 * @size:	portion of page that we want to write to
 776 * @pg_offset:	offset of the new bio or to check whether we are adding
 777 *              a contiguous page to the previous one
 778 * @read_em_generation: generation of the extent_map we are submitting
 779 *			(only used for read)
 780 *
 781 * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
 782 * new one in @bio_ctrl->bbio.
 783 * The mirror number for this IO should already be initialized in
 784 * @bio_ctrl->mirror_num.
 785 */
 786static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl,
 787			       u64 disk_bytenr, struct folio *folio,
 788			       size_t size, unsigned long pg_offset,
 789			       u64 read_em_generation)
 790{
 791	struct btrfs_inode *inode = folio_to_inode(folio);
 792	loff_t file_offset = folio_pos(folio) + pg_offset;
 793
 794	ASSERT(pg_offset + size <= folio_size(folio));
 795	ASSERT(bio_ctrl->end_io_func);
 796
 797	if (bio_ctrl->bbio &&
 798	    !btrfs_bio_is_contig(bio_ctrl, disk_bytenr, file_offset))
 799		submit_one_bio(bio_ctrl);
 800
 801	do {
 802		u32 len = size;
 803
 804		/* Allocate new bio if needed */
 805		if (!bio_ctrl->bbio)
 806			alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset);
 807
 808		/* Cap to the current ordered extent boundary if there is one. */
 809		if (len > bio_ctrl->len_to_oe_boundary) {
 810			ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
 811			ASSERT(is_data_inode(inode));
 812			len = bio_ctrl->len_to_oe_boundary;
 813		}
 814
 815		if (!bio_add_folio(&bio_ctrl->bbio->bio, folio, len, pg_offset)) {
 816			/* bio full: move on to a new one */
 817			submit_one_bio(bio_ctrl);
 818			continue;
 819		}
 820		/*
 821		 * Now that the folio is definitely added to the bio, include its
 822		 * generation in the max generation calculation.
 823		 */
 824		bio_ctrl->generation = max(bio_ctrl->generation, read_em_generation);
 825		bio_ctrl->next_file_offset += len;
 826
 827		if (bio_ctrl->wbc)
 828			wbc_account_cgroup_owner(bio_ctrl->wbc, folio, len);
 829
 830		size -= len;
 831		pg_offset += len;
 832		disk_bytenr += len;
 833		file_offset += len;
 834
 835		/*
 836		 * len_to_oe_boundary defaults to U32_MAX, which isn't folio or
 837		 * sector aligned.  alloc_new_bio() then sets it to the end of
 838		 * our ordered extent for writes into zoned devices.
 839		 *
 840		 * When len_to_oe_boundary is tracking an ordered extent, we
 841		 * trust the ordered extent code to align things properly, and
 842		 * the check above to cap our write to the ordered extent
 843		 * boundary is correct.
 844		 *
 845		 * When len_to_oe_boundary is U32_MAX, the cap above would
 846		 * result in a 4095 byte IO for the last folio right before
 847		 * we hit the bio limit of UINT_MAX.  bio_add_folio() has all
 848		 * the checks required to make sure we don't overflow the bio,
 849		 * and we should just ignore len_to_oe_boundary completely
 850		 * unless we're using it to track an ordered extent.
 851		 *
 852		 * It's pretty hard to make a bio sized U32_MAX, but it can
 853		 * happen when the page cache is able to feed us contiguous
 854		 * folios for large extents.
 855		 */
 856		if (bio_ctrl->len_to_oe_boundary != U32_MAX)
 857			bio_ctrl->len_to_oe_boundary -= len;
 858
 859		/* Ordered extent boundary: move on to a new bio. */
 860		if (bio_ctrl->len_to_oe_boundary == 0)
 861			submit_one_bio(bio_ctrl);
 862	} while (size);
 863}
 864
 865static int attach_extent_buffer_folio(struct extent_buffer *eb,
 866				      struct folio *folio,
 867				      struct btrfs_folio_state *prealloc)
 868{
 869	struct btrfs_fs_info *fs_info = eb->fs_info;
 870	int ret = 0;
 871
 872	/*
 873	 * If the page is mapped to btree inode, we should hold the private
 874	 * lock to prevent race.
 875	 * For cloned or dummy extent buffers, their pages are not mapped and
 876	 * will not race with any other ebs.
 877	 */
 878	if (folio->mapping)
 879		lockdep_assert_held(&folio->mapping->i_private_lock);
 880
 881	if (!btrfs_meta_is_subpage(fs_info)) {
 882		if (!folio_test_private(folio))
 883			folio_attach_private(folio, eb);
 884		else
 885			WARN_ON(folio_get_private(folio) != eb);
 886		return 0;
 887	}
 888
 889	/* Already mapped, just free prealloc */
 890	if (folio_test_private(folio)) {
 891		btrfs_free_folio_state(prealloc);
 892		return 0;
 893	}
 894
 895	if (prealloc)
 896		/* Has preallocated memory for subpage */
 897		folio_attach_private(folio, prealloc);
 898	else
 899		/* Do new allocation to attach subpage */
 900		ret = btrfs_attach_folio_state(fs_info, folio, BTRFS_SUBPAGE_METADATA);
 901	return ret;
 902}
 903
 904int set_folio_extent_mapped(struct folio *folio)
 905{
 906	struct btrfs_fs_info *fs_info;
 907
 908	ASSERT(folio->mapping);
 909
 910	if (folio_test_private(folio))
 911		return 0;
 912
 913	fs_info = folio_to_fs_info(folio);
 914
 915	if (btrfs_is_subpage(fs_info, folio))
 916		return btrfs_attach_folio_state(fs_info, folio, BTRFS_SUBPAGE_DATA);
 917
 918	folio_attach_private(folio, (void *)EXTENT_FOLIO_PRIVATE);
 919	return 0;
 920}
 921
 922void clear_folio_extent_mapped(struct folio *folio)
 923{
 924	struct btrfs_fs_info *fs_info;
 925
 926	ASSERT(folio->mapping);
 927
 928	if (!folio_test_private(folio))
 929		return;
 930
 931	fs_info = folio_to_fs_info(folio);
 932	if (btrfs_is_subpage(fs_info, folio))
 933		return btrfs_detach_folio_state(fs_info, folio, BTRFS_SUBPAGE_DATA);
 934
 935	folio_detach_private(folio);
 936}
 937
 938static struct extent_map *get_extent_map(struct btrfs_inode *inode,
 939					 struct folio *folio, u64 start,
 940					 u64 len, struct extent_map **em_cached)
 941{
 942	struct extent_map *em;
 943
 944	ASSERT(em_cached);
 945
 946	if (*em_cached) {
 947		em = *em_cached;
 948		if (btrfs_extent_map_in_tree(em) && start >= em->start &&
 949		    start < btrfs_extent_map_end(em)) {
 950			refcount_inc(&em->refs);
 951			return em;
 952		}
 953
 954		btrfs_free_extent_map(em);
 955		*em_cached = NULL;
 956	}
 957
 958	em = btrfs_get_extent(inode, folio, start, len);
 959	if (!IS_ERR(em)) {
 960		BUG_ON(*em_cached);
 961		refcount_inc(&em->refs);
 962		*em_cached = em;
 963	}
 964
 965	return em;
 966}
 967
 968static void btrfs_readahead_expand(struct readahead_control *ractl,
 969				   const struct extent_map *em)
 970{
 971	const u64 ra_pos = readahead_pos(ractl);
 972	const u64 ra_end = ra_pos + readahead_length(ractl);
 973	const u64 em_end = em->start + em->len;
 974
 975	/* No expansion for holes and inline extents. */
 976	if (em->disk_bytenr > EXTENT_MAP_LAST_BYTE)
 977		return;
 978
 979	ASSERT(em_end >= ra_pos,
 980	       "extent_map %llu %llu ends before current readahead position %llu",
 981	       em->start, em->len, ra_pos);
 982	if (em_end > ra_end)
 983		readahead_expand(ractl, ra_pos, em_end - ra_pos);
 984}
 985
 986/*
 987 * basic readpage implementation.  Locked extent state structs are inserted
 988 * into the tree that are removed when the IO is done (by the end_io
 989 * handlers)
 990 * XXX JDM: This needs looking at to ensure proper page locking
 991 * return 0 on success, otherwise return error
 992 */
 993static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
 994			     struct btrfs_bio_ctrl *bio_ctrl)
 995{
 996	struct inode *inode = folio->mapping->host;
 997	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
 998	u64 start = folio_pos(folio);
 999	const u64 end = start + folio_size(folio) - 1;
1000	u64 extent_offset;
1001	u64 last_byte = i_size_read(inode);
1002	struct extent_map *em;
1003	int ret = 0;
1004	const size_t blocksize = fs_info->sectorsize;
1005
1006	ret = set_folio_extent_mapped(folio);
1007	if (ret < 0) {
1008		folio_unlock(folio);
1009		return ret;
1010	}
1011
1012	if (folio_contains(folio, last_byte >> PAGE_SHIFT)) {
1013		size_t zero_offset = offset_in_folio(folio, last_byte);
1014
1015		if (zero_offset)
1016			folio_zero_range(folio, zero_offset,
1017					 folio_size(folio) - zero_offset);
1018	}
1019	bio_ctrl->end_io_func = end_bbio_data_read;
1020	begin_folio_read(fs_info, folio);
1021	for (u64 cur = start; cur <= end; cur += blocksize) {
1022		enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1023		unsigned long pg_offset = offset_in_folio(folio, cur);
1024		bool force_bio_submit = false;
1025		u64 disk_bytenr;
1026		u64 block_start;
1027		u64 em_gen;
1028
1029		ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1030		if (cur >= last_byte) {
1031			folio_zero_range(folio, pg_offset, end - cur + 1);
1032			end_folio_read(folio, true, cur, end - cur + 1);
1033			break;
1034		}
1035		if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) {
1036			end_folio_read(folio, true, cur, blocksize);
1037			continue;
1038		}
1039		em = get_extent_map(BTRFS_I(inode), folio, cur, end - cur + 1, em_cached);
1040		if (IS_ERR(em)) {
1041			end_folio_read(folio, false, cur, end + 1 - cur);
1042			return PTR_ERR(em);
1043		}
1044		extent_offset = cur - em->start;
1045		BUG_ON(btrfs_extent_map_end(em) <= cur);
1046		BUG_ON(end < cur);
1047
1048		compress_type = btrfs_extent_map_compression(em);
1049
1050		/*
1051		 * Only expand readahead for extents which are already creating
1052		 * the pages anyway in add_ra_bio_pages, which is compressed
1053		 * extents in the non subpage case.
1054		 */
1055		if (bio_ctrl->ractl &&
1056		    !btrfs_is_subpage(fs_info, folio) &&
1057		    compress_type != BTRFS_COMPRESS_NONE)
1058			btrfs_readahead_expand(bio_ctrl->ractl, em);
1059
1060		if (compress_type != BTRFS_COMPRESS_NONE)
1061			disk_bytenr = em->disk_bytenr;
1062		else
1063			disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
1064
1065		if (em->flags & EXTENT_FLAG_PREALLOC)
1066			block_start = EXTENT_MAP_HOLE;
1067		else
1068			block_start = btrfs_extent_map_block_start(em);
1069
1070		/*
1071		 * If we have a file range that points to a compressed extent
1072		 * and it's followed by a consecutive file range that points
1073		 * to the same compressed extent (possibly with a different
1074		 * offset and/or length, so it either points to the whole extent
1075		 * or only part of it), we must make sure we do not submit a
1076		 * single bio to populate the folios for the 2 ranges because
1077		 * this makes the compressed extent read zero out the folios
1078		 * belonging to the 2nd range. Imagine the following scenario:
1079		 *
1080		 *  File layout
1081		 *  [0 - 8K]                     [8K - 24K]
1082		 *    |                               |
1083		 *    |                               |
1084		 * points to extent X,         points to extent X,
1085		 * offset 4K, length of 8K     offset 0, length 16K
1086		 *
1087		 * [extent X, compressed length = 4K uncompressed length = 16K]
1088		 *
1089		 * If the bio to read the compressed extent covers both ranges,
1090		 * it will decompress extent X into the folios belonging to the
1091		 * first range and then it will stop, zeroing out the remaining
1092		 * folios that belong to the other range that points to extent X.
1093		 * So here we make sure we submit 2 bios, one for the first
1094		 * range and another one for the third range. Both will target
1095		 * the same physical extent from disk, but we can't currently
1096		 * make the compressed bio endio callback populate the folios
1097		 * for both ranges because each compressed bio is tightly
1098		 * coupled with a single extent map, and each range can have
1099		 * an extent map with a different offset value relative to the
1100		 * uncompressed data of our extent and different lengths. This
1101		 * is a corner case so we prioritize correctness over
1102		 * non-optimal behavior (submitting 2 bios for the same extent).
1103		 */
1104		if (compress_type != BTRFS_COMPRESS_NONE &&
1105		    bio_ctrl->last_em_start != U64_MAX &&
1106		    bio_ctrl->last_em_start != em->start)
1107			force_bio_submit = true;
1108
1109		bio_ctrl->last_em_start = em->start;
1110
1111		em_gen = em->generation;
1112		btrfs_free_extent_map(em);
1113		em = NULL;
1114
1115		/* we've found a hole, just zero and go on */
1116		if (block_start == EXTENT_MAP_HOLE) {
1117			folio_zero_range(folio, pg_offset, blocksize);
1118			end_folio_read(folio, true, cur, blocksize);
1119			continue;
1120		}
1121		/* the get_extent function already copied into the folio */
1122		if (block_start == EXTENT_MAP_INLINE) {
1123			end_folio_read(folio, true, cur, blocksize);
1124			continue;
1125		}
1126
1127		if (bio_ctrl->compress_type != compress_type) {
1128			submit_one_bio(bio_ctrl);
1129			bio_ctrl->compress_type = compress_type;
1130		}
1131
1132		if (force_bio_submit)
1133			submit_one_bio(bio_ctrl);
1134		submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize,
1135				    pg_offset, em_gen);
1136	}
1137	return 0;
1138}
1139
1140/*
1141 * Check if we can skip waiting the @ordered extent covering the block at @fileoff.
1142 *
1143 * @fileoff:	Both input and output.
1144 *		Input as the file offset where the check should start at.
1145 *		Output as where the next check should start at,
1146 *		if the function returns true.
1147 *
1148 * Return true if we can skip to @fileoff. The caller needs to check the new
1149 * @fileoff value to make sure it covers the full range, before skipping the
1150 * full OE.
1151 *
1152 * Return false if we must wait for the ordered extent.
1153 */
1154static bool can_skip_one_ordered_range(struct btrfs_inode *inode,
1155				       struct btrfs_ordered_extent *ordered,
1156				       u64 *fileoff)
1157{
1158	const struct btrfs_fs_info *fs_info = inode->root->fs_info;
1159	struct folio *folio;
1160	const u32 blocksize = fs_info->sectorsize;
1161	u64 cur = *fileoff;
1162	bool ret;
1163
1164	folio = filemap_get_folio(inode->vfs_inode.i_mapping, cur >> PAGE_SHIFT);
1165
1166	/*
1167	 * We should have locked the folio(s) for range [start, end], thus
1168	 * there must be a folio and it must be locked.
1169	 */
1170	ASSERT(!IS_ERR(folio));
1171	ASSERT(folio_test_locked(folio));
1172
1173	/*
1174	 * There are several cases for the folio and OE combination:
1175	 *
1176	 * 1) Folio has no private flag
1177	 *    The OE has all its IO done but not yet finished, and folio got
1178	 *    invalidated.
1179	 *
1180	 * Have we have to wait for the OE to finish, as it may contain the
1181	 * to-be-inserted data checksum.
1182	 * Without the data checksum inserted into the csum tree, read will
1183	 * just fail with missing csum.
1184	 */
1185	if (!folio_test_private(folio)) {
1186		ret = false;
1187		goto out;
1188	}
1189
1190	/*
1191	 * 2) The first block is DIRTY.
1192	 *
1193	 * This means the OE is created by some other folios whose file pos is
1194	 * before this one. And since we are holding the folio lock, the writeback
1195	 * of this folio cannot start.
1196	 *
1197	 * We must skip the whole OE, because it will never start until we
1198	 * finished our folio read and unlocked the folio.
1199	 */
1200	if (btrfs_folio_test_dirty(fs_info, folio, cur, blocksize)) {
1201		u64 range_len = umin(folio_next_pos(folio),
1202				    ordered->file_offset + ordered->num_bytes) - cur;
1203
1204		ret = true;
1205		/*
1206		 * At least inside the folio, all the remaining blocks should
1207		 * also be dirty.
1208		 */
1209		ASSERT(btrfs_folio_test_dirty(fs_info, folio, cur, range_len));
1210		*fileoff = ordered->file_offset + ordered->num_bytes;
1211		goto out;
1212	}
1213
1214	/*
1215	 * 3) The first block is uptodate.
1216	 *
1217	 * At least the first block can be skipped, but we are still not fully
1218	 * sure. E.g. if the OE has some other folios in the range that cannot
1219	 * be skipped.
1220	 * So we return true and update @next_ret to the OE/folio boundary.
1221	 */
1222	if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) {
1223		u64 range_len = umin(folio_next_pos(folio),
1224				    ordered->file_offset + ordered->num_bytes) - cur;
1225
1226		/*
1227		 * The whole range to the OE end or folio boundary should also
1228		 * be uptodate.
1229		 */
1230		ASSERT(btrfs_folio_test_uptodate(fs_info, folio, cur, range_len));
1231		ret = true;
1232		*fileoff = cur + range_len;
1233		goto out;
1234	}
1235
1236	/*
1237	 * 4) The first block is not uptodate.
1238	 *
1239	 * This means the folio is invalidated after the writeback was finished,
1240	 * but by some other operations (e.g. block aligned buffered write) the
1241	 * folio is inserted into filemap.
1242	 * Very much the same as case 1).
1243	 */
1244	ret = false;
1245out:
1246	folio_put(folio);
1247	return ret;
1248}
1249
1250static bool can_skip_ordered_extent(struct btrfs_inode *inode,
1251				    struct btrfs_ordered_extent *ordered,
1252				    u64 start, u64 end)
1253{
1254	const u64 range_end = min(end, ordered->file_offset + ordered->num_bytes - 1);
1255	u64 cur = max(start, ordered->file_offset);
1256
1257	while (cur < range_end) {
1258		bool can_skip;
1259
1260		can_skip = can_skip_one_ordered_range(inode, ordered, &cur);
1261		if (!can_skip)
1262			return false;
1263	}
1264	return true;
1265}
1266
1267/*
1268 * Locking helper to make sure we get a stable view of extent maps for the
1269 * involved range.
1270 *
1271 * This is for folio read paths (read and readahead), thus the involved range
1272 * should have all the folios locked.
1273 */
1274static void lock_extents_for_read(struct btrfs_inode *inode, u64 start, u64 end,
1275				  struct extent_state **cached_state)
1276{
1277	u64 cur_pos;
1278
1279	/* Caller must provide a valid @cached_state. */
1280	ASSERT(cached_state);
1281
1282	/* The range must at least be page aligned, as all read paths are folio based. */
1283	ASSERT(IS_ALIGNED(start, PAGE_SIZE));
1284	ASSERT(IS_ALIGNED(end + 1, PAGE_SIZE));
1285
1286again:
1287	btrfs_lock_extent(&inode->io_tree, start, end, cached_state);
1288	cur_pos = start;
1289	while (cur_pos < end) {
1290		struct btrfs_ordered_extent *ordered;
1291
1292		ordered = btrfs_lookup_ordered_range(inode, cur_pos,
1293						     end - cur_pos + 1);
1294		/*
1295		 * No ordered extents in the range, and we hold the extent lock,
1296		 * no one can modify the extent maps in the range, we're safe to return.
1297		 */
1298		if (!ordered)
1299			break;
1300
1301		/* Check if we can skip waiting for the whole OE. */
1302		if (can_skip_ordered_extent(inode, ordered, start, end)) {
1303			cur_pos = min(ordered->file_offset + ordered->num_bytes,
1304				      end + 1);
1305			btrfs_put_ordered_extent(ordered);
1306			continue;
1307		}
1308
1309		/* Now wait for the OE to finish. */
1310		btrfs_unlock_extent(&inode->io_tree, start, end, cached_state);
1311		btrfs_start_ordered_extent_nowriteback(ordered, start, end + 1 - start);
1312		btrfs_put_ordered_extent(ordered);
1313		/* We have unlocked the whole range, restart from the beginning. */
1314		goto again;
1315	}
1316}
1317
1318int btrfs_read_folio(struct file *file, struct folio *folio)
1319{
1320	struct btrfs_inode *inode = folio_to_inode(folio);
1321	const u64 start = folio_pos(folio);
1322	const u64 end = start + folio_size(folio) - 1;
1323	struct extent_state *cached_state = NULL;
1324	struct btrfs_bio_ctrl bio_ctrl = {
1325		.opf = REQ_OP_READ,
1326		.last_em_start = U64_MAX,
1327	};
1328	struct extent_map *em_cached = NULL;
1329	int ret;
1330
1331	lock_extents_for_read(inode, start, end, &cached_state);
1332	ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl);
1333	btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
1334
1335	btrfs_free_extent_map(em_cached);
1336
1337	/*
1338	 * If btrfs_do_readpage() failed we will want to submit the assembled
1339	 * bio to do the cleanup.
1340	 */
1341	submit_one_bio(&bio_ctrl);
1342	return ret;
1343}
1344
1345static void set_delalloc_bitmap(struct folio *folio, unsigned long *delalloc_bitmap,
1346				u64 start, u32 len)
1347{
1348	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1349	const u64 folio_start = folio_pos(folio);
1350	unsigned int start_bit;
1351	unsigned int nbits;
1352
1353	ASSERT(start >= folio_start && start + len <= folio_start + folio_size(folio));
1354	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1355	nbits = len >> fs_info->sectorsize_bits;
1356	ASSERT(bitmap_test_range_all_zero(delalloc_bitmap, start_bit, nbits));
1357	bitmap_set(delalloc_bitmap, start_bit, nbits);
1358}
1359
1360static bool find_next_delalloc_bitmap(struct folio *folio,
1361				      unsigned long *delalloc_bitmap, u64 start,
1362				      u64 *found_start, u32 *found_len)
1363{
1364	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1365	const u64 folio_start = folio_pos(folio);
1366	const unsigned int bitmap_size = btrfs_blocks_per_folio(fs_info, folio);
1367	unsigned int start_bit;
1368	unsigned int first_zero;
1369	unsigned int first_set;
1370
1371	ASSERT(start >= folio_start && start < folio_start + folio_size(folio));
1372
1373	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1374	first_set = find_next_bit(delalloc_bitmap, bitmap_size, start_bit);
1375	if (first_set >= bitmap_size)
1376		return false;
1377
1378	*found_start = folio_start + (first_set << fs_info->sectorsize_bits);
1379	first_zero = find_next_zero_bit(delalloc_bitmap, bitmap_size, first_set);
1380	*found_len = (first_zero - first_set) << fs_info->sectorsize_bits;
1381	return true;
1382}
1383
1384/*
1385 * Do all of the delayed allocation setup.
1386 *
1387 * Return >0 if all the dirty blocks are submitted async (compression) or inlined.
1388 * The @folio should no longer be touched (treat it as already unlocked).
1389 *
1390 * Return 0 if there is still dirty block that needs to be submitted through
1391 * extent_writepage_io().
1392 * bio_ctrl->submit_bitmap will indicate which blocks of the folio should be
1393 * submitted, and @folio is still kept locked.
1394 *
1395 * Return <0 if there is any error hit.
1396 * Any allocated ordered extent range covering this folio will be marked
1397 * finished (IOERR), and @folio is still kept locked.
1398 */
1399static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1400						 struct folio *folio,
1401						 struct btrfs_bio_ctrl *bio_ctrl)
1402{
1403	struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode);
1404	struct writeback_control *wbc = bio_ctrl->wbc;
1405	const bool is_subpage = btrfs_is_subpage(fs_info, folio);
1406	const u64 page_start = folio_pos(folio);
1407	const u64 page_end = page_start + folio_size(folio) - 1;
1408	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1409	unsigned long delalloc_bitmap = 0;
1410	/*
1411	 * Save the last found delalloc end. As the delalloc end can go beyond
1412	 * page boundary, thus we cannot rely on subpage bitmap to locate the
1413	 * last delalloc end.
1414	 */
1415	u64 last_delalloc_end = 0;
1416	/*
1417	 * The range end (exclusive) of the last successfully finished delalloc
1418	 * range.
1419	 * Any range covered by ordered extent must either be manually marked
1420	 * finished (error handling), or has IO submitted (and finish the
1421	 * ordered extent normally).
1422	 *
1423	 * This records the end of ordered extent cleanup if we hit an error.
1424	 */
1425	u64 last_finished_delalloc_end = page_start;
1426	u64 delalloc_start = page_start;
1427	u64 delalloc_end = page_end;
1428	u64 delalloc_to_write = 0;
1429	int ret = 0;
1430	int bit;
1431
1432	/* Save the dirty bitmap as our submission bitmap will be a subset of it. */
1433	if (btrfs_is_subpage(fs_info, folio)) {
1434		ASSERT(blocks_per_folio > 1);
1435		btrfs_get_subpage_dirty_bitmap(fs_info, folio, &bio_ctrl->submit_bitmap);
1436	} else {
1437		bio_ctrl->submit_bitmap = 1;
1438	}
1439
1440	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
1441		u64 start = page_start + (bit << fs_info->sectorsize_bits);
1442
1443		btrfs_folio_set_lock(fs_info, folio, start, fs_info->sectorsize);
1444	}
1445
1446	/* Lock all (subpage) delalloc ranges inside the folio first. */
1447	while (delalloc_start < page_end) {
1448		delalloc_end = page_end;
1449		if (!find_lock_delalloc_range(&inode->vfs_inode, folio,
1450					      &delalloc_start, &delalloc_end)) {
1451			delalloc_start = delalloc_end + 1;
1452			continue;
1453		}
1454		set_delalloc_bitmap(folio, &delalloc_bitmap, delalloc_start,
1455				    min(delalloc_end, page_end) + 1 - delalloc_start);
1456		last_delalloc_end = delalloc_end;
1457		delalloc_start = delalloc_end + 1;
1458	}
1459	delalloc_start = page_start;
1460
1461	if (!last_delalloc_end)
1462		goto out;
1463
1464	/* Run the delalloc ranges for the above locked ranges. */
1465	while (delalloc_start < page_end) {
1466		u64 found_start;
1467		u32 found_len;
1468		bool found;
1469
1470		if (!is_subpage) {
1471			/*
1472			 * For non-subpage case, the found delalloc range must
1473			 * cover this folio and there must be only one locked
1474			 * delalloc range.
1475			 */
1476			found_start = page_start;
1477			found_len = last_delalloc_end + 1 - found_start;
1478			found = true;
1479		} else {
1480			found = find_next_delalloc_bitmap(folio, &delalloc_bitmap,
1481					delalloc_start, &found_start, &found_len);
1482		}
1483		if (!found)
1484			break;
1485		/*
1486		 * The subpage range covers the last sector, the delalloc range may
1487		 * end beyond the folio boundary, use the saved delalloc_end
1488		 * instead.
1489		 */
1490		if (found_start + found_len >= page_end)
1491			found_len = last_delalloc_end + 1 - found_start;
1492
1493		if (ret >= 0) {
1494			/*
1495			 * Some delalloc range may be created by previous folios.
1496			 * Thus we still need to clean up this range during error
1497			 * handling.
1498			 */
1499			last_finished_delalloc_end = found_start;
1500			/* No errors hit so far, run the current delalloc range. */
1501			ret = btrfs_run_delalloc_range(inode, folio,
1502						       found_start,
1503						       found_start + found_len - 1,
1504						       wbc);
1505			if (ret >= 0)
1506				last_finished_delalloc_end = found_start + found_len;
1507			if (unlikely(ret < 0))
1508				btrfs_err_rl(fs_info,
1509"failed to run delalloc range, root=%lld ino=%llu folio=%llu submit_bitmap=%*pbl start=%llu len=%u: %d",
1510					     btrfs_root_id(inode->root),
1511					     btrfs_ino(inode),
1512					     folio_pos(folio),
1513					     blocks_per_folio,
1514					     &bio_ctrl->submit_bitmap,
1515					     found_start, found_len, ret);
1516		} else {
1517			/*
1518			 * We've hit an error during previous delalloc range,
1519			 * have to cleanup the remaining locked ranges.
1520			 */
1521			btrfs_unlock_extent(&inode->io_tree, found_start,
1522					    found_start + found_len - 1, NULL);
1523			unlock_delalloc_folio(&inode->vfs_inode, folio,
1524					      found_start,
1525					      found_start + found_len - 1);
1526		}
1527
1528		/*
1529		 * We have some ranges that's going to be submitted asynchronously
1530		 * (compression or inline).  These range have their own control
1531		 * on when to unlock the pages.  We should not touch them
1532		 * anymore, so clear the range from the submission bitmap.
1533		 */
1534		if (ret > 0) {
1535			unsigned int start_bit = (found_start - page_start) >>
1536						 fs_info->sectorsize_bits;
1537			unsigned int end_bit = (min(page_end + 1, found_start + found_len) -
1538						page_start) >> fs_info->sectorsize_bits;
1539			bitmap_clear(&bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit);
1540		}
1541		/*
1542		 * Above btrfs_run_delalloc_range() may have unlocked the folio,
1543		 * thus for the last range, we cannot touch the folio anymore.
1544		 */
1545		if (found_start + found_len >= last_delalloc_end + 1)
1546			break;
1547
1548		delalloc_start = found_start + found_len;
1549	}
1550	/*
1551	 * It's possible we had some ordered extents created before we hit
1552	 * an error, cleanup non-async successfully created delalloc ranges.
1553	 */
1554	if (unlikely(ret < 0)) {
1555		unsigned int bitmap_size = min(
1556				(last_finished_delalloc_end - page_start) >>
1557				fs_info->sectorsize_bits,
1558				blocks_per_folio);
1559
1560		for_each_set_bit(bit, &bio_ctrl->submit_bitmap, bitmap_size)
1561			btrfs_mark_ordered_io_finished(inode, folio,
1562				page_start + (bit << fs_info->sectorsize_bits),
1563				fs_info->sectorsize, false);
1564		return ret;
1565	}
1566out:
1567	if (last_delalloc_end)
1568		delalloc_end = last_delalloc_end;
1569	else
1570		delalloc_end = page_end;
1571	/*
1572	 * delalloc_end is already one less than the total length, so
1573	 * we don't subtract one from PAGE_SIZE.
1574	 */
1575	delalloc_to_write +=
1576		DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1577
1578	/*
1579	 * If all ranges are submitted asynchronously, we just need to account
1580	 * for them here.
1581	 */
1582	if (bitmap_empty(&bio_ctrl->submit_bitmap, blocks_per_folio)) {
1583		wbc->nr_to_write -= delalloc_to_write;
1584		return 1;
1585	}
1586
1587	if (wbc->nr_to_write < delalloc_to_write) {
1588		int thresh = 8192;
1589
1590		if (delalloc_to_write < thresh * 2)
1591			thresh = delalloc_to_write;
1592		wbc->nr_to_write = min_t(u64, delalloc_to_write,
1593					 thresh);
1594	}
1595
1596	return 0;
1597}
1598
1599/*
1600 * Return 0 if we have submitted or queued the sector for submission.
1601 * Return <0 for critical errors, and the sector will have its dirty flag cleared.
1602 *
1603 * Caller should make sure filepos < i_size and handle filepos >= i_size case.
1604 */
1605static int submit_one_sector(struct btrfs_inode *inode,
1606			     struct folio *folio,
1607			     u64 filepos, struct btrfs_bio_ctrl *bio_ctrl,
1608			     loff_t i_size)
1609{
1610	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1611	struct extent_map *em;
1612	u64 block_start;
1613	u64 disk_bytenr;
1614	u64 extent_offset;
1615	u64 em_end;
1616	const u32 sectorsize = fs_info->sectorsize;
1617
1618	ASSERT(IS_ALIGNED(filepos, sectorsize));
1619
1620	/* @filepos >= i_size case should be handled by the caller. */
1621	ASSERT(filepos < i_size);
1622
1623	em = btrfs_get_extent(inode, NULL, filepos, sectorsize);
1624	if (IS_ERR(em)) {
1625		/*
1626		 * When submission failed, we should still clear the folio dirty.
1627		 * Or the folio will be written back again but without any
1628		 * ordered extent.
1629		 */
1630		btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
1631		btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
1632		btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize);
1633		return PTR_ERR(em);
1634	}
1635
1636	extent_offset = filepos - em->start;
1637	em_end = btrfs_extent_map_end(em);
1638	ASSERT(filepos <= em_end);
1639	ASSERT(IS_ALIGNED(em->start, sectorsize));
1640	ASSERT(IS_ALIGNED(em->len, sectorsize));
1641
1642	block_start = btrfs_extent_map_block_start(em);
1643	disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
1644
1645	ASSERT(!btrfs_extent_map_is_compressed(em));
1646	ASSERT(block_start != EXTENT_MAP_HOLE);
1647	ASSERT(block_start != EXTENT_MAP_INLINE);
1648
1649	btrfs_free_extent_map(em);
1650	em = NULL;
1651
1652	/*
1653	 * Although the PageDirty bit is cleared before entering this
1654	 * function, subpage dirty bit is not cleared.
1655	 * So clear subpage dirty bit here so next time we won't submit
1656	 * a folio for a range already written to disk.
1657	 */
1658	btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
1659	btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
1660	/*
1661	 * Above call should set the whole folio with writeback flag, even
1662	 * just for a single subpage sector.
1663	 * As long as the folio is properly locked and the range is correct,
1664	 * we should always get the folio with writeback flag.
1665	 */
1666	ASSERT(folio_test_writeback(folio));
1667
1668	submit_extent_folio(bio_ctrl, disk_bytenr, folio,
1669			    sectorsize, filepos - folio_pos(folio), 0);
1670	return 0;
1671}
1672
1673/*
1674 * Helper for extent_writepage().  This calls the writepage start hooks,
1675 * and does the loop to map the page into extents and bios.
1676 *
1677 * We return 1 if the IO is started and the page is unlocked,
1678 * 0 if all went well (page still locked)
1679 * < 0 if there were errors (page still locked)
1680 */
1681static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
1682						  struct folio *folio,
1683						  u64 start, u32 len,
1684						  struct btrfs_bio_ctrl *bio_ctrl,
1685						  loff_t i_size)
1686{
1687	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1688	unsigned long range_bitmap = 0;
1689	bool submitted_io = false;
1690	int found_error = 0;
1691	const u64 end = start + len;
1692	const u64 folio_start = folio_pos(folio);
1693	const u64 folio_end = folio_start + folio_size(folio);
1694	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1695	u64 cur;
1696	int bit;
1697	int ret = 0;
1698
1699	ASSERT(start >= folio_start, "start=%llu folio_start=%llu", start, folio_start);
1700	ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu",
1701	       start, len, folio_start, folio_size(folio));
1702
1703	ret = btrfs_writepage_cow_fixup(folio);
1704	if (ret == -EAGAIN) {
1705		/* Fixup worker will requeue */
1706		folio_redirty_for_writepage(bio_ctrl->wbc, folio);
1707		folio_unlock(folio);
1708		return 1;
1709	}
1710	if (ret < 0) {
1711		btrfs_folio_clear_dirty(fs_info, folio, start, len);
1712		btrfs_folio_set_writeback(fs_info, folio, start, len);
1713		btrfs_folio_clear_writeback(fs_info, folio, start, len);
1714		return ret;
1715	}
1716
1717	for (cur = start; cur < end; cur += fs_info->sectorsize)
1718		set_bit((cur - folio_start) >> fs_info->sectorsize_bits, &range_bitmap);
1719	bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap,
1720		   blocks_per_folio);
1721
1722	bio_ctrl->end_io_func = end_bbio_data_write;
1723
1724	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
1725		cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits);
1726
1727		if (cur >= i_size) {
1728			struct btrfs_ordered_extent *ordered;
1729
1730			ordered = btrfs_lookup_first_ordered_range(inode, cur,
1731								   fs_info->sectorsize);
1732			/*
1733			 * We have just run delalloc before getting here, so
1734			 * there must be an ordered extent.
1735			 */
1736			ASSERT(ordered != NULL);
1737			spin_lock(&inode->ordered_tree_lock);
1738			set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
1739			ordered->truncated_len = min(ordered->truncated_len,
1740						     cur - ordered->file_offset);
1741			spin_unlock(&inode->ordered_tree_lock);
1742			btrfs_put_ordered_extent(ordered);
1743
1744			btrfs_mark_ordered_io_finished(inode, folio, cur,
1745						       fs_info->sectorsize, true);
1746			/*
1747			 * This range is beyond i_size, thus we don't need to
1748			 * bother writing back.
1749			 * But we still need to clear the dirty subpage bit, or
1750			 * the next time the folio gets dirtied, we will try to
1751			 * writeback the sectors with subpage dirty bits,
1752			 * causing writeback without ordered extent.
1753			 */
1754			btrfs_folio_clear_dirty(fs_info, folio, cur, fs_info->sectorsize);
1755			continue;
1756		}
1757		ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size);
1758		if (unlikely(ret < 0)) {
1759			/*
1760			 * bio_ctrl may contain a bio crossing several folios.
1761			 * Submit it immediately so that the bio has a chance
1762			 * to finish normally, other than marked as error.
1763			 */
1764			submit_one_bio(bio_ctrl);
1765			/*
1766			 * Failed to grab the extent map which should be very rare.
1767			 * Since there is no bio submitted to finish the ordered
1768			 * extent, we have to manually finish this sector.
1769			 */
1770			btrfs_mark_ordered_io_finished(inode, folio, cur,
1771						       fs_info->sectorsize, false);
1772			if (!found_error)
1773				found_error = ret;
1774			continue;
1775		}
1776		submitted_io = true;
1777	}
1778
1779	/*
1780	 * If we didn't submitted any sector (>= i_size), folio dirty get
1781	 * cleared but PAGECACHE_TAG_DIRTY is not cleared (only cleared
1782	 * by folio_start_writeback() if the folio is not dirty).
1783	 *
1784	 * Here we set writeback and clear for the range. If the full folio
1785	 * is no longer dirty then we clear the PAGECACHE_TAG_DIRTY tag.
1786	 *
1787	 * If we hit any error, the corresponding sector will have its dirty
1788	 * flag cleared and writeback finished, thus no need to handle the error case.
1789	 */
1790	if (!submitted_io && !found_error) {
1791		btrfs_folio_set_writeback(fs_info, folio, start, len);
1792		btrfs_folio_clear_writeback(fs_info, folio, start, len);
1793	}
1794	return found_error;
1795}
1796
1797/*
1798 * the writepage semantics are similar to regular writepage.  extent
1799 * records are inserted to lock ranges in the tree, and as dirty areas
1800 * are found, they are marked writeback.  Then the lock bits are removed
1801 * and the end_io handler clears the writeback ranges
1802 *
1803 * Return 0 if everything goes well.
1804 * Return <0 for error.
1805 */
1806static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl)
1807{
1808	struct btrfs_inode *inode = BTRFS_I(folio->mapping->host);
1809	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1810	int ret;
1811	size_t pg_offset;
1812	loff_t i_size = i_size_read(&inode->vfs_inode);
1813	const pgoff_t end_index = i_size >> PAGE_SHIFT;
1814	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1815
1816	trace_extent_writepage(folio, &inode->vfs_inode, bio_ctrl->wbc);
1817
1818	WARN_ON(!folio_test_locked(folio));
1819
1820	pg_offset = offset_in_folio(folio, i_size);
1821	if (folio->index > end_index ||
1822	   (folio->index == end_index && !pg_offset)) {
1823		folio_invalidate(folio, 0, folio_size(folio));
1824		folio_unlock(folio);
1825		return 0;
1826	}
1827
1828	if (folio_contains(folio, end_index))
1829		folio_zero_range(folio, pg_offset, folio_size(folio) - pg_offset);
1830
1831	/*
1832	 * Default to unlock the whole folio.
1833	 * The proper bitmap can only be initialized until writepage_delalloc().
1834	 */
1835	bio_ctrl->submit_bitmap = (unsigned long)-1;
1836
1837	/*
1838	 * If the page is dirty but without private set, it's marked dirty
1839	 * without informing the fs.
1840	 * Nowadays that is a bug, since the introduction of
1841	 * pin_user_pages*().
1842	 *
1843	 * So here we check if the page has private set to rule out such
1844	 * case.
1845	 * But we also have a long history of relying on the COW fixup,
1846	 * so here we only enable this check for experimental builds until
1847	 * we're sure it's safe.
1848	 */
1849	if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) &&
1850	    unlikely(!folio_test_private(folio))) {
1851		WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
1852		btrfs_err_rl(fs_info,
1853	"root %lld ino %llu folio %llu is marked dirty without notifying the fs",
1854			     btrfs_root_id(inode->root),
1855			     btrfs_ino(inode), folio_pos(folio));
1856		ret = -EUCLEAN;
1857		goto done;
1858	}
1859
1860	ret = set_folio_extent_mapped(folio);
1861	if (ret < 0)
1862		goto done;
1863
1864	ret = writepage_delalloc(inode, folio, bio_ctrl);
1865	if (ret == 1)
1866		return 0;
1867	if (ret)
1868		goto done;
1869
1870	ret = extent_writepage_io(inode, folio, folio_pos(folio),
1871				  folio_size(folio), bio_ctrl, i_size);
1872	if (ret == 1)
1873		return 0;
1874	if (unlikely(ret < 0))
1875		btrfs_err_rl(fs_info,
1876"failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %d",
1877			     btrfs_root_id(inode->root), btrfs_ino(inode),
1878			     folio_pos(folio), blocks_per_folio,
1879			     &bio_ctrl->submit_bitmap, ret);
1880
1881	bio_ctrl->wbc->nr_to_write--;
1882
1883done:
1884	if (ret < 0)
1885		mapping_set_error(folio->mapping, ret);
1886	/*
1887	 * Only unlock ranges that are submitted. As there can be some async
1888	 * submitted ranges inside the folio.
1889	 */
1890	btrfs_folio_end_lock_bitmap(fs_info, folio, bio_ctrl->submit_bitmap);
1891	ASSERT(ret <= 0);
1892	return ret;
1893}
1894
1895/*
1896 * Lock extent buffer status and pages for writeback.
1897 *
1898 * Return %false if the extent buffer doesn't need to be submitted (e.g. the
1899 * extent buffer is not dirty)
1900 * Return %true is the extent buffer is submitted to bio.
1901 */
1902static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1903			  struct writeback_control *wbc)
1904{
1905	struct btrfs_fs_info *fs_info = eb->fs_info;
1906	bool ret = false;
1907
1908	btrfs_tree_lock(eb);
1909	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1910		btrfs_tree_unlock(eb);
1911		if (wbc->sync_mode != WB_SYNC_ALL)
1912			return false;
1913		wait_on_extent_buffer_writeback(eb);
1914		btrfs_tree_lock(eb);
1915	}
1916
1917	/*
1918	 * We need to do this to prevent races in people who check if the eb is
1919	 * under IO since we can end up having no IO bits set for a short period
1920	 * of time.
1921	 */
1922	spin_lock(&eb->refs_lock);
1923	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
1924		XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
1925		unsigned long flags;
1926
1927		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1928		spin_unlock(&eb->refs_lock);
1929
1930		xas_lock_irqsave(&xas, flags);
1931		xas_load(&xas);
1932		xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
1933		xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
1934		xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
1935		xas_unlock_irqrestore(&xas, flags);
1936
1937		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1938		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1939					 -eb->len,
1940					 fs_info->dirty_metadata_batch);
1941		ret = true;
1942	} else {
1943		spin_unlock(&eb->refs_lock);
1944	}
1945	btrfs_tree_unlock(eb);
1946	return ret;
1947}
1948
1949static void set_btree_ioerr(struct extent_buffer *eb)
1950{
1951	struct btrfs_fs_info *fs_info = eb->fs_info;
1952
1953	set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
1954
1955	/*
1956	 * A read may stumble upon this buffer later, make sure that it gets an
1957	 * error and knows there was an error.
1958	 */
1959	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1960
1961	/*
1962	 * We need to set the mapping with the io error as well because a write
1963	 * error will flip the file system readonly, and then syncfs() will
1964	 * return a 0 because we are readonly if we don't modify the err seq for
1965	 * the superblock.
1966	 */
1967	mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
1968
1969	/*
1970	 * If writeback for a btree extent that doesn't belong to a log tree
1971	 * failed, increment the counter transaction->eb_write_errors.
1972	 * We do this because while the transaction is running and before it's
1973	 * committing (when we call filemap_fdata[write|wait]_range against
1974	 * the btree inode), we might have
1975	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1976	 * returns an error or an error happens during writeback, when we're
1977	 * committing the transaction we wouldn't know about it, since the pages
1978	 * can be no longer dirty nor marked anymore for writeback (if a
1979	 * subsequent modification to the extent buffer didn't happen before the
1980	 * transaction commit), which makes filemap_fdata[write|wait]_range not
1981	 * able to find the pages which contain errors at transaction
1982	 * commit time. So if this happens we must abort the transaction,
1983	 * otherwise we commit a super block with btree roots that point to
1984	 * btree nodes/leafs whose content on disk is invalid - either garbage
1985	 * or the content of some node/leaf from a past generation that got
1986	 * cowed or deleted and is no longer valid.
1987	 *
1988	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1989	 * not be enough - we need to distinguish between log tree extents vs
1990	 * non-log tree extents, and the next filemap_fdatawait_range() call
1991	 * will catch and clear such errors in the mapping - and that call might
1992	 * be from a log sync and not from a transaction commit. Also, checking
1993	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1994	 * not done and would not be reliable - the eb might have been released
1995	 * from memory and reading it back again means that flag would not be
1996	 * set (since it's a runtime flag, not persisted on disk).
1997	 *
1998	 * Using the flags below in the btree inode also makes us achieve the
1999	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
2000	 * writeback for all dirty pages and before filemap_fdatawait_range()
2001	 * is called, the writeback for all dirty pages had already finished
2002	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
2003	 * filemap_fdatawait_range() would return success, as it could not know
2004	 * that writeback errors happened (the pages were no longer tagged for
2005	 * writeback).
2006	 */
2007	switch (eb->log_index) {
2008	case -1:
2009		set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
2010		break;
2011	case 0:
2012		set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
2013		break;
2014	case 1:
2015		set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
2016		break;
2017	default:
2018		BUG(); /* unexpected, logic error */
2019	}
2020}
2021
2022static void buffer_tree_set_mark(const struct extent_buffer *eb, xa_mark_t mark)
2023{
2024	struct btrfs_fs_info *fs_info = eb->fs_info;
2025	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
2026	unsigned long flags;
2027
2028	xas_lock_irqsave(&xas, flags);
2029	xas_load(&xas);
2030	xas_set_mark(&xas, mark);
2031	xas_unlock_irqrestore(&xas, flags);
2032}
2033
2034static void buffer_tree_clear_mark(const struct extent_buffer *eb, xa_mark_t mark)
2035{
2036	struct btrfs_fs_info *fs_info = eb->fs_info;
2037	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
2038	unsigned long flags;
2039
2040	xas_lock_irqsave(&xas, flags);
2041	xas_load(&xas);
2042	xas_clear_mark(&xas, mark);
2043	xas_unlock_irqrestore(&xas, flags);
2044}
2045
2046static void buffer_tree_tag_for_writeback(struct btrfs_fs_info *fs_info,
2047					  unsigned long start, unsigned long end)
2048{
2049	XA_STATE(xas, &fs_info->buffer_tree, start);
2050	unsigned int tagged = 0;
2051	void *eb;
2052
2053	xas_lock_irq(&xas);
2054	xas_for_each_marked(&xas, eb, end, PAGECACHE_TAG_DIRTY) {
2055		xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
2056		if (++tagged % XA_CHECK_SCHED)
2057			continue;
2058		xas_pause(&xas);
2059		xas_unlock_irq(&xas);
2060		cond_resched();
2061		xas_lock_irq(&xas);
2062	}
2063	xas_unlock_irq(&xas);
2064}
2065
2066struct eb_batch {
2067	unsigned int nr;
2068	unsigned int cur;
2069	struct extent_buffer *ebs[PAGEVEC_SIZE];
2070};
2071
2072static inline bool eb_batch_add(struct eb_batch *batch, struct extent_buffer *eb)
2073{
2074	batch->ebs[batch->nr++] = eb;
2075	return (batch->nr < PAGEVEC_SIZE);
2076}
2077
2078static inline void eb_batch_init(struct eb_batch *batch)
2079{
2080	batch->nr = 0;
2081	batch->cur = 0;
2082}
2083
2084static inline struct extent_buffer *eb_batch_next(struct eb_batch *batch)
2085{
2086	if (batch->cur >= batch->nr)
2087		return NULL;
2088	return batch->ebs[batch->cur++];
2089}
2090
2091static inline void eb_batch_release(struct eb_batch *batch)
2092{
2093	for (unsigned int i = 0; i < batch->nr; i++)
2094		free_extent_buffer(batch->ebs[i]);
2095	eb_batch_init(batch);
2096}
2097
2098static inline struct extent_buffer *find_get_eb(struct xa_state *xas, unsigned long max,
2099						xa_mark_t mark)
2100{
2101	struct extent_buffer *eb;
2102
2103retry:
2104	eb = xas_find_marked(xas, max, mark);
2105
2106	if (xas_retry(xas, eb))
2107		goto retry;
2108
2109	if (!eb)
2110		return NULL;
2111
2112	if (!refcount_inc_not_zero(&eb->refs)) {
2113		xas_reset(xas);
2114		goto retry;
2115	}
2116
2117	if (unlikely(eb != xas_reload(xas))) {
2118		free_extent_buffer(eb);
2119		xas_reset(xas);
2120		goto retry;
2121	}
2122
2123	return eb;
2124}
2125
2126static unsigned int buffer_tree_get_ebs_tag(struct btrfs_fs_info *fs_info,
2127					    unsigned long *start,
2128					    unsigned long end, xa_mark_t tag,
2129					    struct eb_batch *batch)
2130{
2131	XA_STATE(xas, &fs_info->buffer_tree, *start);
2132	struct extent_buffer *eb;
2133
2134	rcu_read_lock();
2135	while ((eb = find_get_eb(&xas, end, tag)) != NULL) {
2136		if (!eb_batch_add(batch, eb)) {
2137			*start = ((eb->start + eb->len) >> fs_info->nodesize_bits);
2138			goto out;
2139		}
2140	}
2141	if (end == ULONG_MAX)
2142		*start = ULONG_MAX;
2143	else
2144		*start = end + 1;
2145out:
2146	rcu_read_unlock();
2147
2148	return batch->nr;
2149}
2150
2151/*
2152 * The endio specific version which won't touch any unsafe spinlock in endio
2153 * context.
2154 */
2155static struct extent_buffer *find_extent_buffer_nolock(
2156		struct btrfs_fs_info *fs_info, u64 start)
2157{
2158	struct extent_buffer *eb;
2159	unsigned long index = (start >> fs_info->nodesize_bits);
2160
2161	rcu_read_lock();
2162	eb = xa_load(&fs_info->buffer_tree, index);
2163	if (eb && !refcount_inc_not_zero(&eb->refs))
2164		eb = NULL;
2165	rcu_read_unlock();
2166	return eb;
2167}
2168
2169static void end_bbio_meta_write(struct btrfs_bio *bbio)
2170{
2171	struct extent_buffer *eb = bbio->private;
2172	struct folio_iter fi;
2173
2174	if (bbio->bio.bi_status != BLK_STS_OK)
2175		set_btree_ioerr(eb);
2176
2177	bio_for_each_folio_all(fi, &bbio->bio) {
2178		btrfs_meta_folio_clear_writeback(fi.folio, eb);
2179	}
2180
2181	buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK);
2182	clear_and_wake_up_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
2183	bio_put(&bbio->bio);
2184}
2185
2186static void prepare_eb_write(struct extent_buffer *eb)
2187{
2188	u32 nritems;
2189	unsigned long start;
2190	unsigned long end;
2191
2192	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
2193
2194	/* Set btree blocks beyond nritems with 0 to avoid stale content */
2195	nritems = btrfs_header_nritems(eb);
2196	if (btrfs_header_level(eb) > 0) {
2197		end = btrfs_node_key_ptr_offset(eb, nritems);
2198		memzero_extent_buffer(eb, end, eb->len - end);
2199	} else {
2200		/*
2201		 * Leaf:
2202		 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
2203		 */
2204		start = btrfs_item_nr_offset(eb, nritems);
2205		end = btrfs_item_nr_offset(eb, 0);
2206		if (nritems == 0)
2207			end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
2208		else
2209			end += btrfs_item_offset(eb, nritems - 1);
2210		memzero_extent_buffer(eb, start, end - start);
2211	}
2212}
2213
2214static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
2215					    struct writeback_control *wbc)
2216{
2217	struct btrfs_fs_info *fs_info = eb->fs_info;
2218	struct btrfs_bio *bbio;
2219
2220	prepare_eb_write(eb);
2221
2222	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
2223			       REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
2224			       BTRFS_I(fs_info->btree_inode), eb->start,
2225			       end_bbio_meta_write, eb);
2226	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
2227	bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
2228	wbc_init_bio(wbc, &bbio->bio);
2229	for (int i = 0; i < num_extent_folios(eb); i++) {
2230		struct folio *folio = eb->folios[i];
2231		u64 range_start = max_t(u64, eb->start, folio_pos(folio));
2232		u32 range_len = min_t(u64, folio_next_pos(folio),
2233				      eb->start + eb->len) - range_start;
2234
2235		folio_lock(folio);
2236		btrfs_meta_folio_clear_dirty(folio, eb);
2237		btrfs_meta_folio_set_writeback(folio, eb);
2238		if (!folio_test_dirty(folio))
2239			wbc->nr_to_write -= folio_nr_pages(folio);
2240		bio_add_folio_nofail(&bbio->bio, folio, range_len,
2241				     offset_in_folio(folio, range_start));
2242		wbc_account_cgroup_owner(wbc, folio, range_len);
2243		folio_unlock(folio);
2244	}
2245	/*
2246	 * If the fs is already in error status, do not submit any writeback
2247	 * but immediately finish it.
2248	 */
2249	if (unlikely(BTRFS_FS_ERROR(fs_info))) {
2250		btrfs_bio_end_io(bbio, errno_to_blk_status(BTRFS_FS_ERROR(fs_info)));
2251		return;
2252	}
2253	btrfs_submit_bbio(bbio, 0);
2254}
2255
2256/*
2257 * Wait for all eb writeback in the given range to finish.
2258 *
2259 * @fs_info:	The fs_info for this file system.
2260 * @start:	The offset of the range to start waiting on writeback.
2261 * @end:	The end of the range, inclusive. This is meant to be used in
2262 *		conjunction with wait_marked_extents, so this will usually be
2263 *		the_next_eb->start - 1.
2264 */
2265void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start,
2266				      u64 end)
2267{
2268	struct eb_batch batch;
2269	unsigned long start_index = (start >> fs_info->nodesize_bits);
2270	unsigned long end_index = (end >> fs_info->nodesize_bits);
2271
2272	eb_batch_init(&batch);
2273	while (start_index <= end_index) {
2274		struct extent_buffer *eb;
2275		unsigned int nr_ebs;
2276
2277		nr_ebs = buffer_tree_get_ebs_tag(fs_info, &start_index, end_index,
2278						 PAGECACHE_TAG_WRITEBACK, &batch);
2279		if (!nr_ebs)
2280			break;
2281
2282		while ((eb = eb_batch_next(&batch)) != NULL)
2283			wait_on_extent_buffer_writeback(eb);
2284		eb_batch_release(&batch);
2285		cond_resched();
2286	}
2287}
2288
2289int btree_writepages(struct address_space *mapping, struct writeback_control *wbc)
2290{
2291	struct btrfs_eb_write_context ctx = { .wbc = wbc };
2292	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
2293	int ret = 0;
2294	int done = 0;
2295	int nr_to_write_done = 0;
2296	struct eb_batch batch;
2297	unsigned int nr_ebs;
2298	unsigned long index;
2299	unsigned long end;
2300	int scanned = 0;
2301	xa_mark_t tag;
2302
2303	eb_batch_init(&batch);
2304	if (wbc->range_cyclic) {
2305		index = ((mapping->writeback_index << PAGE_SHIFT) >> fs_info->nodesize_bits);
2306		end = -1;
2307
2308		/*
2309		 * Start from the beginning does not need to cycle over the
2310		 * range, mark it as scanned.
2311		 */
2312		scanned = (index == 0);
2313	} else {
2314		index = (wbc->range_start >> fs_info->nodesize_bits);
2315		end = (wbc->range_end >> fs_info->nodesize_bits);
2316
2317		scanned = 1;
2318	}
2319	if (wbc->sync_mode == WB_SYNC_ALL)
2320		tag = PAGECACHE_TAG_TOWRITE;
2321	else
2322		tag = PAGECACHE_TAG_DIRTY;
2323	btrfs_zoned_meta_io_lock(fs_info);
2324retry:
2325	if (wbc->sync_mode == WB_SYNC_ALL)
2326		buffer_tree_tag_for_writeback(fs_info, index, end);
2327	while (!done && !nr_to_write_done && (index <= end) &&
2328	       (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, tag, &batch))) {
2329		struct extent_buffer *eb;
2330
2331		while ((eb = eb_batch_next(&batch)) != NULL) {
2332			ctx.eb = eb;
2333
2334			ret = btrfs_check_meta_write_pointer(eb->fs_info, &ctx);
2335			if (ret) {
2336				if (ret == -EBUSY)
2337					ret = 0;
2338
2339				if (ret) {
2340					done = 1;
2341					break;
2342				}
2343				continue;
2344			}
2345
2346			if (!lock_extent_buffer_for_io(eb, wbc))
2347				continue;
2348
2349			/* Implies write in zoned mode. */
2350			if (ctx.zoned_bg) {
2351				/* Mark the last eb in the block group. */
2352				btrfs_schedule_zone_finish_bg(ctx.zoned_bg, eb);
2353				ctx.zoned_bg->meta_write_pointer += eb->len;
2354			}
2355			write_one_eb(eb, wbc);
2356		}
2357		nr_to_write_done = (wbc->nr_to_write <= 0);
2358		eb_batch_release(&batch);
2359		cond_resched();
2360	}
2361	if (!scanned && !done) {
2362		/*
2363		 * We hit the last page and there is more work to be done: wrap
2364		 * back to the start of the file
2365		 */
2366		scanned = 1;
2367		index = 0;
2368		goto retry;
2369	}
2370	/*
2371	 * If something went wrong, don't allow any metadata write bio to be
2372	 * submitted.
2373	 *
2374	 * This would prevent use-after-free if we had dirty pages not
2375	 * cleaned up, which can still happen by fuzzed images.
2376	 *
2377	 * - Bad extent tree
2378	 *   Allowing existing tree block to be allocated for other trees.
2379	 *
2380	 * - Log tree operations
2381	 *   Exiting tree blocks get allocated to log tree, bumps its
2382	 *   generation, then get cleaned in tree re-balance.
2383	 *   Such tree block will not be written back, since it's clean,
2384	 *   thus no WRITTEN flag set.
2385	 *   And after log writes back, this tree block is not traced by
2386	 *   any dirty extent_io_tree.
2387	 *
2388	 * - Offending tree block gets re-dirtied from its original owner
2389	 *   Since it has bumped generation, no WRITTEN flag, it can be
2390	 *   reused without COWing. This tree block will not be traced
2391	 *   by btrfs_transaction::dirty_pages.
2392	 *
2393	 *   Now such dirty tree block will not be cleaned by any dirty
2394	 *   extent io tree. Thus we don't want to submit such wild eb
2395	 *   if the fs already has error.
2396	 *
2397	 * We can get ret > 0 from submit_extent_folio() indicating how many ebs
2398	 * were submitted. Reset it to 0 to avoid false alerts for the caller.
2399	 */
2400	if (ret > 0)
2401		ret = 0;
2402	if (!ret && BTRFS_FS_ERROR(fs_info))
2403		ret = -EROFS;
2404
2405	if (ctx.zoned_bg)
2406		btrfs_put_block_group(ctx.zoned_bg);
2407	btrfs_zoned_meta_io_unlock(fs_info);
2408	return ret;
2409}
2410
2411/*
2412 * Walk the list of dirty pages of the given address space and write all of them.
2413 *
2414 * @mapping:   address space structure to write
2415 * @wbc:       subtract the number of written pages from *@wbc->nr_to_write
2416 * @bio_ctrl:  holds context for the write, namely the bio
2417 *
2418 * If a page is already under I/O, write_cache_pages() skips it, even
2419 * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
2420 * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
2421 * and msync() need to guarantee that all the data which was dirty at the time
2422 * the call was made get new I/O started against them.  If wbc->sync_mode is
2423 * WB_SYNC_ALL then we were called for data integrity and we must wait for
2424 * existing IO to complete.
2425 */
2426static int extent_write_cache_pages(struct address_space *mapping,
2427			     struct btrfs_bio_ctrl *bio_ctrl)
2428{
2429	struct writeback_control *wbc = bio_ctrl->wbc;
2430	struct inode *inode = mapping->host;
2431	int ret = 0;
2432	int done = 0;
2433	int nr_to_write_done = 0;
2434	struct folio_batch fbatch;
2435	unsigned int nr_folios;
2436	pgoff_t index;
2437	pgoff_t end;		/* Inclusive */
2438	pgoff_t done_index;
2439	int range_whole = 0;
2440	int scanned = 0;
2441	xa_mark_t tag;
2442
2443	/*
2444	 * We have to hold onto the inode so that ordered extents can do their
2445	 * work when the IO finishes.  The alternative to this is failing to add
2446	 * an ordered extent if the igrab() fails there and that is a huge pain
2447	 * to deal with, so instead just hold onto the inode throughout the
2448	 * writepages operation.  If it fails here we are freeing up the inode
2449	 * anyway and we'd rather not waste our time writing out stuff that is
2450	 * going to be truncated anyway.
2451	 */
2452	if (!igrab(inode))
2453		return 0;
2454
2455	folio_batch_init(&fbatch);
2456	if (wbc->range_cyclic) {
2457		index = mapping->writeback_index; /* Start from prev offset */
2458		end = -1;
2459		/*
2460		 * Start from the beginning does not need to cycle over the
2461		 * range, mark it as scanned.
2462		 */
2463		scanned = (index == 0);
2464	} else {
2465		index = wbc->range_start >> PAGE_SHIFT;
2466		end = wbc->range_end >> PAGE_SHIFT;
2467		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2468			range_whole = 1;
2469		scanned = 1;
2470	}
2471
2472	/*
2473	 * We do the tagged writepage as long as the snapshot flush bit is set
2474	 * and we are the first one who do the filemap_flush() on this inode.
2475	 *
2476	 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
2477	 * not race in and drop the bit.
2478	 */
2479	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2480	    test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
2481			       &BTRFS_I(inode)->runtime_flags))
2482		wbc->tagged_writepages = 1;
2483
2484	tag = wbc_to_tag(wbc);
2485retry:
2486	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2487		tag_pages_for_writeback(mapping, index, end);
2488	done_index = index;
2489	while (!done && !nr_to_write_done && (index <= end) &&
2490			(nr_folios = filemap_get_folios_tag(mapping, &index,
2491							end, tag, &fbatch))) {
2492		unsigned i;
2493
2494		for (i = 0; i < nr_folios; i++) {
2495			struct folio *folio = fbatch.folios[i];
2496
2497			done_index = folio_next_index(folio);
2498			/*
2499			 * At this point we hold neither the i_pages lock nor
2500			 * the folio lock: the folio may be truncated or
2501			 * invalidated (changing folio->mapping to NULL).
2502			 */
2503			if (!folio_trylock(folio)) {
2504				submit_write_bio(bio_ctrl, 0);
2505				folio_lock(folio);
2506			}
2507
2508			if (unlikely(folio->mapping != mapping)) {
2509				folio_unlock(folio);
2510				continue;
2511			}
2512
2513			if (!folio_test_dirty(folio)) {
2514				/* Someone wrote it for us. */
2515				folio_unlock(folio);
2516				continue;
2517			}
2518
2519			/*
2520			 * For subpage case, compression can lead to mixed
2521			 * writeback and dirty flags, e.g:
2522			 * 0     32K    64K    96K    128K
2523			 * |     |//////||/////|   |//|
2524			 *
2525			 * In above case, [32K, 96K) is asynchronously submitted
2526			 * for compression, and [124K, 128K) needs to be written back.
2527			 *
2528			 * If we didn't wait writeback for page 64K, [128K, 128K)
2529			 * won't be submitted as the page still has writeback flag
2530			 * and will be skipped in the next check.
2531			 *
2532			 * This mixed writeback and dirty case is only possible for
2533			 * subpage case.
2534			 *
2535			 * TODO: Remove this check after migrating compression to
2536			 * regular submission.
2537			 */
2538			if (wbc->sync_mode != WB_SYNC_NONE ||
2539			    btrfs_is_subpage(inode_to_fs_info(inode), folio)) {
2540				if (folio_test_writeback(folio))
2541					submit_write_bio(bio_ctrl, 0);
2542				folio_wait_writeback(folio);
2543			}
2544
2545			if (folio_test_writeback(folio) ||
2546			    !folio_clear_dirty_for_io(folio)) {
2547				folio_unlock(folio);
2548				continue;
2549			}
2550
2551			ret = extent_writepage(folio, bio_ctrl);
2552			if (ret < 0) {
2553				done = 1;
2554				break;
2555			}
2556
2557			/*
2558			 * The filesystem may choose to bump up nr_to_write.
2559			 * We have to make sure to honor the new nr_to_write
2560			 * at any time.
2561			 */
2562			nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2563					    wbc->nr_to_write <= 0);
2564		}
2565		folio_batch_release(&fbatch);
2566		cond_resched();
2567	}
2568	if (!scanned && !done) {
2569		/*
2570		 * We hit the last page and there is more work to be done: wrap
2571		 * back to the start of the file
2572		 */
2573		scanned = 1;
2574		index = 0;
2575
2576		/*
2577		 * If we're looping we could run into a page that is locked by a
2578		 * writer and that writer could be waiting on writeback for a
2579		 * page in our current bio, and thus deadlock, so flush the
2580		 * write bio here.
2581		 */
2582		submit_write_bio(bio_ctrl, 0);
2583		goto retry;
2584	}
2585
2586	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2587		mapping->writeback_index = done_index;
2588
2589	btrfs_add_delayed_iput(BTRFS_I(inode));
2590	return ret;
2591}
2592
2593/*
2594 * Submit the pages in the range to bio for call sites which delalloc range has
2595 * already been ran (aka, ordered extent inserted) and all pages are still
2596 * locked.
2597 */
2598void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio,
2599			       u64 start, u64 end, struct writeback_control *wbc,
2600			       bool pages_dirty)
2601{
2602	bool found_error = false;
2603	int ret = 0;
2604	struct address_space *mapping = inode->i_mapping;
2605	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2606	const u32 sectorsize = fs_info->sectorsize;
2607	loff_t i_size = i_size_read(inode);
2608	u64 cur = start;
2609	struct btrfs_bio_ctrl bio_ctrl = {
2610		.wbc = wbc,
2611		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2612	};
2613
2614	if (wbc->no_cgroup_owner)
2615		bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2616
2617	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2618
2619	while (cur <= end) {
2620		u64 cur_end;
2621		u32 cur_len;
2622		struct folio *folio;
2623
2624		folio = filemap_get_folio(mapping, cur >> PAGE_SHIFT);
2625
2626		/*
2627		 * This shouldn't happen, the pages are pinned and locked, this
2628		 * code is just in case, but shouldn't actually be run.
2629		 */
2630		if (IS_ERR(folio)) {
2631			cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2632			cur_len = cur_end + 1 - cur;
2633			btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL,
2634						       cur, cur_len, false);
2635			mapping_set_error(mapping, PTR_ERR(folio));
2636			cur = cur_end;
2637			continue;
2638		}
2639
2640		cur_end = min_t(u64, folio_next_pos(folio) - 1, end);
2641		cur_len = cur_end + 1 - cur;
2642
2643		ASSERT(folio_test_locked(folio));
2644		if (pages_dirty && folio != locked_folio)
2645			ASSERT(folio_test_dirty(folio));
2646
2647		/*
2648		 * Set the submission bitmap to submit all sectors.
2649		 * extent_writepage_io() will do the truncation correctly.
2650		 */
2651		bio_ctrl.submit_bitmap = (unsigned long)-1;
2652		ret = extent_writepage_io(BTRFS_I(inode), folio, cur, cur_len,
2653					  &bio_ctrl, i_size);
2654		if (ret == 1)
2655			goto next_page;
2656
2657		if (ret)
2658			mapping_set_error(mapping, ret);
2659		btrfs_folio_end_lock(fs_info, folio, cur, cur_len);
2660		if (ret < 0)
2661			found_error = true;
2662next_page:
2663		folio_put(folio);
2664		cur = cur_end + 1;
2665	}
2666
2667	submit_write_bio(&bio_ctrl, found_error ? ret : 0);
2668}
2669
2670int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
2671{
2672	struct inode *inode = mapping->host;
2673	int ret = 0;
2674	struct btrfs_bio_ctrl bio_ctrl = {
2675		.wbc = wbc,
2676		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2677	};
2678
2679	/*
2680	 * Allow only a single thread to do the reloc work in zoned mode to
2681	 * protect the write pointer updates.
2682	 */
2683	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2684	ret = extent_write_cache_pages(mapping, &bio_ctrl);
2685	submit_write_bio(&bio_ctrl, ret);
2686	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2687	return ret;
2688}
2689
2690void btrfs_readahead(struct readahead_control *rac)
2691{
2692	struct btrfs_bio_ctrl bio_ctrl = {
2693		.opf = REQ_OP_READ | REQ_RAHEAD,
2694		.ractl = rac,
2695		.last_em_start = U64_MAX,
2696	};
2697	struct folio *folio;
2698	struct btrfs_inode *inode = BTRFS_I(rac->mapping->host);
2699	const u64 start = readahead_pos(rac);
2700	const u64 end = start + readahead_length(rac) - 1;
2701	struct extent_state *cached_state = NULL;
2702	struct extent_map *em_cached = NULL;
2703
2704	lock_extents_for_read(inode, start, end, &cached_state);
2705
2706	while ((folio = readahead_folio(rac)) != NULL)
2707		btrfs_do_readpage(folio, &em_cached, &bio_ctrl);
2708
2709	btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state);
2710
2711	if (em_cached)
2712		btrfs_free_extent_map(em_cached);
2713	submit_one_bio(&bio_ctrl);
2714}
2715
2716/*
2717 * basic invalidate_folio code, this waits on any locked or writeback
2718 * ranges corresponding to the folio, and then deletes any extent state
2719 * records from the tree
2720 */
2721int extent_invalidate_folio(struct extent_io_tree *tree,
2722			  struct folio *folio, size_t offset)
2723{
2724	struct extent_state *cached_state = NULL;
2725	u64 start = folio_pos(folio);
2726	u64 end = start + folio_size(folio) - 1;
2727	size_t blocksize = folio_to_fs_info(folio)->sectorsize;
2728
2729	/* This function is only called for the btree inode */
2730	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2731
2732	start += ALIGN(offset, blocksize);
2733	if (start > end)
2734		return 0;
2735
2736	btrfs_lock_extent(tree, start, end, &cached_state);
2737	folio_wait_writeback(folio);
2738
2739	/*
2740	 * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2741	 * so here we only need to unlock the extent range to free any
2742	 * existing extent state.
2743	 */
2744	btrfs_unlock_extent(tree, start, end, &cached_state);
2745	return 0;
2746}
2747
2748/*
2749 * A helper for struct address_space_operations::release_folio, this tests for
2750 * areas of the folio that are locked or under IO and drops the related state
2751 * bits if it is safe to drop the folio.
2752 */
2753static bool try_release_extent_state(struct extent_io_tree *tree,
2754				     struct folio *folio)
2755{
2756	struct extent_state *cached_state = NULL;
2757	u64 start = folio_pos(folio);
2758	u64 end = start + folio_size(folio) - 1;
2759	u32 range_bits;
2760	u32 clear_bits;
2761	bool ret = false;
2762	int ret2;
2763
2764	btrfs_get_range_bits(tree, start, end, &range_bits, &cached_state);
2765
2766	/*
2767	 * We can release the folio if it's locked only for ordered extent
2768	 * completion, since that doesn't require using the folio.
2769	 */
2770	if ((range_bits & EXTENT_LOCKED) &&
2771	    !(range_bits & EXTENT_FINISHING_ORDERED))
2772		goto out;
2773
2774	clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW |
2775		       EXTENT_CTLBITS | EXTENT_QGROUP_RESERVED |
2776		       EXTENT_FINISHING_ORDERED);
2777	/*
2778	 * At this point we can safely clear everything except the locked,
2779	 * nodatasum, delalloc new and finishing ordered bits. The delalloc new
2780	 * bit will be cleared by ordered extent completion.
2781	 */
2782	ret2 = btrfs_clear_extent_bit(tree, start, end, clear_bits, &cached_state);
2783	/*
2784	 * If clear_extent_bit failed for enomem reasons, we can't allow the
2785	 * release to continue.
2786	 */
2787	if (ret2 == 0)
2788		ret = true;
2789out:
2790	btrfs_free_extent_state(cached_state);
2791
2792	return ret;
2793}
2794
2795/*
2796 * a helper for release_folio.  As long as there are no locked extents
2797 * in the range corresponding to the page, both state records and extent
2798 * map records are removed
2799 */
2800bool try_release_extent_mapping(struct folio *folio, gfp_t mask)
2801{
2802	u64 start = folio_pos(folio);
2803	u64 end = start + folio_size(folio) - 1;
2804	struct btrfs_inode *inode = folio_to_inode(folio);
2805	struct extent_io_tree *io_tree = &inode->io_tree;
2806
2807	while (start <= end) {
2808		const u64 cur_gen = btrfs_get_fs_generation(inode->root->fs_info);
2809		const u64 len = end - start + 1;
2810		struct extent_map_tree *extent_tree = &inode->extent_tree;
2811		struct extent_map *em;
2812
2813		write_lock(&extent_tree->lock);
2814		em = btrfs_lookup_extent_mapping(extent_tree, start, len);
2815		if (!em) {
2816			write_unlock(&extent_tree->lock);
2817			break;
2818		}
2819		if ((em->flags & EXTENT_FLAG_PINNED) || em->start != start) {
2820			write_unlock(&extent_tree->lock);
2821			btrfs_free_extent_map(em);
2822			break;
2823		}
2824		if (btrfs_test_range_bit_exists(io_tree, em->start,
2825						btrfs_extent_map_end(em) - 1,
2826						EXTENT_LOCKED))
2827			goto next;
2828		/*
2829		 * If it's not in the list of modified extents, used by a fast
2830		 * fsync, we can remove it. If it's being logged we can safely
2831		 * remove it since fsync took an extra reference on the em.
2832		 */
2833		if (list_empty(&em->list) || (em->flags & EXTENT_FLAG_LOGGING))
2834			goto remove_em;
2835		/*
2836		 * If it's in the list of modified extents, remove it only if
2837		 * its generation is older then the current one, in which case
2838		 * we don't need it for a fast fsync. Otherwise don't remove it,
2839		 * we could be racing with an ongoing fast fsync that could miss
2840		 * the new extent.
2841		 */
2842		if (em->generation >= cur_gen)
2843			goto next;
2844remove_em:
2845		/*
2846		 * We only remove extent maps that are not in the list of
2847		 * modified extents or that are in the list but with a
2848		 * generation lower then the current generation, so there is no
2849		 * need to set the full fsync flag on the inode (it hurts the
2850		 * fsync performance for workloads with a data size that exceeds
2851		 * or is close to the system's memory).
2852		 */
2853		btrfs_remove_extent_mapping(inode, em);
2854		/* Once for the inode's extent map tree. */
2855		btrfs_free_extent_map(em);
2856next:
2857		start = btrfs_extent_map_end(em);
2858		write_unlock(&extent_tree->lock);
2859
2860		/* Once for us, for the lookup_extent_mapping() reference. */
2861		btrfs_free_extent_map(em);
2862
2863		if (need_resched()) {
2864			/*
2865			 * If we need to resched but we can't block just exit
2866			 * and leave any remaining extent maps.
2867			 */
2868			if (!gfpflags_allow_blocking(mask))
2869				break;
2870
2871			cond_resched();
2872		}
2873	}
2874	return try_release_extent_state(io_tree, folio);
2875}
2876
2877static int extent_buffer_under_io(const struct extent_buffer *eb)
2878{
2879	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
2880		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2881}
2882
2883static bool folio_range_has_eb(struct folio *folio)
2884{
2885	struct btrfs_folio_state *bfs;
2886
2887	lockdep_assert_held(&folio->mapping->i_private_lock);
2888
2889	if (folio_test_private(folio)) {
2890		bfs = folio_get_private(folio);
2891		if (atomic_read(&bfs->eb_refs))
2892			return true;
2893	}
2894	return false;
2895}
2896
2897static void detach_extent_buffer_folio(const struct extent_buffer *eb, struct folio *folio)
2898{
2899	struct btrfs_fs_info *fs_info = eb->fs_info;
2900	struct address_space *mapping = folio->mapping;
2901	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
2902
2903	/*
2904	 * For mapped eb, we're going to change the folio private, which should
2905	 * be done under the i_private_lock.
2906	 */
2907	if (mapped)
2908		spin_lock(&mapping->i_private_lock);
2909
2910	if (!folio_test_private(folio)) {
2911		if (mapped)
2912			spin_unlock(&mapping->i_private_lock);
2913		return;
2914	}
2915
2916	if (!btrfs_meta_is_subpage(fs_info)) {
2917		/*
2918		 * We do this since we'll remove the pages after we've removed
2919		 * the eb from the xarray, so we could race and have this page
2920		 * now attached to the new eb.  So only clear folio if it's
2921		 * still connected to this eb.
2922		 */
2923		if (folio_test_private(folio) && folio_get_private(folio) == eb) {
2924			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2925			BUG_ON(folio_test_dirty(folio));
2926			BUG_ON(folio_test_writeback(folio));
2927			/* We need to make sure we haven't be attached to a new eb. */
2928			folio_detach_private(folio);
2929		}
2930		if (mapped)
2931			spin_unlock(&mapping->i_private_lock);
2932		return;
2933	}
2934
2935	/*
2936	 * For subpage, we can have dummy eb with folio private attached.  In
2937	 * this case, we can directly detach the private as such folio is only
2938	 * attached to one dummy eb, no sharing.
2939	 */
2940	if (!mapped) {
2941		btrfs_detach_folio_state(fs_info, folio, BTRFS_SUBPAGE_METADATA);
2942		return;
2943	}
2944
2945	btrfs_folio_dec_eb_refs(fs_info, folio);
2946
2947	/*
2948	 * We can only detach the folio private if there are no other ebs in the
2949	 * page range and no unfinished IO.
2950	 */
2951	if (!folio_range_has_eb(folio))
2952		btrfs_detach_folio_state(fs_info, folio, BTRFS_SUBPAGE_METADATA);
2953
2954	spin_unlock(&mapping->i_private_lock);
2955}
2956
2957/* Release all folios attached to the extent buffer */
2958static void btrfs_release_extent_buffer_folios(const struct extent_buffer *eb)
2959{
2960	ASSERT(!extent_buffer_under_io(eb));
2961
2962	for (int i = 0; i < INLINE_EXTENT_BUFFER_PAGES; i++) {
2963		struct folio *folio = eb->folios[i];
2964
2965		if (!folio)
2966			continue;
2967
2968		detach_extent_buffer_folio(eb, folio);
2969	}
2970}
2971
2972/*
2973 * Helper for releasing the extent buffer.
2974 */
2975static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
2976{
2977	btrfs_release_extent_buffer_folios(eb);
2978	btrfs_leak_debug_del_eb(eb);
2979	kmem_cache_free(extent_buffer_cache, eb);
2980}
2981
2982static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info,
2983						   u64 start)
2984{
2985	struct extent_buffer *eb = NULL;
2986
2987	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
2988	eb->start = start;
2989	eb->len = fs_info->nodesize;
2990	eb->fs_info = fs_info;
2991	init_rwsem(&eb->lock);
2992
2993	btrfs_leak_debug_add_eb(eb);
2994
2995	spin_lock_init(&eb->refs_lock);
2996	refcount_set(&eb->refs, 1);
2997
2998	ASSERT(eb->len <= BTRFS_MAX_METADATA_BLOCKSIZE);
2999
3000	return eb;
3001}
3002
3003/*
3004 * For use in eb allocation error cleanup paths, as btrfs_release_extent_buffer()
3005 * does not call folio_put(), and we need to set the folios to NULL so that
3006 * btrfs_release_extent_buffer() will not detach them a second time.
3007 */
3008static void cleanup_extent_buffer_folios(struct extent_buffer *eb)
3009{
3010	const int num_folios = num_extent_folios(eb);
3011
3012	/* We cannot use num_extent_folios() as loop bound as eb->folios changes. */
3013	for (int i = 0; i < num_folios; i++) {
3014		ASSERT(eb->folios[i]);
3015		detach_extent_buffer_folio(eb, eb->folios[i]);
3016		folio_put(eb->folios[i]);
3017		eb->folios[i] = NULL;
3018	}
3019}
3020
3021struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3022{
3023	struct extent_buffer *new;
3024	int num_folios;
3025	int ret;
3026
3027	new = __alloc_extent_buffer(src->fs_info, src->start);
3028	if (new == NULL)
3029		return NULL;
3030
3031	/*
3032	 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3033	 * btrfs_release_extent_buffer() have different behavior for
3034	 * UNMAPPED subpage extent buffer.
3035	 */
3036	set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
3037
3038	ret = alloc_eb_folio_array(new, false);
3039	if (ret)
3040		goto release_eb;
3041
3042	ASSERT(num_extent_folios(src) == num_extent_folios(new),
3043	       "%d != %d", num_extent_folios(src), num_extent_folios(new));
3044	/* Explicitly use the cached num_extent value from now on. */
3045	num_folios = num_extent_folios(src);
3046	for (int i = 0; i < num_folios; i++) {
3047		struct folio *folio = new->folios[i];
3048
3049		ret = attach_extent_buffer_folio(new, folio, NULL);
3050		if (ret < 0)
3051			goto cleanup_folios;
3052		WARN_ON(folio_test_dirty(folio));
3053	}
3054	for (int i = 0; i < num_folios; i++)
3055		folio_put(new->folios[i]);
3056
3057	copy_extent_buffer_full(new, src);
3058	set_extent_buffer_uptodate(new);
3059
3060	return new;
3061
3062cleanup_folios:
3063	cleanup_extent_buffer_folios(new);
3064release_eb:
3065	btrfs_release_extent_buffer(new);
3066	return NULL;
3067}
3068
3069struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3070						u64 start)
3071{
3072	struct extent_buffer *eb;
3073	int ret;
3074
3075	eb = __alloc_extent_buffer(fs_info, start);
3076	if (!eb)
3077		return NULL;
3078
3079	ret = alloc_eb_folio_array(eb, false);
3080	if (ret)
3081		goto release_eb;
3082
3083	for (int i = 0; i < num_extent_folios(eb); i++) {
3084		ret = attach_extent_buffer_folio(eb, eb->folios[i], NULL);
3085		if (ret < 0)
3086			goto cleanup_folios;
3087	}
3088	for (int i = 0; i < num_extent_folios(eb); i++)
3089		folio_put(eb->folios[i]);
3090
3091	set_extent_buffer_uptodate(eb);
3092	btrfs_set_header_nritems(eb, 0);
3093	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3094
3095	return eb;
3096
3097cleanup_folios:
3098	cleanup_extent_buffer_folios(eb);
3099release_eb:
3100	btrfs_release_extent_buffer(eb);
3101	return NULL;
3102}
3103
3104static void check_buffer_tree_ref(struct extent_buffer *eb)
3105{
3106	int refs;
3107	/*
3108	 * The TREE_REF bit is first set when the extent_buffer is added to the
3109	 * xarray. It is also reset, if unset, when a new reference is created
3110	 * by find_extent_buffer.
3111	 *
3112	 * It is only cleared in two cases: freeing the last non-tree
3113	 * reference to the extent_buffer when its STALE bit is set or
3114	 * calling release_folio when the tree reference is the only reference.
3115	 *
3116	 * In both cases, care is taken to ensure that the extent_buffer's
3117	 * pages are not under io. However, release_folio can be concurrently
3118	 * called with creating new references, which is prone to race
3119	 * conditions between the calls to check_buffer_tree_ref in those
3120	 * codepaths and clearing TREE_REF in try_release_extent_buffer.
3121	 *
3122	 * The actual lifetime of the extent_buffer in the xarray is adequately
3123	 * protected by the refcount, but the TREE_REF bit and its corresponding
3124	 * reference are not. To protect against this class of races, we call
3125	 * check_buffer_tree_ref() from the code paths which trigger io. Note that
3126	 * once io is initiated, TREE_REF can no longer be cleared, so that is
3127	 * the moment at which any such race is best fixed.
3128	 */
3129	refs = refcount_read(&eb->refs);
3130	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3131		return;
3132
3133	spin_lock(&eb->refs_lock);
3134	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3135		refcount_inc(&eb->refs);
3136	spin_unlock(&eb->refs_lock);
3137}
3138
3139static void mark_extent_buffer_accessed(struct extent_buffer *eb)
3140{
3141	check_buffer_tree_ref(eb);
3142
3143	for (int i = 0; i < num_extent_folios(eb); i++)
3144		folio_mark_accessed(eb->folios[i]);
3145}
3146
3147struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3148					 u64 start)
3149{
3150	struct extent_buffer *eb;
3151
3152	eb = find_extent_buffer_nolock(fs_info, start);
3153	if (!eb)
3154		return NULL;
3155	/*
3156	 * Lock our eb's refs_lock to avoid races with free_extent_buffer().
3157	 * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3158	 * another task running free_extent_buffer() might have seen that flag
3159	 * set, eb->refs == 2, that the buffer isn't under IO (dirty and
3160	 * writeback flags not set) and it's still in the tree (flag
3161	 * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3162	 * decrementing the extent buffer's reference count twice.  So here we
3163	 * could race and increment the eb's reference count, clear its stale
3164	 * flag, mark it as dirty and drop our reference before the other task
3165	 * finishes executing free_extent_buffer, which would later result in
3166	 * an attempt to free an extent buffer that is dirty.
3167	 */
3168	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3169		spin_lock(&eb->refs_lock);
3170		spin_unlock(&eb->refs_lock);
3171	}
3172	mark_extent_buffer_accessed(eb);
3173	return eb;
3174}
3175
3176struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3177					u64 start)
3178{
3179#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3180	struct extent_buffer *eb, *exists = NULL;
3181	int ret;
3182
3183	eb = find_extent_buffer(fs_info, start);
3184	if (eb)
3185		return eb;
3186	eb = alloc_dummy_extent_buffer(fs_info, start);
3187	if (!eb)
3188		return ERR_PTR(-ENOMEM);
3189	eb->fs_info = fs_info;
3190again:
3191	xa_lock_irq(&fs_info->buffer_tree);
3192	exists = __xa_cmpxchg(&fs_info->buffer_tree, start >> fs_info->nodesize_bits,
3193			      NULL, eb, GFP_NOFS);
3194	if (xa_is_err(exists)) {
3195		ret = xa_err(exists);
3196		xa_unlock_irq(&fs_info->buffer_tree);
3197		btrfs_release_extent_buffer(eb);
3198		return ERR_PTR(ret);
3199	}
3200	if (exists) {
3201		if (!refcount_inc_not_zero(&exists->refs)) {
3202			/* The extent buffer is being freed, retry. */
3203			xa_unlock_irq(&fs_info->buffer_tree);
3204			goto again;
3205		}
3206		xa_unlock_irq(&fs_info->buffer_tree);
3207		btrfs_release_extent_buffer(eb);
3208		return exists;
3209	}
3210	xa_unlock_irq(&fs_info->buffer_tree);
3211	check_buffer_tree_ref(eb);
3212
3213	return eb;
3214#else
3215	/* Stub to avoid linker error when compiled with optimizations turned off. */
3216	return NULL;
3217#endif
3218}
3219
3220static struct extent_buffer *grab_extent_buffer(struct btrfs_fs_info *fs_info,
3221						struct folio *folio)
3222{
3223	struct extent_buffer *exists;
3224
3225	lockdep_assert_held(&folio->mapping->i_private_lock);
3226
3227	/*
3228	 * For subpage case, we completely rely on xarray to ensure we don't try
3229	 * to insert two ebs for the same bytenr.  So here we always return NULL
3230	 * and just continue.
3231	 */
3232	if (btrfs_meta_is_subpage(fs_info))
3233		return NULL;
3234
3235	/* Page not yet attached to an extent buffer */
3236	if (!folio_test_private(folio))
3237		return NULL;
3238
3239	/*
3240	 * We could have already allocated an eb for this folio and attached one
3241	 * so lets see if we can get a ref on the existing eb, and if we can we
3242	 * know it's good and we can just return that one, else we know we can
3243	 * just overwrite folio private.
3244	 */
3245	exists = folio_get_private(folio);
3246	if (refcount_inc_not_zero(&exists->refs))
3247		return exists;
3248
3249	WARN_ON(folio_test_dirty(folio));
3250	folio_detach_private(folio);
3251	return NULL;
3252}
3253
3254/*
3255 * Validate alignment constraints of eb at logical address @start.
3256 */
3257static bool check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3258{
3259	const u32 nodesize = fs_info->nodesize;
3260
3261	if (unlikely(!IS_ALIGNED(start, fs_info->sectorsize))) {
3262		btrfs_err(fs_info, "bad tree block start %llu", start);
3263		return true;
3264	}
3265
3266	if (unlikely(nodesize < PAGE_SIZE && !IS_ALIGNED(start, nodesize))) {
3267		btrfs_err(fs_info,
3268		"tree block is not nodesize aligned, start %llu nodesize %u",
3269			  start, nodesize);
3270		return true;
3271	}
3272	if (unlikely(nodesize >= PAGE_SIZE && !PAGE_ALIGNED(start))) {
3273		btrfs_err(fs_info,
3274		"tree block is not page aligned, start %llu nodesize %u",
3275			  start, nodesize);
3276		return true;
3277	}
3278	if (unlikely(!IS_ALIGNED(start, nodesize) &&
3279		     !test_and_set_bit(BTRFS_FS_UNALIGNED_TREE_BLOCK, &fs_info->flags))) {
3280		btrfs_warn(fs_info,
3281"tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance",
3282			      start, nodesize);
3283	}
3284	return false;
3285}
3286
3287/*
3288 * Return 0 if eb->folios[i] is attached to btree inode successfully.
3289 * Return >0 if there is already another extent buffer for the range,
3290 * and @found_eb_ret would be updated.
3291 * Return -EAGAIN if the filemap has an existing folio but with different size
3292 * than @eb.
3293 * The caller needs to free the existing folios and retry using the same order.
3294 */
3295static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,
3296				      struct btrfs_folio_state *prealloc,
3297				      struct extent_buffer **found_eb_ret)
3298{
3299
3300	struct btrfs_fs_info *fs_info = eb->fs_info;
3301	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3302	const pgoff_t index = eb->start >> PAGE_SHIFT;
3303	struct folio *existing_folio;
3304	int ret;
3305
3306	ASSERT(found_eb_ret);
3307
3308	/* Caller should ensure the folio exists. */
3309	ASSERT(eb->folios[i]);
3310
3311retry:
3312	existing_folio = NULL;
3313	ret = filemap_add_folio(mapping, eb->folios[i], index + i,
3314				GFP_NOFS | __GFP_NOFAIL);
3315	if (!ret)
3316		goto finish;
3317
3318	existing_folio = filemap_lock_folio(mapping, index + i);
3319	/* The page cache only exists for a very short time, just retry. */
3320	if (IS_ERR(existing_folio))
3321		goto retry;
3322
3323	/* For now, we should only have single-page folios for btree inode. */
3324	ASSERT(folio_nr_pages(existing_folio) == 1);
3325
3326	if (folio_size(existing_folio) != eb->folio_size) {
3327		folio_unlock(existing_folio);
3328		folio_put(existing_folio);
3329		return -EAGAIN;
3330	}
3331
3332finish:
3333	spin_lock(&mapping->i_private_lock);
3334	if (existing_folio && btrfs_meta_is_subpage(fs_info)) {
3335		/* We're going to reuse the existing page, can drop our folio now. */
3336		__free_page(folio_page(eb->folios[i], 0));
3337		eb->folios[i] = existing_folio;
3338	} else if (existing_folio) {
3339		struct extent_buffer *existing_eb;
3340
3341		existing_eb = grab_extent_buffer(fs_info, existing_folio);
3342		if (existing_eb) {
3343			/* The extent buffer still exists, we can use it directly. */
3344			*found_eb_ret = existing_eb;
3345			spin_unlock(&mapping->i_private_lock);
3346			folio_unlock(existing_folio);
3347			folio_put(existing_folio);
3348			return 1;
3349		}
3350		/* The extent buffer no longer exists, we can reuse the folio. */
3351		__free_page(folio_page(eb->folios[i], 0));
3352		eb->folios[i] = existing_folio;
3353	}
3354	eb->folio_size = folio_size(eb->folios[i]);
3355	eb->folio_shift = folio_shift(eb->folios[i]);
3356	/* Should not fail, as we have preallocated the memory. */
3357	ret = attach_extent_buffer_folio(eb, eb->folios[i], prealloc);
3358	ASSERT(!ret);
3359	/*
3360	 * To inform we have an extra eb under allocation, so that
3361	 * detach_extent_buffer_page() won't release the folio private when the
3362	 * eb hasn't been inserted into the xarray yet.
3363	 *
3364	 * The ref will be decreased when the eb releases the page, in
3365	 * detach_extent_buffer_page().  Thus needs no special handling in the
3366	 * error path.
3367	 */
3368	btrfs_folio_inc_eb_refs(fs_info, eb->folios[i]);
3369	spin_unlock(&mapping->i_private_lock);
3370	return 0;
3371}
3372
3373struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3374					  u64 start, u64 owner_root, int level)
3375{
3376	int attached = 0;
3377	struct extent_buffer *eb;
3378	struct extent_buffer *existing_eb = NULL;
3379	struct btrfs_folio_state *prealloc = NULL;
3380	u64 lockdep_owner = owner_root;
3381	bool page_contig = true;
3382	int uptodate = 1;
3383	int ret;
3384
3385	if (check_eb_alignment(fs_info, start))
3386		return ERR_PTR(-EINVAL);
3387
3388#if BITS_PER_LONG == 32
3389	if (start >= MAX_LFS_FILESIZE) {
3390		btrfs_err_rl(fs_info,
3391		"extent buffer %llu is beyond 32bit page cache limit", start);
3392		btrfs_err_32bit_limit(fs_info);
3393		return ERR_PTR(-EOVERFLOW);
3394	}
3395	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3396		btrfs_warn_32bit_limit(fs_info);
3397#endif
3398
3399	eb = find_extent_buffer(fs_info, start);
3400	if (eb)
3401		return eb;
3402
3403	eb = __alloc_extent_buffer(fs_info, start);
3404	if (!eb)
3405		return ERR_PTR(-ENOMEM);
3406
3407	/*
3408	 * The reloc trees are just snapshots, so we need them to appear to be
3409	 * just like any other fs tree WRT lockdep.
3410	 */
3411	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3412		lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3413
3414	btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3415
3416	/*
3417	 * Preallocate folio private for subpage case, so that we won't
3418	 * allocate memory with i_private_lock nor page lock hold.
3419	 *
3420	 * The memory will be freed by attach_extent_buffer_page() or freed
3421	 * manually if we exit earlier.
3422	 */
3423	if (btrfs_meta_is_subpage(fs_info)) {
3424		prealloc = btrfs_alloc_folio_state(fs_info, PAGE_SIZE, BTRFS_SUBPAGE_METADATA);
3425		if (IS_ERR(prealloc)) {
3426			ret = PTR_ERR(prealloc);
3427			goto out;
3428		}
3429	}
3430
3431reallocate:
3432	/* Allocate all pages first. */
3433	ret = alloc_eb_folio_array(eb, true);
3434	if (ret < 0) {
3435		btrfs_free_folio_state(prealloc);
3436		goto out;
3437	}
3438
3439	/* Attach all pages to the filemap. */
3440	for (int i = 0; i < num_extent_folios(eb); i++) {
3441		struct folio *folio;
3442
3443		ret = attach_eb_folio_to_filemap(eb, i, prealloc, &existing_eb);
3444		if (ret > 0) {
3445			ASSERT(existing_eb);
3446			goto out;
3447		}
3448
3449		/*
3450		 * TODO: Special handling for a corner case where the order of
3451		 * folios mismatch between the new eb and filemap.
3452		 *
3453		 * This happens when:
3454		 *
3455		 * - the new eb is using higher order folio
3456		 *
3457		 * - the filemap is still using 0-order folios for the range
3458		 *   This can happen at the previous eb allocation, and we don't
3459		 *   have higher order folio for the call.
3460		 *
3461		 * - the existing eb has already been freed
3462		 *
3463		 * In this case, we have to free the existing folios first, and
3464		 * re-allocate using the same order.
3465		 * Thankfully this is not going to happen yet, as we're still
3466		 * using 0-order folios.
3467		 */
3468		if (unlikely(ret == -EAGAIN)) {
3469			DEBUG_WARN("folio order mismatch between new eb and filemap");
3470			goto reallocate;
3471		}
3472		attached++;
3473
3474		/*
3475		 * Only after attach_eb_folio_to_filemap(), eb->folios[] is
3476		 * reliable, as we may choose to reuse the existing page cache
3477		 * and free the allocated page.
3478		 */
3479		folio = eb->folios[i];
3480		WARN_ON(btrfs_meta_folio_test_dirty(folio, eb));
3481
3482		/*
3483		 * Check if the current page is physically contiguous with previous eb
3484		 * page.
3485		 * At this stage, either we allocated a large folio, thus @i
3486		 * would only be 0, or we fall back to per-page allocation.
3487		 */
3488		if (i && folio_page(eb->folios[i - 1], 0) + 1 != folio_page(folio, 0))
3489			page_contig = false;
3490
3491		if (!btrfs_meta_folio_test_uptodate(folio, eb))
3492			uptodate = 0;
3493
3494		/*
3495		 * We can't unlock the pages just yet since the extent buffer
3496		 * hasn't been properly inserted into the xarray, this opens a
3497		 * race with btree_release_folio() which can free a page while we
3498		 * are still filling in all pages for the buffer and we could crash.
3499		 */
3500	}
3501	if (uptodate)
3502		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3503	/* All pages are physically contiguous, can skip cross page handling. */
3504	if (page_contig)
3505		eb->addr = folio_address(eb->folios[0]) + offset_in_page(eb->start);
3506again:
3507	xa_lock_irq(&fs_info->buffer_tree);
3508	existing_eb = __xa_cmpxchg(&fs_info->buffer_tree,
3509				   start >> fs_info->nodesize_bits, NULL, eb,
3510				   GFP_NOFS);
3511	if (xa_is_err(existing_eb)) {
3512		ret = xa_err(existing_eb);
3513		xa_unlock_irq(&fs_info->buffer_tree);
3514		goto out;
3515	}
3516	if (existing_eb) {
3517		if (!refcount_inc_not_zero(&existing_eb->refs)) {
3518			xa_unlock_irq(&fs_info->buffer_tree);
3519			goto again;
3520		}
3521		xa_unlock_irq(&fs_info->buffer_tree);
3522		goto out;
3523	}
3524	xa_unlock_irq(&fs_info->buffer_tree);
3525
3526	/* add one reference for the tree */
3527	check_buffer_tree_ref(eb);
3528
3529	/*
3530	 * Now it's safe to unlock the pages because any calls to
3531	 * btree_release_folio will correctly detect that a page belongs to a
3532	 * live buffer and won't free them prematurely.
3533	 */
3534	for (int i = 0; i < num_extent_folios(eb); i++) {
3535		folio_unlock(eb->folios[i]);
3536		/*
3537		 * A folio that has been added to an address_space mapping
3538		 * should not continue holding the refcount from its original
3539		 * allocation indefinitely.
3540		 */
3541		folio_put(eb->folios[i]);
3542	}
3543	return eb;
3544
3545out:
3546	WARN_ON(!refcount_dec_and_test(&eb->refs));
3547
3548	/*
3549	 * Any attached folios need to be detached before we unlock them.  This
3550	 * is because when we're inserting our new folios into the mapping, and
3551	 * then attaching our eb to that folio.  If we fail to insert our folio
3552	 * we'll lookup the folio for that index, and grab that EB.  We do not
3553	 * want that to grab this eb, as we're getting ready to free it.  So we
3554	 * have to detach it first and then unlock it.
3555	 *
3556	 * Note: the bounds is num_extent_pages() as we need to go through all slots.
3557	 */
3558	for (int i = 0; i < num_extent_pages(eb); i++) {
3559		struct folio *folio = eb->folios[i];
3560
3561		if (i < attached) {
3562			ASSERT(folio);
3563			detach_extent_buffer_folio(eb, folio);
3564			folio_unlock(folio);
3565		} else if (!folio) {
3566			continue;
3567		}
3568
3569		folio_put(folio);
3570		eb->folios[i] = NULL;
3571	}
3572	btrfs_release_extent_buffer(eb);
3573	if (ret < 0)
3574		return ERR_PTR(ret);
3575	ASSERT(existing_eb);
3576	return existing_eb;
3577}
3578
3579static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3580{
3581	struct extent_buffer *eb =
3582			container_of(head, struct extent_buffer, rcu_head);
3583
3584	kmem_cache_free(extent_buffer_cache, eb);
3585}
3586
3587static int release_extent_buffer(struct extent_buffer *eb)
3588	__releases(&eb->refs_lock)
3589{
3590	lockdep_assert_held(&eb->refs_lock);
3591
3592	if (refcount_dec_and_test(&eb->refs)) {
3593		struct btrfs_fs_info *fs_info = eb->fs_info;
3594
3595		spin_unlock(&eb->refs_lock);
3596
3597		/*
3598		 * We're erasing, theoretically there will be no allocations, so
3599		 * just use GFP_ATOMIC.
3600		 *
3601		 * We use cmpxchg instead of erase because we do not know if
3602		 * this eb is actually in the tree or not, we could be cleaning
3603		 * up an eb that we allocated but never inserted into the tree.
3604		 * Thus use cmpxchg to remove it from the tree if it is there,
3605		 * or leave the other entry if this isn't in the tree.
3606		 *
3607		 * The documentation says that putting a NULL value is the same
3608		 * as erase as long as XA_FLAGS_ALLOC is not set, which it isn't
3609		 * in this case.
3610		 */
3611		xa_cmpxchg_irq(&fs_info->buffer_tree,
3612			       eb->start >> fs_info->nodesize_bits, eb, NULL,
3613			       GFP_ATOMIC);
3614
3615		btrfs_leak_debug_del_eb(eb);
3616		/* Should be safe to release folios at this point. */
3617		btrfs_release_extent_buffer_folios(eb);
3618#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3619		if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3620			kmem_cache_free(extent_buffer_cache, eb);
3621			return 1;
3622		}
3623#endif
3624		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3625		return 1;
3626	}
3627	spin_unlock(&eb->refs_lock);
3628
3629	return 0;
3630}
3631
3632void free_extent_buffer(struct extent_buffer *eb)
3633{
3634	int refs;
3635	if (!eb)
3636		return;
3637
3638	refs = refcount_read(&eb->refs);
3639	while (1) {
3640		if (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)) {
3641			if (refs == 1)
3642				break;
3643		} else if (refs <= 3) {
3644			break;
3645		}
3646
3647		/* Optimization to avoid locking eb->refs_lock. */
3648		if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1))
3649			return;
3650	}
3651
3652	spin_lock(&eb->refs_lock);
3653	if (refcount_read(&eb->refs) == 2 &&
3654	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3655	    !extent_buffer_under_io(eb) &&
3656	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3657		refcount_dec(&eb->refs);
3658
3659	/*
3660	 * I know this is terrible, but it's temporary until we stop tracking
3661	 * the uptodate bits and such for the extent buffers.
3662	 */
3663	release_extent_buffer(eb);
3664}
3665
3666void free_extent_buffer_stale(struct extent_buffer *eb)
3667{
3668	if (!eb)
3669		return;
3670
3671	spin_lock(&eb->refs_lock);
3672	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
3673
3674	if (refcount_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3675	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3676		refcount_dec(&eb->refs);
3677	release_extent_buffer(eb);
3678}
3679
3680static void btree_clear_folio_dirty_tag(struct folio *folio)
3681{
3682	ASSERT(!folio_test_dirty(folio));
3683	ASSERT(folio_test_locked(folio));
3684	xa_lock_irq(&folio->mapping->i_pages);
3685	if (!folio_test_dirty(folio))
3686		__xa_clear_mark(&folio->mapping->i_pages, folio->index,
3687				PAGECACHE_TAG_DIRTY);
3688	xa_unlock_irq(&folio->mapping->i_pages);
3689}
3690
3691void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3692			      struct extent_buffer *eb)
3693{
3694	struct btrfs_fs_info *fs_info = eb->fs_info;
3695
3696	btrfs_assert_tree_write_locked(eb);
3697
3698	if (trans && btrfs_header_generation(eb) != trans->transid)
3699		return;
3700
3701	/*
3702	 * Instead of clearing the dirty flag off of the buffer, mark it as
3703	 * EXTENT_BUFFER_ZONED_ZEROOUT. This allows us to preserve
3704	 * write-ordering in zoned mode, without the need to later re-dirty
3705	 * the extent_buffer.
3706	 *
3707	 * The actual zeroout of the buffer will happen later in
3708	 * btree_csum_one_bio.
3709	 */
3710	if (btrfs_is_zoned(fs_info) && test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3711		set_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags);
3712		return;
3713	}
3714
3715	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
3716		return;
3717
3718	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
3719	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
3720				 fs_info->dirty_metadata_batch);
3721
3722	for (int i = 0; i < num_extent_folios(eb); i++) {
3723		struct folio *folio = eb->folios[i];
3724		bool last;
3725
3726		if (!folio_test_dirty(folio))
3727			continue;
3728		folio_lock(folio);
3729		last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
3730		if (last)
3731			btree_clear_folio_dirty_tag(folio);
3732		folio_unlock(folio);
3733	}
3734	WARN_ON(refcount_read(&eb->refs) == 0);
3735}
3736
3737void set_extent_buffer_dirty(struct extent_buffer *eb)
3738{
3739	bool was_dirty;
3740
3741	check_buffer_tree_ref(eb);
3742
3743	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3744
3745	WARN_ON(refcount_read(&eb->refs) == 0);
3746	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3747	WARN_ON(test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags));
3748
3749	if (!was_dirty) {
3750		bool subpage = btrfs_meta_is_subpage(eb->fs_info);
3751
3752		/*
3753		 * For subpage case, we can have other extent buffers in the
3754		 * same page, and in clear_extent_buffer_dirty() we
3755		 * have to clear page dirty without subpage lock held.
3756		 * This can cause race where our page gets dirty cleared after
3757		 * we just set it.
3758		 *
3759		 * Thankfully, clear_extent_buffer_dirty() has locked
3760		 * its page for other reasons, we can use page lock to prevent
3761		 * the above race.
3762		 */
3763		if (subpage)
3764			folio_lock(eb->folios[0]);
3765		for (int i = 0; i < num_extent_folios(eb); i++)
3766			btrfs_meta_folio_set_dirty(eb->folios[i], eb);
3767		buffer_tree_set_mark(eb, PAGECACHE_TAG_DIRTY);
3768		if (subpage)
3769			folio_unlock(eb->folios[0]);
3770		percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
3771					 eb->len,
3772					 eb->fs_info->dirty_metadata_batch);
3773	}
3774#ifdef CONFIG_BTRFS_DEBUG
3775	for (int i = 0; i < num_extent_folios(eb); i++)
3776		ASSERT(folio_test_dirty(eb->folios[i]));
3777#endif
3778}
3779
3780void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3781{
3782
3783	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3784	for (int i = 0; i < num_extent_folios(eb); i++) {
3785		struct folio *folio = eb->folios[i];
3786
3787		if (!folio)
3788			continue;
3789
3790		btrfs_meta_folio_clear_uptodate(folio, eb);
3791	}
3792}
3793
3794void set_extent_buffer_uptodate(struct extent_buffer *eb)
3795{
3796
3797	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3798	for (int i = 0; i < num_extent_folios(eb); i++)
3799		btrfs_meta_folio_set_uptodate(eb->folios[i], eb);
3800}
3801
3802static void clear_extent_buffer_reading(struct extent_buffer *eb)
3803{
3804	clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags);
3805}
3806
3807static void end_bbio_meta_read(struct btrfs_bio *bbio)
3808{
3809	struct extent_buffer *eb = bbio->private;
3810	bool uptodate = !bbio->bio.bi_status;
3811
3812	/*
3813	 * If the extent buffer is marked UPTODATE before the read operation
3814	 * completes, other calls to read_extent_buffer_pages() will return
3815	 * early without waiting for the read to finish, causing data races.
3816	 */
3817	WARN_ON(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags));
3818
3819	eb->read_mirror = bbio->mirror_num;
3820
3821	if (uptodate &&
3822	    btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
3823		uptodate = false;
3824
3825	if (uptodate)
3826		set_extent_buffer_uptodate(eb);
3827	else
3828		clear_extent_buffer_uptodate(eb);
3829
3830	clear_extent_buffer_reading(eb);
3831	free_extent_buffer(eb);
3832
3833	bio_put(&bbio->bio);
3834}
3835
3836int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num,
3837				    const struct btrfs_tree_parent_check *check)
3838{
3839	struct btrfs_fs_info *fs_info = eb->fs_info;
3840	struct btrfs_bio *bbio;
3841
3842	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3843		return 0;
3844
3845	/*
3846	 * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3847	 * operation, which could potentially still be in flight.  In this case
3848	 * we simply want to return an error.
3849	 */
3850	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3851		return -EIO;
3852
3853	/* Someone else is already reading the buffer, just wait for it. */
3854	if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
3855		return 0;
3856
3857	/*
3858	 * Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above
3859	 * test_and_set_bit(EXTENT_BUFFER_READING), someone else could have
3860	 * started and finished reading the same eb.  In this case, UPTODATE
3861	 * will now be set, and we shouldn't read it in again.
3862	 */
3863	if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
3864		clear_extent_buffer_reading(eb);
3865		return 0;
3866	}
3867
3868	eb->read_mirror = 0;
3869	check_buffer_tree_ref(eb);
3870	refcount_inc(&eb->refs);
3871
3872	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3873			       REQ_OP_READ | REQ_META, BTRFS_I(fs_info->btree_inode),
3874			       eb->start, end_bbio_meta_read, eb);
3875	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
3876	memcpy(&bbio->parent_check, check, sizeof(*check));
3877	for (int i = 0; i < num_extent_folios(eb); i++) {
3878		struct folio *folio = eb->folios[i];
3879		u64 range_start = max_t(u64, eb->start, folio_pos(folio));
3880		u32 range_len = min_t(u64, folio_next_pos(folio),
3881				      eb->start + eb->len) - range_start;
3882
3883		bio_add_folio_nofail(&bbio->bio, folio, range_len,
3884				     offset_in_folio(folio, range_start));
3885	}
3886	btrfs_submit_bbio(bbio, mirror_num);
3887	return 0;
3888}
3889
3890int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num,
3891			     const struct btrfs_tree_parent_check *check)
3892{
3893	int ret;
3894
3895	ret = read_extent_buffer_pages_nowait(eb, mirror_num, check);
3896	if (ret < 0)
3897		return ret;
3898
3899	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
3900	if (unlikely(!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)))
3901		return -EIO;
3902	return 0;
3903}
3904
3905static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
3906			    unsigned long len)
3907{
3908	btrfs_warn(eb->fs_info,
3909		"access to eb bytenr %llu len %u out of range start %lu len %lu",
3910		eb->start, eb->len, start, len);
3911	DEBUG_WARN();
3912
3913	return true;
3914}
3915
3916/*
3917 * Check if the [start, start + len) range is valid before reading/writing
3918 * the eb.
3919 * NOTE: @start and @len are offset inside the eb, not logical address.
3920 *
3921 * Caller should not touch the dst/src memory if this function returns error.
3922 */
3923static inline int check_eb_range(const struct extent_buffer *eb,
3924				 unsigned long start, unsigned long len)
3925{
3926	unsigned long offset;
3927
3928	/* start, start + len should not go beyond eb->len nor overflow */
3929	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
3930		return report_eb_range(eb, start, len);
3931
3932	return false;
3933}
3934
3935void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
3936			unsigned long start, unsigned long len)
3937{
3938	const int unit_size = eb->folio_size;
3939	size_t cur;
3940	size_t offset;
3941	char *dst = (char *)dstv;
3942	unsigned long i = get_eb_folio_index(eb, start);
3943
3944	if (check_eb_range(eb, start, len)) {
3945		/*
3946		 * Invalid range hit, reset the memory, so callers won't get
3947		 * some random garbage for their uninitialized memory.
3948		 */
3949		memset(dstv, 0, len);
3950		return;
3951	}
3952
3953	if (eb->addr) {
3954		memcpy(dstv, eb->addr + start, len);
3955		return;
3956	}
3957
3958	offset = get_eb_offset_in_folio(eb, start);
3959
3960	while (len > 0) {
3961		char *kaddr;
3962
3963		cur = min(len, unit_size - offset);
3964		kaddr = folio_address(eb->folios[i]);
3965		memcpy(dst, kaddr + offset, cur);
3966
3967		dst += cur;
3968		len -= cur;
3969		offset = 0;
3970		i++;
3971	}
3972}
3973
3974int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
3975				       void __user *dstv,
3976				       unsigned long start, unsigned long len)
3977{
3978	const int unit_size = eb->folio_size;
3979	size_t cur;
3980	size_t offset;
3981	char __user *dst = (char __user *)dstv;
3982	unsigned long i = get_eb_folio_index(eb, start);
3983	int ret = 0;
3984
3985	WARN_ON(start > eb->len);
3986	WARN_ON(start + len > eb->start + eb->len);
3987
3988	if (eb->addr) {
3989		if (copy_to_user_nofault(dstv, eb->addr + start, len))
3990			ret = -EFAULT;
3991		return ret;
3992	}
3993
3994	offset = get_eb_offset_in_folio(eb, start);
3995
3996	while (len > 0) {
3997		char *kaddr;
3998
3999		cur = min(len, unit_size - offset);
4000		kaddr = folio_address(eb->folios[i]);
4001		if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
4002			ret = -EFAULT;
4003			break;
4004		}
4005
4006		dst += cur;
4007		len -= cur;
4008		offset = 0;
4009		i++;
4010	}
4011
4012	return ret;
4013}
4014
4015int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4016			 unsigned long start, unsigned long len)
4017{
4018	const int unit_size = eb->folio_size;
4019	size_t cur;
4020	size_t offset;
4021	char *kaddr;
4022	char *ptr = (char *)ptrv;
4023	unsigned long i = get_eb_folio_index(eb, start);
4024	int ret = 0;
4025
4026	if (check_eb_range(eb, start, len))
4027		return -EINVAL;
4028
4029	if (eb->addr)
4030		return memcmp(ptrv, eb->addr + start, len);
4031
4032	offset = get_eb_offset_in_folio(eb, start);
4033
4034	while (len > 0) {
4035		cur = min(len, unit_size - offset);
4036		kaddr = folio_address(eb->folios[i]);
4037		ret = memcmp(ptr, kaddr + offset, cur);
4038		if (ret)
4039			break;
4040
4041		ptr += cur;
4042		len -= cur;
4043		offset = 0;
4044		i++;
4045	}
4046	return ret;
4047}
4048
4049/*
4050 * Check that the extent buffer is uptodate.
4051 *
4052 * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4053 * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4054 */
4055static void assert_eb_folio_uptodate(const struct extent_buffer *eb, int i)
4056{
4057	struct btrfs_fs_info *fs_info = eb->fs_info;
4058	struct folio *folio = eb->folios[i];
4059
4060	ASSERT(folio);
4061
4062	/*
4063	 * If we are using the commit root we could potentially clear a page
4064	 * Uptodate while we're using the extent buffer that we've previously
4065	 * looked up.  We don't want to complain in this case, as the page was
4066	 * valid before, we just didn't write it out.  Instead we want to catch
4067	 * the case where we didn't actually read the block properly, which
4068	 * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4069	 */
4070	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4071		return;
4072
4073	if (btrfs_meta_is_subpage(fs_info)) {
4074		folio = eb->folios[0];
4075		ASSERT(i == 0);
4076		if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, folio,
4077							 eb->start, eb->len)))
4078			btrfs_subpage_dump_bitmap(fs_info, folio, eb->start, eb->len);
4079	} else {
4080		WARN_ON(!folio_test_uptodate(folio));
4081	}
4082}
4083
4084static void __write_extent_buffer(const struct extent_buffer *eb,
4085				  const void *srcv, unsigned long start,
4086				  unsigned long len, bool use_memmove)
4087{
4088	const int unit_size = eb->folio_size;
4089	size_t cur;
4090	size_t offset;
4091	char *kaddr;
4092	const char *src = (const char *)srcv;
4093	unsigned long i = get_eb_folio_index(eb, start);
4094	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
4095	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4096
4097	if (check_eb_range(eb, start, len))
4098		return;
4099
4100	if (eb->addr) {
4101		if (use_memmove)
4102			memmove(eb->addr + start, srcv, len);
4103		else
4104			memcpy(eb->addr + start, srcv, len);
4105		return;
4106	}
4107
4108	offset = get_eb_offset_in_folio(eb, start);
4109
4110	while (len > 0) {
4111		if (check_uptodate)
4112			assert_eb_folio_uptodate(eb, i);
4113
4114		cur = min(len, unit_size - offset);
4115		kaddr = folio_address(eb->folios[i]);
4116		if (use_memmove)
4117			memmove(kaddr + offset, src, cur);
4118		else
4119			memcpy(kaddr + offset, src, cur);
4120
4121		src += cur;
4122		len -= cur;
4123		offset = 0;
4124		i++;
4125	}
4126}
4127
4128void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4129			 unsigned long start, unsigned long len)
4130{
4131	return __write_extent_buffer(eb, srcv, start, len, false);
4132}
4133
4134static void memset_extent_buffer(const struct extent_buffer *eb, int c,
4135				 unsigned long start, unsigned long len)
4136{
4137	const int unit_size = eb->folio_size;
4138	unsigned long cur = start;
4139
4140	if (eb->addr) {
4141		memset(eb->addr + start, c, len);
4142		return;
4143	}
4144
4145	while (cur < start + len) {
4146		unsigned long index = get_eb_folio_index(eb, cur);
4147		unsigned int offset = get_eb_offset_in_folio(eb, cur);
4148		unsigned int cur_len = min(start + len - cur, unit_size - offset);
4149
4150		assert_eb_folio_uptodate(eb, index);
4151		memset(folio_address(eb->folios[index]) + offset, c, cur_len);
4152
4153		cur += cur_len;
4154	}
4155}
4156
4157void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4158			   unsigned long len)
4159{
4160	if (check_eb_range(eb, start, len))
4161		return;
4162	return memset_extent_buffer(eb, 0, start, len);
4163}
4164
4165void copy_extent_buffer_full(const struct extent_buffer *dst,
4166			     const struct extent_buffer *src)
4167{
4168	const int unit_size = src->folio_size;
4169	unsigned long cur = 0;
4170
4171	ASSERT(dst->len == src->len);
4172
4173	while (cur < src->len) {
4174		unsigned long index = get_eb_folio_index(src, cur);
4175		unsigned long offset = get_eb_offset_in_folio(src, cur);
4176		unsigned long cur_len = min(src->len, unit_size - offset);
4177		void *addr = folio_address(src->folios[index]) + offset;
4178
4179		write_extent_buffer(dst, addr, cur, cur_len);
4180
4181		cur += cur_len;
4182	}
4183}
4184
4185void copy_extent_buffer(const struct extent_buffer *dst,
4186			const struct extent_buffer *src,
4187			unsigned long dst_offset, unsigned long src_offset,
4188			unsigned long len)
4189{
4190	const int unit_size = dst->folio_size;
4191	u64 dst_len = dst->len;
4192	size_t cur;
4193	size_t offset;
4194	char *kaddr;
4195	unsigned long i = get_eb_folio_index(dst, dst_offset);
4196
4197	if (check_eb_range(dst, dst_offset, len) ||
4198	    check_eb_range(src, src_offset, len))
4199		return;
4200
4201	WARN_ON(src->len != dst_len);
4202
4203	offset = get_eb_offset_in_folio(dst, dst_offset);
4204
4205	while (len > 0) {
4206		assert_eb_folio_uptodate(dst, i);
4207
4208		cur = min(len, (unsigned long)(unit_size - offset));
4209
4210		kaddr = folio_address(dst->folios[i]);
4211		read_extent_buffer(src, kaddr + offset, src_offset, cur);
4212
4213		src_offset += cur;
4214		len -= cur;
4215		offset = 0;
4216		i++;
4217	}
4218}
4219
4220/*
4221 * Calculate the folio and offset of the byte containing the given bit number.
4222 *
4223 * @eb:           the extent buffer
4224 * @start:        offset of the bitmap item in the extent buffer
4225 * @nr:           bit number
4226 * @folio_index:  return index of the folio in the extent buffer that contains
4227 *                the given bit number
4228 * @folio_offset: return offset into the folio given by folio_index
4229 *
4230 * This helper hides the ugliness of finding the byte in an extent buffer which
4231 * contains a given bit.
4232 */
4233static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4234				    unsigned long start, unsigned long nr,
4235				    unsigned long *folio_index,
4236				    size_t *folio_offset)
4237{
4238	size_t byte_offset = BIT_BYTE(nr);
4239	size_t offset;
4240
4241	/*
4242	 * The byte we want is the offset of the extent buffer + the offset of
4243	 * the bitmap item in the extent buffer + the offset of the byte in the
4244	 * bitmap item.
4245	 */
4246	offset = start + offset_in_eb_folio(eb, eb->start) + byte_offset;
4247
4248	*folio_index = offset >> eb->folio_shift;
4249	*folio_offset = offset_in_eb_folio(eb, offset);
4250}
4251
4252/*
4253 * Determine whether a bit in a bitmap item is set.
4254 *
4255 * @eb:     the extent buffer
4256 * @start:  offset of the bitmap item in the extent buffer
4257 * @nr:     bit number to test
4258 */
4259bool extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4260			    unsigned long nr)
4261{
4262	unsigned long i;
4263	size_t offset;
4264	u8 *kaddr;
4265
4266	eb_bitmap_offset(eb, start, nr, &i, &offset);
4267	assert_eb_folio_uptodate(eb, i);
4268	kaddr = folio_address(eb->folios[i]);
4269	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4270}
4271
4272static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4273{
4274	unsigned long index = get_eb_folio_index(eb, bytenr);
4275
4276	if (check_eb_range(eb, bytenr, 1))
4277		return NULL;
4278	return folio_address(eb->folios[index]) + get_eb_offset_in_folio(eb, bytenr);
4279}
4280
4281/*
4282 * Set an area of a bitmap to 1.
4283 *
4284 * @eb:     the extent buffer
4285 * @start:  offset of the bitmap item in the extent buffer
4286 * @pos:    bit number of the first bit
4287 * @len:    number of bits to set
4288 */
4289void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4290			      unsigned long pos, unsigned long len)
4291{
4292	unsigned int first_byte = start + BIT_BYTE(pos);
4293	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4294	const bool same_byte = (first_byte == last_byte);
4295	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4296	u8 *kaddr;
4297
4298	if (same_byte)
4299		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4300
4301	/* Handle the first byte. */
4302	kaddr = extent_buffer_get_byte(eb, first_byte);
4303	*kaddr |= mask;
4304	if (same_byte)
4305		return;
4306
4307	/* Handle the byte aligned part. */
4308	ASSERT(first_byte + 1 <= last_byte);
4309	memset_extent_buffer(eb, 0xff, first_byte + 1, last_byte - first_byte - 1);
4310
4311	/* Handle the last byte. */
4312	kaddr = extent_buffer_get_byte(eb, last_byte);
4313	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
4314}
4315
4316
4317/*
4318 * Clear an area of a bitmap.
4319 *
4320 * @eb:     the extent buffer
4321 * @start:  offset of the bitmap item in the extent buffer
4322 * @pos:    bit number of the first bit
4323 * @len:    number of bits to clear
4324 */
4325void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4326				unsigned long start, unsigned long pos,
4327				unsigned long len)
4328{
4329	unsigned int first_byte = start + BIT_BYTE(pos);
4330	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4331	const bool same_byte = (first_byte == last_byte);
4332	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4333	u8 *kaddr;
4334
4335	if (same_byte)
4336		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4337
4338	/* Handle the first byte. */
4339	kaddr = extent_buffer_get_byte(eb, first_byte);
4340	*kaddr &= ~mask;
4341	if (same_byte)
4342		return;
4343
4344	/* Handle the byte aligned part. */
4345	ASSERT(first_byte + 1 <= last_byte);
4346	memset_extent_buffer(eb, 0, first_byte + 1, last_byte - first_byte - 1);
4347
4348	/* Handle the last byte. */
4349	kaddr = extent_buffer_get_byte(eb, last_byte);
4350	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
4351}
4352
4353static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4354{
4355	unsigned long distance = (src > dst) ? src - dst : dst - src;
4356	return distance < len;
4357}
4358
4359void memcpy_extent_buffer(const struct extent_buffer *dst,
4360			  unsigned long dst_offset, unsigned long src_offset,
4361			  unsigned long len)
4362{
4363	const int unit_size = dst->folio_size;
4364	unsigned long cur_off = 0;
4365
4366	if (check_eb_range(dst, dst_offset, len) ||
4367	    check_eb_range(dst, src_offset, len))
4368		return;
4369
4370	if (dst->addr) {
4371		const bool use_memmove = areas_overlap(src_offset, dst_offset, len);
4372
4373		if (use_memmove)
4374			memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4375		else
4376			memcpy(dst->addr + dst_offset, dst->addr + src_offset, len);
4377		return;
4378	}
4379
4380	while (cur_off < len) {
4381		unsigned long cur_src = cur_off + src_offset;
4382		unsigned long folio_index = get_eb_folio_index(dst, cur_src);
4383		unsigned long folio_off = get_eb_offset_in_folio(dst, cur_src);
4384		unsigned long cur_len = min(src_offset + len - cur_src,
4385					    unit_size - folio_off);
4386		void *src_addr = folio_address(dst->folios[folio_index]) + folio_off;
4387		const bool use_memmove = areas_overlap(src_offset + cur_off,
4388						       dst_offset + cur_off, cur_len);
4389
4390		__write_extent_buffer(dst, src_addr, dst_offset + cur_off, cur_len,
4391				      use_memmove);
4392		cur_off += cur_len;
4393	}
4394}
4395
4396void memmove_extent_buffer(const struct extent_buffer *dst,
4397			   unsigned long dst_offset, unsigned long src_offset,
4398			   unsigned long len)
4399{
4400	unsigned long dst_end = dst_offset + len - 1;
4401	unsigned long src_end = src_offset + len - 1;
4402
4403	if (check_eb_range(dst, dst_offset, len) ||
4404	    check_eb_range(dst, src_offset, len))
4405		return;
4406
4407	if (dst_offset < src_offset) {
4408		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4409		return;
4410	}
4411
4412	if (dst->addr) {
4413		memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4414		return;
4415	}
4416
4417	while (len > 0) {
4418		unsigned long src_i;
4419		size_t cur;
4420		size_t dst_off_in_folio;
4421		size_t src_off_in_folio;
4422		void *src_addr;
4423		bool use_memmove;
4424
4425		src_i = get_eb_folio_index(dst, src_end);
4426
4427		dst_off_in_folio = get_eb_offset_in_folio(dst, dst_end);
4428		src_off_in_folio = get_eb_offset_in_folio(dst, src_end);
4429
4430		cur = min_t(unsigned long, len, src_off_in_folio + 1);
4431		cur = min(cur, dst_off_in_folio + 1);
4432
4433		src_addr = folio_address(dst->folios[src_i]) + src_off_in_folio -
4434					 cur + 1;
4435		use_memmove = areas_overlap(src_end - cur + 1, dst_end - cur + 1,
4436					    cur);
4437
4438		__write_extent_buffer(dst, src_addr, dst_end - cur + 1, cur,
4439				      use_memmove);
4440
4441		dst_end -= cur;
4442		src_end -= cur;
4443		len -= cur;
4444	}
4445}
4446
4447static int try_release_subpage_extent_buffer(struct folio *folio)
4448{
4449	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
4450	struct extent_buffer *eb;
4451	unsigned long start = (folio_pos(folio) >> fs_info->nodesize_bits);
4452	unsigned long index = start;
4453	unsigned long end = index + (PAGE_SIZE >> fs_info->nodesize_bits) - 1;
4454	int ret;
4455
4456	rcu_read_lock();
4457	xa_for_each_range(&fs_info->buffer_tree, index, eb, start, end) {
4458		/*
4459		 * The same as try_release_extent_buffer(), to ensure the eb
4460		 * won't disappear out from under us.
4461		 */
4462		spin_lock(&eb->refs_lock);
4463		rcu_read_unlock();
4464
4465		if (refcount_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4466			spin_unlock(&eb->refs_lock);
4467			rcu_read_lock();
4468			continue;
4469		}
4470
4471		/*
4472		 * If tree ref isn't set then we know the ref on this eb is a
4473		 * real ref, so just return, this eb will likely be freed soon
4474		 * anyway.
4475		 */
4476		if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4477			spin_unlock(&eb->refs_lock);
4478			rcu_read_lock();
4479			break;
4480		}
4481
4482		/*
4483		 * Here we don't care about the return value, we will always
4484		 * check the folio private at the end.  And
4485		 * release_extent_buffer() will release the refs_lock.
4486		 */
4487		release_extent_buffer(eb);
4488		rcu_read_lock();
4489	}
4490	rcu_read_unlock();
4491
4492	/*
4493	 * Finally to check if we have cleared folio private, as if we have
4494	 * released all ebs in the page, the folio private should be cleared now.
4495	 */
4496	spin_lock(&folio->mapping->i_private_lock);
4497	if (!folio_test_private(folio))
4498		ret = 1;
4499	else
4500		ret = 0;
4501	spin_unlock(&folio->mapping->i_private_lock);
4502	return ret;
4503}
4504
4505int try_release_extent_buffer(struct folio *folio)
4506{
4507	struct extent_buffer *eb;
4508
4509	if (btrfs_meta_is_subpage(folio_to_fs_info(folio)))
4510		return try_release_subpage_extent_buffer(folio);
4511
4512	/*
4513	 * We need to make sure nobody is changing folio private, as we rely on
4514	 * folio private as the pointer to extent buffer.
4515	 */
4516	spin_lock(&folio->mapping->i_private_lock);
4517	if (!folio_test_private(folio)) {
4518		spin_unlock(&folio->mapping->i_private_lock);
4519		return 1;
4520	}
4521
4522	eb = folio_get_private(folio);
4523	BUG_ON(!eb);
4524
4525	/*
4526	 * This is a little awful but should be ok, we need to make sure that
4527	 * the eb doesn't disappear out from under us while we're looking at
4528	 * this page.
4529	 */
4530	spin_lock(&eb->refs_lock);
4531	if (refcount_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4532		spin_unlock(&eb->refs_lock);
4533		spin_unlock(&folio->mapping->i_private_lock);
4534		return 0;
4535	}
4536	spin_unlock(&folio->mapping->i_private_lock);
4537
4538	/*
4539	 * If tree ref isn't set then we know the ref on this eb is a real ref,
4540	 * so just return, this page will likely be freed soon anyway.
4541	 */
4542	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4543		spin_unlock(&eb->refs_lock);
4544		return 0;
4545	}
4546
4547	return release_extent_buffer(eb);
4548}
4549
4550/*
4551 * Attempt to readahead a child block.
4552 *
4553 * @fs_info:	the fs_info
4554 * @bytenr:	bytenr to read
4555 * @owner_root: objectid of the root that owns this eb
4556 * @gen:	generation for the uptodate check, can be 0
4557 * @level:	level for the eb
4558 *
4559 * Attempt to readahead a tree block at @bytenr.  If @gen is 0 then we do a
4560 * normal uptodate check of the eb, without checking the generation.  If we have
4561 * to read the block we will not block on anything.
4562 */
4563void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4564				u64 bytenr, u64 owner_root, u64 gen, int level)
4565{
4566	struct btrfs_tree_parent_check check = {
4567		.level = level,
4568		.transid = gen
4569	};
4570	struct extent_buffer *eb;
4571	int ret;
4572
4573	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4574	if (IS_ERR(eb))
4575		return;
4576
4577	if (btrfs_buffer_uptodate(eb, gen, true)) {
4578		free_extent_buffer(eb);
4579		return;
4580	}
4581
4582	ret = read_extent_buffer_pages_nowait(eb, 0, &check);
4583	if (ret < 0)
4584		free_extent_buffer_stale(eb);
4585	else
4586		free_extent_buffer(eb);
4587}
4588
4589/*
4590 * Readahead a node's child block.
4591 *
4592 * @node:	parent node we're reading from
4593 * @slot:	slot in the parent node for the child we want to read
4594 *
4595 * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4596 * the slot in the node provided.
4597 */
4598void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4599{
4600	btrfs_readahead_tree_block(node->fs_info,
4601				   btrfs_node_blockptr(node, slot),
4602				   btrfs_header_owner(node),
4603				   btrfs_node_ptr_generation(node, slot),
4604				   btrfs_header_level(node) - 1);
4605}