file.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2007 Oracle.  All rights reserved.
   4 */
   5
   6#include <linux/fs.h>
   7#include <linux/pagemap.h>
   8#include <linux/time.h>
   9#include <linux/init.h>
  10#include <linux/string.h>
  11#include <linux/backing-dev.h>
  12#include <linux/falloc.h>
  13#include <linux/writeback.h>
  14#include <linux/compat.h>
  15#include <linux/slab.h>
  16#include <linux/btrfs.h>
  17#include <linux/uio.h>
  18#include <linux/iversion.h>
  19#include <linux/fsverity.h>
  20#include "ctree.h"
  21#include "direct-io.h"
  22#include "disk-io.h"
  23#include "transaction.h"
  24#include "btrfs_inode.h"
  25#include "tree-log.h"
  26#include "locking.h"
  27#include "qgroup.h"
  28#include "compression.h"
  29#include "delalloc-space.h"
  30#include "reflink.h"
  31#include "subpage.h"
  32#include "fs.h"
  33#include "accessors.h"
  34#include "extent-tree.h"
  35#include "file-item.h"
  36#include "ioctl.h"
  37#include "file.h"
  38#include "super.h"
  39#include "print-tree.h"
  40
  41/*
  42 * Unlock folio after btrfs_file_write() is done with it.
  43 */
  44static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio,
  45			     u64 pos, u64 copied)
  46{
  47	u64 block_start = round_down(pos, fs_info->sectorsize);
  48	u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start;
  49
  50	ASSERT(block_len <= U32_MAX);
  51	/*
  52	 * Folio checked is some magic around finding folios that have been
  53	 * modified without going through btrfs_dirty_folio().  Clear it here.
  54	 * There should be no need to mark the pages accessed as
  55	 * prepare_one_folio() should have marked them accessed in
  56	 * prepare_one_folio() via find_or_create_page()
  57	 */
  58	btrfs_folio_clamp_clear_checked(fs_info, folio, block_start, block_len);
  59	folio_unlock(folio);
  60	folio_put(folio);
  61}
  62
  63/*
  64 * After copy_folio_from_iter_atomic(), update the following things for delalloc:
  65 * - Mark newly dirtied folio as DELALLOC in the io tree.
  66 *   Used to advise which range is to be written back.
  67 * - Mark modified folio as Uptodate/Dirty and not needing COW fixup
  68 * - Update inode size for past EOF write
  69 */
  70int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos,
  71		      size_t write_bytes, struct extent_state **cached, bool noreserve)
  72{
  73	struct btrfs_fs_info *fs_info = inode->root->fs_info;
  74	int ret = 0;
  75	u64 num_bytes;
  76	u64 start_pos;
  77	u64 end_of_last_block;
  78	const u64 end_pos = pos + write_bytes;
  79	loff_t isize = i_size_read(&inode->vfs_inode);
  80	unsigned int extra_bits = 0;
  81
  82	if (write_bytes == 0)
  83		return 0;
  84
  85	if (noreserve)
  86		extra_bits |= EXTENT_NORESERVE;
  87
  88	start_pos = round_down(pos, fs_info->sectorsize);
  89	num_bytes = round_up(end_pos - start_pos, fs_info->sectorsize);
  90	ASSERT(num_bytes <= U32_MAX);
  91	ASSERT(folio_pos(folio) <= pos && folio_next_pos(folio) >= end_pos);
  92
  93	end_of_last_block = start_pos + num_bytes - 1;
  94
  95	/*
  96	 * The pages may have already been dirty, clear out old accounting so
  97	 * we can set things up properly
  98	 */
  99	btrfs_clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block,
 100			       EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
 101			       cached);
 102
 103	ret = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
 104					extra_bits, cached);
 105	if (ret)
 106		return ret;
 107
 108	btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes);
 109	btrfs_folio_clamp_clear_checked(fs_info, folio, start_pos, num_bytes);
 110	btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes);
 111
 112	/*
 113	 * we've only changed i_size in ram, and we haven't updated
 114	 * the disk i_size.  There is no need to log the inode
 115	 * at this time.
 116	 */
 117	if (end_pos > isize)
 118		i_size_write(&inode->vfs_inode, end_pos);
 119	return 0;
 120}
 121
 122/*
 123 * this is very complex, but the basic idea is to drop all extents
 124 * in the range start - end.  hint_block is filled in with a block number
 125 * that would be a good hint to the block allocator for this file.
 126 *
 127 * If an extent intersects the range but is not entirely inside the range
 128 * it is either truncated or split.  Anything entirely inside the range
 129 * is deleted from the tree.
 130 *
 131 * Note: the VFS' inode number of bytes is not updated, it's up to the caller
 132 * to deal with that. We set the field 'bytes_found' of the arguments structure
 133 * with the number of allocated bytes found in the target range, so that the
 134 * caller can update the inode's number of bytes in an atomic way when
 135 * replacing extents in a range to avoid races with stat(2).
 136 */
 137int btrfs_drop_extents(struct btrfs_trans_handle *trans,
 138		       struct btrfs_root *root, struct btrfs_inode *inode,
 139		       struct btrfs_drop_extents_args *args)
 140{
 141	struct btrfs_fs_info *fs_info = root->fs_info;
 142	struct extent_buffer *leaf;
 143	struct btrfs_file_extent_item *fi;
 144	struct btrfs_key key;
 145	struct btrfs_key new_key;
 146	u64 ino = btrfs_ino(inode);
 147	u64 search_start = args->start;
 148	u64 disk_bytenr = 0;
 149	u64 num_bytes = 0;
 150	u64 extent_offset = 0;
 151	u64 extent_end = 0;
 152	u64 last_end = args->start;
 153	int del_nr = 0;
 154	int del_slot = 0;
 155	int extent_type;
 156	int recow;
 157	int ret;
 158	int modify_tree = -1;
 159	int update_refs;
 160	int found = 0;
 161	struct btrfs_path *path = args->path;
 162
 163	args->bytes_found = 0;
 164	args->extent_inserted = false;
 165
 166	/* Must always have a path if ->replace_extent is true */
 167	ASSERT(!(args->replace_extent && !args->path));
 168
 169	if (!path) {
 170		path = btrfs_alloc_path();
 171		if (!path) {
 172			ret = -ENOMEM;
 173			goto out;
 174		}
 175	}
 176
 177	if (args->drop_cache)
 178		btrfs_drop_extent_map_range(inode, args->start, args->end - 1, false);
 179
 180	if (data_race(args->start >= inode->disk_i_size) && !args->replace_extent)
 181		modify_tree = 0;
 182
 183	update_refs = (btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID);
 184	while (1) {
 185		recow = 0;
 186		ret = btrfs_lookup_file_extent(trans, root, path, ino,
 187					       search_start, modify_tree);
 188		if (ret < 0)
 189			break;
 190		if (ret > 0 && path->slots[0] > 0 && search_start == args->start) {
 191			leaf = path->nodes[0];
 192			btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
 193			if (key.objectid == ino &&
 194			    key.type == BTRFS_EXTENT_DATA_KEY)
 195				path->slots[0]--;
 196		}
 197		ret = 0;
 198next_slot:
 199		leaf = path->nodes[0];
 200		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 201			if (WARN_ON(del_nr > 0)) {
 202				btrfs_print_leaf(leaf);
 203				ret = -EINVAL;
 204				break;
 205			}
 206			ret = btrfs_next_leaf(root, path);
 207			if (ret < 0)
 208				break;
 209			if (ret > 0) {
 210				ret = 0;
 211				break;
 212			}
 213			leaf = path->nodes[0];
 214			recow = 1;
 215		}
 216
 217		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 218
 219		if (key.objectid > ino)
 220			break;
 221		if (WARN_ON_ONCE(key.objectid < ino) ||
 222		    key.type < BTRFS_EXTENT_DATA_KEY) {
 223			ASSERT(del_nr == 0);
 224			path->slots[0]++;
 225			goto next_slot;
 226		}
 227		if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end)
 228			break;
 229
 230		fi = btrfs_item_ptr(leaf, path->slots[0],
 231				    struct btrfs_file_extent_item);
 232		extent_type = btrfs_file_extent_type(leaf, fi);
 233
 234		if (extent_type == BTRFS_FILE_EXTENT_REG ||
 235		    extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
 236			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
 237			num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
 238			extent_offset = btrfs_file_extent_offset(leaf, fi);
 239			extent_end = key.offset +
 240				btrfs_file_extent_num_bytes(leaf, fi);
 241		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 242			extent_end = key.offset +
 243				btrfs_file_extent_ram_bytes(leaf, fi);
 244		} else {
 245			/* can't happen */
 246			BUG();
 247		}
 248
 249		/*
 250		 * Don't skip extent items representing 0 byte lengths. They
 251		 * used to be created (bug) if while punching holes we hit
 252		 * -ENOSPC condition. So if we find one here, just ensure we
 253		 * delete it, otherwise we would insert a new file extent item
 254		 * with the same key (offset) as that 0 bytes length file
 255		 * extent item in the call to setup_items_for_insert() later
 256		 * in this function.
 257		 */
 258		if (extent_end == key.offset && extent_end >= search_start) {
 259			last_end = extent_end;
 260			goto delete_extent_item;
 261		}
 262
 263		if (extent_end <= search_start) {
 264			path->slots[0]++;
 265			goto next_slot;
 266		}
 267
 268		found = 1;
 269		search_start = max(key.offset, args->start);
 270		if (recow || !modify_tree) {
 271			modify_tree = -1;
 272			btrfs_release_path(path);
 273			continue;
 274		}
 275
 276		/*
 277		 *     | - range to drop - |
 278		 *  | -------- extent -------- |
 279		 */
 280		if (args->start > key.offset && args->end < extent_end) {
 281			if (WARN_ON(del_nr > 0)) {
 282				btrfs_print_leaf(leaf);
 283				ret = -EINVAL;
 284				break;
 285			}
 286			if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 287				ret = -EOPNOTSUPP;
 288				break;
 289			}
 290
 291			memcpy(&new_key, &key, sizeof(new_key));
 292			new_key.offset = args->start;
 293			ret = btrfs_duplicate_item(trans, root, path,
 294						   &new_key);
 295			if (ret == -EAGAIN) {
 296				btrfs_release_path(path);
 297				continue;
 298			}
 299			if (ret < 0)
 300				break;
 301
 302			leaf = path->nodes[0];
 303			fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
 304					    struct btrfs_file_extent_item);
 305			btrfs_set_file_extent_num_bytes(leaf, fi,
 306							args->start - key.offset);
 307
 308			fi = btrfs_item_ptr(leaf, path->slots[0],
 309					    struct btrfs_file_extent_item);
 310
 311			extent_offset += args->start - key.offset;
 312			btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 313			btrfs_set_file_extent_num_bytes(leaf, fi,
 314							extent_end - args->start);
 315
 316			if (update_refs && disk_bytenr > 0) {
 317				struct btrfs_ref ref = {
 318					.action = BTRFS_ADD_DELAYED_REF,
 319					.bytenr = disk_bytenr,
 320					.num_bytes = num_bytes,
 321					.parent = 0,
 322					.owning_root = btrfs_root_id(root),
 323					.ref_root = btrfs_root_id(root),
 324				};
 325				btrfs_init_data_ref(&ref, new_key.objectid,
 326						    args->start - extent_offset,
 327						    0, false);
 328				ret = btrfs_inc_extent_ref(trans, &ref);
 329				if (unlikely(ret)) {
 330					btrfs_abort_transaction(trans, ret);
 331					break;
 332				}
 333			}
 334			key.offset = args->start;
 335		}
 336		/*
 337		 * From here on out we will have actually dropped something, so
 338		 * last_end can be updated.
 339		 */
 340		last_end = extent_end;
 341
 342		/*
 343		 *  | ---- range to drop ----- |
 344		 *      | -------- extent -------- |
 345		 */
 346		if (args->start <= key.offset && args->end < extent_end) {
 347			if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 348				ret = -EOPNOTSUPP;
 349				break;
 350			}
 351
 352			memcpy(&new_key, &key, sizeof(new_key));
 353			new_key.offset = args->end;
 354			btrfs_set_item_key_safe(trans, path, &new_key);
 355
 356			extent_offset += args->end - key.offset;
 357			btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 358			btrfs_set_file_extent_num_bytes(leaf, fi,
 359							extent_end - args->end);
 360			if (update_refs && disk_bytenr > 0)
 361				args->bytes_found += args->end - key.offset;
 362			break;
 363		}
 364
 365		search_start = extent_end;
 366		/*
 367		 *       | ---- range to drop ----- |
 368		 *  | -------- extent -------- |
 369		 */
 370		if (args->start > key.offset && args->end >= extent_end) {
 371			if (WARN_ON(del_nr > 0)) {
 372				btrfs_print_leaf(leaf);
 373				ret = -EINVAL;
 374				break;
 375			}
 376			if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 377				ret = -EOPNOTSUPP;
 378				break;
 379			}
 380
 381			btrfs_set_file_extent_num_bytes(leaf, fi,
 382							args->start - key.offset);
 383			if (update_refs && disk_bytenr > 0)
 384				args->bytes_found += extent_end - args->start;
 385			if (args->end == extent_end)
 386				break;
 387
 388			path->slots[0]++;
 389			goto next_slot;
 390		}
 391
 392		/*
 393		 *  | ---- range to drop ----- |
 394		 *    | ------ extent ------ |
 395		 */
 396		if (args->start <= key.offset && args->end >= extent_end) {
 397delete_extent_item:
 398			if (del_nr == 0) {
 399				del_slot = path->slots[0];
 400				del_nr = 1;
 401			} else {
 402				if (WARN_ON(del_slot + del_nr != path->slots[0])) {
 403					btrfs_print_leaf(leaf);
 404					ret = -EINVAL;
 405					break;
 406				}
 407				del_nr++;
 408			}
 409
 410			if (update_refs &&
 411			    extent_type == BTRFS_FILE_EXTENT_INLINE) {
 412				args->bytes_found += extent_end - key.offset;
 413				extent_end = ALIGN(extent_end,
 414						   fs_info->sectorsize);
 415			} else if (update_refs && disk_bytenr > 0) {
 416				struct btrfs_ref ref = {
 417					.action = BTRFS_DROP_DELAYED_REF,
 418					.bytenr = disk_bytenr,
 419					.num_bytes = num_bytes,
 420					.parent = 0,
 421					.owning_root = btrfs_root_id(root),
 422					.ref_root = btrfs_root_id(root),
 423				};
 424				btrfs_init_data_ref(&ref, key.objectid,
 425						    key.offset - extent_offset,
 426						    0, false);
 427				ret = btrfs_free_extent(trans, &ref);
 428				if (unlikely(ret)) {
 429					btrfs_abort_transaction(trans, ret);
 430					break;
 431				}
 432				args->bytes_found += extent_end - key.offset;
 433			}
 434
 435			if (args->end == extent_end)
 436				break;
 437
 438			if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
 439				path->slots[0]++;
 440				goto next_slot;
 441			}
 442
 443			ret = btrfs_del_items(trans, root, path, del_slot,
 444					      del_nr);
 445			if (unlikely(ret)) {
 446				btrfs_abort_transaction(trans, ret);
 447				break;
 448			}
 449
 450			del_nr = 0;
 451			del_slot = 0;
 452
 453			btrfs_release_path(path);
 454			continue;
 455		}
 456
 457		BUG();
 458	}
 459
 460	if (!ret && del_nr > 0) {
 461		/*
 462		 * Set path->slots[0] to first slot, so that after the delete
 463		 * if items are move off from our leaf to its immediate left or
 464		 * right neighbor leafs, we end up with a correct and adjusted
 465		 * path->slots[0] for our insertion (if args->replace_extent).
 466		 */
 467		path->slots[0] = del_slot;
 468		ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
 469		if (ret)
 470			btrfs_abort_transaction(trans, ret);
 471	}
 472
 473	leaf = path->nodes[0];
 474	/*
 475	 * If btrfs_del_items() was called, it might have deleted a leaf, in
 476	 * which case it unlocked our path, so check path->locks[0] matches a
 477	 * write lock.
 478	 */
 479	if (!ret && args->replace_extent &&
 480	    path->locks[0] == BTRFS_WRITE_LOCK &&
 481	    btrfs_leaf_free_space(leaf) >=
 482	    sizeof(struct btrfs_item) + args->extent_item_size) {
 483
 484		key.objectid = ino;
 485		key.type = BTRFS_EXTENT_DATA_KEY;
 486		key.offset = args->start;
 487		if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
 488			struct btrfs_key slot_key;
 489
 490			btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
 491			if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
 492				path->slots[0]++;
 493		}
 494		btrfs_setup_item_for_insert(trans, root, path, &key,
 495					    args->extent_item_size);
 496		args->extent_inserted = true;
 497	}
 498
 499	if (!args->path)
 500		btrfs_free_path(path);
 501	else if (!args->extent_inserted)
 502		btrfs_release_path(path);
 503out:
 504	args->drop_end = found ? min(args->end, last_end) : args->end;
 505
 506	return ret;
 507}
 508
 509static bool extent_mergeable(struct extent_buffer *leaf, int slot, u64 objectid,
 510			     u64 bytenr, u64 orig_offset, u64 *start, u64 *end)
 511{
 512	struct btrfs_file_extent_item *fi;
 513	struct btrfs_key key;
 514	u64 extent_end;
 515
 516	if (slot < 0 || slot >= btrfs_header_nritems(leaf))
 517		return false;
 518
 519	btrfs_item_key_to_cpu(leaf, &key, slot);
 520	if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
 521		return false;
 522
 523	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
 524	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
 525	    btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
 526	    btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
 527	    btrfs_file_extent_compression(leaf, fi) ||
 528	    btrfs_file_extent_encryption(leaf, fi) ||
 529	    btrfs_file_extent_other_encoding(leaf, fi))
 530		return false;
 531
 532	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
 533	if ((*start && *start != key.offset) || (*end && *end != extent_end))
 534		return false;
 535
 536	*start = key.offset;
 537	*end = extent_end;
 538	return true;
 539}
 540
 541/*
 542 * Mark extent in the range start - end as written.
 543 *
 544 * This changes extent type from 'pre-allocated' to 'regular'. If only
 545 * part of extent is marked as written, the extent will be split into
 546 * two or three.
 547 */
 548int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
 549			      struct btrfs_inode *inode, u64 start, u64 end)
 550{
 551	struct btrfs_root *root = inode->root;
 552	struct extent_buffer *leaf;
 553	BTRFS_PATH_AUTO_FREE(path);
 554	struct btrfs_file_extent_item *fi;
 555	struct btrfs_ref ref = { 0 };
 556	struct btrfs_key key;
 557	struct btrfs_key new_key;
 558	u64 bytenr;
 559	u64 num_bytes;
 560	u64 extent_end;
 561	u64 orig_offset;
 562	u64 other_start;
 563	u64 other_end;
 564	u64 split;
 565	int del_nr = 0;
 566	int del_slot = 0;
 567	int recow;
 568	int ret = 0;
 569	u64 ino = btrfs_ino(inode);
 570
 571	path = btrfs_alloc_path();
 572	if (!path)
 573		return -ENOMEM;
 574again:
 575	recow = 0;
 576	split = start;
 577	key.objectid = ino;
 578	key.type = BTRFS_EXTENT_DATA_KEY;
 579	key.offset = split;
 580
 581	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
 582	if (ret < 0)
 583		goto out;
 584	if (ret > 0 && path->slots[0] > 0)
 585		path->slots[0]--;
 586
 587	leaf = path->nodes[0];
 588	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 589	if (unlikely(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)) {
 590		ret = -EINVAL;
 591		btrfs_abort_transaction(trans, ret);
 592		goto out;
 593	}
 594	fi = btrfs_item_ptr(leaf, path->slots[0],
 595			    struct btrfs_file_extent_item);
 596	if (unlikely(btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC)) {
 597		ret = -EINVAL;
 598		btrfs_abort_transaction(trans, ret);
 599		goto out;
 600	}
 601	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
 602	if (unlikely(key.offset > start || extent_end < end)) {
 603		ret = -EINVAL;
 604		btrfs_abort_transaction(trans, ret);
 605		goto out;
 606	}
 607
 608	bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
 609	num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
 610	orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
 611	memcpy(&new_key, &key, sizeof(new_key));
 612
 613	if (start == key.offset && end < extent_end) {
 614		other_start = 0;
 615		other_end = start;
 616		if (extent_mergeable(leaf, path->slots[0] - 1,
 617				     ino, bytenr, orig_offset,
 618				     &other_start, &other_end)) {
 619			new_key.offset = end;
 620			btrfs_set_item_key_safe(trans, path, &new_key);
 621			fi = btrfs_item_ptr(leaf, path->slots[0],
 622					    struct btrfs_file_extent_item);
 623			btrfs_set_file_extent_generation(leaf, fi,
 624							 trans->transid);
 625			btrfs_set_file_extent_num_bytes(leaf, fi,
 626							extent_end - end);
 627			btrfs_set_file_extent_offset(leaf, fi,
 628						     end - orig_offset);
 629			fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
 630					    struct btrfs_file_extent_item);
 631			btrfs_set_file_extent_generation(leaf, fi,
 632							 trans->transid);
 633			btrfs_set_file_extent_num_bytes(leaf, fi,
 634							end - other_start);
 635			goto out;
 636		}
 637	}
 638
 639	if (start > key.offset && end == extent_end) {
 640		other_start = end;
 641		other_end = 0;
 642		if (extent_mergeable(leaf, path->slots[0] + 1,
 643				     ino, bytenr, orig_offset,
 644				     &other_start, &other_end)) {
 645			fi = btrfs_item_ptr(leaf, path->slots[0],
 646					    struct btrfs_file_extent_item);
 647			btrfs_set_file_extent_num_bytes(leaf, fi,
 648							start - key.offset);
 649			btrfs_set_file_extent_generation(leaf, fi,
 650							 trans->transid);
 651			path->slots[0]++;
 652			new_key.offset = start;
 653			btrfs_set_item_key_safe(trans, path, &new_key);
 654
 655			fi = btrfs_item_ptr(leaf, path->slots[0],
 656					    struct btrfs_file_extent_item);
 657			btrfs_set_file_extent_generation(leaf, fi,
 658							 trans->transid);
 659			btrfs_set_file_extent_num_bytes(leaf, fi,
 660							other_end - start);
 661			btrfs_set_file_extent_offset(leaf, fi,
 662						     start - orig_offset);
 663			goto out;
 664		}
 665	}
 666
 667	while (start > key.offset || end < extent_end) {
 668		if (key.offset == start)
 669			split = end;
 670
 671		new_key.offset = split;
 672		ret = btrfs_duplicate_item(trans, root, path, &new_key);
 673		if (ret == -EAGAIN) {
 674			btrfs_release_path(path);
 675			goto again;
 676		}
 677		if (unlikely(ret < 0)) {
 678			btrfs_abort_transaction(trans, ret);
 679			goto out;
 680		}
 681
 682		leaf = path->nodes[0];
 683		fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
 684				    struct btrfs_file_extent_item);
 685		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
 686		btrfs_set_file_extent_num_bytes(leaf, fi,
 687						split - key.offset);
 688
 689		fi = btrfs_item_ptr(leaf, path->slots[0],
 690				    struct btrfs_file_extent_item);
 691
 692		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
 693		btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
 694		btrfs_set_file_extent_num_bytes(leaf, fi,
 695						extent_end - split);
 696
 697		ref.action = BTRFS_ADD_DELAYED_REF;
 698		ref.bytenr = bytenr;
 699		ref.num_bytes = num_bytes;
 700		ref.parent = 0;
 701		ref.owning_root = btrfs_root_id(root);
 702		ref.ref_root = btrfs_root_id(root);
 703		btrfs_init_data_ref(&ref, ino, orig_offset, 0, false);
 704		ret = btrfs_inc_extent_ref(trans, &ref);
 705		if (unlikely(ret)) {
 706			btrfs_abort_transaction(trans, ret);
 707			goto out;
 708		}
 709
 710		if (split == start) {
 711			key.offset = start;
 712		} else {
 713			if (unlikely(start != key.offset)) {
 714				ret = -EINVAL;
 715				btrfs_abort_transaction(trans, ret);
 716				goto out;
 717			}
 718			path->slots[0]--;
 719			extent_end = end;
 720		}
 721		recow = 1;
 722	}
 723
 724	other_start = end;
 725	other_end = 0;
 726
 727	ref.action = BTRFS_DROP_DELAYED_REF;
 728	ref.bytenr = bytenr;
 729	ref.num_bytes = num_bytes;
 730	ref.parent = 0;
 731	ref.owning_root = btrfs_root_id(root);
 732	ref.ref_root = btrfs_root_id(root);
 733	btrfs_init_data_ref(&ref, ino, orig_offset, 0, false);
 734	if (extent_mergeable(leaf, path->slots[0] + 1,
 735			     ino, bytenr, orig_offset,
 736			     &other_start, &other_end)) {
 737		if (recow) {
 738			btrfs_release_path(path);
 739			goto again;
 740		}
 741		extent_end = other_end;
 742		del_slot = path->slots[0] + 1;
 743		del_nr++;
 744		ret = btrfs_free_extent(trans, &ref);
 745		if (unlikely(ret)) {
 746			btrfs_abort_transaction(trans, ret);
 747			goto out;
 748		}
 749	}
 750	other_start = 0;
 751	other_end = start;
 752	if (extent_mergeable(leaf, path->slots[0] - 1,
 753			     ino, bytenr, orig_offset,
 754			     &other_start, &other_end)) {
 755		if (recow) {
 756			btrfs_release_path(path);
 757			goto again;
 758		}
 759		key.offset = other_start;
 760		del_slot = path->slots[0];
 761		del_nr++;
 762		ret = btrfs_free_extent(trans, &ref);
 763		if (unlikely(ret)) {
 764			btrfs_abort_transaction(trans, ret);
 765			goto out;
 766		}
 767	}
 768	if (del_nr == 0) {
 769		fi = btrfs_item_ptr(leaf, path->slots[0],
 770			   struct btrfs_file_extent_item);
 771		btrfs_set_file_extent_type(leaf, fi,
 772					   BTRFS_FILE_EXTENT_REG);
 773		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
 774	} else {
 775		fi = btrfs_item_ptr(leaf, del_slot - 1,
 776			   struct btrfs_file_extent_item);
 777		btrfs_set_file_extent_type(leaf, fi,
 778					   BTRFS_FILE_EXTENT_REG);
 779		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
 780		btrfs_set_file_extent_num_bytes(leaf, fi,
 781						extent_end - key.offset);
 782
 783		ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
 784		if (unlikely(ret < 0)) {
 785			btrfs_abort_transaction(trans, ret);
 786			goto out;
 787		}
 788	}
 789out:
 790	return ret;
 791}
 792
 793/*
 794 * On error return an unlocked folio and the error value
 795 * On success return a locked folio and 0
 796 */
 797static int prepare_uptodate_folio(struct inode *inode, struct folio *folio, u64 pos,
 798				  u64 len)
 799{
 800	u64 clamp_start = max_t(u64, pos, folio_pos(folio));
 801	u64 clamp_end = min_t(u64, pos + len, folio_next_pos(folio));
 802	const u32 blocksize = inode_to_fs_info(inode)->sectorsize;
 803	int ret = 0;
 804
 805	if (folio_test_uptodate(folio))
 806		return 0;
 807
 808	if (IS_ALIGNED(clamp_start, blocksize) &&
 809	    IS_ALIGNED(clamp_end, blocksize))
 810		return 0;
 811
 812	ret = btrfs_read_folio(NULL, folio);
 813	if (ret)
 814		return ret;
 815	folio_lock(folio);
 816	if (unlikely(!folio_test_uptodate(folio))) {
 817		folio_unlock(folio);
 818		return -EIO;
 819	}
 820
 821	/*
 822	 * Since btrfs_read_folio() will unlock the folio before it returns,
 823	 * there is a window where btrfs_release_folio() can be called to
 824	 * release the page.  Here we check both inode mapping and page
 825	 * private to make sure the page was not released.
 826	 *
 827	 * The private flag check is essential for subpage as we need to store
 828	 * extra bitmap using folio private.
 829	 */
 830	if (folio->mapping != inode->i_mapping || !folio_test_private(folio)) {
 831		folio_unlock(folio);
 832		return -EAGAIN;
 833	}
 834	return 0;
 835}
 836
 837static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait)
 838{
 839	gfp_t gfp;
 840
 841	gfp = btrfs_alloc_write_mask(inode->i_mapping);
 842	if (nowait) {
 843		gfp &= ~__GFP_DIRECT_RECLAIM;
 844		gfp |= GFP_NOWAIT;
 845	}
 846
 847	return gfp;
 848}
 849
 850/*
 851 * Get folio into the page cache and lock it.
 852 */
 853static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_ret,
 854				      loff_t pos, size_t write_bytes,
 855				      bool nowait)
 856{
 857	const pgoff_t index = pos >> PAGE_SHIFT;
 858	gfp_t mask = get_prepare_gfp_flags(inode, nowait);
 859	fgf_t fgp_flags = (nowait ? FGP_WRITEBEGIN | FGP_NOWAIT : FGP_WRITEBEGIN) |
 860			  fgf_set_order(write_bytes);
 861	struct folio *folio;
 862	int ret = 0;
 863
 864again:
 865	folio = __filemap_get_folio(inode->i_mapping, index, fgp_flags, mask);
 866	if (IS_ERR(folio))
 867		return PTR_ERR(folio);
 868
 869	ret = set_folio_extent_mapped(folio);
 870	if (ret < 0) {
 871		folio_unlock(folio);
 872		folio_put(folio);
 873		return ret;
 874	}
 875	ret = prepare_uptodate_folio(inode, folio, pos, write_bytes);
 876	if (ret) {
 877		/* The folio is already unlocked. */
 878		folio_put(folio);
 879		if (!nowait && ret == -EAGAIN) {
 880			ret = 0;
 881			goto again;
 882		}
 883		return ret;
 884	}
 885	*folio_ret = folio;
 886	return 0;
 887}
 888
 889/*
 890 * Locks the extent and properly waits for data=ordered extents to finish
 891 * before allowing the folios to be modified if need.
 892 *
 893 * Return:
 894 * 1 - the extent is locked
 895 * 0 - the extent is not locked, and everything is OK
 896 * -EAGAIN - need to prepare the folios again
 897 */
 898static noinline int
 899lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio,
 900				loff_t pos, size_t write_bytes,
 901				u64 *lockstart, u64 *lockend, bool nowait,
 902				struct extent_state **cached_state)
 903{
 904	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 905	u64 start_pos;
 906	u64 last_pos;
 907	int ret = 0;
 908
 909	start_pos = round_down(pos, fs_info->sectorsize);
 910	last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;
 911
 912	if (start_pos < inode->vfs_inode.i_size) {
 913		struct btrfs_ordered_extent *ordered;
 914
 915		if (nowait) {
 916			if (!btrfs_try_lock_extent(&inode->io_tree, start_pos,
 917						   last_pos, cached_state)) {
 918				folio_unlock(folio);
 919				folio_put(folio);
 920				return -EAGAIN;
 921			}
 922		} else {
 923			btrfs_lock_extent(&inode->io_tree, start_pos, last_pos,
 924					  cached_state);
 925		}
 926
 927		ordered = btrfs_lookup_ordered_range(inode, start_pos,
 928						     last_pos - start_pos + 1);
 929		if (ordered &&
 930		    ordered->file_offset + ordered->num_bytes > start_pos &&
 931		    ordered->file_offset <= last_pos) {
 932			btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos,
 933					    cached_state);
 934			folio_unlock(folio);
 935			folio_put(folio);
 936			btrfs_start_ordered_extent(ordered);
 937			btrfs_put_ordered_extent(ordered);
 938			return -EAGAIN;
 939		}
 940		if (ordered)
 941			btrfs_put_ordered_extent(ordered);
 942
 943		*lockstart = start_pos;
 944		*lockend = last_pos;
 945		ret = 1;
 946	}
 947
 948	/*
 949	 * We should be called after prepare_one_folio() which should have locked
 950	 * all pages in the range.
 951	 */
 952	WARN_ON(!folio_test_locked(folio));
 953
 954	return ret;
 955}
 956
 957/*
 958 * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
 959 *
 960 * @pos:         File offset.
 961 * @write_bytes: The length to write, will be updated to the nocow writeable
 962 *               range.
 963 * @nowait:      Indicate if we can block or not (non-blocking IO context).
 964 *
 965 * This function will flush ordered extents in the range to ensure proper
 966 * nocow checks.
 967 *
 968 * Return:
 969 * > 0          If we can nocow, and updates @write_bytes.
 970 *  0           If we can't do a nocow write.
 971 * -EAGAIN      If we can't do a nocow write because snapshotting of the inode's
 972 *              root is in progress or because we are in a non-blocking IO
 973 *              context and need to block (@nowait is true).
 974 * < 0          If an error happened.
 975 *
 976 * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0.
 977 */
 978int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,
 979			   size_t *write_bytes, bool nowait)
 980{
 981	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 982	struct btrfs_root *root = inode->root;
 983	struct extent_state *cached_state = NULL;
 984	u64 lockstart, lockend;
 985	u64 cur_offset;
 986	int ret = 0;
 987
 988	if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
 989		return 0;
 990
 991	if (!btrfs_drew_try_write_lock(&root->snapshot_lock))
 992		return -EAGAIN;
 993
 994	lockstart = round_down(pos, fs_info->sectorsize);
 995	lockend = round_up(pos + *write_bytes,
 996			   fs_info->sectorsize) - 1;
 997
 998	if (nowait) {
 999		if (!btrfs_try_lock_ordered_range(inode, lockstart, lockend,
1000						  &cached_state)) {
1001			btrfs_drew_write_unlock(&root->snapshot_lock);
1002			return -EAGAIN;
1003		}
1004	} else {
1005		btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend,
1006						   &cached_state);
1007	}
1008
1009	cur_offset = lockstart;
1010	while (cur_offset < lockend) {
1011		u64 num_bytes = lockend - cur_offset + 1;
1012
1013		ret = can_nocow_extent(inode, cur_offset, &num_bytes, NULL, nowait);
1014		if (ret <= 0) {
1015			/*
1016			 * If cur_offset == lockstart it means we haven't found
1017			 * any extent against which we can NOCOW, so unlock the
1018			 * snapshot lock.
1019			 */
1020			if (cur_offset == lockstart)
1021				btrfs_drew_write_unlock(&root->snapshot_lock);
1022			break;
1023		}
1024		cur_offset += num_bytes;
1025	}
1026
1027	btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
1028
1029	/*
1030	 * cur_offset > lockstart means there's at least a partial range we can
1031	 * NOCOW, and that range can cover one or more extents.
1032	 */
1033	if (cur_offset > lockstart) {
1034		*write_bytes = min_t(size_t, *write_bytes, cur_offset - pos);
1035		return 1;
1036	}
1037
1038	return ret;
1039}
1040
1041void btrfs_check_nocow_unlock(struct btrfs_inode *inode)
1042{
1043	btrfs_drew_write_unlock(&inode->root->snapshot_lock);
1044}
1045
1046int btrfs_write_check(struct kiocb *iocb, size_t count)
1047{
1048	struct file *file = iocb->ki_filp;
1049	struct inode *inode = file_inode(file);
1050	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
1051	loff_t pos = iocb->ki_pos;
1052	int ret;
1053	loff_t oldsize;
1054
1055	/*
1056	 * Quickly bail out on NOWAIT writes if we don't have the nodatacow or
1057	 * prealloc flags, as without those flags we always have to COW. We will
1058	 * later check if we can really COW into the target range (using
1059	 * can_nocow_extent() at btrfs_get_blocks_direct_write()).
1060	 */
1061	if ((iocb->ki_flags & IOCB_NOWAIT) &&
1062	    !(BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
1063		return -EAGAIN;
1064
1065	ret = file_remove_privs(file);
1066	if (ret)
1067		return ret;
1068
1069	/*
1070	 * We reserve space for updating the inode when we reserve space for the
1071	 * extent we are going to write, so we will enospc out there.  We don't
1072	 * need to start yet another transaction to update the inode as we will
1073	 * update the inode when we finish writing whatever data we write.
1074	 */
1075	if (!IS_NOCMTIME(inode)) {
1076		inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
1077		inode_inc_iversion(inode);
1078	}
1079
1080	oldsize = i_size_read(inode);
1081	if (pos > oldsize) {
1082		/* Expand hole size to cover write data, preventing empty gap */
1083		loff_t end_pos = round_up(pos + count, fs_info->sectorsize);
1084
1085		ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, end_pos);
1086		if (ret)
1087			return ret;
1088	}
1089
1090	return 0;
1091}
1092
1093static void release_space(struct btrfs_inode *inode, struct extent_changeset *data_reserved,
1094			  u64 start, u64 len, bool only_release_metadata)
1095{
1096	if (len == 0)
1097		return;
1098
1099	if (only_release_metadata) {
1100		btrfs_check_nocow_unlock(inode);
1101		btrfs_delalloc_release_metadata(inode, len, true);
1102	} else {
1103		const struct btrfs_fs_info *fs_info = inode->root->fs_info;
1104
1105		btrfs_delalloc_release_space(inode, data_reserved,
1106					     round_down(start, fs_info->sectorsize),
1107					     len, true);
1108	}
1109}
1110
1111/*
1112 * Reserve data and metadata space for this buffered write range.
1113 *
1114 * Return >0 for the number of bytes reserved, which is always block aligned.
1115 * Return <0 for error.
1116 */
1117static ssize_t reserve_space(struct btrfs_inode *inode,
1118			     struct extent_changeset **data_reserved,
1119			     u64 start, size_t *len, bool nowait,
1120			     bool *only_release_metadata)
1121{
1122	const struct btrfs_fs_info *fs_info = inode->root->fs_info;
1123	const unsigned int block_offset = (start & (fs_info->sectorsize - 1));
1124	size_t reserve_bytes;
1125	int ret;
1126
1127	ret = btrfs_check_data_free_space(inode, data_reserved, start, *len, nowait);
1128	if (ret < 0) {
1129		int can_nocow;
1130
1131		if (nowait && (ret == -ENOSPC || ret == -EAGAIN))
1132			return -EAGAIN;
1133
1134		/*
1135		 * If we don't have to COW at the offset, reserve metadata only.
1136		 * write_bytes may get smaller than requested here.
1137		 */
1138		can_nocow = btrfs_check_nocow_lock(inode, start, len, nowait);
1139		if (can_nocow < 0)
1140			ret = can_nocow;
1141		if (can_nocow > 0)
1142			ret = 0;
1143		if (ret)
1144			return ret;
1145		*only_release_metadata = true;
1146	}
1147
1148	reserve_bytes = round_up(*len + block_offset, fs_info->sectorsize);
1149	WARN_ON(reserve_bytes == 0);
1150	ret = btrfs_delalloc_reserve_metadata(inode, reserve_bytes,
1151					      reserve_bytes, nowait);
1152	if (ret) {
1153		if (!*only_release_metadata)
1154			btrfs_free_reserved_data_space(inode, *data_reserved,
1155						       start, *len);
1156		else
1157			btrfs_check_nocow_unlock(inode);
1158
1159		if (nowait && ret == -ENOSPC)
1160			ret = -EAGAIN;
1161		return ret;
1162	}
1163	return reserve_bytes;
1164}
1165
1166/* Shrink the reserved data and metadata space from @reserved_len to @new_len. */
1167static void shrink_reserved_space(struct btrfs_inode *inode,
1168				  struct extent_changeset *data_reserved,
1169				  u64 reserved_start, u64 reserved_len,
1170				  u64 new_len, bool only_release_metadata)
1171{
1172	const u64 diff = reserved_len - new_len;
1173
1174	ASSERT(new_len <= reserved_len);
1175	btrfs_delalloc_shrink_extents(inode, reserved_len, new_len);
1176	if (only_release_metadata)
1177		btrfs_delalloc_release_metadata(inode, diff, true);
1178	else
1179		btrfs_delalloc_release_space(inode, data_reserved,
1180					     reserved_start + new_len, diff, true);
1181}
1182
1183/* Calculate the maximum amount of bytes we can write into one folio. */
1184static size_t calc_write_bytes(const struct btrfs_inode *inode,
1185			       const struct iov_iter *iter, u64 start)
1186{
1187	const size_t max_folio_size = mapping_max_folio_size(inode->vfs_inode.i_mapping);
1188
1189	return min(max_folio_size - (start & (max_folio_size - 1)),
1190		   iov_iter_count(iter));
1191}
1192
1193/*
1194 * Do the heavy-lifting work to copy one range into one folio of the page cache.
1195 *
1196 * Return > 0 in case we copied all bytes or just some of them.
1197 * Return 0 if no bytes were copied, in which case the caller should retry.
1198 * Return <0 on error.
1199 */
1200static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter,
1201			  struct extent_changeset **data_reserved, u64 start,
1202			  bool nowait)
1203{
1204	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1205	struct extent_state *cached_state = NULL;
1206	size_t write_bytes = calc_write_bytes(inode, iter, start);
1207	size_t copied;
1208	const u64 reserved_start = round_down(start, fs_info->sectorsize);
1209	u64 reserved_len;
1210	struct folio *folio = NULL;
1211	int extents_locked;
1212	u64 lockstart;
1213	u64 lockend;
1214	bool only_release_metadata = false;
1215	const unsigned int bdp_flags = (nowait ? BDP_ASYNC : 0);
1216	int ret;
1217
1218	/*
1219	 * Fault all pages before locking them in prepare_one_folio() to avoid
1220	 * recursive lock.
1221	 */
1222	if (unlikely(fault_in_iov_iter_readable(iter, write_bytes)))
1223		return -EFAULT;
1224	extent_changeset_release(*data_reserved);
1225	ret = reserve_space(inode, data_reserved, start, &write_bytes, nowait,
1226			    &only_release_metadata);
1227	if (ret < 0)
1228		return ret;
1229	reserved_len = ret;
1230	/* Write range must be inside the reserved range. */
1231	ASSERT(reserved_start <= start);
1232	ASSERT(start + write_bytes <= reserved_start + reserved_len);
1233
1234again:
1235	ret = balance_dirty_pages_ratelimited_flags(inode->vfs_inode.i_mapping,
1236						    bdp_flags);
1237	if (ret) {
1238		btrfs_delalloc_release_extents(inode, reserved_len);
1239		release_space(inode, *data_reserved, reserved_start, reserved_len,
1240			      only_release_metadata);
1241		return ret;
1242	}
1243
1244	ret = prepare_one_folio(&inode->vfs_inode, &folio, start, write_bytes, false);
1245	if (ret) {
1246		btrfs_delalloc_release_extents(inode, reserved_len);
1247		release_space(inode, *data_reserved, reserved_start, reserved_len,
1248			      only_release_metadata);
1249		return ret;
1250	}
1251
1252	/*
1253	 * The reserved range goes beyond the current folio, shrink the reserved
1254	 * space to the folio boundary.
1255	 */
1256	if (reserved_start + reserved_len > folio_next_pos(folio)) {
1257		const u64 last_block = folio_next_pos(folio);
1258
1259		shrink_reserved_space(inode, *data_reserved, reserved_start,
1260				      reserved_len, last_block - reserved_start,
1261				      only_release_metadata);
1262		write_bytes = last_block - start;
1263		reserved_len = last_block - reserved_start;
1264	}
1265
1266	extents_locked = lock_and_cleanup_extent_if_need(inode, folio, start,
1267							 write_bytes, &lockstart,
1268							 &lockend, nowait,
1269							 &cached_state);
1270	if (extents_locked < 0) {
1271		if (!nowait && extents_locked == -EAGAIN)
1272			goto again;
1273
1274		btrfs_delalloc_release_extents(inode, reserved_len);
1275		release_space(inode, *data_reserved, reserved_start, reserved_len,
1276			      only_release_metadata);
1277		ret = extents_locked;
1278		return ret;
1279	}
1280
1281	copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start),
1282					     write_bytes, iter);
1283	flush_dcache_folio(folio);
1284
1285	if (unlikely(copied < write_bytes)) {
1286		u64 last_block;
1287
1288		/*
1289		 * The original write range doesn't need an uptodate folio as
1290		 * the range is block aligned. But now a short copy happened.
1291		 * We cannot handle it without an uptodate folio.
1292		 *
1293		 * So just revert the range and we will retry.
1294		 */
1295		if (!folio_test_uptodate(folio)) {
1296			iov_iter_revert(iter, copied);
1297			copied = 0;
1298		}
1299
1300		/* No copied bytes, unlock, release reserved space and exit. */
1301		if (copied == 0) {
1302			if (extents_locked)
1303				btrfs_unlock_extent(&inode->io_tree, lockstart, lockend,
1304						    &cached_state);
1305			else
1306				btrfs_free_extent_state(cached_state);
1307			btrfs_delalloc_release_extents(inode, reserved_len);
1308			release_space(inode, *data_reserved, reserved_start, reserved_len,
1309				      only_release_metadata);
1310			btrfs_drop_folio(fs_info, folio, start, copied);
1311			return 0;
1312		}
1313
1314		/* Release the reserved space beyond the last block. */
1315		last_block = round_up(start + copied, fs_info->sectorsize);
1316
1317		shrink_reserved_space(inode, *data_reserved, reserved_start,
1318				      reserved_len, last_block - reserved_start,
1319				      only_release_metadata);
1320		reserved_len = last_block - reserved_start;
1321	}
1322
1323	ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state,
1324				only_release_metadata);
1325	/*
1326	 * If we have not locked the extent range, because the range's start
1327	 * offset is >= i_size, we might still have a non-NULL cached extent
1328	 * state, acquired while marking the extent range as delalloc through
1329	 * btrfs_dirty_page(). Therefore free any possible cached extent state
1330	 * to avoid a memory leak.
1331	 */
1332	if (extents_locked)
1333		btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
1334	else
1335		btrfs_free_extent_state(cached_state);
1336
1337	btrfs_delalloc_release_extents(inode, reserved_len);
1338	if (ret) {
1339		btrfs_drop_folio(fs_info, folio, start, copied);
1340		release_space(inode, *data_reserved, reserved_start, reserved_len,
1341			      only_release_metadata);
1342		return ret;
1343	}
1344	if (only_release_metadata)
1345		btrfs_check_nocow_unlock(inode);
1346
1347	btrfs_drop_folio(fs_info, folio, start, copied);
1348	return copied;
1349}
1350
1351ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
1352{
1353	struct file *file = iocb->ki_filp;
1354	loff_t pos;
1355	struct inode *inode = file_inode(file);
1356	struct extent_changeset *data_reserved = NULL;
1357	size_t num_written = 0;
1358	ssize_t ret;
1359	loff_t old_isize;
1360	unsigned int ilock_flags = 0;
1361	const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
1362
1363	if (nowait)
1364		ilock_flags |= BTRFS_ILOCK_TRY;
1365
1366	ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags);
1367	if (ret < 0)
1368		return ret;
1369
1370	/*
1371	 * We can only trust the isize with inode lock held, or it can race with
1372	 * other buffered writes and cause incorrect call of
1373	 * pagecache_isize_extended() to overwrite existing data.
1374	 */
1375	old_isize = i_size_read(inode);
1376
1377	ret = generic_write_checks(iocb, iter);
1378	if (ret <= 0)
1379		goto out;
1380
1381	ret = btrfs_write_check(iocb, ret);
1382	if (ret < 0)
1383		goto out;
1384
1385	pos = iocb->ki_pos;
1386	while (iov_iter_count(iter) > 0) {
1387		ret = copy_one_range(BTRFS_I(inode), iter, &data_reserved, pos, nowait);
1388		if (ret < 0)
1389			break;
1390		pos += ret;
1391		num_written += ret;
1392		cond_resched();
1393	}
1394
1395	extent_changeset_free(data_reserved);
1396	if (num_written > 0) {
1397		pagecache_isize_extended(inode, old_isize, iocb->ki_pos);
1398		iocb->ki_pos += num_written;
1399	}
1400out:
1401	btrfs_inode_unlock(BTRFS_I(inode), ilock_flags);
1402	return num_written ? num_written : ret;
1403}
1404
1405static ssize_t btrfs_encoded_write(struct kiocb *iocb, struct iov_iter *from,
1406			const struct btrfs_ioctl_encoded_io_args *encoded)
1407{
1408	struct file *file = iocb->ki_filp;
1409	struct inode *inode = file_inode(file);
1410	loff_t count;
1411	ssize_t ret;
1412
1413	btrfs_inode_lock(BTRFS_I(inode), 0);
1414	count = encoded->len;
1415	ret = generic_write_checks_count(iocb, &count);
1416	if (ret == 0 && count != encoded->len) {
1417		/*
1418		 * The write got truncated by generic_write_checks_count(). We
1419		 * can't do a partial encoded write.
1420		 */
1421		ret = -EFBIG;
1422	}
1423	if (ret || encoded->len == 0)
1424		goto out;
1425
1426	ret = btrfs_write_check(iocb, encoded->len);
1427	if (ret < 0)
1428		goto out;
1429
1430	ret = btrfs_do_encoded_write(iocb, from, encoded);
1431out:
1432	btrfs_inode_unlock(BTRFS_I(inode), 0);
1433	return ret;
1434}
1435
1436ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from,
1437			    const struct btrfs_ioctl_encoded_io_args *encoded)
1438{
1439	struct file *file = iocb->ki_filp;
1440	struct btrfs_inode *inode = BTRFS_I(file_inode(file));
1441	ssize_t num_written, num_sync;
1442
1443	if (unlikely(btrfs_is_shutdown(inode->root->fs_info)))
1444		return -EIO;
1445	/*
1446	 * If the fs flips readonly due to some impossible error, although we
1447	 * have opened a file as writable, we have to stop this write operation
1448	 * to ensure consistency.
1449	 */
1450	if (BTRFS_FS_ERROR(inode->root->fs_info))
1451		return -EROFS;
1452
1453	if (encoded && (iocb->ki_flags & IOCB_NOWAIT))
1454		return -EOPNOTSUPP;
1455
1456	if (encoded) {
1457		num_written = btrfs_encoded_write(iocb, from, encoded);
1458		num_sync = encoded->len;
1459	} else if (iocb->ki_flags & IOCB_DIRECT) {
1460		num_written = btrfs_direct_write(iocb, from);
1461		num_sync = num_written;
1462	} else {
1463		num_written = btrfs_buffered_write(iocb, from);
1464		num_sync = num_written;
1465	}
1466
1467	btrfs_set_inode_last_sub_trans(inode);
1468
1469	if (num_sync > 0) {
1470		num_sync = generic_write_sync(iocb, num_sync);
1471		if (num_sync < 0)
1472			num_written = num_sync;
1473	}
1474
1475	return num_written;
1476}
1477
1478static ssize_t btrfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1479{
1480	return btrfs_do_write_iter(iocb, from, NULL);
1481}
1482
1483int btrfs_release_file(struct inode *inode, struct file *filp)
1484{
1485	struct btrfs_file_private *private = filp->private_data;
1486
1487	if (private) {
1488		kfree(private->filldir_buf);
1489		btrfs_free_extent_state(private->llseek_cached_state);
1490		kfree(private);
1491		filp->private_data = NULL;
1492	}
1493
1494	/*
1495	 * Set by setattr when we are about to truncate a file from a non-zero
1496	 * size to a zero size.  This tries to flush down new bytes that may
1497	 * have been written if the application were using truncate to replace
1498	 * a file in place.
1499	 */
1500	if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
1501			       &BTRFS_I(inode)->runtime_flags))
1502			filemap_flush(inode->i_mapping);
1503	return 0;
1504}
1505
1506static int start_ordered_ops(struct btrfs_inode *inode, loff_t start, loff_t end)
1507{
1508	int ret;
1509	struct blk_plug plug;
1510
1511	/*
1512	 * This is only called in fsync, which would do synchronous writes, so
1513	 * a plug can merge adjacent IOs as much as possible.  Esp. in case of
1514	 * multiple disks using raid profile, a large IO can be split to
1515	 * several segments of stripe length (currently 64K).
1516	 */
1517	blk_start_plug(&plug);
1518	ret = btrfs_fdatawrite_range(inode, start, end);
1519	blk_finish_plug(&plug);
1520
1521	return ret;
1522}
1523
1524static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx)
1525{
1526	struct btrfs_inode *inode = ctx->inode;
1527	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1528
1529	if (btrfs_inode_in_log(inode, btrfs_get_fs_generation(fs_info)) &&
1530	    list_empty(&ctx->ordered_extents))
1531		return true;
1532
1533	/*
1534	 * If we are doing a fast fsync we can not bail out if the inode's
1535	 * last_trans is <= then the last committed transaction, because we only
1536	 * update the last_trans of the inode during ordered extent completion,
1537	 * and for a fast fsync we don't wait for that, we only wait for the
1538	 * writeback to complete.
1539	 */
1540	if (inode->last_trans <= btrfs_get_last_trans_committed(fs_info) &&
1541	    (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) ||
1542	     list_empty(&ctx->ordered_extents)))
1543		return true;
1544
1545	return false;
1546}
1547
1548/*
1549 * fsync call for both files and directories.  This logs the inode into
1550 * the tree log instead of forcing full commits whenever possible.
1551 *
1552 * It needs to call filemap_fdatawait so that all ordered extent updates are
1553 * in the metadata btree are up to date for copying to the log.
1554 *
1555 * It drops the inode mutex before doing the tree log commit.  This is an
1556 * important optimization for directories because holding the mutex prevents
1557 * new operations on the dir while we write to disk.
1558 */
1559int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
1560{
1561	struct dentry *dentry = file_dentry(file);
1562	struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
1563	struct btrfs_root *root = inode->root;
1564	struct btrfs_fs_info *fs_info = root->fs_info;
1565	struct btrfs_trans_handle *trans;
1566	struct btrfs_log_ctx ctx;
1567	int ret = 0, err;
1568	u64 len;
1569	bool full_sync;
1570	bool skip_ilock = false;
1571
1572	if (current->journal_info == BTRFS_TRANS_DIO_WRITE_STUB) {
1573		skip_ilock = true;
1574		current->journal_info = NULL;
1575		btrfs_assert_inode_locked(inode);
1576	}
1577
1578	trace_btrfs_sync_file(file, datasync);
1579
1580	btrfs_init_log_ctx(&ctx, inode);
1581
1582	/*
1583	 * Always set the range to a full range, otherwise we can get into
1584	 * several problems, from missing file extent items to represent holes
1585	 * when not using the NO_HOLES feature, to log tree corruption due to
1586	 * races between hole detection during logging and completion of ordered
1587	 * extents outside the range, to missing checksums due to ordered extents
1588	 * for which we flushed only a subset of their pages.
1589	 */
1590	start = 0;
1591	end = LLONG_MAX;
1592	len = (u64)LLONG_MAX + 1;
1593
1594	/*
1595	 * We write the dirty pages in the range and wait until they complete
1596	 * out of the ->i_mutex. If so, we can flush the dirty pages by
1597	 * multi-task, and make the performance up.  See
1598	 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
1599	 */
1600	ret = start_ordered_ops(inode, start, end);
1601	if (ret)
1602		goto out;
1603
1604	if (skip_ilock)
1605		down_write(&inode->i_mmap_lock);
1606	else
1607		btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
1608
1609	atomic_inc(&root->log_batch);
1610
1611	/*
1612	 * Before we acquired the inode's lock and the mmap lock, someone may
1613	 * have dirtied more pages in the target range. We need to make sure
1614	 * that writeback for any such pages does not start while we are logging
1615	 * the inode, because if it does, any of the following might happen when
1616	 * we are not doing a full inode sync:
1617	 *
1618	 * 1) We log an extent after its writeback finishes but before its
1619	 *    checksums are added to the csum tree, leading to -EIO errors
1620	 *    when attempting to read the extent after a log replay.
1621	 *
1622	 * 2) We can end up logging an extent before its writeback finishes.
1623	 *    Therefore after the log replay we will have a file extent item
1624	 *    pointing to an unwritten extent (and no data checksums as well).
1625	 *
1626	 * So trigger writeback for any eventual new dirty pages and then we
1627	 * wait for all ordered extents to complete below.
1628	 */
1629	ret = start_ordered_ops(inode, start, end);
1630	if (ret) {
1631		if (skip_ilock)
1632			up_write(&inode->i_mmap_lock);
1633		else
1634			btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
1635		goto out;
1636	}
1637
1638	/*
1639	 * Always check for the full sync flag while holding the inode's lock,
1640	 * to avoid races with other tasks. The flag must be either set all the
1641	 * time during logging or always off all the time while logging.
1642	 * We check the flag here after starting delalloc above, because when
1643	 * running delalloc the full sync flag may be set if we need to drop
1644	 * extra extent map ranges due to temporary memory allocation failures.
1645	 */
1646	full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
1647
1648	/*
1649	 * We have to do this here to avoid the priority inversion of waiting on
1650	 * IO of a lower priority task while holding a transaction open.
1651	 *
1652	 * For a full fsync we wait for the ordered extents to complete while
1653	 * for a fast fsync we wait just for writeback to complete, and then
1654	 * attach the ordered extents to the transaction so that a transaction
1655	 * commit waits for their completion, to avoid data loss if we fsync,
1656	 * the current transaction commits before the ordered extents complete
1657	 * and a power failure happens right after that.
1658	 *
1659	 * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the
1660	 * logical address recorded in the ordered extent may change. We need
1661	 * to wait for the IO to stabilize the logical address.
1662	 */
1663	if (full_sync || btrfs_is_zoned(fs_info)) {
1664		ret = btrfs_wait_ordered_range(inode, start, len);
1665		clear_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags);
1666	} else {
1667		/*
1668		 * Get our ordered extents as soon as possible to avoid doing
1669		 * checksum lookups in the csum tree, and use instead the
1670		 * checksums attached to the ordered extents.
1671		 */
1672		btrfs_get_ordered_extents_for_logging(inode, &ctx.ordered_extents);
1673		ret = filemap_fdatawait_range(inode->vfs_inode.i_mapping, start, end);
1674		if (ret)
1675			goto out_release_extents;
1676
1677		/*
1678		 * Check and clear the BTRFS_INODE_COW_WRITE_ERROR now after
1679		 * starting and waiting for writeback, because for buffered IO
1680		 * it may have been set during the end IO callback
1681		 * (end_bbio_data_write() -> btrfs_finish_ordered_extent()) in
1682		 * case an error happened and we need to wait for ordered
1683		 * extents to complete so that any extent maps that point to
1684		 * unwritten locations are dropped and we don't log them.
1685		 */
1686		if (test_and_clear_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags))
1687			ret = btrfs_wait_ordered_range(inode, start, len);
1688	}
1689
1690	if (ret)
1691		goto out_release_extents;
1692
1693	atomic_inc(&root->log_batch);
1694
1695	if (skip_inode_logging(&ctx)) {
1696		/*
1697		 * We've had everything committed since the last time we were
1698		 * modified so clear this flag in case it was set for whatever
1699		 * reason, it's no longer relevant.
1700		 */
1701		clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
1702		/*
1703		 * An ordered extent might have started before and completed
1704		 * already with io errors, in which case the inode was not
1705		 * updated and we end up here. So check the inode's mapping
1706		 * for any errors that might have happened since we last
1707		 * checked called fsync.
1708		 */
1709		ret = filemap_check_wb_err(inode->vfs_inode.i_mapping, file->f_wb_err);
1710		goto out_release_extents;
1711	}
1712
1713	btrfs_init_log_ctx_scratch_eb(&ctx);
1714
1715	/*
1716	 * We use start here because we will need to wait on the IO to complete
1717	 * in btrfs_sync_log, which could require joining a transaction (for
1718	 * example checking cross references in the nocow path).  If we use join
1719	 * here we could get into a situation where we're waiting on IO to
1720	 * happen that is blocked on a transaction trying to commit.  With start
1721	 * we inc the extwriter counter, so we wait for all extwriters to exit
1722	 * before we start blocking joiners.  This comment is to keep somebody
1723	 * from thinking they are super smart and changing this to
1724	 * btrfs_join_transaction *cough*Josef*cough*.
1725	 */
1726	trans = btrfs_start_transaction(root, 0);
1727	if (IS_ERR(trans)) {
1728		ret = PTR_ERR(trans);
1729		goto out_release_extents;
1730	}
1731	trans->in_fsync = true;
1732
1733	ret = btrfs_log_dentry_safe(trans, dentry, &ctx);
1734	/*
1735	 * Scratch eb no longer needed, release before syncing log or commit
1736	 * transaction, to avoid holding unnecessary memory during such long
1737	 * operations.
1738	 */
1739	if (ctx.scratch_eb) {
1740		free_extent_buffer(ctx.scratch_eb);
1741		ctx.scratch_eb = NULL;
1742	}
1743	btrfs_release_log_ctx_extents(&ctx);
1744	if (ret < 0) {
1745		/* Fallthrough and commit/free transaction. */
1746		ret = BTRFS_LOG_FORCE_COMMIT;
1747	}
1748
1749	/* we've logged all the items and now have a consistent
1750	 * version of the file in the log.  It is possible that
1751	 * someone will come in and modify the file, but that's
1752	 * fine because the log is consistent on disk, and we
1753	 * have references to all of the file's extents
1754	 *
1755	 * It is possible that someone will come in and log the
1756	 * file again, but that will end up using the synchronization
1757	 * inside btrfs_sync_log to keep things safe.
1758	 */
1759	if (skip_ilock)
1760		up_write(&inode->i_mmap_lock);
1761	else
1762		btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
1763
1764	if (ret == BTRFS_NO_LOG_SYNC) {
1765		ret = btrfs_end_transaction(trans);
1766		goto out;
1767	}
1768
1769	/* We successfully logged the inode, attempt to sync the log. */
1770	if (!ret) {
1771		ret = btrfs_sync_log(trans, root, &ctx);
1772		if (!ret) {
1773			ret = btrfs_end_transaction(trans);
1774			goto out;
1775		}
1776	}
1777
1778	/*
1779	 * At this point we need to commit the transaction because we had
1780	 * btrfs_need_log_full_commit() or some other error.
1781	 *
1782	 * If we didn't do a full sync we have to stop the trans handle, wait on
1783	 * the ordered extents, start it again and commit the transaction.  If
1784	 * we attempt to wait on the ordered extents here we could deadlock with
1785	 * something like fallocate() that is holding the extent lock trying to
1786	 * start a transaction while some other thread is trying to commit the
1787	 * transaction while we (fsync) are currently holding the transaction
1788	 * open.
1789	 */
1790	if (!full_sync) {
1791		ret = btrfs_end_transaction(trans);
1792		if (ret)
1793			goto out;
1794		ret = btrfs_wait_ordered_range(inode, start, len);
1795		if (ret)
1796			goto out;
1797
1798		/*
1799		 * This is safe to use here because we're only interested in
1800		 * making sure the transaction that had the ordered extents is
1801		 * committed.  We aren't waiting on anything past this point,
1802		 * we're purely getting the transaction and committing it.
1803		 */
1804		trans = btrfs_attach_transaction_barrier(root);
1805		if (IS_ERR(trans)) {
1806			ret = PTR_ERR(trans);
1807
1808			/*
1809			 * We committed the transaction and there's no currently
1810			 * running transaction, this means everything we care
1811			 * about made it to disk and we are done.
1812			 */
1813			if (ret == -ENOENT)
1814				ret = 0;
1815			goto out;
1816		}
1817	}
1818
1819	ret = btrfs_commit_transaction(trans);
1820out:
1821	free_extent_buffer(ctx.scratch_eb);
1822	ASSERT(list_empty(&ctx.list));
1823	ASSERT(list_empty(&ctx.conflict_inodes));
1824	err = file_check_and_advance_wb_err(file);
1825	if (!ret)
1826		ret = err;
1827	return ret > 0 ? -EIO : ret;
1828
1829out_release_extents:
1830	btrfs_release_log_ctx_extents(&ctx);
1831	if (skip_ilock)
1832		up_write(&inode->i_mmap_lock);
1833	else
1834		btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
1835	goto out;
1836}
1837
1838/*
1839 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
1840 * called from a page fault handler when a page is first dirtied. Hence we must
1841 * be careful to check for EOF conditions here. We set the page up correctly
1842 * for a written page which means we get ENOSPC checking when writing into
1843 * holes and correct delalloc and unwritten extent mapping on filesystems that
1844 * support these features.
1845 *
1846 * We are not allowed to take the i_mutex here so we have to play games to
1847 * protect against truncate races as the page could now be beyond EOF.  Because
1848 * truncate_setsize() writes the inode size before removing pages, once we have
1849 * the page lock we can determine safely if the page is beyond EOF. If it is not
1850 * beyond EOF, then the page is guaranteed safe against truncation until we
1851 * unlock the page.
1852 */
1853static vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf)
1854{
1855	struct page *page = vmf->page;
1856	struct folio *folio = page_folio(page);
1857	struct btrfs_inode *inode = BTRFS_I(file_inode(vmf->vma->vm_file));
1858	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1859	struct extent_io_tree *io_tree = &inode->io_tree;
1860	struct btrfs_ordered_extent *ordered;
1861	struct extent_state *cached_state = NULL;
1862	struct extent_changeset *data_reserved = NULL;
1863	unsigned long zero_start;
1864	loff_t size;
1865	size_t fsize = folio_size(folio);
1866	int ret;
1867	bool only_release_metadata = false;
1868	u64 reserved_space;
1869	u64 page_start;
1870	u64 page_end;
1871	u64 end;
1872
1873	reserved_space = fsize;
1874
1875	sb_start_pagefault(inode->vfs_inode.i_sb);
1876	page_start = folio_pos(folio);
1877	page_end = page_start + folio_size(folio) - 1;
1878	end = page_end;
1879
1880	/*
1881	 * Reserving delalloc space after obtaining the page lock can lead to
1882	 * deadlock. For example, if a dirty page is locked by this function
1883	 * and the call to btrfs_delalloc_reserve_space() ends up triggering
1884	 * dirty page write out, then the btrfs_writepages() function could
1885	 * end up waiting indefinitely to get a lock on the page currently
1886	 * being processed by btrfs_page_mkwrite() function.
1887	 */
1888	ret = btrfs_check_data_free_space(inode, &data_reserved, page_start,
1889					  reserved_space, false);
1890	if (ret < 0) {
1891		size_t write_bytes = reserved_space;
1892
1893		if (btrfs_check_nocow_lock(inode, page_start, &write_bytes, false) <= 0)
1894			goto out_noreserve;
1895
1896		only_release_metadata = true;
1897
1898		/*
1899		 * Can't write the whole range, there may be shared extents or
1900		 * holes in the range, bail out with @only_release_metadata set
1901		 * to true so that we unlock the nocow lock before returning the
1902		 * error.
1903		 */
1904		if (write_bytes < reserved_space)
1905			goto out_noreserve;
1906	}
1907	ret = btrfs_delalloc_reserve_metadata(inode, reserved_space,
1908					      reserved_space, false);
1909	if (ret < 0) {
1910		if (!only_release_metadata)
1911			btrfs_free_reserved_data_space(inode, data_reserved,
1912						       page_start, reserved_space);
1913		goto out_noreserve;
1914	}
1915
1916	ret = file_update_time(vmf->vma->vm_file);
1917	if (ret < 0)
1918		goto out;
1919again:
1920	down_read(&inode->i_mmap_lock);
1921	folio_lock(folio);
1922	size = i_size_read(&inode->vfs_inode);
1923
1924	if ((folio->mapping != inode->vfs_inode.i_mapping) ||
1925	    (page_start >= size)) {
1926		/* Page got truncated out from underneath us. */
1927		goto out_unlock;
1928	}
1929	folio_wait_writeback(folio);
1930
1931	btrfs_lock_extent(io_tree, page_start, page_end, &cached_state);
1932	ret = set_folio_extent_mapped(folio);
1933	if (ret < 0) {
1934		btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
1935		goto out_unlock;
1936	}
1937
1938	/*
1939	 * We can't set the delalloc bits if there are pending ordered
1940	 * extents.  Drop our locks and wait for them to finish.
1941	 */
1942	ordered = btrfs_lookup_ordered_range(inode, page_start, fsize);
1943	if (ordered) {
1944		btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
1945		folio_unlock(folio);
1946		up_read(&inode->i_mmap_lock);
1947		btrfs_start_ordered_extent(ordered);
1948		btrfs_put_ordered_extent(ordered);
1949		goto again;
1950	}
1951
1952	if (folio_contains(folio, (size - 1) >> PAGE_SHIFT)) {
1953		reserved_space = round_up(size - page_start, fs_info->sectorsize);
1954		if (reserved_space < fsize) {
1955			const u64 to_free = fsize - reserved_space;
1956
1957			end = page_start + reserved_space - 1;
1958			if (only_release_metadata)
1959				btrfs_delalloc_release_metadata(inode, to_free, true);
1960			else
1961				btrfs_delalloc_release_space(inode, data_reserved,
1962							     end + 1, to_free, true);
1963		}
1964	}
1965
1966	/*
1967	 * page_mkwrite gets called when the page is firstly dirtied after it's
1968	 * faulted in, but write(2) could also dirty a page and set delalloc
1969	 * bits, thus in this case for space account reason, we still need to
1970	 * clear any delalloc bits within this page range since we have to
1971	 * reserve data&meta space before lock_page() (see above comments).
1972	 */
1973	btrfs_clear_extent_bit(io_tree, page_start, end,
1974			       EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
1975			       EXTENT_DEFRAG, &cached_state);
1976
1977	ret = btrfs_set_extent_delalloc(inode, page_start, end, 0, &cached_state);
1978	if (ret < 0) {
1979		btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
1980		goto out_unlock;
1981	}
1982
1983	/* Page is wholly or partially inside EOF. */
1984	if (page_start + folio_size(folio) > size)
1985		zero_start = offset_in_folio(folio, size);
1986	else
1987		zero_start = fsize;
1988
1989	if (zero_start != fsize)
1990		folio_zero_range(folio, zero_start, folio_size(folio) - zero_start);
1991
1992	btrfs_folio_clear_checked(fs_info, folio, page_start, fsize);
1993	btrfs_folio_set_dirty(fs_info, folio, page_start, end + 1 - page_start);
1994	btrfs_folio_set_uptodate(fs_info, folio, page_start, end + 1 - page_start);
1995
1996	btrfs_set_inode_last_sub_trans(inode);
1997
1998	if (only_release_metadata)
1999		btrfs_set_extent_bit(io_tree, page_start, end, EXTENT_NORESERVE,
2000				     &cached_state);
2001
2002	btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
2003	up_read(&inode->i_mmap_lock);
2004
2005	btrfs_delalloc_release_extents(inode, fsize);
2006	if (only_release_metadata)
2007		btrfs_check_nocow_unlock(inode);
2008	sb_end_pagefault(inode->vfs_inode.i_sb);
2009	extent_changeset_free(data_reserved);
2010	return VM_FAULT_LOCKED;
2011
2012out_unlock:
2013	folio_unlock(folio);
2014	up_read(&inode->i_mmap_lock);
2015out:
2016	btrfs_delalloc_release_extents(inode, fsize);
2017	if (only_release_metadata)
2018		btrfs_delalloc_release_metadata(inode, reserved_space, true);
2019	else
2020		btrfs_delalloc_release_space(inode, data_reserved, page_start,
2021					     reserved_space, true);
2022out_noreserve:
2023	if (only_release_metadata)
2024		btrfs_check_nocow_unlock(inode);
2025
2026	sb_end_pagefault(inode->vfs_inode.i_sb);
2027
2028	extent_changeset_free(data_reserved);
2029
2030	if (ret < 0)
2031		return vmf_error(ret);
2032
2033	/* Make the VM retry the fault. */
2034	return VM_FAULT_NOPAGE;
2035}
2036
2037static const struct vm_operations_struct btrfs_file_vm_ops = {
2038	.fault		= filemap_fault,
2039	.map_pages	= filemap_map_pages,
2040	.page_mkwrite	= btrfs_page_mkwrite,
2041};
2042
2043static int btrfs_file_mmap_prepare(struct vm_area_desc *desc)
2044{
2045	struct file *filp = desc->file;
2046	struct address_space *mapping = filp->f_mapping;
2047
2048	if (unlikely(btrfs_is_shutdown(inode_to_fs_info(file_inode(filp)))))
2049		return -EIO;
2050	if (!mapping->a_ops->read_folio)
2051		return -ENOEXEC;
2052
2053	file_accessed(filp);
2054	desc->vm_ops = &btrfs_file_vm_ops;
2055
2056	return 0;
2057}
2058
2059static bool hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
2060			   int slot, u64 start, u64 end)
2061{
2062	struct btrfs_file_extent_item *fi;
2063	struct btrfs_key key;
2064
2065	if (slot < 0 || slot >= btrfs_header_nritems(leaf))
2066		return false;
2067
2068	btrfs_item_key_to_cpu(leaf, &key, slot);
2069	if (key.objectid != btrfs_ino(inode) ||
2070	    key.type != BTRFS_EXTENT_DATA_KEY)
2071		return false;
2072
2073	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2074
2075	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2076		return false;
2077
2078	if (btrfs_file_extent_disk_bytenr(leaf, fi))
2079		return false;
2080
2081	if (key.offset == end)
2082		return true;
2083	if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
2084		return true;
2085	return false;
2086}
2087
2088static int fill_holes(struct btrfs_trans_handle *trans,
2089		struct btrfs_inode *inode,
2090		struct btrfs_path *path, u64 offset, u64 end)
2091{
2092	struct btrfs_fs_info *fs_info = trans->fs_info;
2093	struct btrfs_root *root = inode->root;
2094	struct extent_buffer *leaf;
2095	struct btrfs_file_extent_item *fi;
2096	struct extent_map *hole_em;
2097	struct btrfs_key key;
2098	int ret;
2099
2100	if (btrfs_fs_incompat(fs_info, NO_HOLES))
2101		goto out;
2102
2103	key.objectid = btrfs_ino(inode);
2104	key.type = BTRFS_EXTENT_DATA_KEY;
2105	key.offset = offset;
2106
2107	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2108	if (ret <= 0) {
2109		/*
2110		 * We should have dropped this offset, so if we find it then
2111		 * something has gone horribly wrong.
2112		 */
2113		if (ret == 0)
2114			ret = -EINVAL;
2115		return ret;
2116	}
2117
2118	leaf = path->nodes[0];
2119	if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
2120		u64 num_bytes;
2121
2122		path->slots[0]--;
2123		fi = btrfs_item_ptr(leaf, path->slots[0],
2124				    struct btrfs_file_extent_item);
2125		num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
2126			end - offset;
2127		btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2128		btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2129		btrfs_set_file_extent_offset(leaf, fi, 0);
2130		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2131		goto out;
2132	}
2133
2134	if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
2135		u64 num_bytes;
2136
2137		key.offset = offset;
2138		btrfs_set_item_key_safe(trans, path, &key);
2139		fi = btrfs_item_ptr(leaf, path->slots[0],
2140				    struct btrfs_file_extent_item);
2141		num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
2142			offset;
2143		btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2144		btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2145		btrfs_set_file_extent_offset(leaf, fi, 0);
2146		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2147		goto out;
2148	}
2149	btrfs_release_path(path);
2150
2151	ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset,
2152				       end - offset);
2153	if (ret)
2154		return ret;
2155
2156out:
2157	btrfs_release_path(path);
2158
2159	hole_em = btrfs_alloc_extent_map();
2160	if (!hole_em) {
2161		btrfs_drop_extent_map_range(inode, offset, end - 1, false);
2162		btrfs_set_inode_full_sync(inode);
2163	} else {
2164		hole_em->start = offset;
2165		hole_em->len = end - offset;
2166		hole_em->ram_bytes = hole_em->len;
2167
2168		hole_em->disk_bytenr = EXTENT_MAP_HOLE;
2169		hole_em->disk_num_bytes = 0;
2170		hole_em->generation = trans->transid;
2171
2172		ret = btrfs_replace_extent_map_range(inode, hole_em, true);
2173		btrfs_free_extent_map(hole_em);
2174		if (ret)
2175			btrfs_set_inode_full_sync(inode);
2176	}
2177
2178	return 0;
2179}
2180
2181/*
2182 * Find a hole extent on given inode and change start/len to the end of hole
2183 * extent.(hole/vacuum extent whose em->start <= start &&
2184 *	   em->start + em->len > start)
2185 * When a hole extent is found, return 1 and modify start/len.
2186 */
2187static int find_first_non_hole(struct btrfs_inode *inode, u64 *start, u64 *len)
2188{
2189	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2190	struct extent_map *em;
2191	int ret = 0;
2192
2193	em = btrfs_get_extent(inode, NULL,
2194			      round_down(*start, fs_info->sectorsize),
2195			      round_up(*len, fs_info->sectorsize));
2196	if (IS_ERR(em))
2197		return PTR_ERR(em);
2198
2199	/* Hole or vacuum extent(only exists in no-hole mode) */
2200	if (em->disk_bytenr == EXTENT_MAP_HOLE) {
2201		ret = 1;
2202		*len = em->start + em->len > *start + *len ?
2203		       0 : *start + *len - em->start - em->len;
2204		*start = em->start + em->len;
2205	}
2206	btrfs_free_extent_map(em);
2207	return ret;
2208}
2209
2210/*
2211 * Check if there is no folio in the range.
2212 *
2213 * We cannot utilize filemap_range_has_page() in a filemap with large folios
2214 * as we can hit the following false positive:
2215 *
2216 *        start                            end
2217 *        |                                |
2218 *  |//|//|//|//|  |  |  |  |  |  |  |  |//|//|
2219 *   \         /                         \   /
2220 *    Folio A                            Folio B
2221 *
2222 * That large folio A and B cover the start and end indexes.
2223 * In that case filemap_range_has_page() will always return true, but the above
2224 * case is fine for btrfs_punch_hole_lock_range() usage.
2225 *
2226 * So here we only ensure that no other folios is in the range, excluding the
2227 * head/tail large folio.
2228 */
2229static bool check_range_has_page(struct inode *inode, u64 start, u64 end)
2230{
2231	struct folio_batch fbatch;
2232	bool ret = false;
2233	/*
2234	 * For subpage case, if the range is not at page boundary, we could
2235	 * have pages at the leading/tailing part of the range.
2236	 * This could lead to dead loop since filemap_range_has_page()
2237	 * will always return true.
2238	 * So here we need to do extra page alignment for
2239	 * filemap_range_has_page().
2240	 *
2241	 * And do not decrease page_lockend right now, as it can be 0.
2242	 */
2243	const u64 page_lockstart = round_up(start, PAGE_SIZE);
2244	const u64 page_lockend = round_down(end + 1, PAGE_SIZE);
2245	const pgoff_t start_index = page_lockstart >> PAGE_SHIFT;
2246	const pgoff_t end_index = (page_lockend - 1) >> PAGE_SHIFT;
2247	pgoff_t tmp = start_index;
2248	int found_folios;
2249
2250	/* The same page or adjacent pages. */
2251	if (page_lockend <= page_lockstart)
2252		return false;
2253
2254	folio_batch_init(&fbatch);
2255	found_folios = filemap_get_folios(inode->i_mapping, &tmp, end_index, &fbatch);
2256	for (int i = 0; i < found_folios; i++) {
2257		struct folio *folio = fbatch.folios[i];
2258
2259		/* A large folio begins before the start. Not a target. */
2260		if (folio->index < start_index)
2261			continue;
2262		/* A large folio extends beyond the end. Not a target. */
2263		if (folio_next_index(folio) > end_index)
2264			continue;
2265		/* A folio doesn't cover the head/tail index. Found a target. */
2266		ret = true;
2267		break;
2268	}
2269	folio_batch_release(&fbatch);
2270	return ret;
2271}
2272
2273static void btrfs_punch_hole_lock_range(struct inode *inode,
2274					const u64 lockstart, const u64 lockend,
2275					struct extent_state **cached_state)
2276{
2277	while (1) {
2278		truncate_pagecache_range(inode, lockstart, lockend);
2279
2280		btrfs_lock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2281				  cached_state);
2282		/*
2283		 * We can't have ordered extents in the range, nor dirty/writeback
2284		 * pages, because we have locked the inode's VFS lock in exclusive
2285		 * mode, we have locked the inode's i_mmap_lock in exclusive mode,
2286		 * we have flushed all delalloc in the range and we have waited
2287		 * for any ordered extents in the range to complete.
2288		 * We can race with anyone reading pages from this range, so after
2289		 * locking the range check if we have pages in the range, and if
2290		 * we do, unlock the range and retry.
2291		 */
2292		if (!check_range_has_page(inode, lockstart, lockend))
2293			break;
2294
2295		btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2296				    cached_state);
2297	}
2298
2299	btrfs_assert_inode_range_clean(BTRFS_I(inode), lockstart, lockend);
2300}
2301
2302static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans,
2303				     struct btrfs_inode *inode,
2304				     struct btrfs_path *path,
2305				     struct btrfs_replace_extent_info *extent_info,
2306				     const u64 replace_len,
2307				     const u64 bytes_to_drop)
2308{
2309	struct btrfs_fs_info *fs_info = trans->fs_info;
2310	struct btrfs_root *root = inode->root;
2311	struct btrfs_file_extent_item *extent;
2312	struct extent_buffer *leaf;
2313	struct btrfs_key key;
2314	int slot;
2315	int ret;
2316
2317	if (replace_len == 0)
2318		return 0;
2319
2320	if (extent_info->disk_offset == 0 &&
2321	    btrfs_fs_incompat(fs_info, NO_HOLES)) {
2322		btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2323		return 0;
2324	}
2325
2326	key.objectid = btrfs_ino(inode);
2327	key.type = BTRFS_EXTENT_DATA_KEY;
2328	key.offset = extent_info->file_offset;
2329	ret = btrfs_insert_empty_item(trans, root, path, &key,
2330				      sizeof(struct btrfs_file_extent_item));
2331	if (ret)
2332		return ret;
2333	leaf = path->nodes[0];
2334	slot = path->slots[0];
2335	write_extent_buffer(leaf, extent_info->extent_buf,
2336			    btrfs_item_ptr_offset(leaf, slot),
2337			    sizeof(struct btrfs_file_extent_item));
2338	extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2339	ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE);
2340	btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset);
2341	btrfs_set_file_extent_num_bytes(leaf, extent, replace_len);
2342	if (extent_info->is_new_extent)
2343		btrfs_set_file_extent_generation(leaf, extent, trans->transid);
2344	btrfs_release_path(path);
2345
2346	ret = btrfs_inode_set_file_extent_range(inode, extent_info->file_offset,
2347						replace_len);
2348	if (ret)
2349		return ret;
2350
2351	/* If it's a hole, nothing more needs to be done. */
2352	if (extent_info->disk_offset == 0) {
2353		btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2354		return 0;
2355	}
2356
2357	btrfs_update_inode_bytes(inode, replace_len, bytes_to_drop);
2358
2359	if (extent_info->is_new_extent && extent_info->insertions == 0) {
2360		key.objectid = extent_info->disk_offset;
2361		key.type = BTRFS_EXTENT_ITEM_KEY;
2362		key.offset = extent_info->disk_len;
2363		ret = btrfs_alloc_reserved_file_extent(trans, root,
2364						       btrfs_ino(inode),
2365						       extent_info->file_offset,
2366						       extent_info->qgroup_reserved,
2367						       &key);
2368	} else {
2369		struct btrfs_ref ref = {
2370			.action = BTRFS_ADD_DELAYED_REF,
2371			.bytenr = extent_info->disk_offset,
2372			.num_bytes = extent_info->disk_len,
2373			.owning_root = btrfs_root_id(root),
2374			.ref_root = btrfs_root_id(root),
2375		};
2376		u64 ref_offset;
2377
2378		ref_offset = extent_info->file_offset - extent_info->data_offset;
2379		btrfs_init_data_ref(&ref, btrfs_ino(inode), ref_offset, 0, false);
2380		ret = btrfs_inc_extent_ref(trans, &ref);
2381	}
2382
2383	extent_info->insertions++;
2384
2385	return ret;
2386}
2387
2388/*
2389 * The respective range must have been previously locked, as well as the inode.
2390 * The end offset is inclusive (last byte of the range).
2391 * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
2392 * the file range with an extent.
2393 * When not punching a hole, we don't want to end up in a state where we dropped
2394 * extents without inserting a new one, so we must abort the transaction to avoid
2395 * a corruption.
2396 */
2397int btrfs_replace_file_extents(struct btrfs_inode *inode,
2398			       struct btrfs_path *path, const u64 start,
2399			       const u64 end,
2400			       struct btrfs_replace_extent_info *extent_info,
2401			       struct btrfs_trans_handle **trans_out)
2402{
2403	struct btrfs_drop_extents_args drop_args = { 0 };
2404	struct btrfs_root *root = inode->root;
2405	struct btrfs_fs_info *fs_info = root->fs_info;
2406	u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
2407	u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
2408	struct btrfs_trans_handle *trans = NULL;
2409	struct btrfs_block_rsv rsv;
2410	unsigned int rsv_count;
2411	u64 cur_offset;
2412	u64 len = end - start;
2413	int ret = 0;
2414
2415	if (end <= start)
2416		return -EINVAL;
2417
2418	btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP);
2419	rsv.size = btrfs_calc_insert_metadata_size(fs_info, 1);
2420	rsv.failfast = true;
2421
2422	/*
2423	 * 1 - update the inode
2424	 * 1 - removing the extents in the range
2425	 * 1 - adding the hole extent if no_holes isn't set or if we are
2426	 *     replacing the range with a new extent
2427	 */
2428	if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info)
2429		rsv_count = 3;
2430	else
2431		rsv_count = 2;
2432
2433	trans = btrfs_start_transaction(root, rsv_count);
2434	if (IS_ERR(trans)) {
2435		ret = PTR_ERR(trans);
2436		trans = NULL;
2437		goto out_release;
2438	}
2439
2440	ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, &rsv,
2441				      min_size, false);
2442	if (WARN_ON(ret))
2443		goto out_trans;
2444	trans->block_rsv = &rsv;
2445
2446	cur_offset = start;
2447	drop_args.path = path;
2448	drop_args.end = end + 1;
2449	drop_args.drop_cache = true;
2450	while (cur_offset < end) {
2451		drop_args.start = cur_offset;
2452		ret = btrfs_drop_extents(trans, root, inode, &drop_args);
2453		/* If we are punching a hole decrement the inode's byte count */
2454		if (!extent_info)
2455			btrfs_update_inode_bytes(inode, 0,
2456						 drop_args.bytes_found);
2457		if (ret != -ENOSPC) {
2458			/*
2459			 * The only time we don't want to abort is if we are
2460			 * attempting to clone a partial inline extent, in which
2461			 * case we'll get EOPNOTSUPP.  However if we aren't
2462			 * clone we need to abort no matter what, because if we
2463			 * got EOPNOTSUPP via prealloc then we messed up and
2464			 * need to abort.
2465			 */
2466			if (unlikely(ret &&
2467				     (ret != -EOPNOTSUPP ||
2468				      (extent_info && extent_info->is_new_extent))))
2469				btrfs_abort_transaction(trans, ret);
2470			break;
2471		}
2472
2473		trans->block_rsv = &fs_info->trans_block_rsv;
2474
2475		if (!extent_info && cur_offset < drop_args.drop_end &&
2476		    cur_offset < ino_size) {
2477			ret = fill_holes(trans, inode, path, cur_offset,
2478					 drop_args.drop_end);
2479			if (unlikely(ret)) {
2480				/*
2481				 * If we failed then we didn't insert our hole
2482				 * entries for the area we dropped, so now the
2483				 * fs is corrupted, so we must abort the
2484				 * transaction.
2485				 */
2486				btrfs_abort_transaction(trans, ret);
2487				break;
2488			}
2489		} else if (!extent_info && cur_offset < drop_args.drop_end) {
2490			/*
2491			 * We are past the i_size here, but since we didn't
2492			 * insert holes we need to clear the mapped area so we
2493			 * know to not set disk_i_size in this area until a new
2494			 * file extent is inserted here.
2495			 */
2496			ret = btrfs_inode_clear_file_extent_range(inode,
2497					cur_offset,
2498					drop_args.drop_end - cur_offset);
2499			if (unlikely(ret)) {
2500				/*
2501				 * We couldn't clear our area, so we could
2502				 * presumably adjust up and corrupt the fs, so
2503				 * we need to abort.
2504				 */
2505				btrfs_abort_transaction(trans, ret);
2506				break;
2507			}
2508		}
2509
2510		if (extent_info &&
2511		    drop_args.drop_end > extent_info->file_offset) {
2512			u64 replace_len = drop_args.drop_end -
2513					  extent_info->file_offset;
2514
2515			ret = btrfs_insert_replace_extent(trans, inode,	path,
2516					extent_info, replace_len,
2517					drop_args.bytes_found);
2518			if (unlikely(ret)) {
2519				btrfs_abort_transaction(trans, ret);
2520				break;
2521			}
2522			extent_info->data_len -= replace_len;
2523			extent_info->data_offset += replace_len;
2524			extent_info->file_offset += replace_len;
2525		}
2526
2527		/*
2528		 * We are releasing our handle on the transaction, balance the
2529		 * dirty pages of the btree inode and flush delayed items, and
2530		 * then get a new transaction handle, which may now point to a
2531		 * new transaction in case someone else may have committed the
2532		 * transaction we used to replace/drop file extent items. So
2533		 * bump the inode's iversion and update mtime and ctime except
2534		 * if we are called from a dedupe context. This is because a
2535		 * power failure/crash may happen after the transaction is
2536		 * committed and before we finish replacing/dropping all the
2537		 * file extent items we need.
2538		 */
2539		inode_inc_iversion(&inode->vfs_inode);
2540
2541		if (!extent_info || extent_info->update_times)
2542			inode_set_mtime_to_ts(&inode->vfs_inode,
2543					      inode_set_ctime_current(&inode->vfs_inode));
2544
2545		ret = btrfs_update_inode(trans, inode);
2546		if (ret)
2547			break;
2548
2549		btrfs_end_transaction(trans);
2550		btrfs_btree_balance_dirty(fs_info);
2551
2552		trans = btrfs_start_transaction(root, rsv_count);
2553		if (IS_ERR(trans)) {
2554			ret = PTR_ERR(trans);
2555			trans = NULL;
2556			break;
2557		}
2558
2559		ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
2560					      &rsv, min_size, false);
2561		if (WARN_ON(ret))
2562			break;
2563		trans->block_rsv = &rsv;
2564
2565		cur_offset = drop_args.drop_end;
2566		len = end - cur_offset;
2567		if (!extent_info && len) {
2568			ret = find_first_non_hole(inode, &cur_offset, &len);
2569			if (unlikely(ret < 0))
2570				break;
2571			if (ret && !len) {
2572				ret = 0;
2573				break;
2574			}
2575		}
2576	}
2577
2578	/*
2579	 * If we were cloning, force the next fsync to be a full one since we
2580	 * we replaced (or just dropped in the case of cloning holes when
2581	 * NO_HOLES is enabled) file extent items and did not setup new extent
2582	 * maps for the replacement extents (or holes).
2583	 */
2584	if (extent_info && !extent_info->is_new_extent)
2585		btrfs_set_inode_full_sync(inode);
2586
2587	if (ret)
2588		goto out_trans;
2589
2590	trans->block_rsv = &fs_info->trans_block_rsv;
2591	/*
2592	 * If we are using the NO_HOLES feature we might have had already an
2593	 * hole that overlaps a part of the region [lockstart, lockend] and
2594	 * ends at (or beyond) lockend. Since we have no file extent items to
2595	 * represent holes, drop_end can be less than lockend and so we must
2596	 * make sure we have an extent map representing the existing hole (the
2597	 * call to __btrfs_drop_extents() might have dropped the existing extent
2598	 * map representing the existing hole), otherwise the fast fsync path
2599	 * will not record the existence of the hole region
2600	 * [existing_hole_start, lockend].
2601	 */
2602	if (drop_args.drop_end <= end)
2603		drop_args.drop_end = end + 1;
2604	/*
2605	 * Don't insert file hole extent item if it's for a range beyond eof
2606	 * (because it's useless) or if it represents a 0 bytes range (when
2607	 * cur_offset == drop_end).
2608	 */
2609	if (!extent_info && cur_offset < ino_size &&
2610	    cur_offset < drop_args.drop_end) {
2611		ret = fill_holes(trans, inode, path, cur_offset,
2612				 drop_args.drop_end);
2613		if (unlikely(ret)) {
2614			/* Same comment as above. */
2615			btrfs_abort_transaction(trans, ret);
2616			goto out_trans;
2617		}
2618	} else if (!extent_info && cur_offset < drop_args.drop_end) {
2619		/* See the comment in the loop above for the reasoning here. */
2620		ret = btrfs_inode_clear_file_extent_range(inode, cur_offset,
2621					drop_args.drop_end - cur_offset);
2622		if (unlikely(ret)) {
2623			btrfs_abort_transaction(trans, ret);
2624			goto out_trans;
2625		}
2626
2627	}
2628	if (extent_info) {
2629		ret = btrfs_insert_replace_extent(trans, inode, path,
2630				extent_info, extent_info->data_len,
2631				drop_args.bytes_found);
2632		if (unlikely(ret)) {
2633			btrfs_abort_transaction(trans, ret);
2634			goto out_trans;
2635		}
2636	}
2637
2638out_trans:
2639	if (!trans)
2640		goto out_release;
2641
2642	trans->block_rsv = &fs_info->trans_block_rsv;
2643	if (ret)
2644		btrfs_end_transaction(trans);
2645	else
2646		*trans_out = trans;
2647out_release:
2648	btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL);
2649	return ret;
2650}
2651
2652static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len)
2653{
2654	struct inode *inode = file_inode(file);
2655	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2656	struct btrfs_root *root = BTRFS_I(inode)->root;
2657	struct extent_state *cached_state = NULL;
2658	struct btrfs_path *path;
2659	struct btrfs_trans_handle *trans = NULL;
2660	u64 lockstart;
2661	u64 lockend;
2662	u64 tail_start;
2663	u64 tail_len;
2664	const u64 orig_start = offset;
2665	const u64 orig_end = offset + len - 1;
2666	int ret = 0;
2667	bool same_block;
2668	u64 ino_size;
2669	bool truncated_block = false;
2670	bool updated_inode = false;
2671
2672	btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
2673
2674	ret = btrfs_wait_ordered_range(BTRFS_I(inode), offset, len);
2675	if (ret)
2676		goto out_only_mutex;
2677
2678	ino_size = round_up(inode->i_size, fs_info->sectorsize);
2679	ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
2680	if (ret < 0)
2681		goto out_only_mutex;
2682	if (ret && !len) {
2683		/* Already in a large hole */
2684		ret = 0;
2685		goto out_only_mutex;
2686	}
2687
2688	ret = file_modified(file);
2689	if (ret)
2690		goto out_only_mutex;
2691
2692	lockstart = round_up(offset, fs_info->sectorsize);
2693	lockend = round_down(offset + len, fs_info->sectorsize) - 1;
2694	same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
2695		== (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
2696	/*
2697	 * Only do this if we are in the same block and we aren't doing the
2698	 * entire block.
2699	 */
2700	if (same_block && len < fs_info->sectorsize) {
2701		if (offset < ino_size) {
2702			truncated_block = true;
2703			ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1,
2704						   orig_start, orig_end);
2705		} else {
2706			ret = 0;
2707		}
2708		goto out_only_mutex;
2709	}
2710
2711	/* zero back part of the first block */
2712	if (offset < ino_size) {
2713		truncated_block = true;
2714		ret = btrfs_truncate_block(BTRFS_I(inode), offset, orig_start, orig_end);
2715		if (ret) {
2716			btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
2717			return ret;
2718		}
2719	}
2720
2721	/* Check the aligned pages after the first unaligned page,
2722	 * if offset != orig_start, which means the first unaligned page
2723	 * including several following pages are already in holes,
2724	 * the extra check can be skipped */
2725	if (offset == orig_start) {
2726		/* after truncate page, check hole again */
2727		len = offset + len - lockstart;
2728		offset = lockstart;
2729		ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
2730		if (ret < 0)
2731			goto out_only_mutex;
2732		if (ret && !len) {
2733			ret = 0;
2734			goto out_only_mutex;
2735		}
2736		lockstart = offset;
2737	}
2738
2739	/* Check the tail unaligned part is in a hole */
2740	tail_start = lockend + 1;
2741	tail_len = offset + len - tail_start;
2742	if (tail_len) {
2743		ret = find_first_non_hole(BTRFS_I(inode), &tail_start, &tail_len);
2744		if (unlikely(ret < 0))
2745			goto out_only_mutex;
2746		if (!ret) {
2747			/* zero the front end of the last page */
2748			if (tail_start + tail_len < ino_size) {
2749				truncated_block = true;
2750				ret = btrfs_truncate_block(BTRFS_I(inode),
2751							tail_start + tail_len - 1,
2752							orig_start, orig_end);
2753				if (ret)
2754					goto out_only_mutex;
2755			}
2756		}
2757	}
2758
2759	if (lockend < lockstart) {
2760		ret = 0;
2761		goto out_only_mutex;
2762	}
2763
2764	btrfs_punch_hole_lock_range(inode, lockstart, lockend, &cached_state);
2765
2766	path = btrfs_alloc_path();
2767	if (!path) {
2768		ret = -ENOMEM;
2769		goto out;
2770	}
2771
2772	ret = btrfs_replace_file_extents(BTRFS_I(inode), path, lockstart,
2773					 lockend, NULL, &trans);
2774	btrfs_free_path(path);
2775	if (ret)
2776		goto out;
2777
2778	ASSERT(trans != NULL);
2779	inode_inc_iversion(inode);
2780	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
2781	ret = btrfs_update_inode(trans, BTRFS_I(inode));
2782	updated_inode = true;
2783	btrfs_end_transaction(trans);
2784	btrfs_btree_balance_dirty(fs_info);
2785out:
2786	btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2787			    &cached_state);
2788out_only_mutex:
2789	if (!updated_inode && truncated_block && !ret) {
2790		/*
2791		 * If we only end up zeroing part of a page, we still need to
2792		 * update the inode item, so that all the time fields are
2793		 * updated as well as the necessary btrfs inode in memory fields
2794		 * for detecting, at fsync time, if the inode isn't yet in the
2795		 * log tree or it's there but not up to date.
2796		 */
2797		struct timespec64 now = inode_set_ctime_current(inode);
2798
2799		inode_inc_iversion(inode);
2800		inode_set_mtime_to_ts(inode, now);
2801		trans = btrfs_start_transaction(root, 1);
2802		if (IS_ERR(trans)) {
2803			ret = PTR_ERR(trans);
2804		} else {
2805			int ret2;
2806
2807			ret = btrfs_update_inode(trans, BTRFS_I(inode));
2808			ret2 = btrfs_end_transaction(trans);
2809			if (!ret)
2810				ret = ret2;
2811		}
2812	}
2813	btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
2814	return ret;
2815}
2816
2817/* Helper structure to record which range is already reserved */
2818struct falloc_range {
2819	struct list_head list;
2820	u64 start;
2821	u64 len;
2822};
2823
2824/*
2825 * Helper function to add falloc range
2826 *
2827 * Caller should have locked the larger range of extent containing
2828 * [start, len)
2829 */
2830static int add_falloc_range(struct list_head *head, u64 start, u64 len)
2831{
2832	struct falloc_range *range = NULL;
2833
2834	if (!list_empty(head)) {
2835		/*
2836		 * As fallocate iterates by bytenr order, we only need to check
2837		 * the last range.
2838		 */
2839		range = list_last_entry(head, struct falloc_range, list);
2840		if (range->start + range->len == start) {
2841			range->len += len;
2842			return 0;
2843		}
2844	}
2845
2846	range = kmalloc(sizeof(*range), GFP_KERNEL);
2847	if (!range)
2848		return -ENOMEM;
2849	range->start = start;
2850	range->len = len;
2851	list_add_tail(&range->list, head);
2852	return 0;
2853}
2854
2855static int btrfs_fallocate_update_isize(struct inode *inode,
2856					const u64 end,
2857					const int mode)
2858{
2859	struct btrfs_trans_handle *trans;
2860	struct btrfs_root *root = BTRFS_I(inode)->root;
2861	u64 range_start;
2862	u64 range_end;
2863	int ret;
2864	int ret2;
2865
2866	if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
2867		return 0;
2868
2869	range_start = round_down(i_size_read(inode), root->fs_info->sectorsize);
2870	range_end = round_up(end, root->fs_info->sectorsize);
2871
2872	ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), range_start,
2873						range_end - range_start);
2874	if (ret)
2875		return ret;
2876
2877	trans = btrfs_start_transaction(root, 1);
2878	if (IS_ERR(trans))
2879		return PTR_ERR(trans);
2880
2881	inode_set_ctime_current(inode);
2882	i_size_write(inode, end);
2883	btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
2884	ret = btrfs_update_inode(trans, BTRFS_I(inode));
2885	ret2 = btrfs_end_transaction(trans);
2886
2887	return ret ? ret : ret2;
2888}
2889
2890enum {
2891	RANGE_BOUNDARY_WRITTEN_EXTENT,
2892	RANGE_BOUNDARY_PREALLOC_EXTENT,
2893	RANGE_BOUNDARY_HOLE,
2894};
2895
2896static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode,
2897						 u64 offset)
2898{
2899	const u64 sectorsize = inode->root->fs_info->sectorsize;
2900	struct extent_map *em;
2901	int ret;
2902
2903	offset = round_down(offset, sectorsize);
2904	em = btrfs_get_extent(inode, NULL, offset, sectorsize);
2905	if (IS_ERR(em))
2906		return PTR_ERR(em);
2907
2908	if (em->disk_bytenr == EXTENT_MAP_HOLE)
2909		ret = RANGE_BOUNDARY_HOLE;
2910	else if (em->flags & EXTENT_FLAG_PREALLOC)
2911		ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
2912	else
2913		ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
2914
2915	btrfs_free_extent_map(em);
2916	return ret;
2917}
2918
2919static int btrfs_zero_range(struct inode *inode,
2920			    loff_t offset,
2921			    loff_t len,
2922			    const int mode)
2923{
2924	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2925	struct extent_map *em;
2926	struct extent_changeset *data_reserved = NULL;
2927	int ret;
2928	u64 alloc_hint = 0;
2929	const u64 sectorsize = fs_info->sectorsize;
2930	const u64 orig_start = offset;
2931	const u64 orig_end = offset + len - 1;
2932	u64 alloc_start = round_down(offset, sectorsize);
2933	u64 alloc_end = round_up(offset + len, sectorsize);
2934	u64 bytes_to_reserve = 0;
2935	bool space_reserved = false;
2936
2937	em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start,
2938			      alloc_end - alloc_start);
2939	if (IS_ERR(em)) {
2940		ret = PTR_ERR(em);
2941		goto out;
2942	}
2943
2944	/*
2945	 * Avoid hole punching and extent allocation for some cases. More cases
2946	 * could be considered, but these are unlikely common and we keep things
2947	 * as simple as possible for now. Also, intentionally, if the target
2948	 * range contains one or more prealloc extents together with regular
2949	 * extents and holes, we drop all the existing extents and allocate a
2950	 * new prealloc extent, so that we get a larger contiguous disk extent.
2951	 */
2952	if (em->start <= alloc_start && (em->flags & EXTENT_FLAG_PREALLOC)) {
2953		const u64 em_end = em->start + em->len;
2954
2955		if (em_end >= offset + len) {
2956			/*
2957			 * The whole range is already a prealloc extent,
2958			 * do nothing except updating the inode's i_size if
2959			 * needed.
2960			 */
2961			btrfs_free_extent_map(em);
2962			ret = btrfs_fallocate_update_isize(inode, offset + len,
2963							   mode);
2964			goto out;
2965		}
2966		/*
2967		 * Part of the range is already a prealloc extent, so operate
2968		 * only on the remaining part of the range.
2969		 */
2970		alloc_start = em_end;
2971		ASSERT(IS_ALIGNED(alloc_start, sectorsize));
2972		len = offset + len - alloc_start;
2973		offset = alloc_start;
2974		alloc_hint = btrfs_extent_map_block_start(em) + em->len;
2975	}
2976	btrfs_free_extent_map(em);
2977
2978	if (BTRFS_BYTES_TO_BLKS(fs_info, offset) ==
2979	    BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) {
2980		em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start, sectorsize);
2981		if (IS_ERR(em)) {
2982			ret = PTR_ERR(em);
2983			goto out;
2984		}
2985
2986		if (em->flags & EXTENT_FLAG_PREALLOC) {
2987			btrfs_free_extent_map(em);
2988			ret = btrfs_fallocate_update_isize(inode, offset + len,
2989							   mode);
2990			goto out;
2991		}
2992		if (len < sectorsize && em->disk_bytenr != EXTENT_MAP_HOLE) {
2993			btrfs_free_extent_map(em);
2994			ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1,
2995						   orig_start, orig_end);
2996			if (!ret)
2997				ret = btrfs_fallocate_update_isize(inode,
2998								   offset + len,
2999								   mode);
3000			return ret;
3001		}
3002		btrfs_free_extent_map(em);
3003		alloc_start = round_down(offset, sectorsize);
3004		alloc_end = alloc_start + sectorsize;
3005		goto reserve_space;
3006	}
3007
3008	alloc_start = round_up(offset, sectorsize);
3009	alloc_end = round_down(offset + len, sectorsize);
3010
3011	/*
3012	 * For unaligned ranges, check the pages at the boundaries, they might
3013	 * map to an extent, in which case we need to partially zero them, or
3014	 * they might map to a hole, in which case we need our allocation range
3015	 * to cover them.
3016	 */
3017	if (!IS_ALIGNED(offset, sectorsize)) {
3018		ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3019							    offset);
3020		if (ret < 0)
3021			goto out;
3022		if (ret == RANGE_BOUNDARY_HOLE) {
3023			alloc_start = round_down(offset, sectorsize);
3024			ret = 0;
3025		} else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3026			ret = btrfs_truncate_block(BTRFS_I(inode), offset,
3027						   orig_start, orig_end);
3028			if (ret)
3029				goto out;
3030		} else {
3031			ret = 0;
3032		}
3033	}
3034
3035	if (!IS_ALIGNED(offset + len, sectorsize)) {
3036		ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3037							    offset + len);
3038		if (ret < 0)
3039			goto out;
3040		if (ret == RANGE_BOUNDARY_HOLE) {
3041			alloc_end = round_up(offset + len, sectorsize);
3042			ret = 0;
3043		} else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3044			ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1,
3045						   orig_start, orig_end);
3046			if (ret)
3047				goto out;
3048		} else {
3049			ret = 0;
3050		}
3051	}
3052
3053reserve_space:
3054	if (alloc_start < alloc_end) {
3055		struct extent_state *cached_state = NULL;
3056		const u64 lockstart = alloc_start;
3057		const u64 lockend = alloc_end - 1;
3058
3059		bytes_to_reserve = alloc_end - alloc_start;
3060		ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3061						      bytes_to_reserve);
3062		if (ret < 0)
3063			goto out;
3064		space_reserved = true;
3065		btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3066					    &cached_state);
3067		ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved,
3068						alloc_start, bytes_to_reserve);
3069		if (ret) {
3070			btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart,
3071					    lockend, &cached_state);
3072			goto out;
3073		}
3074		ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
3075						alloc_end - alloc_start,
3076						fs_info->sectorsize,
3077						offset + len, &alloc_hint);
3078		btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3079				    &cached_state);
3080		/* btrfs_prealloc_file_range releases reserved space on error */
3081		if (ret) {
3082			space_reserved = false;
3083			goto out;
3084		}
3085	}
3086	ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
3087 out:
3088	if (ret && space_reserved)
3089		btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3090					       alloc_start, bytes_to_reserve);
3091	extent_changeset_free(data_reserved);
3092
3093	return ret;
3094}
3095
3096static long btrfs_fallocate(struct file *file, int mode,
3097			    loff_t offset, loff_t len)
3098{
3099	struct inode *inode = file_inode(file);
3100	struct extent_state *cached_state = NULL;
3101	struct extent_changeset *data_reserved = NULL;
3102	struct falloc_range *range;
3103	struct falloc_range *tmp;
3104	LIST_HEAD(reserve_list);
3105	u64 cur_offset;
3106	u64 last_byte;
3107	u64 alloc_start;
3108	u64 alloc_end;
3109	u64 alloc_hint = 0;
3110	u64 locked_end;
3111	u64 actual_end = 0;
3112	u64 data_space_needed = 0;
3113	u64 data_space_reserved = 0;
3114	u64 qgroup_reserved = 0;
3115	struct extent_map *em;
3116	int blocksize = BTRFS_I(inode)->root->fs_info->sectorsize;
3117	int ret;
3118
3119	if (unlikely(btrfs_is_shutdown(inode_to_fs_info(inode))))
3120		return -EIO;
3121
3122	/* Do not allow fallocate in ZONED mode */
3123	if (btrfs_is_zoned(inode_to_fs_info(inode)))
3124		return -EOPNOTSUPP;
3125
3126	alloc_start = round_down(offset, blocksize);
3127	alloc_end = round_up(offset + len, blocksize);
3128	cur_offset = alloc_start;
3129
3130	/* Make sure we aren't being give some crap mode */
3131	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3132		     FALLOC_FL_ZERO_RANGE))
3133		return -EOPNOTSUPP;
3134
3135	if (mode & FALLOC_FL_PUNCH_HOLE)
3136		return btrfs_punch_hole(file, offset, len);
3137
3138	btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
3139
3140	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
3141		ret = inode_newsize_ok(inode, offset + len);
3142		if (ret)
3143			goto out;
3144	}
3145
3146	ret = file_modified(file);
3147	if (ret)
3148		goto out;
3149
3150	/*
3151	 * TODO: Move these two operations after we have checked
3152	 * accurate reserved space, or fallocate can still fail but
3153	 * with page truncated or size expanded.
3154	 *
3155	 * But that's a minor problem and won't do much harm BTW.
3156	 */
3157	if (alloc_start > inode->i_size) {
3158		ret = btrfs_cont_expand(BTRFS_I(inode), i_size_read(inode),
3159					alloc_start);
3160		if (ret)
3161			goto out;
3162	} else if (offset + len > inode->i_size) {
3163		/*
3164		 * If we are fallocating from the end of the file onward we
3165		 * need to zero out the end of the block if i_size lands in the
3166		 * middle of a block.
3167		 */
3168		ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size,
3169					   inode->i_size, (u64)-1);
3170		if (ret)
3171			goto out;
3172	}
3173
3174	/*
3175	 * We have locked the inode at the VFS level (in exclusive mode) and we
3176	 * have locked the i_mmap_lock lock (in exclusive mode). Now before
3177	 * locking the file range, flush all dealloc in the range and wait for
3178	 * all ordered extents in the range to complete. After this we can lock
3179	 * the file range and, due to the previous locking we did, we know there
3180	 * can't be more delalloc or ordered extents in the range.
3181	 */
3182	ret = btrfs_wait_ordered_range(BTRFS_I(inode), alloc_start,
3183				       alloc_end - alloc_start);
3184	if (ret)
3185		goto out;
3186
3187	if (mode & FALLOC_FL_ZERO_RANGE) {
3188		ret = btrfs_zero_range(inode, offset, len, mode);
3189		btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
3190		return ret;
3191	}
3192
3193	locked_end = alloc_end - 1;
3194	btrfs_lock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3195			  &cached_state);
3196
3197	btrfs_assert_inode_range_clean(BTRFS_I(inode), alloc_start, locked_end);
3198
3199	/* First, check if we exceed the qgroup limit */
3200	while (cur_offset < alloc_end) {
3201		em = btrfs_get_extent(BTRFS_I(inode), NULL, cur_offset,
3202				      alloc_end - cur_offset);
3203		if (IS_ERR(em)) {
3204			ret = PTR_ERR(em);
3205			break;
3206		}
3207		last_byte = min(btrfs_extent_map_end(em), alloc_end);
3208		actual_end = min_t(u64, btrfs_extent_map_end(em), offset + len);
3209		last_byte = ALIGN(last_byte, blocksize);
3210		if (em->disk_bytenr == EXTENT_MAP_HOLE ||
3211		    (cur_offset >= inode->i_size &&
3212		     !(em->flags & EXTENT_FLAG_PREALLOC))) {
3213			const u64 range_len = last_byte - cur_offset;
3214
3215			ret = add_falloc_range(&reserve_list, cur_offset, range_len);
3216			if (ret < 0) {
3217				btrfs_free_extent_map(em);
3218				break;
3219			}
3220			ret = btrfs_qgroup_reserve_data(BTRFS_I(inode),
3221					&data_reserved, cur_offset, range_len);
3222			if (ret < 0) {
3223				btrfs_free_extent_map(em);
3224				break;
3225			}
3226			qgroup_reserved += range_len;
3227			data_space_needed += range_len;
3228		}
3229		btrfs_free_extent_map(em);
3230		cur_offset = last_byte;
3231	}
3232
3233	if (!ret && data_space_needed > 0) {
3234		/*
3235		 * We are safe to reserve space here as we can't have delalloc
3236		 * in the range, see above.
3237		 */
3238		ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3239						      data_space_needed);
3240		if (!ret)
3241			data_space_reserved = data_space_needed;
3242	}
3243
3244	/*
3245	 * If ret is still 0, means we're OK to fallocate.
3246	 * Or just cleanup the list and exit.
3247	 */
3248	list_for_each_entry_safe(range, tmp, &reserve_list, list) {
3249		if (!ret) {
3250			ret = btrfs_prealloc_file_range(inode, mode,
3251					range->start,
3252					range->len, blocksize,
3253					offset + len, &alloc_hint);
3254			/*
3255			 * btrfs_prealloc_file_range() releases space even
3256			 * if it returns an error.
3257			 */
3258			data_space_reserved -= range->len;
3259			qgroup_reserved -= range->len;
3260		} else if (data_space_reserved > 0) {
3261			btrfs_free_reserved_data_space(BTRFS_I(inode),
3262					       data_reserved, range->start,
3263					       range->len);
3264			data_space_reserved -= range->len;
3265			qgroup_reserved -= range->len;
3266		} else if (qgroup_reserved > 0) {
3267			btrfs_qgroup_free_data(BTRFS_I(inode), data_reserved,
3268					       range->start, range->len, NULL);
3269			qgroup_reserved -= range->len;
3270		}
3271		list_del(&range->list);
3272		kfree(range);
3273	}
3274	if (ret < 0)
3275		goto out_unlock;
3276
3277	/*
3278	 * We didn't need to allocate any more space, but we still extended the
3279	 * size of the file so we need to update i_size and the inode item.
3280	 */
3281	ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
3282out_unlock:
3283	btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3284			    &cached_state);
3285out:
3286	btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
3287	extent_changeset_free(data_reserved);
3288	return ret;
3289}
3290
3291/*
3292 * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range
3293 * that has unflushed and/or flushing delalloc. There might be other adjacent
3294 * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps
3295 * looping while it gets adjacent subranges, and merging them together.
3296 */
3297static bool find_delalloc_subrange(struct btrfs_inode *inode, u64 start, u64 end,
3298				   struct extent_state **cached_state,
3299				   bool *search_io_tree,
3300				   u64 *delalloc_start_ret, u64 *delalloc_end_ret)
3301{
3302	u64 len = end + 1 - start;
3303	u64 delalloc_len = 0;
3304	struct btrfs_ordered_extent *oe;
3305	u64 oe_start;
3306	u64 oe_end;
3307
3308	/*
3309	 * Search the io tree first for EXTENT_DELALLOC. If we find any, it
3310	 * means we have delalloc (dirty pages) for which writeback has not
3311	 * started yet.
3312	 */
3313	if (*search_io_tree) {
3314		spin_lock(&inode->lock);
3315		if (inode->delalloc_bytes > 0) {
3316			spin_unlock(&inode->lock);
3317			*delalloc_start_ret = start;
3318			delalloc_len = btrfs_count_range_bits(&inode->io_tree,
3319							      delalloc_start_ret, end,
3320							      len, EXTENT_DELALLOC, 1,
3321							      cached_state);
3322		} else {
3323			spin_unlock(&inode->lock);
3324		}
3325	}
3326
3327	if (delalloc_len > 0) {
3328		/*
3329		 * If delalloc was found then *delalloc_start_ret has a sector size
3330		 * aligned value (rounded down).
3331		 */
3332		*delalloc_end_ret = *delalloc_start_ret + delalloc_len - 1;
3333
3334		if (*delalloc_start_ret == start) {
3335			/* Delalloc for the whole range, nothing more to do. */
3336			if (*delalloc_end_ret == end)
3337				return true;
3338			/* Else trim our search range for ordered extents. */
3339			start = *delalloc_end_ret + 1;
3340			len = end + 1 - start;
3341		}
3342	} else {
3343		/* No delalloc, future calls don't need to search again. */
3344		*search_io_tree = false;
3345	}
3346
3347	/*
3348	 * Now also check if there's any ordered extent in the range.
3349	 * We do this because:
3350	 *
3351	 * 1) When delalloc is flushed, the file range is locked, we clear the
3352	 *    EXTENT_DELALLOC bit from the io tree and create an extent map and
3353	 *    an ordered extent for the write. So we might just have been called
3354	 *    after delalloc is flushed and before the ordered extent completes
3355	 *    and inserts the new file extent item in the subvolume's btree;
3356	 *
3357	 * 2) We may have an ordered extent created by flushing delalloc for a
3358	 *    subrange that starts before the subrange we found marked with
3359	 *    EXTENT_DELALLOC in the io tree.
3360	 *
3361	 * We could also use the extent map tree to find such delalloc that is
3362	 * being flushed, but using the ordered extents tree is more efficient
3363	 * because it's usually much smaller as ordered extents are removed from
3364	 * the tree once they complete. With the extent maps, we may have them
3365	 * in the extent map tree for a very long time, and they were either
3366	 * created by previous writes or loaded by read operations.
3367	 */
3368	oe = btrfs_lookup_first_ordered_range(inode, start, len);
3369	if (!oe)
3370		return (delalloc_len > 0);
3371
3372	/* The ordered extent may span beyond our search range. */
3373	oe_start = max(oe->file_offset, start);
3374	oe_end = min(oe->file_offset + oe->num_bytes - 1, end);
3375
3376	btrfs_put_ordered_extent(oe);
3377
3378	/* Don't have unflushed delalloc, return the ordered extent range. */
3379	if (delalloc_len == 0) {
3380		*delalloc_start_ret = oe_start;
3381		*delalloc_end_ret = oe_end;
3382		return true;
3383	}
3384
3385	/*
3386	 * We have both unflushed delalloc (io_tree) and an ordered extent.
3387	 * If the ranges are adjacent returned a combined range, otherwise
3388	 * return the leftmost range.
3389	 */
3390	if (oe_start < *delalloc_start_ret) {
3391		if (oe_end < *delalloc_start_ret)
3392			*delalloc_end_ret = oe_end;
3393		*delalloc_start_ret = oe_start;
3394	} else if (*delalloc_end_ret + 1 == oe_start) {
3395		*delalloc_end_ret = oe_end;
3396	}
3397
3398	return true;
3399}
3400
3401/*
3402 * Check if there's delalloc in a given range.
3403 *
3404 * @inode:               The inode.
3405 * @start:               The start offset of the range. It does not need to be
3406 *                       sector size aligned.
3407 * @end:                 The end offset (inclusive value) of the search range.
3408 *                       It does not need to be sector size aligned.
3409 * @cached_state:        Extent state record used for speeding up delalloc
3410 *                       searches in the inode's io_tree. Can be NULL.
3411 * @delalloc_start_ret:  Output argument, set to the start offset of the
3412 *                       subrange found with delalloc (may not be sector size
3413 *                       aligned).
3414 * @delalloc_end_ret:    Output argument, set to he end offset (inclusive value)
3415 *                       of the subrange found with delalloc.
3416 *
3417 * Returns true if a subrange with delalloc is found within the given range, and
3418 * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and
3419 * end offsets of the subrange.
3420 */
3421bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end,
3422				  struct extent_state **cached_state,
3423				  u64 *delalloc_start_ret, u64 *delalloc_end_ret)
3424{
3425	u64 cur_offset = round_down(start, inode->root->fs_info->sectorsize);
3426	u64 prev_delalloc_end = 0;
3427	bool search_io_tree = true;
3428	bool ret = false;
3429
3430	while (cur_offset <= end) {
3431		u64 delalloc_start;
3432		u64 delalloc_end;
3433		bool delalloc;
3434
3435		delalloc = find_delalloc_subrange(inode, cur_offset, end,
3436						  cached_state, &search_io_tree,
3437						  &delalloc_start,
3438						  &delalloc_end);
3439		if (!delalloc)
3440			break;
3441
3442		if (prev_delalloc_end == 0) {
3443			/* First subrange found. */
3444			*delalloc_start_ret = max(delalloc_start, start);
3445			*delalloc_end_ret = delalloc_end;
3446			ret = true;
3447		} else if (delalloc_start == prev_delalloc_end + 1) {
3448			/* Subrange adjacent to the previous one, merge them. */
3449			*delalloc_end_ret = delalloc_end;
3450		} else {
3451			/* Subrange not adjacent to the previous one, exit. */
3452			break;
3453		}
3454
3455		prev_delalloc_end = delalloc_end;
3456		cur_offset = delalloc_end + 1;
3457		cond_resched();
3458	}
3459
3460	return ret;
3461}
3462
3463/*
3464 * Check if there's a hole or delalloc range in a range representing a hole (or
3465 * prealloc extent) found in the inode's subvolume btree.
3466 *
3467 * @inode:      The inode.
3468 * @whence:     Seek mode (SEEK_DATA or SEEK_HOLE).
3469 * @start:      Start offset of the hole region. It does not need to be sector
3470 *              size aligned.
3471 * @end:        End offset (inclusive value) of the hole region. It does not
3472 *              need to be sector size aligned.
3473 * @start_ret:  Return parameter, used to set the start of the subrange in the
3474 *              hole that matches the search criteria (seek mode), if such
3475 *              subrange is found (return value of the function is true).
3476 *              The value returned here may not be sector size aligned.
3477 *
3478 * Returns true if a subrange matching the given seek mode is found, and if one
3479 * is found, it updates @start_ret with the start of the subrange.
3480 */
3481static bool find_desired_extent_in_hole(struct btrfs_inode *inode, int whence,
3482					struct extent_state **cached_state,
3483					u64 start, u64 end, u64 *start_ret)
3484{
3485	u64 delalloc_start;
3486	u64 delalloc_end;
3487	bool delalloc;
3488
3489	delalloc = btrfs_find_delalloc_in_range(inode, start, end, cached_state,
3490						&delalloc_start, &delalloc_end);
3491	if (delalloc && whence == SEEK_DATA) {
3492		*start_ret = delalloc_start;
3493		return true;
3494	}
3495
3496	if (delalloc && whence == SEEK_HOLE) {
3497		/*
3498		 * We found delalloc but it starts after out start offset. So we
3499		 * have a hole between our start offset and the delalloc start.
3500		 */
3501		if (start < delalloc_start) {
3502			*start_ret = start;
3503			return true;
3504		}
3505		/*
3506		 * Delalloc range starts at our start offset.
3507		 * If the delalloc range's length is smaller than our range,
3508		 * then it means we have a hole that starts where the delalloc
3509		 * subrange ends.
3510		 */
3511		if (delalloc_end < end) {
3512			*start_ret = delalloc_end + 1;
3513			return true;
3514		}
3515
3516		/* There's delalloc for the whole range. */
3517		return false;
3518	}
3519
3520	if (!delalloc && whence == SEEK_HOLE) {
3521		*start_ret = start;
3522		return true;
3523	}
3524
3525	/*
3526	 * No delalloc in the range and we are seeking for data. The caller has
3527	 * to iterate to the next extent item in the subvolume btree.
3528	 */
3529	return false;
3530}
3531
3532static loff_t find_desired_extent(struct file *file, loff_t offset, int whence)
3533{
3534	struct btrfs_inode *inode = BTRFS_I(file->f_mapping->host);
3535	struct btrfs_file_private *private;
3536	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3537	struct extent_state *cached_state = NULL;
3538	struct extent_state **delalloc_cached_state;
3539	const loff_t i_size = i_size_read(&inode->vfs_inode);
3540	const u64 ino = btrfs_ino(inode);
3541	struct btrfs_root *root = inode->root;
3542	struct btrfs_path *path;
3543	struct btrfs_key key;
3544	u64 last_extent_end;
3545	u64 lockstart;
3546	u64 lockend;
3547	u64 start;
3548	int ret;
3549	bool found = false;
3550
3551	if (i_size == 0 || offset >= i_size)
3552		return -ENXIO;
3553
3554	/*
3555	 * Quick path. If the inode has no prealloc extents and its number of
3556	 * bytes used matches its i_size, then it can not have holes.
3557	 */
3558	if (whence == SEEK_HOLE &&
3559	    !(inode->flags & BTRFS_INODE_PREALLOC) &&
3560	    inode_get_bytes(&inode->vfs_inode) == i_size)
3561		return i_size;
3562
3563	spin_lock(&inode->lock);
3564	private = file->private_data;
3565	spin_unlock(&inode->lock);
3566
3567	if (private && private->owner_task != current) {
3568		/*
3569		 * Not allocated by us, don't use it as its cached state is used
3570		 * by the task that allocated it and we don't want neither to
3571		 * mess with it nor get incorrect results because it reflects an
3572		 * invalid state for the current task.
3573		 */
3574		private = NULL;
3575	} else if (!private) {
3576		private = kzalloc(sizeof(*private), GFP_KERNEL);
3577		/*
3578		 * No worries if memory allocation failed.
3579		 * The private structure is used only for speeding up multiple
3580		 * lseek SEEK_HOLE/DATA calls to a file when there's delalloc,
3581		 * so everything will still be correct.
3582		 */
3583		if (private) {
3584			bool free = false;
3585
3586			private->owner_task = current;
3587
3588			spin_lock(&inode->lock);
3589			if (file->private_data)
3590				free = true;
3591			else
3592				file->private_data = private;
3593			spin_unlock(&inode->lock);
3594
3595			if (free) {
3596				kfree(private);
3597				private = NULL;
3598			}
3599		}
3600	}
3601
3602	if (private)
3603		delalloc_cached_state = &private->llseek_cached_state;
3604	else
3605		delalloc_cached_state = NULL;
3606
3607	/*
3608	 * offset can be negative, in this case we start finding DATA/HOLE from
3609	 * the very start of the file.
3610	 */
3611	start = max_t(loff_t, 0, offset);
3612
3613	lockstart = round_down(start, fs_info->sectorsize);
3614	lockend = round_up(i_size, fs_info->sectorsize);
3615	if (lockend <= lockstart)
3616		lockend = lockstart + fs_info->sectorsize;
3617	lockend--;
3618
3619	path = btrfs_alloc_path();
3620	if (!path)
3621		return -ENOMEM;
3622	path->reada = READA_FORWARD;
3623
3624	key.objectid = ino;
3625	key.type = BTRFS_EXTENT_DATA_KEY;
3626	key.offset = start;
3627
3628	last_extent_end = lockstart;
3629
3630	btrfs_lock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
3631
3632	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3633	if (ret < 0) {
3634		goto out;
3635	} else if (ret > 0 && path->slots[0] > 0) {
3636		btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
3637		if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
3638			path->slots[0]--;
3639	}
3640
3641	while (start < i_size) {
3642		struct extent_buffer *leaf = path->nodes[0];
3643		struct btrfs_file_extent_item *extent;
3644		u64 extent_end;
3645		u8 type;
3646
3647		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
3648			ret = btrfs_next_leaf(root, path);
3649			if (ret < 0)
3650				goto out;
3651			else if (ret > 0)
3652				break;
3653
3654			leaf = path->nodes[0];
3655		}
3656
3657		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3658		if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
3659			break;
3660
3661		extent_end = btrfs_file_extent_end(path);
3662
3663		/*
3664		 * In the first iteration we may have a slot that points to an
3665		 * extent that ends before our start offset, so skip it.
3666		 */
3667		if (extent_end <= start) {
3668			path->slots[0]++;
3669			continue;
3670		}
3671
3672		/* We have an implicit hole, NO_HOLES feature is likely set. */
3673		if (last_extent_end < key.offset) {
3674			u64 search_start = last_extent_end;
3675			u64 found_start;
3676
3677			/*
3678			 * First iteration, @start matches @offset and it's
3679			 * within the hole.
3680			 */
3681			if (start == offset)
3682				search_start = offset;
3683
3684			found = find_desired_extent_in_hole(inode, whence,
3685							    delalloc_cached_state,
3686							    search_start,
3687							    key.offset - 1,
3688							    &found_start);
3689			if (found) {
3690				start = found_start;
3691				break;
3692			}
3693			/*
3694			 * Didn't find data or a hole (due to delalloc) in the
3695			 * implicit hole range, so need to analyze the extent.
3696			 */
3697		}
3698
3699		extent = btrfs_item_ptr(leaf, path->slots[0],
3700					struct btrfs_file_extent_item);
3701		type = btrfs_file_extent_type(leaf, extent);
3702
3703		/*
3704		 * Can't access the extent's disk_bytenr field if this is an
3705		 * inline extent, since at that offset, it's where the extent
3706		 * data starts.
3707		 */
3708		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
3709		    (type == BTRFS_FILE_EXTENT_REG &&
3710		     btrfs_file_extent_disk_bytenr(leaf, extent) == 0)) {
3711			/*
3712			 * Explicit hole or prealloc extent, search for delalloc.
3713			 * A prealloc extent is treated like a hole.
3714			 */
3715			u64 search_start = key.offset;
3716			u64 found_start;
3717
3718			/*
3719			 * First iteration, @start matches @offset and it's
3720			 * within the hole.
3721			 */
3722			if (start == offset)
3723				search_start = offset;
3724
3725			found = find_desired_extent_in_hole(inode, whence,
3726							    delalloc_cached_state,
3727							    search_start,
3728							    extent_end - 1,
3729							    &found_start);
3730			if (found) {
3731				start = found_start;
3732				break;
3733			}
3734			/*
3735			 * Didn't find data or a hole (due to delalloc) in the
3736			 * implicit hole range, so need to analyze the next
3737			 * extent item.
3738			 */
3739		} else {
3740			/*
3741			 * Found a regular or inline extent.
3742			 * If we are seeking for data, adjust the start offset
3743			 * and stop, we're done.
3744			 */
3745			if (whence == SEEK_DATA) {
3746				start = max_t(u64, key.offset, offset);
3747				found = true;
3748				break;
3749			}
3750			/*
3751			 * Else, we are seeking for a hole, check the next file
3752			 * extent item.
3753			 */
3754		}
3755
3756		start = extent_end;
3757		last_extent_end = extent_end;
3758		path->slots[0]++;
3759		if (fatal_signal_pending(current)) {
3760			ret = -EINTR;
3761			goto out;
3762		}
3763		cond_resched();
3764	}
3765
3766	/* We have an implicit hole from the last extent found up to i_size. */
3767	if (!found && start < i_size) {
3768		found = find_desired_extent_in_hole(inode, whence,
3769						    delalloc_cached_state, start,
3770						    i_size - 1, &start);
3771		if (!found)
3772			start = i_size;
3773	}
3774
3775out:
3776	btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
3777	btrfs_free_path(path);
3778
3779	if (ret < 0)
3780		return ret;
3781
3782	if (whence == SEEK_DATA && start >= i_size)
3783		return -ENXIO;
3784
3785	return min_t(loff_t, start, i_size);
3786}
3787
3788static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
3789{
3790	struct inode *inode = file->f_mapping->host;
3791
3792	switch (whence) {
3793	default:
3794		return generic_file_llseek(file, offset, whence);
3795	case SEEK_DATA:
3796	case SEEK_HOLE:
3797		btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_SHARED);
3798		offset = find_desired_extent(file, offset, whence);
3799		btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_SHARED);
3800		break;
3801	}
3802
3803	if (offset < 0)
3804		return offset;
3805
3806	return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
3807}
3808
3809static int btrfs_file_open(struct inode *inode, struct file *filp)
3810{
3811	int ret;
3812
3813	if (unlikely(btrfs_is_shutdown(inode_to_fs_info(inode))))
3814		return -EIO;
3815
3816	filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
3817
3818	ret = fsverity_file_open(inode, filp);
3819	if (ret)
3820		return ret;
3821	return generic_file_open(inode, filp);
3822}
3823
3824static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
3825{
3826	ssize_t ret = 0;
3827
3828	if (unlikely(btrfs_is_shutdown(inode_to_fs_info(file_inode(iocb->ki_filp)))))
3829		return -EIO;
3830
3831	if (iocb->ki_flags & IOCB_DIRECT) {
3832		ret = btrfs_direct_read(iocb, to);
3833		if (ret < 0 || !iov_iter_count(to) ||
3834		    iocb->ki_pos >= i_size_read(file_inode(iocb->ki_filp)))
3835			return ret;
3836	}
3837
3838	return filemap_read(iocb, to, ret);
3839}
3840
3841static ssize_t btrfs_file_splice_read(struct file *in, loff_t *ppos,
3842				      struct pipe_inode_info *pipe,
3843				      size_t len, unsigned int flags)
3844{
3845	if (unlikely(btrfs_is_shutdown(inode_to_fs_info(file_inode(in)))))
3846		return -EIO;
3847
3848	return filemap_splice_read(in, ppos, pipe, len, flags);
3849}
3850
3851const struct file_operations btrfs_file_operations = {
3852	.llseek		= btrfs_file_llseek,
3853	.read_iter      = btrfs_file_read_iter,
3854	.splice_read	= btrfs_file_splice_read,
3855	.write_iter	= btrfs_file_write_iter,
3856	.splice_write	= iter_file_splice_write,
3857	.mmap_prepare	= btrfs_file_mmap_prepare,
3858	.open		= btrfs_file_open,
3859	.release	= btrfs_release_file,
3860	.get_unmapped_area = thp_get_unmapped_area,
3861	.fsync		= btrfs_sync_file,
3862	.fallocate	= btrfs_fallocate,
3863	.unlocked_ioctl	= btrfs_ioctl,
3864#ifdef CONFIG_COMPAT
3865	.compat_ioctl	= btrfs_compat_ioctl,
3866#endif
3867	.remap_file_range = btrfs_remap_file_range,
3868	.uring_cmd	= btrfs_uring_cmd,
3869	.fop_flags	= FOP_BUFFER_RASYNC | FOP_BUFFER_WASYNC,
3870};
3871
3872int btrfs_fdatawrite_range(struct btrfs_inode *inode, loff_t start, loff_t end)
3873{
3874	struct address_space *mapping = inode->vfs_inode.i_mapping;
3875	int ret;
3876
3877	/*
3878	 * So with compression we will find and lock a dirty page and clear the
3879	 * first one as dirty, setup an async extent, and immediately return
3880	 * with the entire range locked but with nobody actually marked with
3881	 * writeback.  So we can't just filemap_write_and_wait_range() and
3882	 * expect it to work since it will just kick off a thread to do the
3883	 * actual work.  So we need to call filemap_fdatawrite_range _again_
3884	 * since it will wait on the page lock, which won't be unlocked until
3885	 * after the pages have been marked as writeback and so we're good to go
3886	 * from there.  We have to do this otherwise we'll miss the ordered
3887	 * extents and that results in badness.  Please Josef, do not think you
3888	 * know better and pull this out at some point in the future, it is
3889	 * right and you are wrong.
3890	 */
3891	ret = filemap_fdatawrite_range(mapping, start, end);
3892	if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags))
3893		ret = filemap_fdatawrite_range(mapping, start, end);
3894
3895	return ret;
3896}