FILE
inode.c
File preview with syntax highlighting.
linux-6.19.14/fs/btrfs/inode.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2007 Oracle. All rights reserved. 4 */ 5 6#include <crypto/hash.h> 7#include <linux/kernel.h> 8#include <linux/bio.h> 9#include <linux/blk-cgroup.h> 10#include <linux/file.h> 11#include <linux/fs.h> 12#include <linux/fs_struct.h> 13#include <linux/pagemap.h> 14#include <linux/highmem.h> 15#include <linux/time.h> 16#include <linux/init.h> 17#include <linux/string.h> 18#include <linux/backing-dev.h> 19#include <linux/writeback.h> 20#include <linux/compat.h> 21#include <linux/xattr.h> 22#include <linux/posix_acl.h> 23#include <linux/falloc.h> 24#include <linux/slab.h> 25#include <linux/ratelimit.h> 26#include <linux/btrfs.h> 27#include <linux/blkdev.h> 28#include <linux/posix_acl_xattr.h> 29#include <linux/uio.h> 30#include <linux/magic.h> 31#include <linux/iversion.h> 32#include <linux/swap.h> 33#include <linux/migrate.h> 34#include <linux/sched/mm.h> 35#include <linux/iomap.h> 36#include <linux/unaligned.h> 37#include <linux/fsverity.h> 38#include "misc.h" 39#include "ctree.h" 40#include "disk-io.h" 41#include "transaction.h" 42#include "btrfs_inode.h" 43#include "ordered-data.h" 44#include "xattr.h" 45#include "tree-log.h" 46#include "bio.h" 47#include "compression.h" 48#include "locking.h" 49#include "props.h" 50#include "qgroup.h" 51#include "delalloc-space.h" 52#include "block-group.h" 53#include "space-info.h" 54#include "zoned.h" 55#include "subpage.h" 56#include "inode-item.h" 57#include "fs.h" 58#include "accessors.h" 59#include "extent-tree.h" 60#include "root-tree.h" 61#include "defrag.h" 62#include "dir-item.h" 63#include "file-item.h" 64#include "uuid-tree.h" 65#include "ioctl.h" 66#include "file.h" 67#include "acl.h" 68#include "relocation.h" 69#include "verity.h" 70#include "super.h" 71#include "orphan.h" 72#include "backref.h" 73#include "raid-stripe-tree.h" 74#include "fiemap.h" 75#include "delayed-inode.h" 76 77#define COW_FILE_RANGE_KEEP_LOCKED (1UL << 0) 78#define COW_FILE_RANGE_NO_INLINE (1UL << 1) 79 80struct btrfs_iget_args { 81 u64 ino; 82 struct btrfs_root *root; 83}; 84 85struct btrfs_rename_ctx { 86 /* Output field. Stores the index number of the old directory entry. */ 87 u64 index; 88}; 89 90/* 91 * Used by data_reloc_print_warning_inode() to pass needed info for filename 92 * resolution and output of error message. 93 */ 94struct data_reloc_warn { 95 struct btrfs_path path; 96 struct btrfs_fs_info *fs_info; 97 u64 extent_item_size; 98 u64 logical; 99 int mirror_num; 100}; 101 102/* 103 * For the file_extent_tree, we want to hold the inode lock when we lookup and 104 * update the disk_i_size, but lockdep will complain because our io_tree we hold 105 * the tree lock and get the inode lock when setting delalloc. These two things 106 * are unrelated, so make a class for the file_extent_tree so we don't get the 107 * two locking patterns mixed up. 108 */ 109static struct lock_class_key file_extent_tree_class; 110 111static const struct inode_operations btrfs_dir_inode_operations; 112static const struct inode_operations btrfs_symlink_inode_operations; 113static const struct inode_operations btrfs_special_inode_operations; 114static const struct inode_operations btrfs_file_inode_operations; 115static const struct address_space_operations btrfs_aops; 116static const struct file_operations btrfs_dir_file_operations; 117 118static struct kmem_cache *btrfs_inode_cachep; 119 120static int btrfs_setsize(struct inode *inode, struct iattr *attr); 121static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback); 122 123static noinline int run_delalloc_cow(struct btrfs_inode *inode, 124 struct folio *locked_folio, u64 start, 125 u64 end, struct writeback_control *wbc, 126 bool pages_dirty); 127 128static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes, 129 u64 root, void *warn_ctx) 130{ 131 struct data_reloc_warn *warn = warn_ctx; 132 struct btrfs_fs_info *fs_info = warn->fs_info; 133 struct extent_buffer *eb; 134 struct btrfs_inode_item *inode_item; 135 struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; 136 struct btrfs_root *local_root; 137 struct btrfs_key key; 138 unsigned int nofs_flag; 139 u32 nlink; 140 int ret; 141 142 local_root = btrfs_get_fs_root(fs_info, root, true); 143 if (IS_ERR(local_root)) { 144 ret = PTR_ERR(local_root); 145 goto err; 146 } 147 148 /* This makes the path point to (inum INODE_ITEM ioff). */ 149 key.objectid = inum; 150 key.type = BTRFS_INODE_ITEM_KEY; 151 key.offset = 0; 152 153 ret = btrfs_search_slot(NULL, local_root, &key, &warn->path, 0, 0); 154 if (ret) { 155 btrfs_put_root(local_root); 156 btrfs_release_path(&warn->path); 157 goto err; 158 } 159 160 eb = warn->path.nodes[0]; 161 inode_item = btrfs_item_ptr(eb, warn->path.slots[0], struct btrfs_inode_item); 162 nlink = btrfs_inode_nlink(eb, inode_item); 163 btrfs_release_path(&warn->path); 164 165 nofs_flag = memalloc_nofs_save(); 166 ipath = init_ipath(4096, local_root, &warn->path); 167 memalloc_nofs_restore(nofs_flag); 168 if (IS_ERR(ipath)) { 169 btrfs_put_root(local_root); 170 ret = PTR_ERR(ipath); 171 ipath = NULL; 172 /* 173 * -ENOMEM, not a critical error, just output an generic error 174 * without filename. 175 */ 176 btrfs_warn(fs_info, 177"checksum error at logical %llu mirror %u root %llu, inode %llu offset %llu", 178 warn->logical, warn->mirror_num, root, inum, offset); 179 return ret; 180 } 181 ret = paths_from_inode(inum, ipath); 182 if (ret < 0) { 183 btrfs_put_root(local_root); 184 goto err; 185 } 186 187 /* 188 * We deliberately ignore the bit ipath might have been too small to 189 * hold all of the paths here 190 */ 191 for (int i = 0; i < ipath->fspath->elem_cnt; i++) { 192 btrfs_warn(fs_info, 193"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu length %u links %u (path: %s)", 194 warn->logical, warn->mirror_num, root, inum, offset, 195 fs_info->sectorsize, nlink, 196 (char *)(unsigned long)ipath->fspath->val[i]); 197 } 198 199 btrfs_put_root(local_root); 200 return 0; 201 202err: 203 btrfs_warn(fs_info, 204"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu, path resolving failed with ret=%d", 205 warn->logical, warn->mirror_num, root, inum, offset, ret); 206 207 return ret; 208} 209 210/* 211 * Do extra user-friendly error output (e.g. lookup all the affected files). 212 * 213 * Return true if we succeeded doing the backref lookup. 214 * Return false if such lookup failed, and has to fallback to the old error message. 215 */ 216static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off, 217 const u8 *csum, const u8 *csum_expected, 218 int mirror_num) 219{ 220 struct btrfs_fs_info *fs_info = inode->root->fs_info; 221 struct btrfs_path path = { 0 }; 222 struct btrfs_key found_key = { 0 }; 223 struct extent_buffer *eb; 224 struct btrfs_extent_item *ei; 225 const u32 csum_size = fs_info->csum_size; 226 u64 logical; 227 u64 flags; 228 u32 item_size; 229 int ret; 230 231 mutex_lock(&fs_info->reloc_mutex); 232 logical = btrfs_get_reloc_bg_bytenr(fs_info); 233 mutex_unlock(&fs_info->reloc_mutex); 234 235 if (logical == U64_MAX) { 236 btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation"); 237 btrfs_warn_rl(fs_info, 238"csum failed root %lld ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", 239 btrfs_root_id(inode->root), btrfs_ino(inode), file_off, 240 BTRFS_CSUM_FMT_VALUE(csum_size, csum), 241 BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), 242 mirror_num); 243 return; 244 } 245 246 logical += file_off; 247 btrfs_warn_rl(fs_info, 248"csum failed root %lld ino %llu off %llu logical %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", 249 btrfs_root_id(inode->root), 250 btrfs_ino(inode), file_off, logical, 251 BTRFS_CSUM_FMT_VALUE(csum_size, csum), 252 BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), 253 mirror_num); 254 255 ret = extent_from_logical(fs_info, logical, &path, &found_key, &flags); 256 if (ret < 0) { 257 btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %d", 258 logical, ret); 259 btrfs_release_path(&path); 260 return; 261 } 262 eb = path.nodes[0]; 263 ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item); 264 item_size = btrfs_item_size(eb, path.slots[0]); 265 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { 266 unsigned long ptr = 0; 267 u64 ref_root; 268 u8 ref_level; 269 270 while (true) { 271 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei, 272 item_size, &ref_root, 273 &ref_level); 274 if (ret < 0) { 275 btrfs_warn_rl(fs_info, 276 "failed to resolve tree backref for logical %llu: %d", 277 logical, ret); 278 break; 279 } 280 if (ret > 0) 281 break; 282 283 btrfs_warn_rl(fs_info, 284"csum error at logical %llu mirror %u: metadata %s (level %d) in tree %llu", 285 logical, mirror_num, 286 (ref_level ? "node" : "leaf"), 287 ref_level, ref_root); 288 } 289 btrfs_release_path(&path); 290 } else { 291 struct btrfs_backref_walk_ctx ctx = { 0 }; 292 struct data_reloc_warn reloc_warn = { 0 }; 293 294 btrfs_release_path(&path); 295 296 ctx.bytenr = found_key.objectid; 297 ctx.extent_item_pos = logical - found_key.objectid; 298 ctx.fs_info = fs_info; 299 300 reloc_warn.logical = logical; 301 reloc_warn.extent_item_size = found_key.offset; 302 reloc_warn.mirror_num = mirror_num; 303 reloc_warn.fs_info = fs_info; 304 305 iterate_extent_inodes(&ctx, true, 306 data_reloc_print_warning_inode, &reloc_warn); 307 } 308} 309 310static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode, 311 u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num) 312{ 313 struct btrfs_root *root = inode->root; 314 const u32 csum_size = root->fs_info->csum_size; 315 316 /* For data reloc tree, it's better to do a backref lookup instead. */ 317 if (btrfs_is_data_reloc_root(root)) 318 return print_data_reloc_error(inode, logical_start, csum, 319 csum_expected, mirror_num); 320 321 /* Output without objectid, which is more meaningful */ 322 if (btrfs_root_id(root) >= BTRFS_LAST_FREE_OBJECTID) { 323 btrfs_warn_rl(root->fs_info, 324"csum failed root %lld ino %lld off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", 325 btrfs_root_id(root), btrfs_ino(inode), 326 logical_start, 327 BTRFS_CSUM_FMT_VALUE(csum_size, csum), 328 BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), 329 mirror_num); 330 } else { 331 btrfs_warn_rl(root->fs_info, 332"csum failed root %llu ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", 333 btrfs_root_id(root), btrfs_ino(inode), 334 logical_start, 335 BTRFS_CSUM_FMT_VALUE(csum_size, csum), 336 BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), 337 mirror_num); 338 } 339} 340 341/* 342 * Lock inode i_rwsem based on arguments passed. 343 * 344 * ilock_flags can have the following bit set: 345 * 346 * BTRFS_ILOCK_SHARED - acquire a shared lock on the inode 347 * BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt 348 * return -EAGAIN 349 * BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock 350 */ 351int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags) 352{ 353 if (ilock_flags & BTRFS_ILOCK_SHARED) { 354 if (ilock_flags & BTRFS_ILOCK_TRY) { 355 if (!inode_trylock_shared(&inode->vfs_inode)) 356 return -EAGAIN; 357 else 358 return 0; 359 } 360 inode_lock_shared(&inode->vfs_inode); 361 } else { 362 if (ilock_flags & BTRFS_ILOCK_TRY) { 363 if (!inode_trylock(&inode->vfs_inode)) 364 return -EAGAIN; 365 else 366 return 0; 367 } 368 inode_lock(&inode->vfs_inode); 369 } 370 if (ilock_flags & BTRFS_ILOCK_MMAP) 371 down_write(&inode->i_mmap_lock); 372 return 0; 373} 374 375/* 376 * Unlock inode i_rwsem. 377 * 378 * ilock_flags should contain the same bits set as passed to btrfs_inode_lock() 379 * to decide whether the lock acquired is shared or exclusive. 380 */ 381void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags) 382{ 383 if (ilock_flags & BTRFS_ILOCK_MMAP) 384 up_write(&inode->i_mmap_lock); 385 if (ilock_flags & BTRFS_ILOCK_SHARED) 386 inode_unlock_shared(&inode->vfs_inode); 387 else 388 inode_unlock(&inode->vfs_inode); 389} 390 391/* 392 * Cleanup all submitted ordered extents in specified range to handle errors 393 * from the btrfs_run_delalloc_range() callback. 394 * 395 * NOTE: caller must ensure that when an error happens, it can not call 396 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING 397 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata 398 * to be released, which we want to happen only when finishing the ordered 399 * extent (btrfs_finish_ordered_io()). 400 */ 401static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, 402 u64 offset, u64 bytes) 403{ 404 pgoff_t index = offset >> PAGE_SHIFT; 405 const pgoff_t end_index = (offset + bytes - 1) >> PAGE_SHIFT; 406 struct folio *folio; 407 408 while (index <= end_index) { 409 folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); 410 if (IS_ERR(folio)) { 411 index++; 412 continue; 413 } 414 415 index = folio_next_index(folio); 416 /* 417 * Here we just clear all Ordered bits for every page in the 418 * range, then btrfs_mark_ordered_io_finished() will handle 419 * the ordered extent accounting for the range. 420 */ 421 btrfs_folio_clamp_clear_ordered(inode->root->fs_info, folio, 422 offset, bytes); 423 folio_put(folio); 424 } 425 426 return btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, false); 427} 428 429static int btrfs_dirty_inode(struct btrfs_inode *inode); 430 431static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, 432 struct btrfs_new_inode_args *args) 433{ 434 int ret; 435 436 if (args->default_acl) { 437 ret = __btrfs_set_acl(trans, args->inode, args->default_acl, 438 ACL_TYPE_DEFAULT); 439 if (ret) 440 return ret; 441 } 442 if (args->acl) { 443 ret = __btrfs_set_acl(trans, args->inode, args->acl, ACL_TYPE_ACCESS); 444 if (ret) 445 return ret; 446 } 447 if (!args->default_acl && !args->acl) 448 cache_no_acl(args->inode); 449 return btrfs_xattr_security_init(trans, args->inode, args->dir, 450 &args->dentry->d_name); 451} 452 453/* 454 * this does all the hard work for inserting an inline extent into 455 * the btree. The caller should have done a btrfs_drop_extents so that 456 * no overlapping inline items exist in the btree 457 */ 458static int insert_inline_extent(struct btrfs_trans_handle *trans, 459 struct btrfs_path *path, 460 struct btrfs_inode *inode, bool extent_inserted, 461 size_t size, size_t compressed_size, 462 int compress_type, 463 struct folio *compressed_folio, 464 bool update_i_size) 465{ 466 struct btrfs_root *root = inode->root; 467 struct extent_buffer *leaf; 468 const u32 sectorsize = trans->fs_info->sectorsize; 469 char *kaddr; 470 unsigned long ptr; 471 struct btrfs_file_extent_item *ei; 472 int ret; 473 size_t cur_size = size; 474 u64 i_size; 475 476 /* 477 * The decompressed size must still be no larger than a sector. Under 478 * heavy race, we can have size == 0 passed in, but that shouldn't be a 479 * big deal and we can continue the insertion. 480 */ 481 ASSERT(size <= sectorsize); 482 483 /* 484 * The compressed size also needs to be no larger than a page. 485 * That's also why we only need one folio as the parameter. 486 */ 487 if (compressed_folio) { 488 ASSERT(compressed_size <= sectorsize); 489 ASSERT(compressed_size <= PAGE_SIZE); 490 } else { 491 ASSERT(compressed_size == 0); 492 } 493 494 if (compressed_size && compressed_folio) 495 cur_size = compressed_size; 496 497 if (!extent_inserted) { 498 struct btrfs_key key; 499 size_t datasize; 500 501 key.objectid = btrfs_ino(inode); 502 key.type = BTRFS_EXTENT_DATA_KEY; 503 key.offset = 0; 504 505 datasize = btrfs_file_extent_calc_inline_size(cur_size); 506 ret = btrfs_insert_empty_item(trans, root, path, &key, 507 datasize); 508 if (ret) 509 goto fail; 510 } 511 leaf = path->nodes[0]; 512 ei = btrfs_item_ptr(leaf, path->slots[0], 513 struct btrfs_file_extent_item); 514 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 515 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); 516 btrfs_set_file_extent_encryption(leaf, ei, 0); 517 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 518 btrfs_set_file_extent_ram_bytes(leaf, ei, size); 519 ptr = btrfs_file_extent_inline_start(ei); 520 521 if (compress_type != BTRFS_COMPRESS_NONE) { 522 kaddr = kmap_local_folio(compressed_folio, 0); 523 write_extent_buffer(leaf, kaddr, ptr, compressed_size); 524 kunmap_local(kaddr); 525 526 btrfs_set_file_extent_compression(leaf, ei, 527 compress_type); 528 } else { 529 struct folio *folio; 530 531 folio = filemap_get_folio(inode->vfs_inode.i_mapping, 0); 532 ASSERT(!IS_ERR(folio)); 533 btrfs_set_file_extent_compression(leaf, ei, 0); 534 kaddr = kmap_local_folio(folio, 0); 535 write_extent_buffer(leaf, kaddr, ptr, size); 536 kunmap_local(kaddr); 537 folio_put(folio); 538 } 539 btrfs_release_path(path); 540 541 /* 542 * We align size to sectorsize for inline extents just for simplicity 543 * sake. 544 */ 545 ret = btrfs_inode_set_file_extent_range(inode, 0, 546 ALIGN(size, root->fs_info->sectorsize)); 547 if (ret) 548 goto fail; 549 550 /* 551 * We're an inline extent, so nobody can extend the file past i_size 552 * without locking a page we already have locked. 553 * 554 * We must do any i_size and inode updates before we unlock the pages. 555 * Otherwise we could end up racing with unlink. 556 */ 557 i_size = i_size_read(&inode->vfs_inode); 558 if (update_i_size && size > i_size) { 559 i_size_write(&inode->vfs_inode, size); 560 i_size = size; 561 } 562 inode->disk_i_size = i_size; 563 564fail: 565 return ret; 566} 567 568static bool can_cow_file_range_inline(struct btrfs_inode *inode, 569 u64 offset, u64 size, 570 size_t compressed_size) 571{ 572 struct btrfs_fs_info *fs_info = inode->root->fs_info; 573 u64 data_len = (compressed_size ?: size); 574 575 /* Inline extents must start at offset 0. */ 576 if (offset != 0) 577 return false; 578 579 /* 580 * Even for bs > ps cases, cow_file_range_inline() can only accept a 581 * single folio. 582 * 583 * This can be problematic and cause access beyond page boundary if a 584 * page sized folio is passed into that function. 585 * And encoded write is doing exactly that. 586 * So here limits the inlined extent size to PAGE_SIZE. 587 */ 588 if (size > PAGE_SIZE || compressed_size > PAGE_SIZE) 589 return false; 590 591 /* Inline extents are limited to sectorsize. */ 592 if (size > fs_info->sectorsize) 593 return false; 594 595 /* We do not allow a non-compressed extent to be as large as block size. */ 596 if (data_len >= fs_info->sectorsize) 597 return false; 598 599 /* We cannot exceed the maximum inline data size. */ 600 if (data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) 601 return false; 602 603 /* We cannot exceed the user specified max_inline size. */ 604 if (data_len > fs_info->max_inline) 605 return false; 606 607 /* Inline extents must be the entirety of the file. */ 608 if (size < i_size_read(&inode->vfs_inode)) 609 return false; 610 611 /* Encrypted file cannot be inlined. */ 612 if (IS_ENCRYPTED(&inode->vfs_inode)) 613 return false; 614 615 return true; 616} 617 618/* 619 * conditionally insert an inline extent into the file. This 620 * does the checks required to make sure the data is small enough 621 * to fit as an inline extent. 622 * 623 * If being used directly, you must have already checked we're allowed to cow 624 * the range by getting true from can_cow_file_range_inline(). 625 */ 626static noinline int __cow_file_range_inline(struct btrfs_inode *inode, 627 u64 size, size_t compressed_size, 628 int compress_type, 629 struct folio *compressed_folio, 630 bool update_i_size) 631{ 632 struct btrfs_drop_extents_args drop_args = { 0 }; 633 struct btrfs_root *root = inode->root; 634 struct btrfs_fs_info *fs_info = root->fs_info; 635 struct btrfs_trans_handle *trans = NULL; 636 u64 data_len = (compressed_size ?: size); 637 int ret; 638 struct btrfs_path *path; 639 640 path = btrfs_alloc_path(); 641 if (!path) { 642 ret = -ENOMEM; 643 goto out; 644 } 645 646 trans = btrfs_join_transaction(root); 647 if (IS_ERR(trans)) { 648 ret = PTR_ERR(trans); 649 trans = NULL; 650 goto out; 651 } 652 trans->block_rsv = &inode->block_rsv; 653 654 drop_args.path = path; 655 drop_args.start = 0; 656 drop_args.end = fs_info->sectorsize; 657 drop_args.drop_cache = true; 658 drop_args.replace_extent = true; 659 drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(data_len); 660 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 661 if (unlikely(ret)) { 662 btrfs_abort_transaction(trans, ret); 663 goto out; 664 } 665 666 ret = insert_inline_extent(trans, path, inode, drop_args.extent_inserted, 667 size, compressed_size, compress_type, 668 compressed_folio, update_i_size); 669 if (unlikely(ret && ret != -ENOSPC)) { 670 btrfs_abort_transaction(trans, ret); 671 goto out; 672 } else if (ret == -ENOSPC) { 673 ret = 1; 674 goto out; 675 } 676 677 btrfs_update_inode_bytes(inode, size, drop_args.bytes_found); 678 ret = btrfs_update_inode(trans, inode); 679 if (unlikely(ret && ret != -ENOSPC)) { 680 btrfs_abort_transaction(trans, ret); 681 goto out; 682 } else if (ret == -ENOSPC) { 683 ret = 1; 684 goto out; 685 } 686 687 btrfs_set_inode_full_sync(inode); 688out: 689 /* 690 * Don't forget to free the reserved space, as for inlined extent 691 * it won't count as data extent, free them directly here. 692 * And at reserve time, it's always aligned to page size, so 693 * just free one page here. 694 * 695 * If we fallback to non-inline (ret == 1) due to -ENOSPC, then we need 696 * to keep the data reservation. 697 */ 698 if (ret <= 0) 699 btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL); 700 btrfs_free_path(path); 701 if (trans) 702 btrfs_end_transaction(trans); 703 return ret; 704} 705 706static noinline int cow_file_range_inline(struct btrfs_inode *inode, 707 struct folio *locked_folio, 708 u64 offset, u64 end, 709 size_t compressed_size, 710 int compress_type, 711 struct folio *compressed_folio, 712 bool update_i_size) 713{ 714 struct extent_state *cached = NULL; 715 unsigned long clear_flags = EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 716 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING | EXTENT_LOCKED; 717 u64 size = min_t(u64, i_size_read(&inode->vfs_inode), end + 1); 718 int ret; 719 720 if (!can_cow_file_range_inline(inode, offset, size, compressed_size)) 721 return 1; 722 723 btrfs_lock_extent(&inode->io_tree, offset, end, &cached); 724 ret = __cow_file_range_inline(inode, size, compressed_size, 725 compress_type, compressed_folio, 726 update_i_size); 727 if (ret > 0) { 728 btrfs_unlock_extent(&inode->io_tree, offset, end, &cached); 729 return ret; 730 } 731 732 /* 733 * In the successful case (ret == 0 here), cow_file_range will return 1. 734 * 735 * Quite a bit further up the callstack in extent_writepage(), ret == 1 736 * is treated as a short circuited success and does not unlock the folio, 737 * so we must do it here. 738 * 739 * In the failure case, the locked_folio does get unlocked by 740 * btrfs_folio_end_all_writers, which asserts that it is still locked 741 * at that point, so we must *not* unlock it here. 742 * 743 * The other two callsites in compress_file_range do not have a 744 * locked_folio, so they are not relevant to this logic. 745 */ 746 if (ret == 0) 747 locked_folio = NULL; 748 749 extent_clear_unlock_delalloc(inode, offset, end, locked_folio, &cached, 750 clear_flags, PAGE_UNLOCK | 751 PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); 752 return ret; 753} 754 755struct async_extent { 756 u64 start; 757 u64 ram_size; 758 u64 compressed_size; 759 struct folio **folios; 760 unsigned long nr_folios; 761 int compress_type; 762 struct list_head list; 763}; 764 765struct async_chunk { 766 struct btrfs_inode *inode; 767 struct folio *locked_folio; 768 u64 start; 769 u64 end; 770 blk_opf_t write_flags; 771 struct list_head extents; 772 struct cgroup_subsys_state *blkcg_css; 773 struct btrfs_work work; 774 struct async_cow *async_cow; 775}; 776 777struct async_cow { 778 atomic_t num_chunks; 779 struct async_chunk chunks[]; 780}; 781 782static noinline int add_async_extent(struct async_chunk *cow, 783 u64 start, u64 ram_size, 784 u64 compressed_size, 785 struct folio **folios, 786 unsigned long nr_folios, 787 int compress_type) 788{ 789 struct async_extent *async_extent; 790 791 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS); 792 if (!async_extent) 793 return -ENOMEM; 794 async_extent->start = start; 795 async_extent->ram_size = ram_size; 796 async_extent->compressed_size = compressed_size; 797 async_extent->folios = folios; 798 async_extent->nr_folios = nr_folios; 799 async_extent->compress_type = compress_type; 800 list_add_tail(&async_extent->list, &cow->extents); 801 return 0; 802} 803 804/* 805 * Check if the inode needs to be submitted to compression, based on mount 806 * options, defragmentation, properties or heuristics. 807 */ 808static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, 809 u64 end) 810{ 811 struct btrfs_fs_info *fs_info = inode->root->fs_info; 812 813 if (!btrfs_inode_can_compress(inode)) { 814 DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode)); 815 return 0; 816 } 817 818 /* Defrag ioctl takes precedence over mount options and properties. */ 819 if (inode->defrag_compress == BTRFS_DEFRAG_DONT_COMPRESS) 820 return 0; 821 if (BTRFS_COMPRESS_NONE < inode->defrag_compress && 822 inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) 823 return 1; 824 /* force compress */ 825 if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) 826 return 1; 827 /* bad compression ratios */ 828 if (inode->flags & BTRFS_INODE_NOCOMPRESS) 829 return 0; 830 if (btrfs_test_opt(fs_info, COMPRESS) || 831 inode->flags & BTRFS_INODE_COMPRESS || 832 inode->prop_compress) 833 return btrfs_compress_heuristic(inode, start, end); 834 return 0; 835} 836 837static inline void inode_should_defrag(struct btrfs_inode *inode, 838 u64 start, u64 end, u64 num_bytes, u32 small_write) 839{ 840 /* If this is a small write inside eof, kick off a defrag */ 841 if (num_bytes < small_write && 842 (start > 0 || end + 1 < inode->disk_i_size)) 843 btrfs_add_inode_defrag(inode, small_write); 844} 845 846static int extent_range_clear_dirty_for_io(struct btrfs_inode *inode, u64 start, u64 end) 847{ 848 const pgoff_t end_index = end >> PAGE_SHIFT; 849 struct folio *folio; 850 int ret = 0; 851 852 for (pgoff_t index = start >> PAGE_SHIFT; index <= end_index; index++) { 853 folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); 854 if (IS_ERR(folio)) { 855 if (!ret) 856 ret = PTR_ERR(folio); 857 continue; 858 } 859 btrfs_folio_clamp_clear_dirty(inode->root->fs_info, folio, start, 860 end + 1 - start); 861 folio_put(folio); 862 } 863 return ret; 864} 865 866/* 867 * Work queue call back to started compression on a file and pages. 868 * 869 * This is done inside an ordered work queue, and the compression is spread 870 * across many cpus. The actual IO submission is step two, and the ordered work 871 * queue takes care of making sure that happens in the same order things were 872 * put onto the queue by writepages and friends. 873 * 874 * If this code finds it can't get good compression, it puts an entry onto the 875 * work queue to write the uncompressed bytes. This makes sure that both 876 * compressed inodes and uncompressed inodes are written in the same order that 877 * the flusher thread sent them down. 878 */ 879static void compress_file_range(struct btrfs_work *work) 880{ 881 struct async_chunk *async_chunk = 882 container_of(work, struct async_chunk, work); 883 struct btrfs_inode *inode = async_chunk->inode; 884 struct btrfs_fs_info *fs_info = inode->root->fs_info; 885 struct address_space *mapping = inode->vfs_inode.i_mapping; 886 const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; 887 const u32 min_folio_size = btrfs_min_folio_size(fs_info); 888 u64 blocksize = fs_info->sectorsize; 889 u64 start = async_chunk->start; 890 u64 end = async_chunk->end; 891 u64 actual_end; 892 u64 i_size; 893 int ret = 0; 894 struct folio **folios = NULL; 895 unsigned long nr_folios; 896 unsigned long total_compressed = 0; 897 unsigned long total_in = 0; 898 unsigned int loff; 899 int i; 900 int compress_type = fs_info->compress_type; 901 int compress_level = fs_info->compress_level; 902 903 if (unlikely(btrfs_is_shutdown(fs_info))) 904 goto cleanup_and_bail_uncompressed; 905 906 inode_should_defrag(inode, start, end, end - start + 1, SZ_16K); 907 908 /* 909 * We need to call clear_page_dirty_for_io on each page in the range. 910 * Otherwise applications with the file mmap'd can wander in and change 911 * the page contents while we are compressing them. 912 */ 913 ret = extent_range_clear_dirty_for_io(inode, start, end); 914 915 /* 916 * All the folios should have been locked thus no failure. 917 * 918 * And even if some folios are missing, btrfs_compress_folios() 919 * would handle them correctly, so here just do an ASSERT() check for 920 * early logic errors. 921 */ 922 ASSERT(ret == 0); 923 924 /* 925 * We need to save i_size before now because it could change in between 926 * us evaluating the size and assigning it. This is because we lock and 927 * unlock the page in truncate and fallocate, and then modify the i_size 928 * later on. 929 * 930 * The barriers are to emulate READ_ONCE, remove that once i_size_read 931 * does that for us. 932 */ 933 barrier(); 934 i_size = i_size_read(&inode->vfs_inode); 935 barrier(); 936 actual_end = min_t(u64, i_size, end + 1); 937again: 938 folios = NULL; 939 nr_folios = (end >> min_folio_shift) - (start >> min_folio_shift) + 1; 940 nr_folios = min_t(unsigned long, nr_folios, BTRFS_MAX_COMPRESSED >> min_folio_shift); 941 942 /* 943 * we don't want to send crud past the end of i_size through 944 * compression, that's just a waste of CPU time. So, if the 945 * end of the file is before the start of our current 946 * requested range of bytes, we bail out to the uncompressed 947 * cleanup code that can deal with all of this. 948 * 949 * It isn't really the fastest way to fix things, but this is a 950 * very uncommon corner. 951 */ 952 if (actual_end <= start) 953 goto cleanup_and_bail_uncompressed; 954 955 total_compressed = actual_end - start; 956 957 /* 958 * Skip compression for a small file range(<=blocksize) that 959 * isn't an inline extent, since it doesn't save disk space at all. 960 */ 961 if (total_compressed <= blocksize && 962 (start > 0 || end + 1 < inode->disk_i_size)) 963 goto cleanup_and_bail_uncompressed; 964 965 total_compressed = min_t(unsigned long, total_compressed, 966 BTRFS_MAX_UNCOMPRESSED); 967 total_in = 0; 968 ret = 0; 969 970 /* 971 * We do compression for mount -o compress and when the inode has not 972 * been flagged as NOCOMPRESS. This flag can change at any time if we 973 * discover bad compression ratios. 974 */ 975 if (!inode_need_compress(inode, start, end)) 976 goto cleanup_and_bail_uncompressed; 977 978 folios = kcalloc(nr_folios, sizeof(struct folio *), GFP_NOFS); 979 if (!folios) { 980 /* 981 * Memory allocation failure is not a fatal error, we can fall 982 * back to uncompressed code. 983 */ 984 goto cleanup_and_bail_uncompressed; 985 } 986 987 if (0 < inode->defrag_compress && inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) { 988 compress_type = inode->defrag_compress; 989 compress_level = inode->defrag_compress_level; 990 } else if (inode->prop_compress) { 991 compress_type = inode->prop_compress; 992 } 993 994 /* Compression level is applied here. */ 995 ret = btrfs_compress_folios(compress_type, compress_level, 996 inode, start, folios, &nr_folios, &total_in, 997 &total_compressed); 998 if (ret) 999 goto mark_incompressible; 1000 1001 /* 1002 * Zero the tail end of the last folio, as we might be sending it down 1003 * to disk. 1004 */ 1005 loff = (total_compressed & (min_folio_size - 1)); 1006 if (loff) 1007 folio_zero_range(folios[nr_folios - 1], loff, min_folio_size - loff); 1008 1009 /* 1010 * Try to create an inline extent. 1011 * 1012 * If we didn't compress the entire range, try to create an uncompressed 1013 * inline extent, else a compressed one. 1014 * 1015 * Check cow_file_range() for why we don't even try to create inline 1016 * extent for the subpage case. 1017 */ 1018 if (total_in < actual_end) 1019 ret = cow_file_range_inline(inode, NULL, start, end, 0, 1020 BTRFS_COMPRESS_NONE, NULL, false); 1021 else 1022 ret = cow_file_range_inline(inode, NULL, start, end, total_compressed, 1023 compress_type, folios[0], false); 1024 if (ret <= 0) { 1025 if (ret < 0) 1026 mapping_set_error(mapping, -EIO); 1027 goto free_pages; 1028 } 1029 1030 /* 1031 * We aren't doing an inline extent. Round the compressed size up to a 1032 * block size boundary so the allocator does sane things. 1033 */ 1034 total_compressed = ALIGN(total_compressed, blocksize); 1035 1036 /* 1037 * One last check to make sure the compression is really a win, compare 1038 * the page count read with the blocks on disk, compression must free at 1039 * least one sector. 1040 */ 1041 total_in = round_up(total_in, fs_info->sectorsize); 1042 if (total_compressed + blocksize > total_in) 1043 goto mark_incompressible; 1044 1045 /* 1046 * The async work queues will take care of doing actual allocation on 1047 * disk for these compressed pages, and will submit the bios. 1048 */ 1049 ret = add_async_extent(async_chunk, start, total_in, total_compressed, folios, 1050 nr_folios, compress_type); 1051 BUG_ON(ret); 1052 if (start + total_in < end) { 1053 start += total_in; 1054 cond_resched(); 1055 goto again; 1056 } 1057 return; 1058 1059mark_incompressible: 1060 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && !inode->prop_compress) 1061 inode->flags |= BTRFS_INODE_NOCOMPRESS; 1062cleanup_and_bail_uncompressed: 1063 ret = add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0, 1064 BTRFS_COMPRESS_NONE); 1065 BUG_ON(ret); 1066free_pages: 1067 if (folios) { 1068 for (i = 0; i < nr_folios; i++) { 1069 WARN_ON(folios[i]->mapping); 1070 btrfs_free_compr_folio(folios[i]); 1071 } 1072 kfree(folios); 1073 } 1074} 1075 1076static void free_async_extent_pages(struct async_extent *async_extent) 1077{ 1078 int i; 1079 1080 if (!async_extent->folios) 1081 return; 1082 1083 for (i = 0; i < async_extent->nr_folios; i++) { 1084 WARN_ON(async_extent->folios[i]->mapping); 1085 btrfs_free_compr_folio(async_extent->folios[i]); 1086 } 1087 kfree(async_extent->folios); 1088 async_extent->nr_folios = 0; 1089 async_extent->folios = NULL; 1090} 1091 1092static void submit_uncompressed_range(struct btrfs_inode *inode, 1093 struct async_extent *async_extent, 1094 struct folio *locked_folio) 1095{ 1096 u64 start = async_extent->start; 1097 u64 end = async_extent->start + async_extent->ram_size - 1; 1098 int ret; 1099 struct writeback_control wbc = { 1100 .sync_mode = WB_SYNC_ALL, 1101 .range_start = start, 1102 .range_end = end, 1103 .no_cgroup_owner = 1, 1104 }; 1105 1106 wbc_attach_fdatawrite_inode(&wbc, &inode->vfs_inode); 1107 ret = run_delalloc_cow(inode, locked_folio, start, end, 1108 &wbc, false); 1109 wbc_detach_inode(&wbc); 1110 if (ret < 0) { 1111 if (locked_folio) 1112 btrfs_folio_end_lock(inode->root->fs_info, locked_folio, 1113 start, async_extent->ram_size); 1114 btrfs_err_rl(inode->root->fs_info, 1115 "%s failed, root=%llu inode=%llu start=%llu len=%llu: %d", 1116 __func__, btrfs_root_id(inode->root), 1117 btrfs_ino(inode), start, async_extent->ram_size, ret); 1118 } 1119} 1120 1121static void submit_one_async_extent(struct async_chunk *async_chunk, 1122 struct async_extent *async_extent, 1123 u64 *alloc_hint) 1124{ 1125 struct btrfs_inode *inode = async_chunk->inode; 1126 struct extent_io_tree *io_tree = &inode->io_tree; 1127 struct btrfs_root *root = inode->root; 1128 struct btrfs_fs_info *fs_info = root->fs_info; 1129 struct btrfs_ordered_extent *ordered; 1130 struct btrfs_file_extent file_extent; 1131 struct btrfs_key ins; 1132 struct folio *locked_folio = NULL; 1133 struct extent_state *cached = NULL; 1134 struct extent_map *em; 1135 int ret = 0; 1136 bool free_pages = false; 1137 u64 start = async_extent->start; 1138 u64 end = async_extent->start + async_extent->ram_size - 1; 1139 1140 if (async_chunk->blkcg_css) 1141 kthread_associate_blkcg(async_chunk->blkcg_css); 1142 1143 /* 1144 * If async_chunk->locked_folio is in the async_extent range, we need to 1145 * handle it. 1146 */ 1147 if (async_chunk->locked_folio) { 1148 u64 locked_folio_start = folio_pos(async_chunk->locked_folio); 1149 u64 locked_folio_end = locked_folio_start + 1150 folio_size(async_chunk->locked_folio) - 1; 1151 1152 if (!(start >= locked_folio_end || end <= locked_folio_start)) 1153 locked_folio = async_chunk->locked_folio; 1154 } 1155 1156 if (async_extent->compress_type == BTRFS_COMPRESS_NONE) { 1157 ASSERT(!async_extent->folios); 1158 ASSERT(async_extent->nr_folios == 0); 1159 submit_uncompressed_range(inode, async_extent, locked_folio); 1160 free_pages = true; 1161 goto done; 1162 } 1163 1164 ret = btrfs_reserve_extent(root, async_extent->ram_size, 1165 async_extent->compressed_size, 1166 async_extent->compressed_size, 1167 0, *alloc_hint, &ins, true, true); 1168 if (ret) { 1169 /* 1170 * We can't reserve contiguous space for the compressed size. 1171 * Unlikely, but it's possible that we could have enough 1172 * non-contiguous space for the uncompressed size instead. So 1173 * fall back to uncompressed. 1174 */ 1175 submit_uncompressed_range(inode, async_extent, locked_folio); 1176 free_pages = true; 1177 goto done; 1178 } 1179 1180 btrfs_lock_extent(io_tree, start, end, &cached); 1181 1182 /* Here we're doing allocation and writeback of the compressed pages */ 1183 file_extent.disk_bytenr = ins.objectid; 1184 file_extent.disk_num_bytes = ins.offset; 1185 file_extent.ram_bytes = async_extent->ram_size; 1186 file_extent.num_bytes = async_extent->ram_size; 1187 file_extent.offset = 0; 1188 file_extent.compression = async_extent->compress_type; 1189 1190 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED); 1191 if (IS_ERR(em)) { 1192 ret = PTR_ERR(em); 1193 goto out_free_reserve; 1194 } 1195 btrfs_free_extent_map(em); 1196 1197 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent, 1198 1U << BTRFS_ORDERED_COMPRESSED); 1199 if (IS_ERR(ordered)) { 1200 btrfs_drop_extent_map_range(inode, start, end, false); 1201 ret = PTR_ERR(ordered); 1202 goto out_free_reserve; 1203 } 1204 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1205 1206 /* Clear dirty, set writeback and unlock the pages. */ 1207 extent_clear_unlock_delalloc(inode, start, end, 1208 NULL, &cached, EXTENT_LOCKED | EXTENT_DELALLOC, 1209 PAGE_UNLOCK | PAGE_START_WRITEBACK); 1210 btrfs_submit_compressed_write(ordered, 1211 async_extent->folios, /* compressed_folios */ 1212 async_extent->nr_folios, 1213 async_chunk->write_flags, true); 1214 *alloc_hint = ins.objectid + ins.offset; 1215done: 1216 if (async_chunk->blkcg_css) 1217 kthread_associate_blkcg(NULL); 1218 if (free_pages) 1219 free_async_extent_pages(async_extent); 1220 kfree(async_extent); 1221 return; 1222 1223out_free_reserve: 1224 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1225 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true); 1226 mapping_set_error(inode->vfs_inode.i_mapping, -EIO); 1227 extent_clear_unlock_delalloc(inode, start, end, 1228 NULL, &cached, 1229 EXTENT_LOCKED | EXTENT_DELALLOC | 1230 EXTENT_DELALLOC_NEW | 1231 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, 1232 PAGE_UNLOCK | PAGE_START_WRITEBACK | 1233 PAGE_END_WRITEBACK); 1234 free_async_extent_pages(async_extent); 1235 if (async_chunk->blkcg_css) 1236 kthread_associate_blkcg(NULL); 1237 btrfs_debug(fs_info, 1238"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d", 1239 btrfs_root_id(root), btrfs_ino(inode), start, 1240 async_extent->ram_size, ret); 1241 kfree(async_extent); 1242} 1243 1244u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, 1245 u64 num_bytes) 1246{ 1247 struct extent_map_tree *em_tree = &inode->extent_tree; 1248 struct extent_map *em; 1249 u64 alloc_hint = 0; 1250 1251 read_lock(&em_tree->lock); 1252 em = btrfs_search_extent_mapping(em_tree, start, num_bytes); 1253 if (em) { 1254 /* 1255 * if block start isn't an actual block number then find the 1256 * first block in this inode and use that as a hint. If that 1257 * block is also bogus then just don't worry about it. 1258 */ 1259 if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { 1260 btrfs_free_extent_map(em); 1261 em = btrfs_search_extent_mapping(em_tree, 0, 0); 1262 if (em && em->disk_bytenr < EXTENT_MAP_LAST_BYTE) 1263 alloc_hint = btrfs_extent_map_block_start(em); 1264 if (em) 1265 btrfs_free_extent_map(em); 1266 } else { 1267 alloc_hint = btrfs_extent_map_block_start(em); 1268 btrfs_free_extent_map(em); 1269 } 1270 } 1271 read_unlock(&em_tree->lock); 1272 1273 return alloc_hint; 1274} 1275 1276/* 1277 * when extent_io.c finds a delayed allocation range in the file, 1278 * the call backs end up in this code. The basic idea is to 1279 * allocate extents on disk for the range, and create ordered data structs 1280 * in ram to track those extents. 1281 * 1282 * locked_folio is the folio that writepage had locked already. We use 1283 * it to make sure we don't do extra locks or unlocks. 1284 * 1285 * When this function fails, it unlocks all folios except @locked_folio. 1286 * 1287 * When this function successfully creates an inline extent, it returns 1 and 1288 * unlocks all folios including locked_folio and starts I/O on them. 1289 * (In reality inline extents are limited to a single block, so locked_folio is 1290 * the only folio handled anyway). 1291 * 1292 * When this function succeed and creates a normal extent, the folio locking 1293 * status depends on the passed in flags: 1294 * 1295 * - If COW_FILE_RANGE_KEEP_LOCKED flag is set, all folios are kept locked. 1296 * - Else all folios except for @locked_folio are unlocked. 1297 * 1298 * When a failure happens in the second or later iteration of the 1299 * while-loop, the ordered extents created in previous iterations are cleaned up. 1300 */ 1301static noinline int cow_file_range(struct btrfs_inode *inode, 1302 struct folio *locked_folio, u64 start, 1303 u64 end, u64 *done_offset, 1304 unsigned long flags) 1305{ 1306 struct btrfs_root *root = inode->root; 1307 struct btrfs_fs_info *fs_info = root->fs_info; 1308 struct extent_state *cached = NULL; 1309 u64 alloc_hint = 0; 1310 u64 orig_start = start; 1311 u64 num_bytes; 1312 u64 cur_alloc_size = 0; 1313 u64 min_alloc_size; 1314 u64 blocksize = fs_info->sectorsize; 1315 struct btrfs_key ins; 1316 struct extent_map *em; 1317 unsigned clear_bits; 1318 unsigned long page_ops; 1319 int ret = 0; 1320 1321 if (unlikely(btrfs_is_shutdown(fs_info))) { 1322 ret = -EIO; 1323 goto out_unlock; 1324 } 1325 1326 if (btrfs_is_free_space_inode(inode)) { 1327 ret = -EINVAL; 1328 goto out_unlock; 1329 } 1330 1331 num_bytes = ALIGN(end - start + 1, blocksize); 1332 num_bytes = max(blocksize, num_bytes); 1333 ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); 1334 1335 inode_should_defrag(inode, start, end, num_bytes, SZ_64K); 1336 1337 if (!(flags & COW_FILE_RANGE_NO_INLINE)) { 1338 /* lets try to make an inline extent */ 1339 ret = cow_file_range_inline(inode, locked_folio, start, end, 0, 1340 BTRFS_COMPRESS_NONE, NULL, false); 1341 if (ret <= 0) { 1342 /* 1343 * We succeeded, return 1 so the caller knows we're done 1344 * with this page and already handled the IO. 1345 * 1346 * If there was an error then cow_file_range_inline() has 1347 * already done the cleanup. 1348 */ 1349 if (ret == 0) 1350 ret = 1; 1351 goto done; 1352 } 1353 } 1354 1355 alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes); 1356 1357 /* 1358 * We're not doing compressed IO, don't unlock the first page (which 1359 * the caller expects to stay locked), don't clear any dirty bits and 1360 * don't set any writeback bits. 1361 * 1362 * Do set the Ordered (Private2) bit so we know this page was properly 1363 * setup for writepage. 1364 */ 1365 page_ops = ((flags & COW_FILE_RANGE_KEEP_LOCKED) ? 0 : PAGE_UNLOCK); 1366 page_ops |= PAGE_SET_ORDERED; 1367 1368 /* 1369 * Relocation relies on the relocated extents to have exactly the same 1370 * size as the original extents. Normally writeback for relocation data 1371 * extents follows a NOCOW path because relocation preallocates the 1372 * extents. However, due to an operation such as scrub turning a block 1373 * group to RO mode, it may fallback to COW mode, so we must make sure 1374 * an extent allocated during COW has exactly the requested size and can 1375 * not be split into smaller extents, otherwise relocation breaks and 1376 * fails during the stage where it updates the bytenr of file extent 1377 * items. 1378 */ 1379 if (btrfs_is_data_reloc_root(root)) 1380 min_alloc_size = num_bytes; 1381 else 1382 min_alloc_size = fs_info->sectorsize; 1383 1384 while (num_bytes > 0) { 1385 struct btrfs_ordered_extent *ordered; 1386 struct btrfs_file_extent file_extent; 1387 1388 ret = btrfs_reserve_extent(root, num_bytes, num_bytes, 1389 min_alloc_size, 0, alloc_hint, 1390 &ins, true, true); 1391 if (ret == -EAGAIN) { 1392 /* 1393 * btrfs_reserve_extent only returns -EAGAIN for zoned 1394 * file systems, which is an indication that there are 1395 * no active zones to allocate from at the moment. 1396 * 1397 * If this is the first loop iteration, wait for at 1398 * least one zone to finish before retrying the 1399 * allocation. Otherwise ask the caller to write out 1400 * the already allocated blocks before coming back to 1401 * us, or return -ENOSPC if it can't handle retries. 1402 */ 1403 ASSERT(btrfs_is_zoned(fs_info)); 1404 if (start == orig_start) { 1405 wait_on_bit_io(&inode->root->fs_info->flags, 1406 BTRFS_FS_NEED_ZONE_FINISH, 1407 TASK_UNINTERRUPTIBLE); 1408 continue; 1409 } 1410 if (done_offset) { 1411 /* 1412 * Move @end to the end of the processed range, 1413 * and exit the loop to unlock the processed extents. 1414 */ 1415 end = start - 1; 1416 ret = 0; 1417 break; 1418 } 1419 ret = -ENOSPC; 1420 } 1421 if (ret < 0) 1422 goto out_unlock; 1423 cur_alloc_size = ins.offset; 1424 1425 file_extent.disk_bytenr = ins.objectid; 1426 file_extent.disk_num_bytes = ins.offset; 1427 file_extent.num_bytes = ins.offset; 1428 file_extent.ram_bytes = ins.offset; 1429 file_extent.offset = 0; 1430 file_extent.compression = BTRFS_COMPRESS_NONE; 1431 1432 /* 1433 * Locked range will be released either during error clean up or 1434 * after the whole range is finished. 1435 */ 1436 btrfs_lock_extent(&inode->io_tree, start, start + cur_alloc_size - 1, 1437 &cached); 1438 1439 em = btrfs_create_io_em(inode, start, &file_extent, 1440 BTRFS_ORDERED_REGULAR); 1441 if (IS_ERR(em)) { 1442 btrfs_unlock_extent(&inode->io_tree, start, 1443 start + cur_alloc_size - 1, &cached); 1444 ret = PTR_ERR(em); 1445 goto out_reserve; 1446 } 1447 btrfs_free_extent_map(em); 1448 1449 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent, 1450 1U << BTRFS_ORDERED_REGULAR); 1451 if (IS_ERR(ordered)) { 1452 btrfs_unlock_extent(&inode->io_tree, start, 1453 start + cur_alloc_size - 1, &cached); 1454 ret = PTR_ERR(ordered); 1455 goto out_drop_extent_cache; 1456 } 1457 1458 if (btrfs_is_data_reloc_root(root)) { 1459 ret = btrfs_reloc_clone_csums(ordered); 1460 1461 /* 1462 * Only drop cache here, and process as normal. 1463 * 1464 * We must not allow extent_clear_unlock_delalloc() 1465 * at out_unlock label to free meta of this ordered 1466 * extent, as its meta should be freed by 1467 * btrfs_finish_ordered_io(). 1468 * 1469 * So we must continue until @start is increased to 1470 * skip current ordered extent. 1471 */ 1472 if (ret) 1473 btrfs_drop_extent_map_range(inode, start, 1474 start + cur_alloc_size - 1, 1475 false); 1476 } 1477 btrfs_put_ordered_extent(ordered); 1478 1479 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1480 1481 if (num_bytes < cur_alloc_size) 1482 num_bytes = 0; 1483 else 1484 num_bytes -= cur_alloc_size; 1485 alloc_hint = ins.objectid + ins.offset; 1486 start += cur_alloc_size; 1487 cur_alloc_size = 0; 1488 1489 /* 1490 * btrfs_reloc_clone_csums() error, since start is increased 1491 * extent_clear_unlock_delalloc() at out_unlock label won't 1492 * free metadata of current ordered extent, we're OK to exit. 1493 */ 1494 if (ret) 1495 goto out_unlock; 1496 } 1497 extent_clear_unlock_delalloc(inode, orig_start, end, locked_folio, &cached, 1498 EXTENT_LOCKED | EXTENT_DELALLOC, page_ops); 1499done: 1500 if (done_offset) 1501 *done_offset = end; 1502 return ret; 1503 1504out_drop_extent_cache: 1505 btrfs_drop_extent_map_range(inode, start, start + cur_alloc_size - 1, false); 1506out_reserve: 1507 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 1508 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true); 1509out_unlock: 1510 /* 1511 * Now, we have three regions to clean up: 1512 * 1513 * |-------(1)----|---(2)---|-------------(3)----------| 1514 * `- orig_start `- start `- start + cur_alloc_size `- end 1515 * 1516 * We process each region below. 1517 */ 1518 1519 /* 1520 * For the range (1). We have already instantiated the ordered extents 1521 * for this region, thus we need to cleanup those ordered extents. 1522 * EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV 1523 * are also handled by the ordered extents cleanup. 1524 * 1525 * So here we only clear EXTENT_LOCKED and EXTENT_DELALLOC flag, and 1526 * finish the writeback of the involved folios, which will be never submitted. 1527 */ 1528 if (orig_start < start) { 1529 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC; 1530 page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; 1531 1532 if (!locked_folio) 1533 mapping_set_error(inode->vfs_inode.i_mapping, ret); 1534 1535 btrfs_cleanup_ordered_extents(inode, orig_start, start - orig_start); 1536 extent_clear_unlock_delalloc(inode, orig_start, start - 1, 1537 locked_folio, NULL, clear_bits, page_ops); 1538 } 1539 1540 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | 1541 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; 1542 page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; 1543 1544 /* 1545 * For the range (2). If we reserved an extent for our delalloc range 1546 * (or a subrange) and failed to create the respective ordered extent, 1547 * then it means that when we reserved the extent we decremented the 1548 * extent's size from the data space_info's bytes_may_use counter and 1549 * incremented the space_info's bytes_reserved counter by the same 1550 * amount. We must make sure extent_clear_unlock_delalloc() does not try 1551 * to decrement again the data space_info's bytes_may_use counter, 1552 * therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV. 1553 */ 1554 if (cur_alloc_size) { 1555 extent_clear_unlock_delalloc(inode, start, 1556 start + cur_alloc_size - 1, 1557 locked_folio, &cached, clear_bits, 1558 page_ops); 1559 btrfs_qgroup_free_data(inode, NULL, start, cur_alloc_size, NULL); 1560 } 1561 1562 /* 1563 * For the range (3). We never touched the region. In addition to the 1564 * clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data 1565 * space_info's bytes_may_use counter, reserved in 1566 * btrfs_check_data_free_space(). 1567 */ 1568 if (start + cur_alloc_size < end) { 1569 clear_bits |= EXTENT_CLEAR_DATA_RESV; 1570 extent_clear_unlock_delalloc(inode, start + cur_alloc_size, 1571 end, locked_folio, 1572 &cached, clear_bits, page_ops); 1573 btrfs_qgroup_free_data(inode, NULL, start + cur_alloc_size, 1574 end - start - cur_alloc_size + 1, NULL); 1575 } 1576 btrfs_err(fs_info, 1577"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%llu: %d", 1578 __func__, btrfs_root_id(inode->root), 1579 btrfs_ino(inode), orig_start, end + 1 - orig_start, 1580 start, cur_alloc_size, ret); 1581 return ret; 1582} 1583 1584/* 1585 * Phase two of compressed writeback. This is the ordered portion of the code, 1586 * which only gets called in the order the work was queued. We walk all the 1587 * async extents created by compress_file_range and send them down to the disk. 1588 * 1589 * If called with @do_free == true then it'll try to finish the work and free 1590 * the work struct eventually. 1591 */ 1592static noinline void submit_compressed_extents(struct btrfs_work *work, bool do_free) 1593{ 1594 struct async_chunk *async_chunk = container_of(work, struct async_chunk, 1595 work); 1596 struct btrfs_fs_info *fs_info = btrfs_work_owner(work); 1597 struct async_extent *async_extent; 1598 unsigned long nr_pages; 1599 u64 alloc_hint = 0; 1600 1601 if (do_free) { 1602 struct async_cow *async_cow; 1603 1604 btrfs_add_delayed_iput(async_chunk->inode); 1605 if (async_chunk->blkcg_css) 1606 css_put(async_chunk->blkcg_css); 1607 1608 async_cow = async_chunk->async_cow; 1609 if (atomic_dec_and_test(&async_cow->num_chunks)) 1610 kvfree(async_cow); 1611 return; 1612 } 1613 1614 nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >> 1615 PAGE_SHIFT; 1616 1617 while (!list_empty(&async_chunk->extents)) { 1618 async_extent = list_first_entry(&async_chunk->extents, 1619 struct async_extent, list); 1620 list_del(&async_extent->list); 1621 submit_one_async_extent(async_chunk, async_extent, &alloc_hint); 1622 } 1623 1624 /* atomic_sub_return implies a barrier */ 1625 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < 1626 5 * SZ_1M) 1627 cond_wake_up_nomb(&fs_info->async_submit_wait); 1628} 1629 1630static bool run_delalloc_compressed(struct btrfs_inode *inode, 1631 struct folio *locked_folio, u64 start, 1632 u64 end, struct writeback_control *wbc) 1633{ 1634 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1635 struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc); 1636 struct async_cow *ctx; 1637 struct async_chunk *async_chunk; 1638 unsigned long nr_pages; 1639 u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K); 1640 int i; 1641 unsigned nofs_flag; 1642 const blk_opf_t write_flags = wbc_to_write_flags(wbc); 1643 1644 nofs_flag = memalloc_nofs_save(); 1645 ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL); 1646 memalloc_nofs_restore(nofs_flag); 1647 if (!ctx) 1648 return false; 1649 1650 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags); 1651 1652 async_chunk = ctx->chunks; 1653 atomic_set(&ctx->num_chunks, num_chunks); 1654 1655 for (i = 0; i < num_chunks; i++) { 1656 u64 cur_end = min(end, start + SZ_512K - 1); 1657 1658 /* 1659 * igrab is called higher up in the call chain, take only the 1660 * lightweight reference for the callback lifetime 1661 */ 1662 ihold(&inode->vfs_inode); 1663 async_chunk[i].async_cow = ctx; 1664 async_chunk[i].inode = inode; 1665 async_chunk[i].start = start; 1666 async_chunk[i].end = cur_end; 1667 async_chunk[i].write_flags = write_flags; 1668 INIT_LIST_HEAD(&async_chunk[i].extents); 1669 1670 /* 1671 * The locked_folio comes all the way from writepage and its 1672 * the original folio we were actually given. As we spread 1673 * this large delalloc region across multiple async_chunk 1674 * structs, only the first struct needs a pointer to 1675 * locked_folio. 1676 * 1677 * This way we don't need racey decisions about who is supposed 1678 * to unlock it. 1679 */ 1680 if (locked_folio) { 1681 /* 1682 * Depending on the compressibility, the pages might or 1683 * might not go through async. We want all of them to 1684 * be accounted against wbc once. Let's do it here 1685 * before the paths diverge. wbc accounting is used 1686 * only for foreign writeback detection and doesn't 1687 * need full accuracy. Just account the whole thing 1688 * against the first page. 1689 */ 1690 wbc_account_cgroup_owner(wbc, locked_folio, 1691 cur_end - start); 1692 async_chunk[i].locked_folio = locked_folio; 1693 locked_folio = NULL; 1694 } else { 1695 async_chunk[i].locked_folio = NULL; 1696 } 1697 1698 if (blkcg_css != blkcg_root_css) { 1699 css_get(blkcg_css); 1700 async_chunk[i].blkcg_css = blkcg_css; 1701 async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT; 1702 } else { 1703 async_chunk[i].blkcg_css = NULL; 1704 } 1705 1706 btrfs_init_work(&async_chunk[i].work, compress_file_range, 1707 submit_compressed_extents); 1708 1709 nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE); 1710 atomic_add(nr_pages, &fs_info->async_delalloc_pages); 1711 1712 btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work); 1713 1714 start = cur_end + 1; 1715 } 1716 return true; 1717} 1718 1719/* 1720 * Run the delalloc range from start to end, and write back any dirty pages 1721 * covered by the range. 1722 */ 1723static noinline int run_delalloc_cow(struct btrfs_inode *inode, 1724 struct folio *locked_folio, u64 start, 1725 u64 end, struct writeback_control *wbc, 1726 bool pages_dirty) 1727{ 1728 u64 done_offset = end; 1729 int ret; 1730 1731 while (start <= end) { 1732 ret = cow_file_range(inode, locked_folio, start, end, 1733 &done_offset, COW_FILE_RANGE_KEEP_LOCKED); 1734 if (ret) 1735 return ret; 1736 extent_write_locked_range(&inode->vfs_inode, locked_folio, 1737 start, done_offset, wbc, pages_dirty); 1738 start = done_offset + 1; 1739 } 1740 1741 return 1; 1742} 1743 1744static int fallback_to_cow(struct btrfs_inode *inode, 1745 struct folio *locked_folio, const u64 start, 1746 const u64 end) 1747{ 1748 const bool is_space_ino = btrfs_is_free_space_inode(inode); 1749 const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root); 1750 const u64 range_bytes = end + 1 - start; 1751 struct extent_io_tree *io_tree = &inode->io_tree; 1752 struct extent_state *cached_state = NULL; 1753 u64 range_start = start; 1754 u64 count; 1755 int ret; 1756 1757 /* 1758 * If EXTENT_NORESERVE is set it means that when the buffered write was 1759 * made we had not enough available data space and therefore we did not 1760 * reserve data space for it, since we though we could do NOCOW for the 1761 * respective file range (either there is prealloc extent or the inode 1762 * has the NOCOW bit set). 1763 * 1764 * However when we need to fallback to COW mode (because for example the 1765 * block group for the corresponding extent was turned to RO mode by a 1766 * scrub or relocation) we need to do the following: 1767 * 1768 * 1) We increment the bytes_may_use counter of the data space info. 1769 * If COW succeeds, it allocates a new data extent and after doing 1770 * that it decrements the space info's bytes_may_use counter and 1771 * increments its bytes_reserved counter by the same amount (we do 1772 * this at btrfs_add_reserved_bytes()). So we need to increment the 1773 * bytes_may_use counter to compensate (when space is reserved at 1774 * buffered write time, the bytes_may_use counter is incremented); 1775 * 1776 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so 1777 * that if the COW path fails for any reason, it decrements (through 1778 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the 1779 * data space info, which we incremented in the step above. 1780 * 1781 * If we need to fallback to cow and the inode corresponds to a free 1782 * space cache inode or an inode of the data relocation tree, we must 1783 * also increment bytes_may_use of the data space_info for the same 1784 * reason. Space caches and relocated data extents always get a prealloc 1785 * extent for them, however scrub or balance may have set the block 1786 * group that contains that extent to RO mode and therefore force COW 1787 * when starting writeback. 1788 */ 1789 btrfs_lock_extent(io_tree, start, end, &cached_state); 1790 count = btrfs_count_range_bits(io_tree, &range_start, end, range_bytes, 1791 EXTENT_NORESERVE, 0, NULL); 1792 if (count > 0 || is_space_ino || is_reloc_ino) { 1793 u64 bytes = count; 1794 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1795 struct btrfs_space_info *sinfo = fs_info->data_sinfo; 1796 1797 if (is_space_ino || is_reloc_ino) 1798 bytes = range_bytes; 1799 1800 spin_lock(&sinfo->lock); 1801 btrfs_space_info_update_bytes_may_use(sinfo, bytes); 1802 spin_unlock(&sinfo->lock); 1803 1804 if (count > 0) 1805 btrfs_clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, 1806 &cached_state); 1807 } 1808 btrfs_unlock_extent(io_tree, start, end, &cached_state); 1809 1810 /* 1811 * Don't try to create inline extents, as a mix of inline extent that 1812 * is written out and unlocked directly and a normal NOCOW extent 1813 * doesn't work. 1814 * 1815 * And here we do not unlock the folio after a successful run. 1816 * The folios will be unlocked after everything is finished, or by error handling. 1817 * 1818 * This is to ensure error handling won't need to clear dirty/ordered flags without 1819 * a locked folio, which can race with writeback. 1820 */ 1821 ret = cow_file_range(inode, locked_folio, start, end, NULL, 1822 COW_FILE_RANGE_NO_INLINE | COW_FILE_RANGE_KEEP_LOCKED); 1823 ASSERT(ret != 1); 1824 return ret; 1825} 1826 1827struct can_nocow_file_extent_args { 1828 /* Input fields. */ 1829 1830 /* Start file offset of the range we want to NOCOW. */ 1831 u64 start; 1832 /* End file offset (inclusive) of the range we want to NOCOW. */ 1833 u64 end; 1834 bool writeback_path; 1835 /* 1836 * Free the path passed to can_nocow_file_extent() once it's not needed 1837 * anymore. 1838 */ 1839 bool free_path; 1840 1841 /* 1842 * Output fields. Only set when can_nocow_file_extent() returns 1. 1843 * The expected file extent for the NOCOW write. 1844 */ 1845 struct btrfs_file_extent file_extent; 1846}; 1847 1848/* 1849 * Check if we can NOCOW the file extent that the path points to. 1850 * This function may return with the path released, so the caller should check 1851 * if path->nodes[0] is NULL or not if it needs to use the path afterwards. 1852 * 1853 * Returns: < 0 on error 1854 * 0 if we can not NOCOW 1855 * 1 if we can NOCOW 1856 */ 1857static int can_nocow_file_extent(struct btrfs_path *path, 1858 struct btrfs_key *key, 1859 struct btrfs_inode *inode, 1860 struct can_nocow_file_extent_args *args) 1861{ 1862 const bool is_freespace_inode = btrfs_is_free_space_inode(inode); 1863 struct extent_buffer *leaf = path->nodes[0]; 1864 struct btrfs_root *root = inode->root; 1865 struct btrfs_file_extent_item *fi; 1866 struct btrfs_root *csum_root; 1867 u64 io_start; 1868 u64 extent_end; 1869 u8 extent_type; 1870 int can_nocow = 0; 1871 int ret = 0; 1872 bool nowait = path->nowait; 1873 1874 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 1875 extent_type = btrfs_file_extent_type(leaf, fi); 1876 1877 if (extent_type == BTRFS_FILE_EXTENT_INLINE) 1878 goto out; 1879 1880 if (!(inode->flags & BTRFS_INODE_NODATACOW) && 1881 extent_type == BTRFS_FILE_EXTENT_REG) 1882 goto out; 1883 1884 /* 1885 * If the extent was created before the generation where the last snapshot 1886 * for its subvolume was created, then this implies the extent is shared, 1887 * hence we must COW. 1888 */ 1889 if (btrfs_file_extent_generation(leaf, fi) <= 1890 btrfs_root_last_snapshot(&root->root_item)) 1891 goto out; 1892 1893 /* An explicit hole, must COW. */ 1894 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) 1895 goto out; 1896 1897 /* Compressed/encrypted/encoded extents must be COWed. */ 1898 if (btrfs_file_extent_compression(leaf, fi) || 1899 btrfs_file_extent_encryption(leaf, fi) || 1900 btrfs_file_extent_other_encoding(leaf, fi)) 1901 goto out; 1902 1903 extent_end = btrfs_file_extent_end(path); 1904 1905 args->file_extent.disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); 1906 args->file_extent.disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); 1907 args->file_extent.ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); 1908 args->file_extent.offset = btrfs_file_extent_offset(leaf, fi); 1909 args->file_extent.compression = btrfs_file_extent_compression(leaf, fi); 1910 1911 /* 1912 * The following checks can be expensive, as they need to take other 1913 * locks and do btree or rbtree searches, so release the path to avoid 1914 * blocking other tasks for too long. 1915 */ 1916 btrfs_release_path(path); 1917 1918 ret = btrfs_cross_ref_exist(inode, key->offset - args->file_extent.offset, 1919 args->file_extent.disk_bytenr, path); 1920 WARN_ON_ONCE(ret > 0 && is_freespace_inode); 1921 if (ret != 0) 1922 goto out; 1923 1924 if (args->free_path) { 1925 /* 1926 * We don't need the path anymore, plus through the 1927 * btrfs_lookup_csums_list() call below we will end up allocating 1928 * another path. So free the path to avoid unnecessary extra 1929 * memory usage. 1930 */ 1931 btrfs_free_path(path); 1932 path = NULL; 1933 } 1934 1935 /* If there are pending snapshots for this root, we must COW. */ 1936 if (args->writeback_path && !is_freespace_inode && 1937 atomic_read(&root->snapshot_force_cow)) 1938 goto out; 1939 1940 args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start; 1941 args->file_extent.offset += args->start - key->offset; 1942 io_start = args->file_extent.disk_bytenr + args->file_extent.offset; 1943 1944 /* 1945 * Force COW if csums exist in the range. This ensures that csums for a 1946 * given extent are either valid or do not exist. 1947 */ 1948 1949 csum_root = btrfs_csum_root(root->fs_info, io_start); 1950 ret = btrfs_lookup_csums_list(csum_root, io_start, 1951 io_start + args->file_extent.num_bytes - 1, 1952 NULL, nowait); 1953 WARN_ON_ONCE(ret > 0 && is_freespace_inode); 1954 if (ret != 0) 1955 goto out; 1956 1957 can_nocow = 1; 1958 out: 1959 if (args->free_path && path) 1960 btrfs_free_path(path); 1961 1962 return ret < 0 ? ret : can_nocow; 1963} 1964 1965static int nocow_one_range(struct btrfs_inode *inode, struct folio *locked_folio, 1966 struct extent_state **cached, 1967 struct can_nocow_file_extent_args *nocow_args, 1968 u64 file_pos, bool is_prealloc) 1969{ 1970 struct btrfs_ordered_extent *ordered; 1971 const u64 len = nocow_args->file_extent.num_bytes; 1972 const u64 end = file_pos + len - 1; 1973 int ret = 0; 1974 1975 btrfs_lock_extent(&inode->io_tree, file_pos, end, cached); 1976 1977 if (is_prealloc) { 1978 struct extent_map *em; 1979 1980 em = btrfs_create_io_em(inode, file_pos, &nocow_args->file_extent, 1981 BTRFS_ORDERED_PREALLOC); 1982 if (IS_ERR(em)) { 1983 ret = PTR_ERR(em); 1984 goto error; 1985 } 1986 btrfs_free_extent_map(em); 1987 } 1988 1989 ordered = btrfs_alloc_ordered_extent(inode, file_pos, &nocow_args->file_extent, 1990 is_prealloc 1991 ? (1U << BTRFS_ORDERED_PREALLOC) 1992 : (1U << BTRFS_ORDERED_NOCOW)); 1993 if (IS_ERR(ordered)) { 1994 if (is_prealloc) 1995 btrfs_drop_extent_map_range(inode, file_pos, end, false); 1996 ret = PTR_ERR(ordered); 1997 goto error; 1998 } 1999 2000 if (btrfs_is_data_reloc_root(inode->root)) 2001 /* 2002 * Errors are handled later, as we must prevent 2003 * extent_clear_unlock_delalloc() in error handler from freeing 2004 * metadata of the created ordered extent. 2005 */ 2006 ret = btrfs_reloc_clone_csums(ordered); 2007 btrfs_put_ordered_extent(ordered); 2008 2009 if (ret < 0) 2010 goto error; 2011 extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached, 2012 EXTENT_LOCKED | EXTENT_DELALLOC | 2013 EXTENT_CLEAR_DATA_RESV, 2014 PAGE_SET_ORDERED); 2015 return ret; 2016 2017error: 2018 btrfs_cleanup_ordered_extents(inode, file_pos, len); 2019 extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached, 2020 EXTENT_LOCKED | EXTENT_DELALLOC | 2021 EXTENT_CLEAR_DATA_RESV, 2022 PAGE_UNLOCK | PAGE_START_WRITEBACK | 2023 PAGE_END_WRITEBACK); 2024 btrfs_err(inode->root->fs_info, 2025 "%s failed, root=%lld inode=%llu start=%llu len=%llu: %d", 2026 __func__, btrfs_root_id(inode->root), btrfs_ino(inode), 2027 file_pos, len, ret); 2028 return ret; 2029} 2030 2031/* 2032 * When nocow writeback calls back. This checks for snapshots or COW copies 2033 * of the extents that exist in the file, and COWs the file as required. 2034 * 2035 * If no cow copies or snapshots exist, we write directly to the existing 2036 * blocks on disk 2037 */ 2038static noinline int run_delalloc_nocow(struct btrfs_inode *inode, 2039 struct folio *locked_folio, 2040 const u64 start, const u64 end) 2041{ 2042 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2043 struct btrfs_root *root = inode->root; 2044 struct btrfs_path *path = NULL; 2045 u64 cow_start = (u64)-1; 2046 /* 2047 * If not 0, represents the inclusive end of the last fallback_to_cow() 2048 * range. Only for error handling. 2049 * 2050 * The same for nocow_end, it's to avoid double cleaning up the range 2051 * already cleaned by nocow_one_range(). 2052 */ 2053 u64 cow_end = 0; 2054 u64 nocow_end = 0; 2055 u64 cur_offset = start; 2056 int ret; 2057 bool check_prev = true; 2058 u64 ino = btrfs_ino(inode); 2059 struct can_nocow_file_extent_args nocow_args = { 0 }; 2060 /* The range that has ordered extent(s). */ 2061 u64 oe_cleanup_start; 2062 u64 oe_cleanup_len = 0; 2063 /* The range that is untouched. */ 2064 u64 untouched_start; 2065 u64 untouched_len = 0; 2066 2067 /* 2068 * Normally on a zoned device we're only doing COW writes, but in case 2069 * of relocation on a zoned filesystem serializes I/O so that we're only 2070 * writing sequentially and can end up here as well. 2071 */ 2072 ASSERT(!btrfs_is_zoned(fs_info) || btrfs_is_data_reloc_root(root)); 2073 2074 if (unlikely(btrfs_is_shutdown(fs_info))) { 2075 ret = -EIO; 2076 goto error; 2077 } 2078 path = btrfs_alloc_path(); 2079 if (!path) { 2080 ret = -ENOMEM; 2081 goto error; 2082 } 2083 2084 nocow_args.end = end; 2085 nocow_args.writeback_path = true; 2086 2087 while (cur_offset <= end) { 2088 struct btrfs_block_group *nocow_bg = NULL; 2089 struct btrfs_key found_key; 2090 struct btrfs_file_extent_item *fi; 2091 struct extent_buffer *leaf; 2092 struct extent_state *cached_state = NULL; 2093 u64 extent_end; 2094 int extent_type; 2095 2096 ret = btrfs_lookup_file_extent(NULL, root, path, ino, 2097 cur_offset, 0); 2098 if (ret < 0) 2099 goto error; 2100 2101 /* 2102 * If there is no extent for our range when doing the initial 2103 * search, then go back to the previous slot as it will be the 2104 * one containing the search offset 2105 */ 2106 if (ret > 0 && path->slots[0] > 0 && check_prev) { 2107 leaf = path->nodes[0]; 2108 btrfs_item_key_to_cpu(leaf, &found_key, 2109 path->slots[0] - 1); 2110 if (found_key.objectid == ino && 2111 found_key.type == BTRFS_EXTENT_DATA_KEY) 2112 path->slots[0]--; 2113 } 2114 check_prev = false; 2115next_slot: 2116 /* Go to next leaf if we have exhausted the current one */ 2117 leaf = path->nodes[0]; 2118 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 2119 ret = btrfs_next_leaf(root, path); 2120 if (ret < 0) 2121 goto error; 2122 if (ret > 0) 2123 break; 2124 leaf = path->nodes[0]; 2125 } 2126 2127 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 2128 2129 /* Didn't find anything for our INO */ 2130 if (found_key.objectid > ino) 2131 break; 2132 /* 2133 * Keep searching until we find an EXTENT_ITEM or there are no 2134 * more extents for this inode 2135 */ 2136 if (WARN_ON_ONCE(found_key.objectid < ino) || 2137 found_key.type < BTRFS_EXTENT_DATA_KEY) { 2138 path->slots[0]++; 2139 goto next_slot; 2140 } 2141 2142 /* Found key is not EXTENT_DATA_KEY or starts after req range */ 2143 if (found_key.type > BTRFS_EXTENT_DATA_KEY || 2144 found_key.offset > end) 2145 break; 2146 2147 /* 2148 * If the found extent starts after requested offset, then 2149 * adjust cur_offset to be right before this extent begins. 2150 */ 2151 if (found_key.offset > cur_offset) { 2152 if (cow_start == (u64)-1) 2153 cow_start = cur_offset; 2154 cur_offset = found_key.offset; 2155 goto next_slot; 2156 } 2157 2158 /* 2159 * Found extent which begins before our range and potentially 2160 * intersect it 2161 */ 2162 fi = btrfs_item_ptr(leaf, path->slots[0], 2163 struct btrfs_file_extent_item); 2164 extent_type = btrfs_file_extent_type(leaf, fi); 2165 /* If this is triggered then we have a memory corruption. */ 2166 ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES); 2167 if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) { 2168 ret = -EUCLEAN; 2169 goto error; 2170 } 2171 extent_end = btrfs_file_extent_end(path); 2172 2173 /* 2174 * If the extent we got ends before our current offset, skip to 2175 * the next extent. 2176 */ 2177 if (extent_end <= cur_offset) { 2178 path->slots[0]++; 2179 goto next_slot; 2180 } 2181 2182 nocow_args.start = cur_offset; 2183 ret = can_nocow_file_extent(path, &found_key, inode, &nocow_args); 2184 if (ret < 0) 2185 goto error; 2186 if (ret == 0) 2187 goto must_cow; 2188 2189 ret = 0; 2190 nocow_bg = btrfs_inc_nocow_writers(fs_info, 2191 nocow_args.file_extent.disk_bytenr + 2192 nocow_args.file_extent.offset); 2193 if (!nocow_bg) { 2194must_cow: 2195 /* 2196 * If we can't perform NOCOW writeback for the range, 2197 * then record the beginning of the range that needs to 2198 * be COWed. It will be written out before the next 2199 * NOCOW range if we find one, or when exiting this 2200 * loop. 2201 */ 2202 if (cow_start == (u64)-1) 2203 cow_start = cur_offset; 2204 cur_offset = extent_end; 2205 if (cur_offset > end) 2206 break; 2207 if (!path->nodes[0]) 2208 continue; 2209 path->slots[0]++; 2210 goto next_slot; 2211 } 2212 2213 /* 2214 * COW range from cow_start to found_key.offset - 1. As the key 2215 * will contain the beginning of the first extent that can be 2216 * NOCOW, following one which needs to be COW'ed 2217 */ 2218 if (cow_start != (u64)-1) { 2219 ret = fallback_to_cow(inode, locked_folio, cow_start, 2220 found_key.offset - 1); 2221 if (ret) { 2222 cow_end = found_key.offset - 1; 2223 btrfs_dec_nocow_writers(nocow_bg); 2224 goto error; 2225 } 2226 cow_start = (u64)-1; 2227 } 2228 2229 ret = nocow_one_range(inode, locked_folio, &cached_state, 2230 &nocow_args, cur_offset, 2231 extent_type == BTRFS_FILE_EXTENT_PREALLOC); 2232 btrfs_dec_nocow_writers(nocow_bg); 2233 if (ret < 0) { 2234 nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1; 2235 goto error; 2236 } 2237 cur_offset = extent_end; 2238 } 2239 btrfs_release_path(path); 2240 2241 if (cur_offset <= end && cow_start == (u64)-1) 2242 cow_start = cur_offset; 2243 2244 if (cow_start != (u64)-1) { 2245 ret = fallback_to_cow(inode, locked_folio, cow_start, end); 2246 if (ret) { 2247 cow_end = end; 2248 goto error; 2249 } 2250 cow_start = (u64)-1; 2251 } 2252 2253 /* 2254 * Everything is finished without an error, can unlock the folios now. 2255 * 2256 * No need to touch the io tree range nor set folio ordered flag, as 2257 * fallback_to_cow() and nocow_one_range() have already handled them. 2258 */ 2259 extent_clear_unlock_delalloc(inode, start, end, locked_folio, NULL, 0, PAGE_UNLOCK); 2260 2261 btrfs_free_path(path); 2262 return 0; 2263 2264error: 2265 if (cow_start == (u64)-1) { 2266 /* 2267 * case a) 2268 * start cur_offset end 2269 * | OE cleanup | Untouched | 2270 * 2271 * We finished a fallback_to_cow() or nocow_one_range() call, 2272 * but failed to check the next range. 2273 * 2274 * or 2275 * start cur_offset nocow_end end 2276 * | OE cleanup | Skip | Untouched | 2277 * 2278 * nocow_one_range() failed, the range [cur_offset, nocow_end] is 2279 * already cleaned up. 2280 */ 2281 oe_cleanup_start = start; 2282 oe_cleanup_len = cur_offset - start; 2283 if (nocow_end) 2284 untouched_start = nocow_end + 1; 2285 else 2286 untouched_start = cur_offset; 2287 untouched_len = end + 1 - untouched_start; 2288 } else if (cow_start != (u64)-1 && cow_end == 0) { 2289 /* 2290 * case b) 2291 * start cow_start cur_offset end 2292 * | OE cleanup | Untouched | 2293 * 2294 * We got a range that needs COW, but before we hit the next NOCOW range, 2295 * thus [cow_start, cur_offset) doesn't yet have any OE. 2296 */ 2297 oe_cleanup_start = start; 2298 oe_cleanup_len = cow_start - start; 2299 untouched_start = cow_start; 2300 untouched_len = end + 1 - untouched_start; 2301 } else { 2302 /* 2303 * case c) 2304 * start cow_start cow_end end 2305 * | OE cleanup | Skip | Untouched | 2306 * 2307 * fallback_to_cow() failed, and fallback_to_cow() will do the 2308 * cleanup for its range, we shouldn't touch the range 2309 * [cow_start, cow_end]. 2310 */ 2311 ASSERT(cow_start != (u64)-1 && cow_end != 0); 2312 oe_cleanup_start = start; 2313 oe_cleanup_len = cow_start - start; 2314 untouched_start = cow_end + 1; 2315 untouched_len = end + 1 - untouched_start; 2316 } 2317 2318 if (oe_cleanup_len) { 2319 const u64 oe_cleanup_end = oe_cleanup_start + oe_cleanup_len - 1; 2320 btrfs_cleanup_ordered_extents(inode, oe_cleanup_start, oe_cleanup_len); 2321 extent_clear_unlock_delalloc(inode, oe_cleanup_start, oe_cleanup_end, 2322 locked_folio, NULL, 2323 EXTENT_LOCKED | EXTENT_DELALLOC, 2324 PAGE_UNLOCK | PAGE_START_WRITEBACK | 2325 PAGE_END_WRITEBACK); 2326 } 2327 2328 if (untouched_len) { 2329 struct extent_state *cached = NULL; 2330 const u64 untouched_end = untouched_start + untouched_len - 1; 2331 2332 /* 2333 * We need to lock the extent here because we're clearing DELALLOC and 2334 * we're not locked at this point. 2335 */ 2336 btrfs_lock_extent(&inode->io_tree, untouched_start, untouched_end, &cached); 2337 extent_clear_unlock_delalloc(inode, untouched_start, untouched_end, 2338 locked_folio, &cached, 2339 EXTENT_LOCKED | EXTENT_DELALLOC | 2340 EXTENT_DEFRAG | 2341 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | 2342 PAGE_START_WRITEBACK | 2343 PAGE_END_WRITEBACK); 2344 btrfs_qgroup_free_data(inode, NULL, untouched_start, untouched_len, NULL); 2345 } 2346 btrfs_free_path(path); 2347 btrfs_err(fs_info, 2348"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu oe_cleanup=%llu oe_cleanup_len=%llu untouched_start=%llu untouched_len=%llu: %d", 2349 __func__, btrfs_root_id(inode->root), btrfs_ino(inode), 2350 start, end + 1 - start, cur_offset, oe_cleanup_start, oe_cleanup_len, 2351 untouched_start, untouched_len, ret); 2352 return ret; 2353} 2354 2355static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end) 2356{ 2357 if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) { 2358 if (inode->defrag_bytes && 2359 btrfs_test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG)) 2360 return false; 2361 return true; 2362 } 2363 return false; 2364} 2365 2366/* 2367 * Function to process delayed allocation (create CoW) for ranges which are 2368 * being touched for the first time. 2369 */ 2370int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio, 2371 u64 start, u64 end, struct writeback_control *wbc) 2372{ 2373 const bool zoned = btrfs_is_zoned(inode->root->fs_info); 2374 int ret; 2375 2376 /* 2377 * The range must cover part of the @locked_folio, or a return of 1 2378 * can confuse the caller. 2379 */ 2380 ASSERT(!(end <= folio_pos(locked_folio) || 2381 start >= folio_next_pos(locked_folio))); 2382 2383 if (should_nocow(inode, start, end)) { 2384 ret = run_delalloc_nocow(inode, locked_folio, start, end); 2385 return ret; 2386 } 2387 2388 if (btrfs_inode_can_compress(inode) && 2389 inode_need_compress(inode, start, end) && 2390 run_delalloc_compressed(inode, locked_folio, start, end, wbc)) 2391 return 1; 2392 2393 if (zoned) 2394 ret = run_delalloc_cow(inode, locked_folio, start, end, wbc, 2395 true); 2396 else 2397 ret = cow_file_range(inode, locked_folio, start, end, NULL, 0); 2398 return ret; 2399} 2400 2401void btrfs_split_delalloc_extent(struct btrfs_inode *inode, 2402 struct extent_state *orig, u64 split) 2403{ 2404 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2405 u64 size; 2406 2407 lockdep_assert_held(&inode->io_tree.lock); 2408 2409 /* not delalloc, ignore it */ 2410 if (!(orig->state & EXTENT_DELALLOC)) 2411 return; 2412 2413 size = orig->end - orig->start + 1; 2414 if (size > fs_info->max_extent_size) { 2415 u32 num_extents; 2416 u64 new_size; 2417 2418 /* 2419 * See the explanation in btrfs_merge_delalloc_extent, the same 2420 * applies here, just in reverse. 2421 */ 2422 new_size = orig->end - split + 1; 2423 num_extents = count_max_extents(fs_info, new_size); 2424 new_size = split - orig->start; 2425 num_extents += count_max_extents(fs_info, new_size); 2426 if (count_max_extents(fs_info, size) >= num_extents) 2427 return; 2428 } 2429 2430 spin_lock(&inode->lock); 2431 btrfs_mod_outstanding_extents(inode, 1); 2432 spin_unlock(&inode->lock); 2433} 2434 2435/* 2436 * Handle merged delayed allocation extents so we can keep track of new extents 2437 * that are just merged onto old extents, such as when we are doing sequential 2438 * writes, so we can properly account for the metadata space we'll need. 2439 */ 2440void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new, 2441 struct extent_state *other) 2442{ 2443 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2444 u64 new_size, old_size; 2445 u32 num_extents; 2446 2447 lockdep_assert_held(&inode->io_tree.lock); 2448 2449 /* not delalloc, ignore it */ 2450 if (!(other->state & EXTENT_DELALLOC)) 2451 return; 2452 2453 if (new->start > other->start) 2454 new_size = new->end - other->start + 1; 2455 else 2456 new_size = other->end - new->start + 1; 2457 2458 /* we're not bigger than the max, unreserve the space and go */ 2459 if (new_size <= fs_info->max_extent_size) { 2460 spin_lock(&inode->lock); 2461 btrfs_mod_outstanding_extents(inode, -1); 2462 spin_unlock(&inode->lock); 2463 return; 2464 } 2465 2466 /* 2467 * We have to add up either side to figure out how many extents were 2468 * accounted for before we merged into one big extent. If the number of 2469 * extents we accounted for is <= the amount we need for the new range 2470 * then we can return, otherwise drop. Think of it like this 2471 * 2472 * [ 4k][MAX_SIZE] 2473 * 2474 * So we've grown the extent by a MAX_SIZE extent, this would mean we 2475 * need 2 outstanding extents, on one side we have 1 and the other side 2476 * we have 1 so they are == and we can return. But in this case 2477 * 2478 * [MAX_SIZE+4k][MAX_SIZE+4k] 2479 * 2480 * Each range on their own accounts for 2 extents, but merged together 2481 * they are only 3 extents worth of accounting, so we need to drop in 2482 * this case. 2483 */ 2484 old_size = other->end - other->start + 1; 2485 num_extents = count_max_extents(fs_info, old_size); 2486 old_size = new->end - new->start + 1; 2487 num_extents += count_max_extents(fs_info, old_size); 2488 if (count_max_extents(fs_info, new_size) >= num_extents) 2489 return; 2490 2491 spin_lock(&inode->lock); 2492 btrfs_mod_outstanding_extents(inode, -1); 2493 spin_unlock(&inode->lock); 2494} 2495 2496static void btrfs_add_delalloc_inode(struct btrfs_inode *inode) 2497{ 2498 struct btrfs_root *root = inode->root; 2499 struct btrfs_fs_info *fs_info = root->fs_info; 2500 2501 spin_lock(&root->delalloc_lock); 2502 ASSERT(list_empty(&inode->delalloc_inodes)); 2503 list_add_tail(&inode->delalloc_inodes, &root->delalloc_inodes); 2504 root->nr_delalloc_inodes++; 2505 if (root->nr_delalloc_inodes == 1) { 2506 spin_lock(&fs_info->delalloc_root_lock); 2507 ASSERT(list_empty(&root->delalloc_root)); 2508 list_add_tail(&root->delalloc_root, &fs_info->delalloc_roots); 2509 spin_unlock(&fs_info->delalloc_root_lock); 2510 } 2511 spin_unlock(&root->delalloc_lock); 2512} 2513 2514void btrfs_del_delalloc_inode(struct btrfs_inode *inode) 2515{ 2516 struct btrfs_root *root = inode->root; 2517 struct btrfs_fs_info *fs_info = root->fs_info; 2518 2519 lockdep_assert_held(&root->delalloc_lock); 2520 2521 /* 2522 * We may be called after the inode was already deleted from the list, 2523 * namely in the transaction abort path btrfs_destroy_delalloc_inodes(), 2524 * and then later through btrfs_clear_delalloc_extent() while the inode 2525 * still has ->delalloc_bytes > 0. 2526 */ 2527 if (!list_empty(&inode->delalloc_inodes)) { 2528 list_del_init(&inode->delalloc_inodes); 2529 root->nr_delalloc_inodes--; 2530 if (!root->nr_delalloc_inodes) { 2531 ASSERT(list_empty(&root->delalloc_inodes)); 2532 spin_lock(&fs_info->delalloc_root_lock); 2533 ASSERT(!list_empty(&root->delalloc_root)); 2534 list_del_init(&root->delalloc_root); 2535 spin_unlock(&fs_info->delalloc_root_lock); 2536 } 2537 } 2538} 2539 2540/* 2541 * Properly track delayed allocation bytes in the inode and to maintain the 2542 * list of inodes that have pending delalloc work to be done. 2543 */ 2544void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state, 2545 u32 bits) 2546{ 2547 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2548 2549 lockdep_assert_held(&inode->io_tree.lock); 2550 2551 if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)) 2552 WARN_ON(1); 2553 /* 2554 * set_bit and clear bit hooks normally require _irqsave/restore 2555 * but in this case, we are only testing for the DELALLOC 2556 * bit, which is only set or cleared with irqs on 2557 */ 2558 if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) { 2559 u64 len = state->end + 1 - state->start; 2560 u64 prev_delalloc_bytes; 2561 u32 num_extents = count_max_extents(fs_info, len); 2562 2563 spin_lock(&inode->lock); 2564 btrfs_mod_outstanding_extents(inode, num_extents); 2565 spin_unlock(&inode->lock); 2566 2567 /* For sanity tests */ 2568 if (btrfs_is_testing(fs_info)) 2569 return; 2570 2571 percpu_counter_add_batch(&fs_info->delalloc_bytes, len, 2572 fs_info->delalloc_batch); 2573 spin_lock(&inode->lock); 2574 prev_delalloc_bytes = inode->delalloc_bytes; 2575 inode->delalloc_bytes += len; 2576 if (bits & EXTENT_DEFRAG) 2577 inode->defrag_bytes += len; 2578 spin_unlock(&inode->lock); 2579 2580 /* 2581 * We don't need to be under the protection of the inode's lock, 2582 * because we are called while holding the inode's io_tree lock 2583 * and are therefore protected against concurrent calls of this 2584 * function and btrfs_clear_delalloc_extent(). 2585 */ 2586 if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0) 2587 btrfs_add_delalloc_inode(inode); 2588 } 2589 2590 if (!(state->state & EXTENT_DELALLOC_NEW) && 2591 (bits & EXTENT_DELALLOC_NEW)) { 2592 spin_lock(&inode->lock); 2593 inode->new_delalloc_bytes += state->end + 1 - state->start; 2594 spin_unlock(&inode->lock); 2595 } 2596} 2597 2598/* 2599 * Once a range is no longer delalloc this function ensures that proper 2600 * accounting happens. 2601 */ 2602void btrfs_clear_delalloc_extent(struct btrfs_inode *inode, 2603 struct extent_state *state, u32 bits) 2604{ 2605 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2606 u64 len = state->end + 1 - state->start; 2607 u32 num_extents = count_max_extents(fs_info, len); 2608 2609 lockdep_assert_held(&inode->io_tree.lock); 2610 2611 if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) { 2612 spin_lock(&inode->lock); 2613 inode->defrag_bytes -= len; 2614 spin_unlock(&inode->lock); 2615 } 2616 2617 /* 2618 * set_bit and clear bit hooks normally require _irqsave/restore 2619 * but in this case, we are only testing for the DELALLOC 2620 * bit, which is only set or cleared with irqs on 2621 */ 2622 if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) { 2623 struct btrfs_root *root = inode->root; 2624 u64 new_delalloc_bytes; 2625 2626 spin_lock(&inode->lock); 2627 btrfs_mod_outstanding_extents(inode, -num_extents); 2628 spin_unlock(&inode->lock); 2629 2630 /* 2631 * We don't reserve metadata space for space cache inodes so we 2632 * don't need to call delalloc_release_metadata if there is an 2633 * error. 2634 */ 2635 if (bits & EXTENT_CLEAR_META_RESV && 2636 root != fs_info->tree_root) 2637 btrfs_delalloc_release_metadata(inode, len, true); 2638 2639 /* For sanity tests. */ 2640 if (btrfs_is_testing(fs_info)) 2641 return; 2642 2643 if (!btrfs_is_data_reloc_root(root) && 2644 !btrfs_is_free_space_inode(inode) && 2645 !(state->state & EXTENT_NORESERVE) && 2646 (bits & EXTENT_CLEAR_DATA_RESV)) 2647 btrfs_free_reserved_data_space_noquota(inode, len); 2648 2649 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, 2650 fs_info->delalloc_batch); 2651 spin_lock(&inode->lock); 2652 inode->delalloc_bytes -= len; 2653 new_delalloc_bytes = inode->delalloc_bytes; 2654 spin_unlock(&inode->lock); 2655 2656 /* 2657 * We don't need to be under the protection of the inode's lock, 2658 * because we are called while holding the inode's io_tree lock 2659 * and are therefore protected against concurrent calls of this 2660 * function and btrfs_set_delalloc_extent(). 2661 */ 2662 if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) { 2663 spin_lock(&root->delalloc_lock); 2664 btrfs_del_delalloc_inode(inode); 2665 spin_unlock(&root->delalloc_lock); 2666 } 2667 } 2668 2669 if ((state->state & EXTENT_DELALLOC_NEW) && 2670 (bits & EXTENT_DELALLOC_NEW)) { 2671 spin_lock(&inode->lock); 2672 ASSERT(inode->new_delalloc_bytes >= len); 2673 inode->new_delalloc_bytes -= len; 2674 if (bits & EXTENT_ADD_INODE_BYTES) 2675 inode_add_bytes(&inode->vfs_inode, len); 2676 spin_unlock(&inode->lock); 2677 } 2678} 2679 2680/* 2681 * given a list of ordered sums record them in the inode. This happens 2682 * at IO completion time based on sums calculated at bio submission time. 2683 */ 2684static int add_pending_csums(struct btrfs_trans_handle *trans, 2685 struct list_head *list) 2686{ 2687 struct btrfs_ordered_sum *sum; 2688 struct btrfs_root *csum_root = NULL; 2689 int ret; 2690 2691 list_for_each_entry(sum, list, list) { 2692 trans->adding_csums = true; 2693 if (!csum_root) 2694 csum_root = btrfs_csum_root(trans->fs_info, 2695 sum->logical); 2696 ret = btrfs_csum_file_blocks(trans, csum_root, sum); 2697 trans->adding_csums = false; 2698 if (ret) 2699 return ret; 2700 } 2701 return 0; 2702} 2703 2704static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode, 2705 const u64 start, 2706 const u64 len, 2707 struct extent_state **cached_state) 2708{ 2709 u64 search_start = start; 2710 const u64 end = start + len - 1; 2711 2712 while (search_start < end) { 2713 const u64 search_len = end - search_start + 1; 2714 struct extent_map *em; 2715 u64 em_len; 2716 int ret = 0; 2717 2718 em = btrfs_get_extent(inode, NULL, search_start, search_len); 2719 if (IS_ERR(em)) 2720 return PTR_ERR(em); 2721 2722 if (em->disk_bytenr != EXTENT_MAP_HOLE) 2723 goto next; 2724 2725 em_len = em->len; 2726 if (em->start < search_start) 2727 em_len -= search_start - em->start; 2728 if (em_len > search_len) 2729 em_len = search_len; 2730 2731 ret = btrfs_set_extent_bit(&inode->io_tree, search_start, 2732 search_start + em_len - 1, 2733 EXTENT_DELALLOC_NEW, cached_state); 2734next: 2735 search_start = btrfs_extent_map_end(em); 2736 btrfs_free_extent_map(em); 2737 if (ret) 2738 return ret; 2739 } 2740 return 0; 2741} 2742 2743int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, 2744 unsigned int extra_bits, 2745 struct extent_state **cached_state) 2746{ 2747 WARN_ON(PAGE_ALIGNED(end)); 2748 2749 if (start >= i_size_read(&inode->vfs_inode) && 2750 !(inode->flags & BTRFS_INODE_PREALLOC)) { 2751 /* 2752 * There can't be any extents following eof in this case so just 2753 * set the delalloc new bit for the range directly. 2754 */ 2755 extra_bits |= EXTENT_DELALLOC_NEW; 2756 } else { 2757 int ret; 2758 2759 ret = btrfs_find_new_delalloc_bytes(inode, start, 2760 end + 1 - start, 2761 cached_state); 2762 if (ret) 2763 return ret; 2764 } 2765 2766 return btrfs_set_extent_bit(&inode->io_tree, start, end, 2767 EXTENT_DELALLOC | extra_bits, cached_state); 2768} 2769 2770/* see btrfs_writepage_start_hook for details on why this is required */ 2771struct btrfs_writepage_fixup { 2772 struct folio *folio; 2773 struct btrfs_inode *inode; 2774 struct btrfs_work work; 2775}; 2776 2777static void btrfs_writepage_fixup_worker(struct btrfs_work *work) 2778{ 2779 struct btrfs_writepage_fixup *fixup = 2780 container_of(work, struct btrfs_writepage_fixup, work); 2781 struct btrfs_ordered_extent *ordered; 2782 struct extent_state *cached_state = NULL; 2783 struct extent_changeset *data_reserved = NULL; 2784 struct folio *folio = fixup->folio; 2785 struct btrfs_inode *inode = fixup->inode; 2786 struct btrfs_fs_info *fs_info = inode->root->fs_info; 2787 u64 page_start = folio_pos(folio); 2788 u64 page_end = folio_next_pos(folio) - 1; 2789 int ret = 0; 2790 bool free_delalloc_space = true; 2791 2792 /* 2793 * This is similar to page_mkwrite, we need to reserve the space before 2794 * we take the folio lock. 2795 */ 2796 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, 2797 folio_size(folio)); 2798again: 2799 folio_lock(folio); 2800 2801 /* 2802 * Before we queued this fixup, we took a reference on the folio. 2803 * folio->mapping may go NULL, but it shouldn't be moved to a different 2804 * address space. 2805 */ 2806 if (!folio->mapping || !folio_test_dirty(folio) || 2807 !folio_test_checked(folio)) { 2808 /* 2809 * Unfortunately this is a little tricky, either 2810 * 2811 * 1) We got here and our folio had already been dealt with and 2812 * we reserved our space, thus ret == 0, so we need to just 2813 * drop our space reservation and bail. This can happen the 2814 * first time we come into the fixup worker, or could happen 2815 * while waiting for the ordered extent. 2816 * 2) Our folio was already dealt with, but we happened to get an 2817 * ENOSPC above from the btrfs_delalloc_reserve_space. In 2818 * this case we obviously don't have anything to release, but 2819 * because the folio was already dealt with we don't want to 2820 * mark the folio with an error, so make sure we're resetting 2821 * ret to 0. This is why we have this check _before_ the ret 2822 * check, because we do not want to have a surprise ENOSPC 2823 * when the folio was already properly dealt with. 2824 */ 2825 if (!ret) { 2826 btrfs_delalloc_release_extents(inode, folio_size(folio)); 2827 btrfs_delalloc_release_space(inode, data_reserved, 2828 page_start, folio_size(folio), 2829 true); 2830 } 2831 ret = 0; 2832 goto out_page; 2833 } 2834 2835 /* 2836 * We can't mess with the folio state unless it is locked, so now that 2837 * it is locked bail if we failed to make our space reservation. 2838 */ 2839 if (ret) 2840 goto out_page; 2841 2842 btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state); 2843 2844 /* already ordered? We're done */ 2845 if (folio_test_ordered(folio)) 2846 goto out_reserved; 2847 2848 ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); 2849 if (ordered) { 2850 btrfs_unlock_extent(&inode->io_tree, page_start, page_end, 2851 &cached_state); 2852 folio_unlock(folio); 2853 btrfs_start_ordered_extent(ordered); 2854 btrfs_put_ordered_extent(ordered); 2855 goto again; 2856 } 2857 2858 ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0, 2859 &cached_state); 2860 if (ret) 2861 goto out_reserved; 2862 2863 /* 2864 * Everything went as planned, we're now the owner of a dirty page with 2865 * delayed allocation bits set and space reserved for our COW 2866 * destination. 2867 * 2868 * The page was dirty when we started, nothing should have cleaned it. 2869 */ 2870 BUG_ON(!folio_test_dirty(folio)); 2871 free_delalloc_space = false; 2872out_reserved: 2873 btrfs_delalloc_release_extents(inode, PAGE_SIZE); 2874 if (free_delalloc_space) 2875 btrfs_delalloc_release_space(inode, data_reserved, page_start, 2876 PAGE_SIZE, true); 2877 btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); 2878out_page: 2879 if (ret) { 2880 /* 2881 * We hit ENOSPC or other errors. Update the mapping and page 2882 * to reflect the errors and clean the page. 2883 */ 2884 mapping_set_error(folio->mapping, ret); 2885 btrfs_mark_ordered_io_finished(inode, folio, page_start, 2886 folio_size(folio), !ret); 2887 folio_clear_dirty_for_io(folio); 2888 } 2889 btrfs_folio_clear_checked(fs_info, folio, page_start, PAGE_SIZE); 2890 folio_unlock(folio); 2891 folio_put(folio); 2892 kfree(fixup); 2893 extent_changeset_free(data_reserved); 2894 /* 2895 * As a precaution, do a delayed iput in case it would be the last iput 2896 * that could need flushing space. Recursing back to fixup worker would 2897 * deadlock. 2898 */ 2899 btrfs_add_delayed_iput(inode); 2900} 2901 2902/* 2903 * There are a few paths in the higher layers of the kernel that directly 2904 * set the folio dirty bit without asking the filesystem if it is a 2905 * good idea. This causes problems because we want to make sure COW 2906 * properly happens and the data=ordered rules are followed. 2907 * 2908 * In our case any range that doesn't have the ORDERED bit set 2909 * hasn't been properly setup for IO. We kick off an async process 2910 * to fix it up. The async helper will wait for ordered extents, set 2911 * the delalloc bit and make it safe to write the folio. 2912 */ 2913int btrfs_writepage_cow_fixup(struct folio *folio) 2914{ 2915 struct inode *inode = folio->mapping->host; 2916 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); 2917 struct btrfs_writepage_fixup *fixup; 2918 2919 /* This folio has ordered extent covering it already */ 2920 if (folio_test_ordered(folio)) 2921 return 0; 2922 2923 /* 2924 * For experimental build, we error out instead of EAGAIN. 2925 * 2926 * We should not hit such out-of-band dirty folios anymore. 2927 */ 2928 if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL)) { 2929 DEBUG_WARN(); 2930 btrfs_err_rl(fs_info, 2931 "root %lld ino %llu folio %llu is marked dirty without notifying the fs", 2932 btrfs_root_id(BTRFS_I(inode)->root), 2933 btrfs_ino(BTRFS_I(inode)), 2934 folio_pos(folio)); 2935 return -EUCLEAN; 2936 } 2937 2938 /* 2939 * folio_checked is set below when we create a fixup worker for this 2940 * folio, don't try to create another one if we're already 2941 * folio_test_checked. 2942 * 2943 * The extent_io writepage code will redirty the foio if we send back 2944 * EAGAIN. 2945 */ 2946 if (folio_test_checked(folio)) 2947 return -EAGAIN; 2948 2949 fixup = kzalloc(sizeof(*fixup), GFP_NOFS); 2950 if (!fixup) 2951 return -EAGAIN; 2952 2953 /* 2954 * We are already holding a reference to this inode from 2955 * write_cache_pages. We need to hold it because the space reservation 2956 * takes place outside of the folio lock, and we can't trust 2957 * folio->mapping outside of the folio lock. 2958 */ 2959 ihold(inode); 2960 btrfs_folio_set_checked(fs_info, folio, folio_pos(folio), folio_size(folio)); 2961 folio_get(folio); 2962 btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL); 2963 fixup->folio = folio; 2964 fixup->inode = BTRFS_I(inode); 2965 btrfs_queue_work(fs_info->fixup_workers, &fixup->work); 2966 2967 return -EAGAIN; 2968} 2969 2970static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, 2971 struct btrfs_inode *inode, u64 file_pos, 2972 struct btrfs_file_extent_item *stack_fi, 2973 const bool update_inode_bytes, 2974 u64 qgroup_reserved) 2975{ 2976 struct btrfs_root *root = inode->root; 2977 const u64 sectorsize = root->fs_info->sectorsize; 2978 BTRFS_PATH_AUTO_FREE(path); 2979 struct extent_buffer *leaf; 2980 struct btrfs_key ins; 2981 u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi); 2982 u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi); 2983 u64 offset = btrfs_stack_file_extent_offset(stack_fi); 2984 u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi); 2985 u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi); 2986 struct btrfs_drop_extents_args drop_args = { 0 }; 2987 int ret; 2988 2989 path = btrfs_alloc_path(); 2990 if (!path) 2991 return -ENOMEM; 2992 2993 /* 2994 * we may be replacing one extent in the tree with another. 2995 * The new extent is pinned in the extent map, and we don't want 2996 * to drop it from the cache until it is completely in the btree. 2997 * 2998 * So, tell btrfs_drop_extents to leave this extent in the cache. 2999 * the caller is expected to unpin it and allow it to be merged 3000 * with the others. 3001 */ 3002 drop_args.path = path; 3003 drop_args.start = file_pos; 3004 drop_args.end = file_pos + num_bytes; 3005 drop_args.replace_extent = true; 3006 drop_args.extent_item_size = sizeof(*stack_fi); 3007 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 3008 if (ret) 3009 goto out; 3010 3011 if (!drop_args.extent_inserted) { 3012 ins.objectid = btrfs_ino(inode); 3013 ins.type = BTRFS_EXTENT_DATA_KEY; 3014 ins.offset = file_pos; 3015 3016 ret = btrfs_insert_empty_item(trans, root, path, &ins, 3017 sizeof(*stack_fi)); 3018 if (ret) 3019 goto out; 3020 } 3021 leaf = path->nodes[0]; 3022 btrfs_set_stack_file_extent_generation(stack_fi, trans->transid); 3023 write_extent_buffer(leaf, stack_fi, 3024 btrfs_item_ptr_offset(leaf, path->slots[0]), 3025 sizeof(struct btrfs_file_extent_item)); 3026 3027 btrfs_release_path(path); 3028 3029 /* 3030 * If we dropped an inline extent here, we know the range where it is 3031 * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the 3032 * number of bytes only for that range containing the inline extent. 3033 * The remaining of the range will be processed when clearing the 3034 * EXTENT_DELALLOC_BIT bit through the ordered extent completion. 3035 */ 3036 if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) { 3037 u64 inline_size = round_down(drop_args.bytes_found, sectorsize); 3038 3039 inline_size = drop_args.bytes_found - inline_size; 3040 btrfs_update_inode_bytes(inode, sectorsize, inline_size); 3041 drop_args.bytes_found -= inline_size; 3042 num_bytes -= sectorsize; 3043 } 3044 3045 if (update_inode_bytes) 3046 btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found); 3047 3048 ins.objectid = disk_bytenr; 3049 ins.type = BTRFS_EXTENT_ITEM_KEY; 3050 ins.offset = disk_num_bytes; 3051 3052 ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes); 3053 if (ret) 3054 goto out; 3055 3056 ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode), 3057 file_pos - offset, 3058 qgroup_reserved, &ins); 3059out: 3060 return ret; 3061} 3062 3063static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, 3064 u64 start, u64 len) 3065{ 3066 struct btrfs_block_group *cache; 3067 3068 cache = btrfs_lookup_block_group(fs_info, start); 3069 ASSERT(cache); 3070 3071 spin_lock(&cache->lock); 3072 cache->delalloc_bytes -= len; 3073 spin_unlock(&cache->lock); 3074 3075 btrfs_put_block_group(cache); 3076} 3077 3078static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, 3079 struct btrfs_ordered_extent *oe) 3080{ 3081 struct btrfs_file_extent_item stack_fi; 3082 bool update_inode_bytes; 3083 u64 num_bytes = oe->num_bytes; 3084 u64 ram_bytes = oe->ram_bytes; 3085 3086 memset(&stack_fi, 0, sizeof(stack_fi)); 3087 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG); 3088 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr); 3089 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, 3090 oe->disk_num_bytes); 3091 btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset); 3092 if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags)) 3093 num_bytes = oe->truncated_len; 3094 btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes); 3095 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes); 3096 btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type); 3097 /* Encryption and other encoding is reserved and all 0 */ 3098 3099 /* 3100 * For delalloc, when completing an ordered extent we update the inode's 3101 * bytes when clearing the range in the inode's io tree, so pass false 3102 * as the argument 'update_inode_bytes' to insert_reserved_file_extent(), 3103 * except if the ordered extent was truncated. 3104 */ 3105 update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) || 3106 test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) || 3107 test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags); 3108 3109 return insert_reserved_file_extent(trans, oe->inode, 3110 oe->file_offset, &stack_fi, 3111 update_inode_bytes, oe->qgroup_rsv); 3112} 3113 3114/* 3115 * As ordered data IO finishes, this gets called so we can finish 3116 * an ordered extent if the range of bytes in the file it covers are 3117 * fully written. 3118 */ 3119int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) 3120{ 3121 struct btrfs_inode *inode = ordered_extent->inode; 3122 struct btrfs_root *root = inode->root; 3123 struct btrfs_fs_info *fs_info = root->fs_info; 3124 struct btrfs_trans_handle *trans = NULL; 3125 struct extent_io_tree *io_tree = &inode->io_tree; 3126 struct extent_state *cached_state = NULL; 3127 u64 start, end; 3128 int compress_type = 0; 3129 int ret = 0; 3130 u64 logical_len = ordered_extent->num_bytes; 3131 bool freespace_inode; 3132 bool truncated = false; 3133 bool clear_reserved_extent = true; 3134 unsigned int clear_bits = EXTENT_DEFRAG; 3135 3136 start = ordered_extent->file_offset; 3137 end = start + ordered_extent->num_bytes - 1; 3138 3139 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3140 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && 3141 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) && 3142 !test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags)) 3143 clear_bits |= EXTENT_DELALLOC_NEW; 3144 3145 freespace_inode = btrfs_is_free_space_inode(inode); 3146 if (!freespace_inode) 3147 btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent); 3148 3149 if (unlikely(test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags))) { 3150 ret = -EIO; 3151 goto out; 3152 } 3153 3154 ret = btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr, 3155 ordered_extent->disk_num_bytes); 3156 if (ret) 3157 goto out; 3158 3159 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { 3160 truncated = true; 3161 logical_len = ordered_extent->truncated_len; 3162 /* Truncated the entire extent, don't bother adding */ 3163 if (!logical_len) 3164 goto out; 3165 } 3166 3167 /* 3168 * If it's a COW write we need to lock the extent range as we will be 3169 * inserting/replacing file extent items and unpinning an extent map. 3170 * This must be taken before joining a transaction, as it's a higher 3171 * level lock (like the inode's VFS lock), otherwise we can run into an 3172 * ABBA deadlock with other tasks (transactions work like a lock, 3173 * depending on their current state). 3174 */ 3175 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 3176 clear_bits |= EXTENT_LOCKED | EXTENT_FINISHING_ORDERED; 3177 btrfs_lock_extent_bits(io_tree, start, end, 3178 EXTENT_LOCKED | EXTENT_FINISHING_ORDERED, 3179 &cached_state); 3180 } 3181 3182 if (freespace_inode) 3183 trans = btrfs_join_transaction_spacecache(root); 3184 else 3185 trans = btrfs_join_transaction(root); 3186 if (IS_ERR(trans)) { 3187 ret = PTR_ERR(trans); 3188 trans = NULL; 3189 goto out; 3190 } 3191 3192 trans->block_rsv = &inode->block_rsv; 3193 3194 ret = btrfs_insert_raid_extent(trans, ordered_extent); 3195 if (unlikely(ret)) { 3196 btrfs_abort_transaction(trans, ret); 3197 goto out; 3198 } 3199 3200 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { 3201 /* Logic error */ 3202 ASSERT(list_empty(&ordered_extent->list)); 3203 if (unlikely(!list_empty(&ordered_extent->list))) { 3204 ret = -EINVAL; 3205 btrfs_abort_transaction(trans, ret); 3206 goto out; 3207 } 3208 3209 btrfs_inode_safe_disk_i_size_write(inode, 0); 3210 ret = btrfs_update_inode_fallback(trans, inode); 3211 if (unlikely(ret)) { 3212 /* -ENOMEM or corruption */ 3213 btrfs_abort_transaction(trans, ret); 3214 } 3215 goto out; 3216 } 3217 3218 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) 3219 compress_type = ordered_extent->compress_type; 3220 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3221 BUG_ON(compress_type); 3222 ret = btrfs_mark_extent_written(trans, inode, 3223 ordered_extent->file_offset, 3224 ordered_extent->file_offset + 3225 logical_len); 3226 btrfs_zoned_release_data_reloc_bg(fs_info, ordered_extent->disk_bytenr, 3227 ordered_extent->disk_num_bytes); 3228 } else { 3229 BUG_ON(root == fs_info->tree_root); 3230 ret = insert_ordered_extent_file_extent(trans, ordered_extent); 3231 if (!ret) { 3232 clear_reserved_extent = false; 3233 btrfs_release_delalloc_bytes(fs_info, 3234 ordered_extent->disk_bytenr, 3235 ordered_extent->disk_num_bytes); 3236 } 3237 } 3238 if (unlikely(ret < 0)) { 3239 btrfs_abort_transaction(trans, ret); 3240 goto out; 3241 } 3242 3243 ret = btrfs_unpin_extent_cache(inode, ordered_extent->file_offset, 3244 ordered_extent->num_bytes, trans->transid); 3245 if (unlikely(ret < 0)) { 3246 btrfs_abort_transaction(trans, ret); 3247 goto out; 3248 } 3249 3250 ret = add_pending_csums(trans, &ordered_extent->list); 3251 if (unlikely(ret)) { 3252 btrfs_abort_transaction(trans, ret); 3253 goto out; 3254 } 3255 3256 /* 3257 * If this is a new delalloc range, clear its new delalloc flag to 3258 * update the inode's number of bytes. This needs to be done first 3259 * before updating the inode item. 3260 */ 3261 if ((clear_bits & EXTENT_DELALLOC_NEW) && 3262 !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) 3263 btrfs_clear_extent_bit(&inode->io_tree, start, end, 3264 EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES, 3265 &cached_state); 3266 3267 btrfs_inode_safe_disk_i_size_write(inode, 0); 3268 ret = btrfs_update_inode_fallback(trans, inode); 3269 if (unlikely(ret)) { /* -ENOMEM or corruption */ 3270 btrfs_abort_transaction(trans, ret); 3271 goto out; 3272 } 3273out: 3274 btrfs_clear_extent_bit(&inode->io_tree, start, end, clear_bits, 3275 &cached_state); 3276 3277 if (trans) 3278 btrfs_end_transaction(trans); 3279 3280 if (ret || truncated) { 3281 /* 3282 * If we failed to finish this ordered extent for any reason we 3283 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered 3284 * extent, and mark the inode with the error if it wasn't 3285 * already set. Any error during writeback would have already 3286 * set the mapping error, so we need to set it if we're the ones 3287 * marking this ordered extent as failed. 3288 */ 3289 if (ret) 3290 btrfs_mark_ordered_extent_error(ordered_extent); 3291 3292 /* 3293 * Drop extent maps for the part of the extent we didn't write. 3294 * 3295 * We have an exception here for the free_space_inode, this is 3296 * because when we do btrfs_get_extent() on the free space inode 3297 * we will search the commit root. If this is a new block group 3298 * we won't find anything, and we will trip over the assert in 3299 * writepage where we do ASSERT(em->block_start != 3300 * EXTENT_MAP_HOLE). 3301 * 3302 * Theoretically we could also skip this for any NOCOW extent as 3303 * we don't mess with the extent map tree in the NOCOW case, but 3304 * for now simply skip this if we are the free space inode. 3305 */ 3306 if (!btrfs_is_free_space_inode(inode)) { 3307 u64 unwritten_start = start; 3308 3309 if (truncated) 3310 unwritten_start += logical_len; 3311 3312 btrfs_drop_extent_map_range(inode, unwritten_start, 3313 end, false); 3314 } 3315 3316 /* 3317 * If the ordered extent had an IOERR or something else went 3318 * wrong we need to return the space for this ordered extent 3319 * back to the allocator. We only free the extent in the 3320 * truncated case if we didn't write out the extent at all. 3321 * 3322 * If we made it past insert_reserved_file_extent before we 3323 * errored out then we don't need to do this as the accounting 3324 * has already been done. 3325 */ 3326 if ((ret || !logical_len) && 3327 clear_reserved_extent && 3328 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && 3329 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { 3330 /* 3331 * Discard the range before returning it back to the 3332 * free space pool 3333 */ 3334 if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC)) 3335 btrfs_discard_extent(fs_info, 3336 ordered_extent->disk_bytenr, 3337 ordered_extent->disk_num_bytes, 3338 NULL); 3339 btrfs_free_reserved_extent(fs_info, 3340 ordered_extent->disk_bytenr, 3341 ordered_extent->disk_num_bytes, true); 3342 /* 3343 * Actually free the qgroup rsv which was released when 3344 * the ordered extent was created. 3345 */ 3346 btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(inode->root), 3347 ordered_extent->qgroup_rsv, 3348 BTRFS_QGROUP_RSV_DATA); 3349 } 3350 } 3351 3352 /* 3353 * This needs to be done to make sure anybody waiting knows we are done 3354 * updating everything for this ordered extent. 3355 */ 3356 btrfs_remove_ordered_extent(inode, ordered_extent); 3357 3358 /* once for us */ 3359 btrfs_put_ordered_extent(ordered_extent); 3360 /* once for the tree */ 3361 btrfs_put_ordered_extent(ordered_extent); 3362 3363 return ret; 3364} 3365 3366int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered) 3367{ 3368 if (btrfs_is_zoned(ordered->inode->root->fs_info) && 3369 !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) && 3370 list_empty(&ordered->bioc_list)) 3371 btrfs_finish_ordered_zoned(ordered); 3372 return btrfs_finish_one_ordered(ordered); 3373} 3374 3375/* 3376 * Calculate the checksum of an fs block at physical memory address @paddr, 3377 * and save the result to @dest. 3378 * 3379 * The folio containing @paddr must be large enough to contain a full fs block. 3380 */ 3381void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info, 3382 const phys_addr_t paddr, u8 *dest) 3383{ 3384 struct folio *folio = page_folio(phys_to_page(paddr)); 3385 const u32 blocksize = fs_info->sectorsize; 3386 const u32 step = min(blocksize, PAGE_SIZE); 3387 const u32 nr_steps = blocksize / step; 3388 phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; 3389 3390 /* The full block must be inside the folio. */ 3391 ASSERT(offset_in_folio(folio, paddr) + blocksize <= folio_size(folio)); 3392 3393 for (int i = 0; i < nr_steps; i++) { 3394 u32 pindex = offset_in_folio(folio, paddr + i * step) >> PAGE_SHIFT; 3395 3396 /* 3397 * For bs <= ps cases, we will only run the loop once, so the offset 3398 * inside the page will only added to paddrs[0]. 3399 * 3400 * For bs > ps cases, the block must be page aligned, thus offset 3401 * inside the page will always be 0. 3402 */ 3403 paddrs[i] = page_to_phys(folio_page(folio, pindex)) + offset_in_page(paddr); 3404 } 3405 return btrfs_calculate_block_csum_pages(fs_info, paddrs, dest); 3406} 3407 3408/* 3409 * Calculate the checksum of a fs block backed by multiple noncontiguous pages 3410 * at @paddrs[] and save the result to @dest. 3411 * 3412 * The folio containing @paddr must be large enough to contain a full fs block. 3413 */ 3414void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info, 3415 const phys_addr_t paddrs[], u8 *dest) 3416{ 3417 const u32 blocksize = fs_info->sectorsize; 3418 const u32 step = min(blocksize, PAGE_SIZE); 3419 const u32 nr_steps = blocksize / step; 3420 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); 3421 3422 shash->tfm = fs_info->csum_shash; 3423 crypto_shash_init(shash); 3424 for (int i = 0; i < nr_steps; i++) { 3425 const phys_addr_t paddr = paddrs[i]; 3426 void *kaddr; 3427 3428 ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE); 3429 kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr); 3430 crypto_shash_update(shash, kaddr, step); 3431 kunmap_local(kaddr); 3432 } 3433 crypto_shash_final(shash, dest); 3434} 3435 3436/* 3437 * Verify the checksum for a single sector without any extra action that depend 3438 * on the type of I/O. 3439 * 3440 * @kaddr must be a properly kmapped address. 3441 */ 3442int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum, 3443 const u8 * const csum_expected) 3444{ 3445 btrfs_calculate_block_csum_folio(fs_info, paddr, csum); 3446 if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0)) 3447 return -EIO; 3448 return 0; 3449} 3450 3451/* 3452 * Verify the checksum of a single data sector, which can be scattered at 3453 * different noncontiguous pages. 3454 * 3455 * @bbio: btrfs_io_bio which contains the csum 3456 * @dev: device the sector is on 3457 * @bio_offset: offset to the beginning of the bio (in bytes) 3458 * @paddrs: physical addresses which back the fs block 3459 * 3460 * Check if the checksum on a data block is valid. When a checksum mismatch is 3461 * detected, report the error and fill the corrupted range with zero. 3462 * 3463 * Return %true if the sector is ok or had no checksum to start with, else %false. 3464 */ 3465bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, 3466 u32 bio_offset, const phys_addr_t paddrs[]) 3467{ 3468 struct btrfs_inode *inode = bbio->inode; 3469 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3470 const u32 blocksize = fs_info->sectorsize; 3471 const u32 step = min(blocksize, PAGE_SIZE); 3472 const u32 nr_steps = blocksize / step; 3473 u64 file_offset = bbio->file_offset + bio_offset; 3474 u64 end = file_offset + blocksize - 1; 3475 u8 *csum_expected; 3476 u8 csum[BTRFS_CSUM_SIZE]; 3477 3478 if (!bbio->csum) 3479 return true; 3480 3481 if (btrfs_is_data_reloc_root(inode->root) && 3482 btrfs_test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM, 3483 NULL)) { 3484 /* Skip the range without csum for data reloc inode */ 3485 btrfs_clear_extent_bit(&inode->io_tree, file_offset, end, 3486 EXTENT_NODATASUM, NULL); 3487 return true; 3488 } 3489 3490 csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) * 3491 fs_info->csum_size; 3492 btrfs_calculate_block_csum_pages(fs_info, paddrs, csum); 3493 if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0)) 3494 goto zeroit; 3495 return true; 3496 3497zeroit: 3498 btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected, 3499 bbio->mirror_num); 3500 if (dev) 3501 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); 3502 for (int i = 0; i < nr_steps; i++) 3503 memzero_page(phys_to_page(paddrs[i]), offset_in_page(paddrs[i]), step); 3504 return false; 3505} 3506 3507/* 3508 * Perform a delayed iput on @inode. 3509 * 3510 * @inode: The inode we want to perform iput on 3511 * 3512 * This function uses the generic vfs_inode::i_count to track whether we should 3513 * just decrement it (in case it's > 1) or if this is the last iput then link 3514 * the inode to the delayed iput machinery. Delayed iputs are processed at 3515 * transaction commit time/superblock commit/cleaner kthread. 3516 */ 3517void btrfs_add_delayed_iput(struct btrfs_inode *inode) 3518{ 3519 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3520 unsigned long flags; 3521 3522 if (atomic_add_unless(&inode->vfs_inode.i_count, -1, 1)) 3523 return; 3524 3525 WARN_ON_ONCE(test_bit(BTRFS_FS_STATE_NO_DELAYED_IPUT, &fs_info->fs_state)); 3526 atomic_inc(&fs_info->nr_delayed_iputs); 3527 /* 3528 * Need to be irq safe here because we can be called from either an irq 3529 * context (see bio.c and btrfs_put_ordered_extent()) or a non-irq 3530 * context. 3531 */ 3532 spin_lock_irqsave(&fs_info->delayed_iput_lock, flags); 3533 ASSERT(list_empty(&inode->delayed_iput)); 3534 list_add_tail(&inode->delayed_iput, &fs_info->delayed_iputs); 3535 spin_unlock_irqrestore(&fs_info->delayed_iput_lock, flags); 3536 if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags)) 3537 wake_up_process(fs_info->cleaner_kthread); 3538} 3539 3540static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info, 3541 struct btrfs_inode *inode) 3542{ 3543 list_del_init(&inode->delayed_iput); 3544 spin_unlock_irq(&fs_info->delayed_iput_lock); 3545 iput(&inode->vfs_inode); 3546 if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) 3547 wake_up(&fs_info->delayed_iputs_wait); 3548 spin_lock_irq(&fs_info->delayed_iput_lock); 3549} 3550 3551static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info, 3552 struct btrfs_inode *inode) 3553{ 3554 if (!list_empty(&inode->delayed_iput)) { 3555 spin_lock_irq(&fs_info->delayed_iput_lock); 3556 if (!list_empty(&inode->delayed_iput)) 3557 run_delayed_iput_locked(fs_info, inode); 3558 spin_unlock_irq(&fs_info->delayed_iput_lock); 3559 } 3560} 3561 3562void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) 3563{ 3564 /* 3565 * btrfs_put_ordered_extent() can run in irq context (see bio.c), which 3566 * calls btrfs_add_delayed_iput() and that needs to lock 3567 * fs_info->delayed_iput_lock. So we need to disable irqs here to 3568 * prevent a deadlock. 3569 */ 3570 spin_lock_irq(&fs_info->delayed_iput_lock); 3571 while (!list_empty(&fs_info->delayed_iputs)) { 3572 struct btrfs_inode *inode; 3573 3574 inode = list_first_entry(&fs_info->delayed_iputs, 3575 struct btrfs_inode, delayed_iput); 3576 run_delayed_iput_locked(fs_info, inode); 3577 if (need_resched()) { 3578 spin_unlock_irq(&fs_info->delayed_iput_lock); 3579 cond_resched(); 3580 spin_lock_irq(&fs_info->delayed_iput_lock); 3581 } 3582 } 3583 spin_unlock_irq(&fs_info->delayed_iput_lock); 3584} 3585 3586/* 3587 * Wait for flushing all delayed iputs 3588 * 3589 * @fs_info: the filesystem 3590 * 3591 * This will wait on any delayed iputs that are currently running with KILLABLE 3592 * set. Once they are all done running we will return, unless we are killed in 3593 * which case we return EINTR. This helps in user operations like fallocate etc 3594 * that might get blocked on the iputs. 3595 * 3596 * Return EINTR if we were killed, 0 if nothing's pending 3597 */ 3598int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) 3599{ 3600 int ret = wait_event_killable(fs_info->delayed_iputs_wait, 3601 atomic_read(&fs_info->nr_delayed_iputs) == 0); 3602 if (ret) 3603 return -EINTR; 3604 return 0; 3605} 3606 3607/* 3608 * This creates an orphan entry for the given inode in case something goes wrong 3609 * in the middle of an unlink. 3610 */ 3611int btrfs_orphan_add(struct btrfs_trans_handle *trans, 3612 struct btrfs_inode *inode) 3613{ 3614 int ret; 3615 3616 ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode)); 3617 if (unlikely(ret && ret != -EEXIST)) { 3618 btrfs_abort_transaction(trans, ret); 3619 return ret; 3620 } 3621 3622 return 0; 3623} 3624 3625/* 3626 * We have done the delete so we can go ahead and remove the orphan item for 3627 * this particular inode. 3628 */ 3629static int btrfs_orphan_del(struct btrfs_trans_handle *trans, 3630 struct btrfs_inode *inode) 3631{ 3632 return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode)); 3633} 3634 3635/* 3636 * this cleans up any orphans that may be left on the list from the last use 3637 * of this root. 3638 */ 3639int btrfs_orphan_cleanup(struct btrfs_root *root) 3640{ 3641 struct btrfs_fs_info *fs_info = root->fs_info; 3642 BTRFS_PATH_AUTO_FREE(path); 3643 struct extent_buffer *leaf; 3644 struct btrfs_key key, found_key; 3645 struct btrfs_trans_handle *trans; 3646 u64 last_objectid = 0; 3647 int ret = 0, nr_unlink = 0; 3648 3649 if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state)) 3650 return 0; 3651 3652 path = btrfs_alloc_path(); 3653 if (!path) { 3654 ret = -ENOMEM; 3655 goto out; 3656 } 3657 path->reada = READA_BACK; 3658 3659 key.objectid = BTRFS_ORPHAN_OBJECTID; 3660 key.type = BTRFS_ORPHAN_ITEM_KEY; 3661 key.offset = (u64)-1; 3662 3663 while (1) { 3664 struct btrfs_inode *inode; 3665 3666 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 3667 if (ret < 0) 3668 goto out; 3669 3670 /* 3671 * if ret == 0 means we found what we were searching for, which 3672 * is weird, but possible, so only screw with path if we didn't 3673 * find the key and see if we have stuff that matches 3674 */ 3675 if (ret > 0) { 3676 ret = 0; 3677 if (path->slots[0] == 0) 3678 break; 3679 path->slots[0]--; 3680 } 3681 3682 /* pull out the item */ 3683 leaf = path->nodes[0]; 3684 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 3685 3686 /* make sure the item matches what we want */ 3687 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) 3688 break; 3689 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) 3690 break; 3691 3692 /* release the path since we're done with it */ 3693 btrfs_release_path(path); 3694 3695 /* 3696 * this is where we are basically btrfs_lookup, without the 3697 * crossing root thing. we store the inode number in the 3698 * offset of the orphan item. 3699 */ 3700 3701 if (found_key.offset == last_objectid) { 3702 /* 3703 * We found the same inode as before. This means we were 3704 * not able to remove its items via eviction triggered 3705 * by an iput(). A transaction abort may have happened, 3706 * due to -ENOSPC for example, so try to grab the error 3707 * that lead to a transaction abort, if any. 3708 */ 3709 btrfs_err(fs_info, 3710 "Error removing orphan entry, stopping orphan cleanup"); 3711 ret = BTRFS_FS_ERROR(fs_info) ?: -EINVAL; 3712 goto out; 3713 } 3714 3715 last_objectid = found_key.offset; 3716 3717 found_key.objectid = found_key.offset; 3718 found_key.type = BTRFS_INODE_ITEM_KEY; 3719 found_key.offset = 0; 3720 inode = btrfs_iget(last_objectid, root); 3721 if (IS_ERR(inode)) { 3722 ret = PTR_ERR(inode); 3723 inode = NULL; 3724 if (ret != -ENOENT) 3725 goto out; 3726 } 3727 3728 if (!inode && root == fs_info->tree_root) { 3729 struct btrfs_root *dead_root; 3730 int is_dead_root = 0; 3731 3732 /* 3733 * This is an orphan in the tree root. Currently these 3734 * could come from 2 sources: 3735 * a) a root (snapshot/subvolume) deletion in progress 3736 * b) a free space cache inode 3737 * We need to distinguish those two, as the orphan item 3738 * for a root must not get deleted before the deletion 3739 * of the snapshot/subvolume's tree completes. 3740 * 3741 * btrfs_find_orphan_roots() ran before us, which has 3742 * found all deleted roots and loaded them into 3743 * fs_info->fs_roots_radix. So here we can find if an 3744 * orphan item corresponds to a deleted root by looking 3745 * up the root from that radix tree. 3746 */ 3747 3748 spin_lock(&fs_info->fs_roots_radix_lock); 3749 dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, 3750 (unsigned long)found_key.objectid); 3751 if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) 3752 is_dead_root = 1; 3753 spin_unlock(&fs_info->fs_roots_radix_lock); 3754 3755 if (is_dead_root) { 3756 /* prevent this orphan from being found again */ 3757 key.offset = found_key.objectid - 1; 3758 continue; 3759 } 3760 3761 } 3762 3763 /* 3764 * If we have an inode with links, there are a couple of 3765 * possibilities: 3766 * 3767 * 1. We were halfway through creating fsverity metadata for the 3768 * file. In that case, the orphan item represents incomplete 3769 * fsverity metadata which must be cleaned up with 3770 * btrfs_drop_verity_items and deleting the orphan item. 3771 3772 * 2. Old kernels (before v3.12) used to create an 3773 * orphan item for truncate indicating that there were possibly 3774 * extent items past i_size that needed to be deleted. In v3.12, 3775 * truncate was changed to update i_size in sync with the extent 3776 * items, but the (useless) orphan item was still created. Since 3777 * v4.18, we don't create the orphan item for truncate at all. 3778 * 3779 * So, this item could mean that we need to do a truncate, but 3780 * only if this filesystem was last used on a pre-v3.12 kernel 3781 * and was not cleanly unmounted. The odds of that are quite 3782 * slim, and it's a pain to do the truncate now, so just delete 3783 * the orphan item. 3784 * 3785 * It's also possible that this orphan item was supposed to be 3786 * deleted but wasn't. The inode number may have been reused, 3787 * but either way, we can delete the orphan item. 3788 */ 3789 if (!inode || inode->vfs_inode.i_nlink) { 3790 if (inode) { 3791 ret = btrfs_drop_verity_items(inode); 3792 iput(&inode->vfs_inode); 3793 inode = NULL; 3794 if (ret) 3795 goto out; 3796 } 3797 trans = btrfs_start_transaction(root, 1); 3798 if (IS_ERR(trans)) { 3799 ret = PTR_ERR(trans); 3800 goto out; 3801 } 3802 btrfs_debug(fs_info, "auto deleting %Lu", 3803 found_key.objectid); 3804 ret = btrfs_del_orphan_item(trans, root, 3805 found_key.objectid); 3806 btrfs_end_transaction(trans); 3807 if (ret) 3808 goto out; 3809 continue; 3810 } 3811 3812 nr_unlink++; 3813 3814 /* this will do delete_inode and everything for us */ 3815 iput(&inode->vfs_inode); 3816 } 3817 /* release the path since we're done with it */ 3818 btrfs_release_path(path); 3819 3820 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { 3821 trans = btrfs_join_transaction(root); 3822 if (!IS_ERR(trans)) 3823 btrfs_end_transaction(trans); 3824 } 3825 3826 if (nr_unlink) 3827 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); 3828 3829out: 3830 if (ret) 3831 btrfs_err(fs_info, "could not do orphan cleanup %d", ret); 3832 return ret; 3833} 3834 3835/* 3836 * Look ahead in the leaf for xattrs. If we don't find any then we know there 3837 * can't be any ACLs. 3838 * 3839 * @leaf: the eb leaf where to search 3840 * @slot: the slot the inode is in 3841 * @objectid: the objectid of the inode 3842 * 3843 * Return true if there is xattr/ACL, false otherwise. 3844 */ 3845static noinline bool acls_after_inode_item(struct extent_buffer *leaf, 3846 int slot, u64 objectid, 3847 int *first_xattr_slot) 3848{ 3849 u32 nritems = btrfs_header_nritems(leaf); 3850 struct btrfs_key found_key; 3851 static u64 xattr_access = 0; 3852 static u64 xattr_default = 0; 3853 int scanned = 0; 3854 3855 if (!xattr_access) { 3856 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, 3857 strlen(XATTR_NAME_POSIX_ACL_ACCESS)); 3858 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, 3859 strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); 3860 } 3861 3862 slot++; 3863 *first_xattr_slot = -1; 3864 while (slot < nritems) { 3865 btrfs_item_key_to_cpu(leaf, &found_key, slot); 3866 3867 /* We found a different objectid, there must be no ACLs. */ 3868 if (found_key.objectid != objectid) 3869 return false; 3870 3871 /* We found an xattr, assume we've got an ACL. */ 3872 if (found_key.type == BTRFS_XATTR_ITEM_KEY) { 3873 if (*first_xattr_slot == -1) 3874 *first_xattr_slot = slot; 3875 if (found_key.offset == xattr_access || 3876 found_key.offset == xattr_default) 3877 return true; 3878 } 3879 3880 /* 3881 * We found a key greater than an xattr key, there can't be any 3882 * ACLs later on. 3883 */ 3884 if (found_key.type > BTRFS_XATTR_ITEM_KEY) 3885 return false; 3886 3887 slot++; 3888 scanned++; 3889 3890 /* 3891 * The item order goes like: 3892 * - inode 3893 * - inode backrefs 3894 * - xattrs 3895 * - extents, 3896 * 3897 * so if there are lots of hard links to an inode there can be 3898 * a lot of backrefs. Don't waste time searching too hard, 3899 * this is just an optimization. 3900 */ 3901 if (scanned >= 8) 3902 break; 3903 } 3904 /* 3905 * We hit the end of the leaf before we found an xattr or something 3906 * larger than an xattr. We have to assume the inode has ACLs. 3907 */ 3908 if (*first_xattr_slot == -1) 3909 *first_xattr_slot = slot; 3910 return true; 3911} 3912 3913static int btrfs_init_file_extent_tree(struct btrfs_inode *inode) 3914{ 3915 struct btrfs_fs_info *fs_info = inode->root->fs_info; 3916 3917 if (WARN_ON_ONCE(inode->file_extent_tree)) 3918 return 0; 3919 if (btrfs_fs_incompat(fs_info, NO_HOLES)) 3920 return 0; 3921 if (!S_ISREG(inode->vfs_inode.i_mode)) 3922 return 0; 3923 if (btrfs_is_free_space_inode(inode)) 3924 return 0; 3925 3926 inode->file_extent_tree = kmalloc(sizeof(struct extent_io_tree), GFP_KERNEL); 3927 if (!inode->file_extent_tree) 3928 return -ENOMEM; 3929 3930 btrfs_extent_io_tree_init(fs_info, inode->file_extent_tree, 3931 IO_TREE_INODE_FILE_EXTENT); 3932 /* Lockdep class is set only for the file extent tree. */ 3933 lockdep_set_class(&inode->file_extent_tree->lock, &file_extent_tree_class); 3934 3935 return 0; 3936} 3937 3938static int btrfs_add_inode_to_root(struct btrfs_inode *inode, bool prealloc) 3939{ 3940 struct btrfs_root *root = inode->root; 3941 struct btrfs_inode *existing; 3942 const u64 ino = btrfs_ino(inode); 3943 int ret; 3944 3945 if (inode_unhashed(&inode->vfs_inode)) 3946 return 0; 3947 3948 if (prealloc) { 3949 ret = xa_reserve(&root->inodes, ino, GFP_NOFS); 3950 if (ret) 3951 return ret; 3952 } 3953 3954 existing = xa_store(&root->inodes, ino, inode, GFP_ATOMIC); 3955 3956 if (xa_is_err(existing)) { 3957 ret = xa_err(existing); 3958 ASSERT(ret != -EINVAL); 3959 ASSERT(ret != -ENOMEM); 3960 return ret; 3961 } else if (existing) { 3962 WARN_ON(!(inode_state_read_once(&existing->vfs_inode) & (I_WILL_FREE | I_FREEING))); 3963 } 3964 3965 return 0; 3966} 3967 3968/* 3969 * Read a locked inode from the btree into the in-memory inode and add it to 3970 * its root list/tree. 3971 * 3972 * On failure clean up the inode. 3973 */ 3974static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path *path) 3975{ 3976 struct btrfs_root *root = inode->root; 3977 struct btrfs_fs_info *fs_info = root->fs_info; 3978 struct extent_buffer *leaf; 3979 struct btrfs_inode_item *inode_item; 3980 struct inode *vfs_inode = &inode->vfs_inode; 3981 struct btrfs_key location; 3982 unsigned long ptr; 3983 int maybe_acls; 3984 u32 rdev; 3985 int ret; 3986 bool filled = false; 3987 int first_xattr_slot; 3988 3989 ret = btrfs_fill_inode(inode, &rdev); 3990 if (!ret) 3991 filled = true; 3992 3993 ASSERT(path); 3994 3995 btrfs_get_inode_key(inode, &location); 3996 3997 ret = btrfs_lookup_inode(NULL, root, path, &location, 0); 3998 if (ret) { 3999 /* 4000 * ret > 0 can come from btrfs_search_slot called by 4001 * btrfs_lookup_inode(), this means the inode was not found. 4002 */ 4003 if (ret > 0) 4004 ret = -ENOENT; 4005 goto out; 4006 } 4007 4008 leaf = path->nodes[0]; 4009 4010 if (filled) 4011 goto cache_index; 4012 4013 inode_item = btrfs_item_ptr(leaf, path->slots[0], 4014 struct btrfs_inode_item); 4015 vfs_inode->i_mode = btrfs_inode_mode(leaf, inode_item); 4016 set_nlink(vfs_inode, btrfs_inode_nlink(leaf, inode_item)); 4017 i_uid_write(vfs_inode, btrfs_inode_uid(leaf, inode_item)); 4018 i_gid_write(vfs_inode, btrfs_inode_gid(leaf, inode_item)); 4019 btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item)); 4020 4021 inode_set_atime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->atime), 4022 btrfs_timespec_nsec(leaf, &inode_item->atime)); 4023 4024 inode_set_mtime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->mtime), 4025 btrfs_timespec_nsec(leaf, &inode_item->mtime)); 4026 4027 inode_set_ctime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->ctime), 4028 btrfs_timespec_nsec(leaf, &inode_item->ctime)); 4029 4030 inode->i_otime_sec = btrfs_timespec_sec(leaf, &inode_item->otime); 4031 inode->i_otime_nsec = btrfs_timespec_nsec(leaf, &inode_item->otime); 4032 4033 inode_set_bytes(vfs_inode, btrfs_inode_nbytes(leaf, inode_item)); 4034 inode->generation = btrfs_inode_generation(leaf, inode_item); 4035 inode->last_trans = btrfs_inode_transid(leaf, inode_item); 4036 4037 inode_set_iversion_queried(vfs_inode, btrfs_inode_sequence(leaf, inode_item)); 4038 vfs_inode->i_generation = inode->generation; 4039 vfs_inode->i_rdev = 0; 4040 rdev = btrfs_inode_rdev(leaf, inode_item); 4041 4042 if (S_ISDIR(vfs_inode->i_mode)) 4043 inode->index_cnt = (u64)-1; 4044 4045 btrfs_inode_split_flags(btrfs_inode_flags(leaf, inode_item), 4046 &inode->flags, &inode->ro_flags); 4047 btrfs_update_inode_mapping_flags(inode); 4048 btrfs_set_inode_mapping_order(inode); 4049 4050cache_index: 4051 /* 4052 * If we were modified in the current generation and evicted from memory 4053 * and then re-read we need to do a full sync since we don't have any 4054 * idea about which extents were modified before we were evicted from 4055 * cache. 4056 * 4057 * This is required for both inode re-read from disk and delayed inode 4058 * in the delayed_nodes xarray. 4059 */ 4060 if (inode->last_trans == btrfs_get_fs_generation(fs_info)) 4061 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); 4062 4063 /* 4064 * We don't persist the id of the transaction where an unlink operation 4065 * against the inode was last made. So here we assume the inode might 4066 * have been evicted, and therefore the exact value of last_unlink_trans 4067 * lost, and set it to last_trans to avoid metadata inconsistencies 4068 * between the inode and its parent if the inode is fsync'ed and the log 4069 * replayed. For example, in the scenario: 4070 * 4071 * touch mydir/foo 4072 * ln mydir/foo mydir/bar 4073 * sync 4074 * unlink mydir/bar 4075 * echo 2 > /proc/sys/vm/drop_caches # evicts inode 4076 * xfs_io -c fsync mydir/foo 4077 * <power failure> 4078 * mount fs, triggers fsync log replay 4079 * 4080 * We must make sure that when we fsync our inode foo we also log its 4081 * parent inode, otherwise after log replay the parent still has the 4082 * dentry with the "bar" name but our inode foo has a link count of 1 4083 * and doesn't have an inode ref with the name "bar" anymore. 4084 * 4085 * Setting last_unlink_trans to last_trans is a pessimistic approach, 4086 * but it guarantees correctness at the expense of occasional full 4087 * transaction commits on fsync if our inode is a directory, or if our 4088 * inode is not a directory, logging its parent unnecessarily. 4089 */ 4090 inode->last_unlink_trans = inode->last_trans; 4091 4092 /* 4093 * Same logic as for last_unlink_trans. We don't persist the generation 4094 * of the last transaction where this inode was used for a reflink 4095 * operation, so after eviction and reloading the inode we must be 4096 * pessimistic and assume the last transaction that modified the inode. 4097 */ 4098 inode->last_reflink_trans = inode->last_trans; 4099 4100 path->slots[0]++; 4101 if (vfs_inode->i_nlink != 1 || 4102 path->slots[0] >= btrfs_header_nritems(leaf)) 4103 goto cache_acl; 4104 4105 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); 4106 if (location.objectid != btrfs_ino(inode)) 4107 goto cache_acl; 4108 4109 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); 4110 if (location.type == BTRFS_INODE_REF_KEY) { 4111 struct btrfs_inode_ref *ref; 4112 4113 ref = (struct btrfs_inode_ref *)ptr; 4114 inode->dir_index = btrfs_inode_ref_index(leaf, ref); 4115 } else if (location.type == BTRFS_INODE_EXTREF_KEY) { 4116 struct btrfs_inode_extref *extref; 4117 4118 extref = (struct btrfs_inode_extref *)ptr; 4119 inode->dir_index = btrfs_inode_extref_index(leaf, extref); 4120 } 4121cache_acl: 4122 /* 4123 * try to precache a NULL acl entry for files that don't have 4124 * any xattrs or acls 4125 */ 4126 maybe_acls = acls_after_inode_item(leaf, path->slots[0], 4127 btrfs_ino(inode), &first_xattr_slot); 4128 if (first_xattr_slot != -1) { 4129 path->slots[0] = first_xattr_slot; 4130 ret = btrfs_load_inode_props(inode, path); 4131 if (ret) 4132 btrfs_err(fs_info, 4133 "error loading props for ino %llu (root %llu): %d", 4134 btrfs_ino(inode), btrfs_root_id(root), ret); 4135 } 4136 4137 /* 4138 * We don't need the path anymore, so release it to avoid holding a read 4139 * lock on a leaf while calling btrfs_init_file_extent_tree(), which can 4140 * allocate memory that triggers reclaim (GFP_KERNEL) and cause a locking 4141 * dependency. 4142 */ 4143 btrfs_release_path(path); 4144 4145 ret = btrfs_init_file_extent_tree(inode); 4146 if (ret) 4147 goto out; 4148 btrfs_inode_set_file_extent_range(inode, 0, 4149 round_up(i_size_read(vfs_inode), fs_info->sectorsize)); 4150 4151 if (!maybe_acls) 4152 cache_no_acl(vfs_inode); 4153 4154 switch (vfs_inode->i_mode & S_IFMT) { 4155 case S_IFREG: 4156 vfs_inode->i_mapping->a_ops = &btrfs_aops; 4157 vfs_inode->i_fop = &btrfs_file_operations; 4158 vfs_inode->i_op = &btrfs_file_inode_operations; 4159 break; 4160 case S_IFDIR: 4161 vfs_inode->i_fop = &btrfs_dir_file_operations; 4162 vfs_inode->i_op = &btrfs_dir_inode_operations; 4163 break; 4164 case S_IFLNK: 4165 vfs_inode->i_op = &btrfs_symlink_inode_operations; 4166 inode_nohighmem(vfs_inode); 4167 vfs_inode->i_mapping->a_ops = &btrfs_aops; 4168 break; 4169 default: 4170 vfs_inode->i_op = &btrfs_special_inode_operations; 4171 init_special_inode(vfs_inode, vfs_inode->i_mode, rdev); 4172 break; 4173 } 4174 4175 btrfs_sync_inode_flags_to_i_flags(inode); 4176 4177 ret = btrfs_add_inode_to_root(inode, true); 4178 if (ret) 4179 goto out; 4180 4181 return 0; 4182out: 4183 /* 4184 * We may have a read locked leaf and iget_failed() triggers inode 4185 * eviction which needs to release the delayed inode and that needs 4186 * to lock the delayed inode's mutex. This can cause a ABBA deadlock 4187 * with a task running delayed items, as that require first locking 4188 * the delayed inode's mutex and then modifying its subvolume btree. 4189 * So release the path before iget_failed(). 4190 */ 4191 btrfs_release_path(path); 4192 iget_failed(vfs_inode); 4193 return ret; 4194} 4195 4196/* 4197 * given a leaf and an inode, copy the inode fields into the leaf 4198 */ 4199static void fill_inode_item(struct btrfs_trans_handle *trans, 4200 struct extent_buffer *leaf, 4201 struct btrfs_inode_item *item, 4202 struct inode *inode) 4203{ 4204 u64 flags; 4205 4206 btrfs_set_inode_uid(leaf, item, i_uid_read(inode)); 4207 btrfs_set_inode_gid(leaf, item, i_gid_read(inode)); 4208 btrfs_set_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size); 4209 btrfs_set_inode_mode(leaf, item, inode->i_mode); 4210 btrfs_set_inode_nlink(leaf, item, inode->i_nlink); 4211 4212 btrfs_set_timespec_sec(leaf, &item->atime, inode_get_atime_sec(inode)); 4213 btrfs_set_timespec_nsec(leaf, &item->atime, inode_get_atime_nsec(inode)); 4214 4215 btrfs_set_timespec_sec(leaf, &item->mtime, inode_get_mtime_sec(inode)); 4216 btrfs_set_timespec_nsec(leaf, &item->mtime, inode_get_mtime_nsec(inode)); 4217 4218 btrfs_set_timespec_sec(leaf, &item->ctime, inode_get_ctime_sec(inode)); 4219 btrfs_set_timespec_nsec(leaf, &item->ctime, inode_get_ctime_nsec(inode)); 4220 4221 btrfs_set_timespec_sec(leaf, &item->otime, BTRFS_I(inode)->i_otime_sec); 4222 btrfs_set_timespec_nsec(leaf, &item->otime, BTRFS_I(inode)->i_otime_nsec); 4223 4224 btrfs_set_inode_nbytes(leaf, item, inode_get_bytes(inode)); 4225 btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation); 4226 btrfs_set_inode_sequence(leaf, item, inode_peek_iversion(inode)); 4227 btrfs_set_inode_transid(leaf, item, trans->transid); 4228 btrfs_set_inode_rdev(leaf, item, inode->i_rdev); 4229 flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags, 4230 BTRFS_I(inode)->ro_flags); 4231 btrfs_set_inode_flags(leaf, item, flags); 4232 btrfs_set_inode_block_group(leaf, item, 0); 4233} 4234 4235/* 4236 * copy everything in the in-memory inode into the btree. 4237 */ 4238static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, 4239 struct btrfs_inode *inode) 4240{ 4241 struct btrfs_inode_item *inode_item; 4242 BTRFS_PATH_AUTO_FREE(path); 4243 struct extent_buffer *leaf; 4244 struct btrfs_key key; 4245 int ret; 4246 4247 path = btrfs_alloc_path(); 4248 if (!path) 4249 return -ENOMEM; 4250 4251 btrfs_get_inode_key(inode, &key); 4252 ret = btrfs_lookup_inode(trans, inode->root, path, &key, 1); 4253 if (ret) { 4254 if (ret > 0) 4255 ret = -ENOENT; 4256 return ret; 4257 } 4258 4259 leaf = path->nodes[0]; 4260 inode_item = btrfs_item_ptr(leaf, path->slots[0], 4261 struct btrfs_inode_item); 4262 4263 fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode); 4264 btrfs_set_inode_last_trans(trans, inode); 4265 return 0; 4266} 4267 4268/* 4269 * copy everything in the in-memory inode into the btree. 4270 */ 4271int btrfs_update_inode(struct btrfs_trans_handle *trans, 4272 struct btrfs_inode *inode) 4273{ 4274 struct btrfs_root *root = inode->root; 4275 struct btrfs_fs_info *fs_info = root->fs_info; 4276 int ret; 4277 4278 /* 4279 * If the inode is a free space inode, we can deadlock during commit 4280 * if we put it into the delayed code. 4281 * 4282 * The data relocation inode should also be directly updated 4283 * without delay 4284 */ 4285 if (!btrfs_is_free_space_inode(inode) 4286 && !btrfs_is_data_reloc_root(root) 4287 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { 4288 btrfs_update_root_times(trans, root); 4289 4290 ret = btrfs_delayed_update_inode(trans, inode); 4291 if (!ret) 4292 btrfs_set_inode_last_trans(trans, inode); 4293 return ret; 4294 } 4295 4296 return btrfs_update_inode_item(trans, inode); 4297} 4298 4299int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, 4300 struct btrfs_inode *inode) 4301{ 4302 int ret; 4303 4304 ret = btrfs_update_inode(trans, inode); 4305 if (ret == -ENOSPC) 4306 return btrfs_update_inode_item(trans, inode); 4307 return ret; 4308} 4309 4310static void update_time_after_link_or_unlink(struct btrfs_inode *dir) 4311{ 4312 struct timespec64 now; 4313 4314 /* 4315 * If we are replaying a log tree, we do not want to update the mtime 4316 * and ctime of the parent directory with the current time, since the 4317 * log replay procedure is responsible for setting them to their correct 4318 * values (the ones it had when the fsync was done). 4319 */ 4320 if (test_bit(BTRFS_FS_LOG_RECOVERING, &dir->root->fs_info->flags)) 4321 return; 4322 4323 now = inode_set_ctime_current(&dir->vfs_inode); 4324 inode_set_mtime_to_ts(&dir->vfs_inode, now); 4325} 4326 4327/* 4328 * unlink helper that gets used here in inode.c and in the tree logging 4329 * recovery code. It remove a link in a directory with a given name, and 4330 * also drops the back refs in the inode to the directory 4331 */ 4332static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4333 struct btrfs_inode *dir, 4334 struct btrfs_inode *inode, 4335 const struct fscrypt_str *name, 4336 struct btrfs_rename_ctx *rename_ctx) 4337{ 4338 struct btrfs_root *root = dir->root; 4339 struct btrfs_fs_info *fs_info = root->fs_info; 4340 struct btrfs_path *path; 4341 int ret = 0; 4342 struct btrfs_dir_item *di; 4343 u64 index; 4344 u64 ino = btrfs_ino(inode); 4345 u64 dir_ino = btrfs_ino(dir); 4346 4347 path = btrfs_alloc_path(); 4348 if (!path) 4349 return -ENOMEM; 4350 4351 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, name, -1); 4352 if (IS_ERR_OR_NULL(di)) { 4353 btrfs_free_path(path); 4354 return di ? PTR_ERR(di) : -ENOENT; 4355 } 4356 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4357 /* 4358 * Down the call chains below we'll also need to allocate a path, so no 4359 * need to hold on to this one for longer than necessary. 4360 */ 4361 btrfs_free_path(path); 4362 if (ret) 4363 return ret; 4364 4365 /* 4366 * If we don't have dir index, we have to get it by looking up 4367 * the inode ref, since we get the inode ref, remove it directly, 4368 * it is unnecessary to do delayed deletion. 4369 * 4370 * But if we have dir index, needn't search inode ref to get it. 4371 * Since the inode ref is close to the inode item, it is better 4372 * that we delay to delete it, and just do this deletion when 4373 * we update the inode item. 4374 */ 4375 if (inode->dir_index) { 4376 ret = btrfs_delayed_delete_inode_ref(inode); 4377 if (!ret) { 4378 index = inode->dir_index; 4379 goto skip_backref; 4380 } 4381 } 4382 4383 ret = btrfs_del_inode_ref(trans, root, name, ino, dir_ino, &index); 4384 if (unlikely(ret)) { 4385 btrfs_crit(fs_info, 4386 "failed to delete reference to %.*s, root %llu inode %llu parent %llu", 4387 name->len, name->name, btrfs_root_id(root), ino, dir_ino); 4388 btrfs_abort_transaction(trans, ret); 4389 return ret; 4390 } 4391skip_backref: 4392 if (rename_ctx) 4393 rename_ctx->index = index; 4394 4395 ret = btrfs_delete_delayed_dir_index(trans, dir, index); 4396 if (unlikely(ret)) { 4397 btrfs_abort_transaction(trans, ret); 4398 return ret; 4399 } 4400 4401 /* 4402 * If we are in a rename context, we don't need to update anything in the 4403 * log. That will be done later during the rename by btrfs_log_new_name(). 4404 * Besides that, doing it here would only cause extra unnecessary btree 4405 * operations on the log tree, increasing latency for applications. 4406 */ 4407 if (!rename_ctx) { 4408 btrfs_del_inode_ref_in_log(trans, name, inode, dir); 4409 btrfs_del_dir_entries_in_log(trans, name, dir, index); 4410 } 4411 4412 /* 4413 * If we have a pending delayed iput we could end up with the final iput 4414 * being run in btrfs-cleaner context. If we have enough of these built 4415 * up we can end up burning a lot of time in btrfs-cleaner without any 4416 * way to throttle the unlinks. Since we're currently holding a ref on 4417 * the inode we can run the delayed iput here without any issues as the 4418 * final iput won't be done until after we drop the ref we're currently 4419 * holding. 4420 */ 4421 btrfs_run_delayed_iput(fs_info, inode); 4422 4423 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name->len * 2); 4424 inode_inc_iversion(&inode->vfs_inode); 4425 inode_set_ctime_current(&inode->vfs_inode); 4426 inode_inc_iversion(&dir->vfs_inode); 4427 update_time_after_link_or_unlink(dir); 4428 4429 return btrfs_update_inode(trans, dir); 4430} 4431 4432int btrfs_unlink_inode(struct btrfs_trans_handle *trans, 4433 struct btrfs_inode *dir, struct btrfs_inode *inode, 4434 const struct fscrypt_str *name) 4435{ 4436 int ret; 4437 4438 ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL); 4439 if (!ret) { 4440 drop_nlink(&inode->vfs_inode); 4441 ret = btrfs_update_inode(trans, inode); 4442 } 4443 return ret; 4444} 4445 4446/* 4447 * helper to start transaction for unlink and rmdir. 4448 * 4449 * unlink and rmdir are special in btrfs, they do not always free space, so 4450 * if we cannot make our reservations the normal way try and see if there is 4451 * plenty of slack room in the global reserve to migrate, otherwise we cannot 4452 * allow the unlink to occur. 4453 */ 4454static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir) 4455{ 4456 struct btrfs_root *root = dir->root; 4457 4458 return btrfs_start_transaction_fallback_global_rsv(root, 4459 BTRFS_UNLINK_METADATA_UNITS); 4460} 4461 4462static int btrfs_unlink(struct inode *dir, struct dentry *dentry) 4463{ 4464 struct btrfs_trans_handle *trans; 4465 struct inode *inode = d_inode(dentry); 4466 int ret; 4467 struct fscrypt_name fname; 4468 4469 ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname); 4470 if (ret) 4471 return ret; 4472 4473 /* This needs to handle no-key deletions later on */ 4474 4475 trans = __unlink_start_trans(BTRFS_I(dir)); 4476 if (IS_ERR(trans)) { 4477 ret = PTR_ERR(trans); 4478 goto fscrypt_free; 4479 } 4480 4481 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4482 false); 4483 4484 ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), 4485 &fname.disk_name); 4486 if (ret) 4487 goto end_trans; 4488 4489 if (inode->i_nlink == 0) { 4490 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 4491 if (ret) 4492 goto end_trans; 4493 } 4494 4495end_trans: 4496 btrfs_end_transaction(trans); 4497 btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info); 4498fscrypt_free: 4499 fscrypt_free_filename(&fname); 4500 return ret; 4501} 4502 4503static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, 4504 struct btrfs_inode *dir, struct dentry *dentry) 4505{ 4506 struct btrfs_root *root = dir->root; 4507 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); 4508 BTRFS_PATH_AUTO_FREE(path); 4509 struct extent_buffer *leaf; 4510 struct btrfs_dir_item *di; 4511 struct btrfs_key key; 4512 u64 index; 4513 int ret; 4514 u64 objectid; 4515 u64 dir_ino = btrfs_ino(dir); 4516 struct fscrypt_name fname; 4517 4518 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname); 4519 if (ret) 4520 return ret; 4521 4522 /* This needs to handle no-key deletions later on */ 4523 4524 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { 4525 objectid = btrfs_root_id(inode->root); 4526 } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 4527 objectid = inode->ref_root_id; 4528 } else { 4529 WARN_ON(1); 4530 fscrypt_free_filename(&fname); 4531 return -EINVAL; 4532 } 4533 4534 path = btrfs_alloc_path(); 4535 if (!path) { 4536 ret = -ENOMEM; 4537 goto out; 4538 } 4539 4540 di = btrfs_lookup_dir_item(trans, root, path, dir_ino, 4541 &fname.disk_name, -1); 4542 if (IS_ERR_OR_NULL(di)) { 4543 ret = di ? PTR_ERR(di) : -ENOENT; 4544 goto out; 4545 } 4546 4547 leaf = path->nodes[0]; 4548 btrfs_dir_item_key_to_cpu(leaf, di, &key); 4549 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); 4550 ret = btrfs_delete_one_dir_name(trans, root, path, di); 4551 if (unlikely(ret)) { 4552 btrfs_abort_transaction(trans, ret); 4553 goto out; 4554 } 4555 btrfs_release_path(path); 4556 4557 /* 4558 * This is a placeholder inode for a subvolume we didn't have a 4559 * reference to at the time of the snapshot creation. In the meantime 4560 * we could have renamed the real subvol link into our snapshot, so 4561 * depending on btrfs_del_root_ref to return -ENOENT here is incorrect. 4562 * Instead simply lookup the dir_index_item for this entry so we can 4563 * remove it. Otherwise we know we have a ref to the root and we can 4564 * call btrfs_del_root_ref, and it _shouldn't_ fail. 4565 */ 4566 if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { 4567 di = btrfs_search_dir_index_item(root, path, dir_ino, &fname.disk_name); 4568 if (IS_ERR(di)) { 4569 ret = PTR_ERR(di); 4570 btrfs_abort_transaction(trans, ret); 4571 goto out; 4572 } 4573 4574 leaf = path->nodes[0]; 4575 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 4576 index = key.offset; 4577 btrfs_release_path(path); 4578 } else { 4579 ret = btrfs_del_root_ref(trans, objectid, 4580 btrfs_root_id(root), dir_ino, 4581 &index, &fname.disk_name); 4582 if (unlikely(ret)) { 4583 btrfs_abort_transaction(trans, ret); 4584 goto out; 4585 } 4586 } 4587 4588 ret = btrfs_delete_delayed_dir_index(trans, dir, index); 4589 if (unlikely(ret)) { 4590 btrfs_abort_transaction(trans, ret); 4591 goto out; 4592 } 4593 4594 btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2); 4595 inode_inc_iversion(&dir->vfs_inode); 4596 inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode)); 4597 ret = btrfs_update_inode_fallback(trans, dir); 4598 if (ret) 4599 btrfs_abort_transaction(trans, ret); 4600out: 4601 fscrypt_free_filename(&fname); 4602 return ret; 4603} 4604 4605/* 4606 * Helper to check if the subvolume references other subvolumes or if it's 4607 * default. 4608 */ 4609static noinline int may_destroy_subvol(struct btrfs_root *root) 4610{ 4611 struct btrfs_fs_info *fs_info = root->fs_info; 4612 BTRFS_PATH_AUTO_FREE(path); 4613 struct btrfs_dir_item *di; 4614 struct btrfs_key key; 4615 struct fscrypt_str name = FSTR_INIT("default", 7); 4616 u64 dir_id; 4617 int ret; 4618 4619 path = btrfs_alloc_path(); 4620 if (!path) 4621 return -ENOMEM; 4622 4623 /* Make sure this root isn't set as the default subvol */ 4624 dir_id = btrfs_super_root_dir(fs_info->super_copy); 4625 di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path, 4626 dir_id, &name, 0); 4627 if (di && !IS_ERR(di)) { 4628 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key); 4629 if (key.objectid == btrfs_root_id(root)) { 4630 ret = -EPERM; 4631 btrfs_err(fs_info, 4632 "deleting default subvolume %llu is not allowed", 4633 key.objectid); 4634 return ret; 4635 } 4636 btrfs_release_path(path); 4637 } 4638 4639 key.objectid = btrfs_root_id(root); 4640 key.type = BTRFS_ROOT_REF_KEY; 4641 key.offset = (u64)-1; 4642 4643 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 4644 if (ret < 0) 4645 return ret; 4646 if (unlikely(ret == 0)) { 4647 /* 4648 * Key with offset -1 found, there would have to exist a root 4649 * with such id, but this is out of valid range. 4650 */ 4651 return -EUCLEAN; 4652 } 4653 4654 ret = 0; 4655 if (path->slots[0] > 0) { 4656 path->slots[0]--; 4657 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); 4658 if (key.objectid == btrfs_root_id(root) && key.type == BTRFS_ROOT_REF_KEY) 4659 ret = -ENOTEMPTY; 4660 } 4661 4662 return ret; 4663} 4664 4665/* Delete all dentries for inodes belonging to the root */ 4666static void btrfs_prune_dentries(struct btrfs_root *root) 4667{ 4668 struct btrfs_fs_info *fs_info = root->fs_info; 4669 struct btrfs_inode *inode; 4670 u64 min_ino = 0; 4671 4672 if (!BTRFS_FS_ERROR(fs_info)) 4673 WARN_ON(btrfs_root_refs(&root->root_item) != 0); 4674 4675 inode = btrfs_find_first_inode(root, min_ino); 4676 while (inode) { 4677 if (icount_read(&inode->vfs_inode) > 1) 4678 d_prune_aliases(&inode->vfs_inode); 4679 4680 min_ino = btrfs_ino(inode) + 1; 4681 /* 4682 * btrfs_drop_inode() will have it removed from the inode 4683 * cache when its usage count hits zero. 4684 */ 4685 iput(&inode->vfs_inode); 4686 cond_resched(); 4687 inode = btrfs_find_first_inode(root, min_ino); 4688 } 4689} 4690 4691int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry) 4692{ 4693 struct btrfs_root *root = dir->root; 4694 struct btrfs_fs_info *fs_info = root->fs_info; 4695 struct inode *inode = d_inode(dentry); 4696 struct btrfs_root *dest = BTRFS_I(inode)->root; 4697 struct btrfs_trans_handle *trans; 4698 struct btrfs_block_rsv block_rsv; 4699 u64 root_flags; 4700 u64 qgroup_reserved = 0; 4701 int ret; 4702 4703 down_write(&fs_info->subvol_sem); 4704 4705 /* 4706 * Don't allow to delete a subvolume with send in progress. This is 4707 * inside the inode lock so the error handling that has to drop the bit 4708 * again is not run concurrently. 4709 */ 4710 spin_lock(&dest->root_item_lock); 4711 if (dest->send_in_progress) { 4712 spin_unlock(&dest->root_item_lock); 4713 btrfs_warn(fs_info, 4714 "attempt to delete subvolume %llu during send", 4715 btrfs_root_id(dest)); 4716 ret = -EPERM; 4717 goto out_up_write; 4718 } 4719 if (atomic_read(&dest->nr_swapfiles)) { 4720 spin_unlock(&dest->root_item_lock); 4721 btrfs_warn(fs_info, 4722 "attempt to delete subvolume %llu with active swapfile", 4723 btrfs_root_id(dest)); 4724 ret = -EPERM; 4725 goto out_up_write; 4726 } 4727 root_flags = btrfs_root_flags(&dest->root_item); 4728 btrfs_set_root_flags(&dest->root_item, 4729 root_flags | BTRFS_ROOT_SUBVOL_DEAD); 4730 spin_unlock(&dest->root_item_lock); 4731 4732 ret = may_destroy_subvol(dest); 4733 if (ret) 4734 goto out_undead; 4735 4736 btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); 4737 /* 4738 * One for dir inode, 4739 * two for dir entries, 4740 * two for root ref/backref. 4741 */ 4742 ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); 4743 if (ret) 4744 goto out_undead; 4745 qgroup_reserved = block_rsv.qgroup_rsv_reserved; 4746 4747 trans = btrfs_start_transaction(root, 0); 4748 if (IS_ERR(trans)) { 4749 ret = PTR_ERR(trans); 4750 goto out_release; 4751 } 4752 btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved); 4753 qgroup_reserved = 0; 4754 trans->block_rsv = &block_rsv; 4755 trans->bytes_reserved = block_rsv.size; 4756 4757 btrfs_record_snapshot_destroy(trans, dir); 4758 4759 ret = btrfs_unlink_subvol(trans, dir, dentry); 4760 if (unlikely(ret)) { 4761 btrfs_abort_transaction(trans, ret); 4762 goto out_end_trans; 4763 } 4764 4765 ret = btrfs_record_root_in_trans(trans, dest); 4766 if (unlikely(ret)) { 4767 btrfs_abort_transaction(trans, ret); 4768 goto out_end_trans; 4769 } 4770 4771 memset(&dest->root_item.drop_progress, 0, 4772 sizeof(dest->root_item.drop_progress)); 4773 btrfs_set_root_drop_level(&dest->root_item, 0); 4774 btrfs_set_root_refs(&dest->root_item, 0); 4775 4776 if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) { 4777 ret = btrfs_insert_orphan_item(trans, 4778 fs_info->tree_root, 4779 btrfs_root_id(dest)); 4780 if (unlikely(ret)) { 4781 btrfs_abort_transaction(trans, ret); 4782 goto out_end_trans; 4783 } 4784 } 4785 4786 ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid, 4787 BTRFS_UUID_KEY_SUBVOL, btrfs_root_id(dest)); 4788 if (unlikely(ret && ret != -ENOENT)) { 4789 btrfs_abort_transaction(trans, ret); 4790 goto out_end_trans; 4791 } 4792 if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) { 4793 ret = btrfs_uuid_tree_remove(trans, 4794 dest->root_item.received_uuid, 4795 BTRFS_UUID_KEY_RECEIVED_SUBVOL, 4796 btrfs_root_id(dest)); 4797 if (unlikely(ret && ret != -ENOENT)) { 4798 btrfs_abort_transaction(trans, ret); 4799 goto out_end_trans; 4800 } 4801 } 4802 4803 free_anon_bdev(dest->anon_dev); 4804 dest->anon_dev = 0; 4805out_end_trans: 4806 trans->block_rsv = NULL; 4807 trans->bytes_reserved = 0; 4808 ret = btrfs_end_transaction(trans); 4809 inode->i_flags |= S_DEAD; 4810out_release: 4811 btrfs_block_rsv_release(fs_info, &block_rsv, (u64)-1, NULL); 4812 if (qgroup_reserved) 4813 btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved); 4814out_undead: 4815 if (ret) { 4816 spin_lock(&dest->root_item_lock); 4817 root_flags = btrfs_root_flags(&dest->root_item); 4818 btrfs_set_root_flags(&dest->root_item, 4819 root_flags & ~BTRFS_ROOT_SUBVOL_DEAD); 4820 spin_unlock(&dest->root_item_lock); 4821 } 4822out_up_write: 4823 up_write(&fs_info->subvol_sem); 4824 if (!ret) { 4825 d_invalidate(dentry); 4826 btrfs_prune_dentries(dest); 4827 ASSERT(dest->send_in_progress == 0); 4828 } 4829 4830 return ret; 4831} 4832 4833static int btrfs_rmdir(struct inode *vfs_dir, struct dentry *dentry) 4834{ 4835 struct btrfs_inode *dir = BTRFS_I(vfs_dir); 4836 struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); 4837 struct btrfs_fs_info *fs_info = inode->root->fs_info; 4838 int ret = 0; 4839 struct btrfs_trans_handle *trans; 4840 struct fscrypt_name fname; 4841 4842 if (inode->vfs_inode.i_size > BTRFS_EMPTY_DIR_SIZE) 4843 return -ENOTEMPTY; 4844 if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { 4845 if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) { 4846 btrfs_err(fs_info, 4847 "extent tree v2 doesn't support snapshot deletion yet"); 4848 return -EOPNOTSUPP; 4849 } 4850 return btrfs_delete_subvolume(dir, dentry); 4851 } 4852 4853 ret = fscrypt_setup_filename(vfs_dir, &dentry->d_name, 1, &fname); 4854 if (ret) 4855 return ret; 4856 4857 /* This needs to handle no-key deletions later on */ 4858 4859 trans = __unlink_start_trans(dir); 4860 if (IS_ERR(trans)) { 4861 ret = PTR_ERR(trans); 4862 goto out_notrans; 4863 } 4864 4865 /* 4866 * Propagate the last_unlink_trans value of the deleted dir to its 4867 * parent directory. This is to prevent an unrecoverable log tree in the 4868 * case we do something like this: 4869 * 1) create dir foo 4870 * 2) create snapshot under dir foo 4871 * 3) delete the snapshot 4872 * 4) rmdir foo 4873 * 5) mkdir foo 4874 * 6) fsync foo or some file inside foo 4875 * 4876 * This is because we can't unlink other roots when replaying the dir 4877 * deletes for directory foo. 4878 */ 4879 if (inode->last_unlink_trans >= trans->transid) 4880 btrfs_record_snapshot_destroy(trans, dir); 4881 4882 if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 4883 ret = btrfs_unlink_subvol(trans, dir, dentry); 4884 goto out; 4885 } 4886 4887 ret = btrfs_orphan_add(trans, inode); 4888 if (ret) 4889 goto out; 4890 4891 /* now the directory is empty */ 4892 ret = btrfs_unlink_inode(trans, dir, inode, &fname.disk_name); 4893 if (!ret) 4894 btrfs_i_size_write(inode, 0); 4895out: 4896 btrfs_end_transaction(trans); 4897out_notrans: 4898 btrfs_btree_balance_dirty(fs_info); 4899 fscrypt_free_filename(&fname); 4900 4901 return ret; 4902} 4903 4904static bool is_inside_block(u64 bytenr, u64 blockstart, u32 blocksize) 4905{ 4906 ASSERT(IS_ALIGNED(blockstart, blocksize), "blockstart=%llu blocksize=%u", 4907 blockstart, blocksize); 4908 4909 if (blockstart <= bytenr && bytenr <= blockstart + blocksize - 1) 4910 return true; 4911 return false; 4912} 4913 4914static int truncate_block_zero_beyond_eof(struct btrfs_inode *inode, u64 start) 4915{ 4916 const pgoff_t index = (start >> PAGE_SHIFT); 4917 struct address_space *mapping = inode->vfs_inode.i_mapping; 4918 struct folio *folio; 4919 u64 zero_start; 4920 u64 zero_end; 4921 int ret = 0; 4922 4923again: 4924 folio = filemap_lock_folio(mapping, index); 4925 /* No folio present. */ 4926 if (IS_ERR(folio)) 4927 return 0; 4928 4929 if (!folio_test_uptodate(folio)) { 4930 ret = btrfs_read_folio(NULL, folio); 4931 folio_lock(folio); 4932 if (folio->mapping != mapping) { 4933 folio_unlock(folio); 4934 folio_put(folio); 4935 goto again; 4936 } 4937 if (unlikely(!folio_test_uptodate(folio))) { 4938 ret = -EIO; 4939 goto out_unlock; 4940 } 4941 } 4942 folio_wait_writeback(folio); 4943 4944 /* 4945 * We do not need to lock extents nor wait for OE, as it's already 4946 * beyond EOF. 4947 */ 4948 4949 zero_start = max_t(u64, folio_pos(folio), start); 4950 zero_end = folio_next_pos(folio); 4951 folio_zero_range(folio, zero_start - folio_pos(folio), 4952 zero_end - zero_start); 4953 4954out_unlock: 4955 folio_unlock(folio); 4956 folio_put(folio); 4957 return ret; 4958} 4959 4960/* 4961 * Handle the truncation of a fs block. 4962 * 4963 * @inode - inode that we're zeroing 4964 * @offset - the file offset of the block to truncate 4965 * The value must be inside [@start, @end], and the function will do 4966 * extra checks if the block that covers @offset needs to be zeroed. 4967 * @start - the start file offset of the range we want to zero 4968 * @end - the end (inclusive) file offset of the range we want to zero. 4969 * 4970 * If the range is not block aligned, read out the folio that covers @offset, 4971 * and if needed zero blocks that are inside the folio and covered by [@start, @end). 4972 * If @start or @end + 1 lands inside a block, that block will be marked dirty 4973 * for writeback. 4974 * 4975 * This is utilized by hole punch, zero range, file expansion. 4976 */ 4977int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 end) 4978{ 4979 struct btrfs_fs_info *fs_info = inode->root->fs_info; 4980 struct address_space *mapping = inode->vfs_inode.i_mapping; 4981 struct extent_io_tree *io_tree = &inode->io_tree; 4982 struct btrfs_ordered_extent *ordered; 4983 struct extent_state *cached_state = NULL; 4984 struct extent_changeset *data_reserved = NULL; 4985 bool only_release_metadata = false; 4986 u32 blocksize = fs_info->sectorsize; 4987 pgoff_t index = (offset >> PAGE_SHIFT); 4988 struct folio *folio; 4989 gfp_t mask = btrfs_alloc_write_mask(mapping); 4990 int ret = 0; 4991 const bool in_head_block = is_inside_block(offset, round_down(start, blocksize), 4992 blocksize); 4993 const bool in_tail_block = is_inside_block(offset, round_down(end, blocksize), 4994 blocksize); 4995 bool need_truncate_head = false; 4996 bool need_truncate_tail = false; 4997 u64 zero_start; 4998 u64 zero_end; 4999 u64 block_start; 5000 u64 block_end; 5001 5002 /* @offset should be inside the range. */ 5003 ASSERT(start <= offset && offset <= end, "offset=%llu start=%llu end=%llu", 5004 offset, start, end); 5005 5006 /* The range is aligned at both ends. */ 5007 if (IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize)) { 5008 /* 5009 * For block size < page size case, we may have polluted blocks 5010 * beyond EOF. So we also need to zero them out. 5011 */ 5012 if (end == (u64)-1 && blocksize < PAGE_SIZE) 5013 ret = truncate_block_zero_beyond_eof(inode, start); 5014 goto out; 5015 } 5016 5017 /* 5018 * @offset may not be inside the head nor tail block. In that case we 5019 * don't need to do anything. 5020 */ 5021 if (!in_head_block && !in_tail_block) 5022 goto out; 5023 5024 /* 5025 * Skip the truncation if the range in the target block is already aligned. 5026 * The seemingly complex check will also handle the same block case. 5027 */ 5028 if (in_head_block && !IS_ALIGNED(start, blocksize)) 5029 need_truncate_head = true; 5030 if (in_tail_block && !IS_ALIGNED(end + 1, blocksize)) 5031 need_truncate_tail = true; 5032 if (!need_truncate_head && !need_truncate_tail) 5033 goto out; 5034 5035 block_start = round_down(offset, blocksize); 5036 block_end = block_start + blocksize - 1; 5037 5038 ret = btrfs_check_data_free_space(inode, &data_reserved, block_start, 5039 blocksize, false); 5040 if (ret < 0) { 5041 size_t write_bytes = blocksize; 5042 5043 if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) { 5044 /* For nocow case, no need to reserve data space. */ 5045 ASSERT(write_bytes == blocksize, "write_bytes=%zu blocksize=%u", 5046 write_bytes, blocksize); 5047 only_release_metadata = true; 5048 } else { 5049 goto out; 5050 } 5051 } 5052 ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false); 5053 if (ret < 0) { 5054 if (!only_release_metadata) 5055 btrfs_free_reserved_data_space(inode, data_reserved, 5056 block_start, blocksize); 5057 goto out; 5058 } 5059again: 5060 folio = __filemap_get_folio(mapping, index, 5061 FGP_LOCK | FGP_ACCESSED | FGP_CREAT, mask); 5062 if (IS_ERR(folio)) { 5063 if (only_release_metadata) 5064 btrfs_delalloc_release_metadata(inode, blocksize, true); 5065 else 5066 btrfs_delalloc_release_space(inode, data_reserved, 5067 block_start, blocksize, true); 5068 btrfs_delalloc_release_extents(inode, blocksize); 5069 ret = PTR_ERR(folio); 5070 goto out; 5071 } 5072 5073 if (!folio_test_uptodate(folio)) { 5074 ret = btrfs_read_folio(NULL, folio); 5075 folio_lock(folio); 5076 if (folio->mapping != mapping) { 5077 folio_unlock(folio); 5078 folio_put(folio); 5079 goto again; 5080 } 5081 if (unlikely(!folio_test_uptodate(folio))) { 5082 ret = -EIO; 5083 goto out_unlock; 5084 } 5085 } 5086 5087 /* 5088 * We unlock the page after the io is completed and then re-lock it 5089 * above. release_folio() could have come in between that and cleared 5090 * folio private, but left the page in the mapping. Set the page mapped 5091 * here to make sure it's properly set for the subpage stuff. 5092 */ 5093 ret = set_folio_extent_mapped(folio); 5094 if (ret < 0) 5095 goto out_unlock; 5096 5097 folio_wait_writeback(folio); 5098 5099 btrfs_lock_extent(io_tree, block_start, block_end, &cached_state); 5100 5101 ordered = btrfs_lookup_ordered_extent(inode, block_start); 5102 if (ordered) { 5103 btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); 5104 folio_unlock(folio); 5105 folio_put(folio); 5106 btrfs_start_ordered_extent(ordered); 5107 btrfs_put_ordered_extent(ordered); 5108 goto again; 5109 } 5110 5111 btrfs_clear_extent_bit(&inode->io_tree, block_start, block_end, 5112 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 5113 &cached_state); 5114 5115 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, 5116 &cached_state); 5117 if (ret) { 5118 btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); 5119 goto out_unlock; 5120 } 5121 5122 if (end == (u64)-1) { 5123 /* 5124 * We're truncating beyond EOF, the remaining blocks normally are 5125 * already holes thus no need to zero again, but it's possible for 5126 * fs block size < page size cases to have memory mapped writes 5127 * to pollute ranges beyond EOF. 5128 * 5129 * In that case although such polluted blocks beyond EOF will 5130 * not reach disk, it still affects our page caches. 5131 */ 5132 zero_start = max_t(u64, folio_pos(folio), start); 5133 zero_end = min_t(u64, folio_next_pos(folio) - 1, end); 5134 } else { 5135 zero_start = max_t(u64, block_start, start); 5136 zero_end = min_t(u64, block_end, end); 5137 } 5138 folio_zero_range(folio, zero_start - folio_pos(folio), 5139 zero_end - zero_start + 1); 5140 5141 btrfs_folio_clear_checked(fs_info, folio, block_start, 5142 block_end + 1 - block_start); 5143 btrfs_folio_set_dirty(fs_info, folio, block_start, 5144 block_end + 1 - block_start); 5145 5146 if (only_release_metadata) 5147 btrfs_set_extent_bit(&inode->io_tree, block_start, block_end, 5148 EXTENT_NORESERVE, &cached_state); 5149 5150 btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); 5151 5152out_unlock: 5153 if (ret) { 5154 if (only_release_metadata) 5155 btrfs_delalloc_release_metadata(inode, blocksize, true); 5156 else 5157 btrfs_delalloc_release_space(inode, data_reserved, 5158 block_start, blocksize, true); 5159 } 5160 btrfs_delalloc_release_extents(inode, blocksize); 5161 folio_unlock(folio); 5162 folio_put(folio); 5163out: 5164 if (only_release_metadata) 5165 btrfs_check_nocow_unlock(inode); 5166 extent_changeset_free(data_reserved); 5167 return ret; 5168} 5169 5170static int maybe_insert_hole(struct btrfs_inode *inode, u64 offset, u64 len) 5171{ 5172 struct btrfs_root *root = inode->root; 5173 struct btrfs_fs_info *fs_info = root->fs_info; 5174 struct btrfs_trans_handle *trans; 5175 struct btrfs_drop_extents_args drop_args = { 0 }; 5176 int ret; 5177 5178 /* 5179 * If NO_HOLES is enabled, we don't need to do anything. 5180 * Later, up in the call chain, either btrfs_set_inode_last_sub_trans() 5181 * or btrfs_update_inode() will be called, which guarantee that the next 5182 * fsync will know this inode was changed and needs to be logged. 5183 */ 5184 if (btrfs_fs_incompat(fs_info, NO_HOLES)) 5185 return 0; 5186 5187 /* 5188 * 1 - for the one we're dropping 5189 * 1 - for the one we're adding 5190 * 1 - for updating the inode. 5191 */ 5192 trans = btrfs_start_transaction(root, 3); 5193 if (IS_ERR(trans)) 5194 return PTR_ERR(trans); 5195 5196 drop_args.start = offset; 5197 drop_args.end = offset + len; 5198 drop_args.drop_cache = true; 5199 5200 ret = btrfs_drop_extents(trans, root, inode, &drop_args); 5201 if (unlikely(ret)) { 5202 btrfs_abort_transaction(trans, ret); 5203 btrfs_end_transaction(trans); 5204 return ret; 5205 } 5206 5207 ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len); 5208 if (ret) { 5209 btrfs_abort_transaction(trans, ret); 5210 } else { 5211 btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found); 5212 btrfs_update_inode(trans, inode); 5213 } 5214 btrfs_end_transaction(trans); 5215 return ret; 5216} 5217 5218/* 5219 * This function puts in dummy file extents for the area we're creating a hole 5220 * for. So if we are truncating this file to a larger size we need to insert 5221 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for 5222 * the range between oldsize and size 5223 */ 5224int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) 5225{ 5226 struct btrfs_root *root = inode->root; 5227 struct btrfs_fs_info *fs_info = root->fs_info; 5228 struct extent_io_tree *io_tree = &inode->io_tree; 5229 struct extent_map *em = NULL; 5230 struct extent_state *cached_state = NULL; 5231 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); 5232 u64 block_end = ALIGN(size, fs_info->sectorsize); 5233 u64 last_byte; 5234 u64 cur_offset; 5235 u64 hole_size; 5236 int ret = 0; 5237 5238 /* 5239 * If our size started in the middle of a block we need to zero out the 5240 * rest of the block before we expand the i_size, otherwise we could 5241 * expose stale data. 5242 */ 5243 ret = btrfs_truncate_block(inode, oldsize, oldsize, -1); 5244 if (ret) 5245 return ret; 5246 5247 if (size <= hole_start) 5248 return 0; 5249 5250 btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1, 5251 &cached_state); 5252 cur_offset = hole_start; 5253 while (1) { 5254 em = btrfs_get_extent(inode, NULL, cur_offset, block_end - cur_offset); 5255 if (IS_ERR(em)) { 5256 ret = PTR_ERR(em); 5257 em = NULL; 5258 break; 5259 } 5260 last_byte = min(btrfs_extent_map_end(em), block_end); 5261 last_byte = ALIGN(last_byte, fs_info->sectorsize); 5262 hole_size = last_byte - cur_offset; 5263 5264 if (!(em->flags & EXTENT_FLAG_PREALLOC)) { 5265 struct extent_map *hole_em; 5266 5267 ret = maybe_insert_hole(inode, cur_offset, hole_size); 5268 if (ret) 5269 break; 5270 5271 ret = btrfs_inode_set_file_extent_range(inode, 5272 cur_offset, hole_size); 5273 if (ret) 5274 break; 5275 5276 hole_em = btrfs_alloc_extent_map(); 5277 if (!hole_em) { 5278 btrfs_drop_extent_map_range(inode, cur_offset, 5279 cur_offset + hole_size - 1, 5280 false); 5281 btrfs_set_inode_full_sync(inode); 5282 goto next; 5283 } 5284 hole_em->start = cur_offset; 5285 hole_em->len = hole_size; 5286 5287 hole_em->disk_bytenr = EXTENT_MAP_HOLE; 5288 hole_em->disk_num_bytes = 0; 5289 hole_em->ram_bytes = hole_size; 5290 hole_em->generation = btrfs_get_fs_generation(fs_info); 5291 5292 ret = btrfs_replace_extent_map_range(inode, hole_em, true); 5293 btrfs_free_extent_map(hole_em); 5294 } else { 5295 ret = btrfs_inode_set_file_extent_range(inode, 5296 cur_offset, hole_size); 5297 if (ret) 5298 break; 5299 } 5300next: 5301 btrfs_free_extent_map(em); 5302 em = NULL; 5303 cur_offset = last_byte; 5304 if (cur_offset >= block_end) 5305 break; 5306 } 5307 btrfs_free_extent_map(em); 5308 btrfs_unlock_extent(io_tree, hole_start, block_end - 1, &cached_state); 5309 return ret; 5310} 5311 5312static int btrfs_setsize(struct inode *inode, struct iattr *attr) 5313{ 5314 struct btrfs_root *root = BTRFS_I(inode)->root; 5315 struct btrfs_trans_handle *trans; 5316 loff_t oldsize = i_size_read(inode); 5317 loff_t newsize = attr->ia_size; 5318 int mask = attr->ia_valid; 5319 int ret; 5320 5321 /* 5322 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a 5323 * special case where we need to update the times despite not having 5324 * these flags set. For all other operations the VFS set these flags 5325 * explicitly if it wants a timestamp update. 5326 */ 5327 if (newsize != oldsize) { 5328 inode_inc_iversion(inode); 5329 if (!(mask & (ATTR_CTIME | ATTR_MTIME))) { 5330 inode_set_mtime_to_ts(inode, 5331 inode_set_ctime_current(inode)); 5332 } 5333 } 5334 5335 if (newsize > oldsize) { 5336 /* 5337 * Don't do an expanding truncate while snapshotting is ongoing. 5338 * This is to ensure the snapshot captures a fully consistent 5339 * state of this file - if the snapshot captures this expanding 5340 * truncation, it must capture all writes that happened before 5341 * this truncation. 5342 */ 5343 btrfs_drew_write_lock(&root->snapshot_lock); 5344 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize); 5345 if (ret) { 5346 btrfs_drew_write_unlock(&root->snapshot_lock); 5347 return ret; 5348 } 5349 5350 trans = btrfs_start_transaction(root, 1); 5351 if (IS_ERR(trans)) { 5352 btrfs_drew_write_unlock(&root->snapshot_lock); 5353 return PTR_ERR(trans); 5354 } 5355 5356 i_size_write(inode, newsize); 5357 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 5358 pagecache_isize_extended(inode, oldsize, newsize); 5359 ret = btrfs_update_inode(trans, BTRFS_I(inode)); 5360 btrfs_drew_write_unlock(&root->snapshot_lock); 5361 btrfs_end_transaction(trans); 5362 } else { 5363 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); 5364 5365 if (btrfs_is_zoned(fs_info)) { 5366 ret = btrfs_wait_ordered_range(BTRFS_I(inode), 5367 ALIGN(newsize, fs_info->sectorsize), 5368 (u64)-1); 5369 if (ret) 5370 return ret; 5371 } 5372 5373 /* 5374 * We're truncating a file that used to have good data down to 5375 * zero. Make sure any new writes to the file get on disk 5376 * on close. 5377 */ 5378 if (newsize == 0) 5379 set_bit(BTRFS_INODE_FLUSH_ON_CLOSE, 5380 &BTRFS_I(inode)->runtime_flags); 5381 5382 truncate_setsize(inode, newsize); 5383 5384 inode_dio_wait(inode); 5385 5386 ret = btrfs_truncate(BTRFS_I(inode), newsize == oldsize); 5387 if (ret && inode->i_nlink) { 5388 int ret2; 5389 5390 /* 5391 * Truncate failed, so fix up the in-memory size. We 5392 * adjusted disk_i_size down as we removed extents, so 5393 * wait for disk_i_size to be stable and then update the 5394 * in-memory size to match. 5395 */ 5396 ret2 = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1); 5397 if (ret2) 5398 return ret2; 5399 i_size_write(inode, BTRFS_I(inode)->disk_i_size); 5400 } 5401 } 5402 5403 return ret; 5404} 5405 5406static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, 5407 struct iattr *attr) 5408{ 5409 struct inode *inode = d_inode(dentry); 5410 struct btrfs_root *root = BTRFS_I(inode)->root; 5411 int ret; 5412 5413 if (btrfs_root_readonly(root)) 5414 return -EROFS; 5415 5416 ret = setattr_prepare(idmap, dentry, attr); 5417 if (ret) 5418 return ret; 5419 5420 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 5421 ret = btrfs_setsize(inode, attr); 5422 if (ret) 5423 return ret; 5424 } 5425 5426 if (attr->ia_valid) { 5427 setattr_copy(idmap, inode, attr); 5428 inode_inc_iversion(inode); 5429 ret = btrfs_dirty_inode(BTRFS_I(inode)); 5430 5431 if (!ret && attr->ia_valid & ATTR_MODE) 5432 ret = posix_acl_chmod(idmap, dentry, inode->i_mode); 5433 } 5434 5435 return ret; 5436} 5437 5438/* 5439 * While truncating the inode pages during eviction, we get the VFS 5440 * calling btrfs_invalidate_folio() against each folio of the inode. This 5441 * is slow because the calls to btrfs_invalidate_folio() result in a 5442 * huge amount of calls to lock_extent() and clear_extent_bit(), 5443 * which keep merging and splitting extent_state structures over and over, 5444 * wasting lots of time. 5445 * 5446 * Therefore if the inode is being evicted, let btrfs_invalidate_folio() 5447 * skip all those expensive operations on a per folio basis and do only 5448 * the ordered io finishing, while we release here the extent_map and 5449 * extent_state structures, without the excessive merging and splitting. 5450 */ 5451static void evict_inode_truncate_pages(struct inode *inode) 5452{ 5453 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 5454 struct rb_node *node; 5455 5456 ASSERT(inode_state_read_once(inode) & I_FREEING); 5457 truncate_inode_pages_final(&inode->i_data); 5458 5459 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); 5460 5461 /* 5462 * Keep looping until we have no more ranges in the io tree. 5463 * We can have ongoing bios started by readahead that have 5464 * their endio callback (extent_io.c:end_bio_extent_readpage) 5465 * still in progress (unlocked the pages in the bio but did not yet 5466 * unlocked the ranges in the io tree). Therefore this means some 5467 * ranges can still be locked and eviction started because before 5468 * submitting those bios, which are executed by a separate task (work 5469 * queue kthread), inode references (inode->i_count) were not taken 5470 * (which would be dropped in the end io callback of each bio). 5471 * Therefore here we effectively end up waiting for those bios and 5472 * anyone else holding locked ranges without having bumped the inode's 5473 * reference count - if we don't do it, when they access the inode's 5474 * io_tree to unlock a range it may be too late, leading to an 5475 * use-after-free issue. 5476 */ 5477 spin_lock(&io_tree->lock); 5478 while (!RB_EMPTY_ROOT(&io_tree->state)) { 5479 struct extent_state *state; 5480 struct extent_state *cached_state = NULL; 5481 u64 start; 5482 u64 end; 5483 unsigned state_flags; 5484 5485 node = rb_first(&io_tree->state); 5486 state = rb_entry(node, struct extent_state, rb_node); 5487 start = state->start; 5488 end = state->end; 5489 state_flags = state->state; 5490 spin_unlock(&io_tree->lock); 5491 5492 btrfs_lock_extent(io_tree, start, end, &cached_state); 5493 5494 /* 5495 * If still has DELALLOC flag, the extent didn't reach disk, 5496 * and its reserved space won't be freed by delayed_ref. 5497 * So we need to free its reserved space here. 5498 * (Refer to comment in btrfs_invalidate_folio, case 2) 5499 * 5500 * Note, end is the bytenr of last byte, so we need + 1 here. 5501 */ 5502 if (state_flags & EXTENT_DELALLOC) 5503 btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start, 5504 end - start + 1, NULL); 5505 5506 btrfs_clear_extent_bit(io_tree, start, end, 5507 EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING, 5508 &cached_state); 5509 5510 cond_resched(); 5511 spin_lock(&io_tree->lock); 5512 } 5513 spin_unlock(&io_tree->lock); 5514} 5515 5516static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root, 5517 struct btrfs_block_rsv *rsv) 5518{ 5519 struct btrfs_fs_info *fs_info = root->fs_info; 5520 struct btrfs_trans_handle *trans; 5521 u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1); 5522 int ret; 5523 5524 /* 5525 * Eviction should be taking place at some place safe because of our 5526 * delayed iputs. However the normal flushing code will run delayed 5527 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock. 5528 * 5529 * We reserve the delayed_refs_extra here again because we can't use 5530 * btrfs_start_transaction(root, 0) for the same deadlocky reason as 5531 * above. We reserve our extra bit here because we generate a ton of 5532 * delayed refs activity by truncating. 5533 * 5534 * BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can, 5535 * if we fail to make this reservation we can re-try without the 5536 * delayed_refs_extra so we can make some forward progress. 5537 */ 5538 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size + delayed_refs_extra, 5539 BTRFS_RESERVE_FLUSH_EVICT); 5540 if (ret) { 5541 ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size, 5542 BTRFS_RESERVE_FLUSH_EVICT); 5543 if (ret) { 5544 btrfs_warn(fs_info, 5545 "could not allocate space for delete; will truncate on mount"); 5546 return ERR_PTR(-ENOSPC); 5547 } 5548 delayed_refs_extra = 0; 5549 } 5550 5551 trans = btrfs_join_transaction(root); 5552 if (IS_ERR(trans)) 5553 return trans; 5554 5555 if (delayed_refs_extra) { 5556 trans->block_rsv = &fs_info->trans_block_rsv; 5557 trans->bytes_reserved = delayed_refs_extra; 5558 btrfs_block_rsv_migrate(rsv, trans->block_rsv, 5559 delayed_refs_extra, true); 5560 } 5561 return trans; 5562} 5563 5564void btrfs_evict_inode(struct inode *inode) 5565{ 5566 struct btrfs_fs_info *fs_info; 5567 struct btrfs_trans_handle *trans; 5568 struct btrfs_root *root = BTRFS_I(inode)->root; 5569 struct btrfs_block_rsv rsv; 5570 int ret; 5571 5572 trace_btrfs_inode_evict(inode); 5573 5574 if (!root) { 5575 fsverity_cleanup_inode(inode); 5576 clear_inode(inode); 5577 return; 5578 } 5579 5580 fs_info = inode_to_fs_info(inode); 5581 evict_inode_truncate_pages(inode); 5582 5583 if (inode->i_nlink && 5584 ((btrfs_root_refs(&root->root_item) != 0 && 5585 btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID) || 5586 btrfs_is_free_space_inode(BTRFS_I(inode)))) 5587 goto out; 5588 5589 if (is_bad_inode(inode)) 5590 goto out; 5591 5592 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) 5593 goto out; 5594 5595 if (inode->i_nlink > 0) { 5596 BUG_ON(btrfs_root_refs(&root->root_item) != 0 && 5597 btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID); 5598 goto out; 5599 } 5600 5601 /* 5602 * This makes sure the inode item in tree is uptodate and the space for 5603 * the inode update is released. 5604 */ 5605 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); 5606 if (ret) 5607 goto out; 5608 5609 /* 5610 * This drops any pending insert or delete operations we have for this 5611 * inode. We could have a delayed dir index deletion queued up, but 5612 * we're removing the inode completely so that'll be taken care of in 5613 * the truncate. 5614 */ 5615 btrfs_kill_delayed_inode_items(BTRFS_I(inode)); 5616 5617 btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); 5618 rsv.size = btrfs_calc_metadata_size(fs_info, 1); 5619 rsv.failfast = true; 5620 5621 btrfs_i_size_write(BTRFS_I(inode), 0); 5622 5623 while (1) { 5624 struct btrfs_truncate_control control = { 5625 .inode = BTRFS_I(inode), 5626 .ino = btrfs_ino(BTRFS_I(inode)), 5627 .new_size = 0, 5628 .min_type = 0, 5629 }; 5630 5631 trans = evict_refill_and_join(root, &rsv); 5632 if (IS_ERR(trans)) 5633 goto out_release; 5634 5635 trans->block_rsv = &rsv; 5636 5637 ret = btrfs_truncate_inode_items(trans, root, &control); 5638 trans->block_rsv = &fs_info->trans_block_rsv; 5639 btrfs_end_transaction(trans); 5640 /* 5641 * We have not added new delayed items for our inode after we 5642 * have flushed its delayed items, so no need to throttle on 5643 * delayed items. However we have modified extent buffers. 5644 */ 5645 btrfs_btree_balance_dirty_nodelay(fs_info); 5646 if (ret && ret != -ENOSPC && ret != -EAGAIN) 5647 goto out_release; 5648 else if (!ret) 5649 break; 5650 } 5651 5652 /* 5653 * Errors here aren't a big deal, it just means we leave orphan items in 5654 * the tree. They will be cleaned up on the next mount. If the inode 5655 * number gets reused, cleanup deletes the orphan item without doing 5656 * anything, and unlink reuses the existing orphan item. 5657 * 5658 * If it turns out that we are dropping too many of these, we might want 5659 * to add a mechanism for retrying these after a commit. 5660 */ 5661 trans = evict_refill_and_join(root, &rsv); 5662 if (!IS_ERR(trans)) { 5663 trans->block_rsv = &rsv; 5664 btrfs_orphan_del(trans, BTRFS_I(inode)); 5665 trans->block_rsv = &fs_info->trans_block_rsv; 5666 btrfs_end_transaction(trans); 5667 } 5668 5669out_release: 5670 btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL); 5671out: 5672 /* 5673 * If we didn't successfully delete, the orphan item will still be in 5674 * the tree and we'll retry on the next mount. Again, we might also want 5675 * to retry these periodically in the future. 5676 */ 5677 btrfs_remove_delayed_node(BTRFS_I(inode)); 5678 fsverity_cleanup_inode(inode); 5679 clear_inode(inode); 5680} 5681 5682/* 5683 * Return the key found in the dir entry in the location pointer, fill @type 5684 * with BTRFS_FT_*, and return 0. 5685 * 5686 * If no dir entries were found, returns -ENOENT. 5687 * If found a corrupted location in dir entry, returns -EUCLEAN. 5688 */ 5689static int btrfs_inode_by_name(struct btrfs_inode *dir, struct dentry *dentry, 5690 struct btrfs_key *location, u8 *type) 5691{ 5692 struct btrfs_dir_item *di; 5693 BTRFS_PATH_AUTO_FREE(path); 5694 struct btrfs_root *root = dir->root; 5695 int ret = 0; 5696 struct fscrypt_name fname; 5697 5698 path = btrfs_alloc_path(); 5699 if (!path) 5700 return -ENOMEM; 5701 5702 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname); 5703 if (ret < 0) 5704 return ret; 5705 /* 5706 * fscrypt_setup_filename() should never return a positive value, but 5707 * gcc on sparc/parisc thinks it can, so assert that doesn't happen. 5708 */ 5709 ASSERT(ret == 0); 5710 5711 /* This needs to handle no-key deletions later on */ 5712 5713 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), 5714 &fname.disk_name, 0); 5715 if (IS_ERR_OR_NULL(di)) { 5716 ret = di ? PTR_ERR(di) : -ENOENT; 5717 goto out; 5718 } 5719 5720 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); 5721 if (unlikely(location->type != BTRFS_INODE_ITEM_KEY && 5722 location->type != BTRFS_ROOT_ITEM_KEY)) { 5723 ret = -EUCLEAN; 5724 btrfs_warn(root->fs_info, 5725"%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location " BTRFS_KEY_FMT ")", 5726 __func__, fname.disk_name.name, btrfs_ino(dir), 5727 BTRFS_KEY_FMT_VALUE(location)); 5728 } 5729 if (!ret) 5730 *type = btrfs_dir_ftype(path->nodes[0], di); 5731out: 5732 fscrypt_free_filename(&fname); 5733 return ret; 5734} 5735 5736/* 5737 * when we hit a tree root in a directory, the btrfs part of the inode 5738 * needs to be changed to reflect the root directory of the tree root. This 5739 * is kind of like crossing a mount point. 5740 */ 5741static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, 5742 struct btrfs_inode *dir, 5743 struct dentry *dentry, 5744 struct btrfs_key *location, 5745 struct btrfs_root **sub_root) 5746{ 5747 BTRFS_PATH_AUTO_FREE(path); 5748 struct btrfs_root *new_root; 5749 struct btrfs_root_ref *ref; 5750 struct extent_buffer *leaf; 5751 struct btrfs_key key; 5752 int ret; 5753 int err = 0; 5754 struct fscrypt_name fname; 5755 5756 ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 0, &fname); 5757 if (ret) 5758 return ret; 5759 5760 path = btrfs_alloc_path(); 5761 if (!path) { 5762 err = -ENOMEM; 5763 goto out; 5764 } 5765 5766 err = -ENOENT; 5767 key.objectid = btrfs_root_id(dir->root); 5768 key.type = BTRFS_ROOT_REF_KEY; 5769 key.offset = location->objectid; 5770 5771 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); 5772 if (ret) { 5773 if (ret < 0) 5774 err = ret; 5775 goto out; 5776 } 5777 5778 leaf = path->nodes[0]; 5779 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); 5780 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) || 5781 btrfs_root_ref_name_len(leaf, ref) != fname.disk_name.len) 5782 goto out; 5783 5784 ret = memcmp_extent_buffer(leaf, fname.disk_name.name, 5785 (unsigned long)(ref + 1), fname.disk_name.len); 5786 if (ret) 5787 goto out; 5788 5789 btrfs_release_path(path); 5790 5791 new_root = btrfs_get_fs_root(fs_info, location->objectid, true); 5792 if (IS_ERR(new_root)) { 5793 err = PTR_ERR(new_root); 5794 goto out; 5795 } 5796 5797 *sub_root = new_root; 5798 location->objectid = btrfs_root_dirid(&new_root->root_item); 5799 location->type = BTRFS_INODE_ITEM_KEY; 5800 location->offset = 0; 5801 err = 0; 5802out: 5803 fscrypt_free_filename(&fname); 5804 return err; 5805} 5806 5807 5808 5809static void btrfs_del_inode_from_root(struct btrfs_inode *inode) 5810{ 5811 struct btrfs_root *root = inode->root; 5812 struct btrfs_inode *entry; 5813 bool empty = false; 5814 5815 xa_lock(&root->inodes); 5816 /* 5817 * This btrfs_inode is being freed and has already been unhashed at this 5818 * point. It's possible that another btrfs_inode has already been 5819 * allocated for the same inode and inserted itself into the root, so 5820 * don't delete it in that case. 5821 * 5822 * Note that this shouldn't need to allocate memory, so the gfp flags 5823 * don't really matter. 5824 */ 5825 entry = __xa_cmpxchg(&root->inodes, btrfs_ino(inode), inode, NULL, 5826 GFP_ATOMIC); 5827 if (entry == inode) 5828 empty = xa_empty(&root->inodes); 5829 xa_unlock(&root->inodes); 5830 5831 if (empty && btrfs_root_refs(&root->root_item) == 0) { 5832 xa_lock(&root->inodes); 5833 empty = xa_empty(&root->inodes); 5834 xa_unlock(&root->inodes); 5835 if (empty) 5836 btrfs_add_dead_root(root); 5837 } 5838} 5839 5840 5841static int btrfs_init_locked_inode(struct inode *inode, void *p) 5842{ 5843 struct btrfs_iget_args *args = p; 5844 5845 btrfs_set_inode_number(BTRFS_I(inode), args->ino); 5846 BTRFS_I(inode)->root = btrfs_grab_root(args->root); 5847 5848 if (args->root && args->root == args->root->fs_info->tree_root && 5849 args->ino != BTRFS_BTREE_INODE_OBJECTID) 5850 set_bit(BTRFS_INODE_FREE_SPACE_INODE, 5851 &BTRFS_I(inode)->runtime_flags); 5852 return 0; 5853} 5854 5855static int btrfs_find_actor(struct inode *inode, void *opaque) 5856{ 5857 struct btrfs_iget_args *args = opaque; 5858 5859 return args->ino == btrfs_ino(BTRFS_I(inode)) && 5860 args->root == BTRFS_I(inode)->root; 5861} 5862 5863static struct btrfs_inode *btrfs_iget_locked(u64 ino, struct btrfs_root *root) 5864{ 5865 struct inode *inode; 5866 struct btrfs_iget_args args; 5867 unsigned long hashval = btrfs_inode_hash(ino, root); 5868 5869 args.ino = ino; 5870 args.root = root; 5871 5872 inode = iget5_locked_rcu(root->fs_info->sb, hashval, btrfs_find_actor, 5873 btrfs_init_locked_inode, 5874 (void *)&args); 5875 if (!inode) 5876 return NULL; 5877 return BTRFS_I(inode); 5878} 5879 5880/* 5881 * Get an inode object given its inode number and corresponding root. Path is 5882 * preallocated to prevent recursing back to iget through allocator. 5883 */ 5884struct btrfs_inode *btrfs_iget_path(u64 ino, struct btrfs_root *root, 5885 struct btrfs_path *path) 5886{ 5887 struct btrfs_inode *inode; 5888 int ret; 5889 5890 inode = btrfs_iget_locked(ino, root); 5891 if (!inode) 5892 return ERR_PTR(-ENOMEM); 5893 5894 if (!(inode_state_read_once(&inode->vfs_inode) & I_NEW)) 5895 return inode; 5896 5897 ret = btrfs_read_locked_inode(inode, path); 5898 if (ret) 5899 return ERR_PTR(ret); 5900 5901 unlock_new_inode(&inode->vfs_inode); 5902 return inode; 5903} 5904 5905/* 5906 * Get an inode object given its inode number and corresponding root. 5907 */ 5908struct btrfs_inode *btrfs_iget(u64 ino, struct btrfs_root *root) 5909{ 5910 struct btrfs_inode *inode; 5911 struct btrfs_path *path; 5912 int ret; 5913 5914 inode = btrfs_iget_locked(ino, root); 5915 if (!inode) 5916 return ERR_PTR(-ENOMEM); 5917 5918 if (!(inode_state_read_once(&inode->vfs_inode) & I_NEW)) 5919 return inode; 5920 5921 path = btrfs_alloc_path(); 5922 if (!path) { 5923 iget_failed(&inode->vfs_inode); 5924 return ERR_PTR(-ENOMEM); 5925 } 5926 5927 ret = btrfs_read_locked_inode(inode, path); 5928 btrfs_free_path(path); 5929 if (ret) 5930 return ERR_PTR(ret); 5931 5932 if (S_ISDIR(inode->vfs_inode.i_mode)) 5933 inode->vfs_inode.i_opflags |= IOP_FASTPERM_MAY_EXEC; 5934 unlock_new_inode(&inode->vfs_inode); 5935 return inode; 5936} 5937 5938static struct btrfs_inode *new_simple_dir(struct inode *dir, 5939 struct btrfs_key *key, 5940 struct btrfs_root *root) 5941{ 5942 struct timespec64 ts; 5943 struct inode *vfs_inode; 5944 struct btrfs_inode *inode; 5945 5946 vfs_inode = new_inode(dir->i_sb); 5947 if (!vfs_inode) 5948 return ERR_PTR(-ENOMEM); 5949 5950 inode = BTRFS_I(vfs_inode); 5951 inode->root = btrfs_grab_root(root); 5952 inode->ref_root_id = key->objectid; 5953 set_bit(BTRFS_INODE_ROOT_STUB, &inode->runtime_flags); 5954 set_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags); 5955 5956 btrfs_set_inode_number(inode, BTRFS_EMPTY_SUBVOL_DIR_OBJECTID); 5957 /* 5958 * We only need lookup, the rest is read-only and there's no inode 5959 * associated with the dentry 5960 */ 5961 vfs_inode->i_op = &simple_dir_inode_operations; 5962 vfs_inode->i_opflags &= ~IOP_XATTR; 5963 vfs_inode->i_fop = &simple_dir_operations; 5964 vfs_inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; 5965 5966 ts = inode_set_ctime_current(vfs_inode); 5967 inode_set_mtime_to_ts(vfs_inode, ts); 5968 inode_set_atime_to_ts(vfs_inode, inode_get_atime(dir)); 5969 inode->i_otime_sec = ts.tv_sec; 5970 inode->i_otime_nsec = ts.tv_nsec; 5971 5972 vfs_inode->i_uid = dir->i_uid; 5973 vfs_inode->i_gid = dir->i_gid; 5974 5975 return inode; 5976} 5977 5978static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN); 5979static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE); 5980static_assert(BTRFS_FT_DIR == FT_DIR); 5981static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV); 5982static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV); 5983static_assert(BTRFS_FT_FIFO == FT_FIFO); 5984static_assert(BTRFS_FT_SOCK == FT_SOCK); 5985static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK); 5986 5987static inline u8 btrfs_inode_type(const struct btrfs_inode *inode) 5988{ 5989 return fs_umode_to_ftype(inode->vfs_inode.i_mode); 5990} 5991 5992struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) 5993{ 5994 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); 5995 struct btrfs_inode *inode; 5996 struct btrfs_root *root = BTRFS_I(dir)->root; 5997 struct btrfs_root *sub_root = root; 5998 struct btrfs_key location = { 0 }; 5999 u8 di_type = 0; 6000 int ret = 0; 6001 6002 if (dentry->d_name.len > BTRFS_NAME_LEN) 6003 return ERR_PTR(-ENAMETOOLONG); 6004 6005 ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, &location, &di_type); 6006 if (ret < 0) 6007 return ERR_PTR(ret); 6008 6009 if (location.type == BTRFS_INODE_ITEM_KEY) { 6010 inode = btrfs_iget(location.objectid, root); 6011 if (IS_ERR(inode)) 6012 return ERR_CAST(inode); 6013 6014 /* Do extra check against inode mode with di_type */ 6015 if (unlikely(btrfs_inode_type(inode) != di_type)) { 6016 btrfs_crit(fs_info, 6017"inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u", 6018 inode->vfs_inode.i_mode, btrfs_inode_type(inode), 6019 di_type); 6020 iput(&inode->vfs_inode); 6021 return ERR_PTR(-EUCLEAN); 6022 } 6023 return &inode->vfs_inode; 6024 } 6025 6026 ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry, 6027 &location, &sub_root); 6028 if (ret < 0) { 6029 if (ret != -ENOENT) 6030 inode = ERR_PTR(ret); 6031 else 6032 inode = new_simple_dir(dir, &location, root); 6033 } else { 6034 inode = btrfs_iget(location.objectid, sub_root); 6035 btrfs_put_root(sub_root); 6036 6037 if (IS_ERR(inode)) 6038 return ERR_CAST(inode); 6039 6040 down_read(&fs_info->cleanup_work_sem); 6041 if (!sb_rdonly(inode->vfs_inode.i_sb)) 6042 ret = btrfs_orphan_cleanup(sub_root); 6043 up_read(&fs_info->cleanup_work_sem); 6044 if (ret) { 6045 iput(&inode->vfs_inode); 6046 inode = ERR_PTR(ret); 6047 } 6048 } 6049 6050 if (IS_ERR(inode)) 6051 return ERR_CAST(inode); 6052 6053 return &inode->vfs_inode; 6054} 6055 6056static int btrfs_dentry_delete(const struct dentry *dentry) 6057{ 6058 struct btrfs_root *root; 6059 struct inode *inode = d_inode(dentry); 6060 6061 if (!inode && !IS_ROOT(dentry)) 6062 inode = d_inode(dentry->d_parent); 6063 6064 if (inode) { 6065 root = BTRFS_I(inode)->root; 6066 if (btrfs_root_refs(&root->root_item) == 0) 6067 return 1; 6068 6069 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 6070 return 1; 6071 } 6072 return 0; 6073} 6074 6075static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, 6076 unsigned int flags) 6077{ 6078 struct inode *inode = btrfs_lookup_dentry(dir, dentry); 6079 6080 if (inode == ERR_PTR(-ENOENT)) 6081 inode = NULL; 6082 return d_splice_alias(inode, dentry); 6083} 6084 6085/* 6086 * Find the highest existing sequence number in a directory and then set the 6087 * in-memory index_cnt variable to the first free sequence number. 6088 */ 6089static int btrfs_set_inode_index_count(struct btrfs_inode *inode) 6090{ 6091 struct btrfs_root *root = inode->root; 6092 struct btrfs_key key, found_key; 6093 BTRFS_PATH_AUTO_FREE(path); 6094 struct extent_buffer *leaf; 6095 int ret; 6096 6097 key.objectid = btrfs_ino(inode); 6098 key.type = BTRFS_DIR_INDEX_KEY; 6099 key.offset = (u64)-1; 6100 6101 path = btrfs_alloc_path(); 6102 if (!path) 6103 return -ENOMEM; 6104 6105 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 6106 if (ret < 0) 6107 return ret; 6108 /* FIXME: we should be able to handle this */ 6109 if (ret == 0) 6110 return ret; 6111 6112 if (path->slots[0] == 0) { 6113 inode->index_cnt = BTRFS_DIR_START_INDEX; 6114 return 0; 6115 } 6116 6117 path->slots[0]--; 6118 6119 leaf = path->nodes[0]; 6120 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 6121 6122 if (found_key.objectid != btrfs_ino(inode) || 6123 found_key.type != BTRFS_DIR_INDEX_KEY) { 6124 inode->index_cnt = BTRFS_DIR_START_INDEX; 6125 return 0; 6126 } 6127 6128 inode->index_cnt = found_key.offset + 1; 6129 6130 return 0; 6131} 6132 6133static int btrfs_get_dir_last_index(struct btrfs_inode *dir, u64 *index) 6134{ 6135 int ret = 0; 6136 6137 btrfs_inode_lock(dir, 0); 6138 if (dir->index_cnt == (u64)-1) { 6139 ret = btrfs_inode_delayed_dir_index_count(dir); 6140 if (ret) { 6141 ret = btrfs_set_inode_index_count(dir); 6142 if (ret) 6143 goto out; 6144 } 6145 } 6146 6147 /* index_cnt is the index number of next new entry, so decrement it. */ 6148 *index = dir->index_cnt - 1; 6149out: 6150 btrfs_inode_unlock(dir, 0); 6151 6152 return ret; 6153} 6154 6155/* 6156 * All this infrastructure exists because dir_emit can fault, and we are holding 6157 * the tree lock when doing readdir. For now just allocate a buffer and copy 6158 * our information into that, and then dir_emit from the buffer. This is 6159 * similar to what NFS does, only we don't keep the buffer around in pagecache 6160 * because I'm afraid I'll mess that up. Long term we need to make filldir do 6161 * copy_to_user_inatomic so we don't have to worry about page faulting under the 6162 * tree lock. 6163 */ 6164static int btrfs_opendir(struct inode *inode, struct file *file) 6165{ 6166 struct btrfs_file_private *private; 6167 u64 last_index; 6168 int ret; 6169 6170 ret = btrfs_get_dir_last_index(BTRFS_I(inode), &last_index); 6171 if (ret) 6172 return ret; 6173 6174 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL); 6175 if (!private) 6176 return -ENOMEM; 6177 private->last_index = last_index; 6178 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); 6179 if (!private->filldir_buf) { 6180 kfree(private); 6181 return -ENOMEM; 6182 } 6183 file->private_data = private; 6184 return 0; 6185} 6186 6187static loff_t btrfs_dir_llseek(struct file *file, loff_t offset, int whence) 6188{ 6189 struct btrfs_file_private *private = file->private_data; 6190 int ret; 6191 6192 ret = btrfs_get_dir_last_index(BTRFS_I(file_inode(file)), 6193 &private->last_index); 6194 if (ret) 6195 return ret; 6196 6197 return generic_file_llseek(file, offset, whence); 6198} 6199 6200struct dir_entry { 6201 u64 ino; 6202 u64 offset; 6203 unsigned type; 6204 int name_len; 6205}; 6206 6207static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) 6208{ 6209 while (entries--) { 6210 struct dir_entry *entry = addr; 6211 char *name = (char *)(entry + 1); 6212 6213 ctx->pos = get_unaligned(&entry->offset); 6214 if (!dir_emit(ctx, name, get_unaligned(&entry->name_len), 6215 get_unaligned(&entry->ino), 6216 get_unaligned(&entry->type))) 6217 return 1; 6218 addr += sizeof(struct dir_entry) + 6219 get_unaligned(&entry->name_len); 6220 ctx->pos++; 6221 } 6222 return 0; 6223} 6224 6225static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) 6226{ 6227 struct inode *inode = file_inode(file); 6228 struct btrfs_root *root = BTRFS_I(inode)->root; 6229 struct btrfs_file_private *private = file->private_data; 6230 struct btrfs_dir_item *di; 6231 struct btrfs_key key; 6232 struct btrfs_key found_key; 6233 BTRFS_PATH_AUTO_FREE(path); 6234 void *addr; 6235 LIST_HEAD(ins_list); 6236 LIST_HEAD(del_list); 6237 int ret; 6238 char *name_ptr; 6239 int name_len; 6240 int entries = 0; 6241 int total_len = 0; 6242 bool put = false; 6243 struct btrfs_key location; 6244 6245 if (!dir_emit_dots(file, ctx)) 6246 return 0; 6247 6248 path = btrfs_alloc_path(); 6249 if (!path) 6250 return -ENOMEM; 6251 6252 addr = private->filldir_buf; 6253 path->reada = READA_FORWARD; 6254 6255 put = btrfs_readdir_get_delayed_items(BTRFS_I(inode), private->last_index, 6256 &ins_list, &del_list); 6257 6258again: 6259 key.type = BTRFS_DIR_INDEX_KEY; 6260 key.offset = ctx->pos; 6261 key.objectid = btrfs_ino(BTRFS_I(inode)); 6262 6263 btrfs_for_each_slot(root, &key, &found_key, path, ret) { 6264 struct dir_entry *entry; 6265 struct extent_buffer *leaf = path->nodes[0]; 6266 u8 ftype; 6267 6268 if (found_key.objectid != key.objectid) 6269 break; 6270 if (found_key.type != BTRFS_DIR_INDEX_KEY) 6271 break; 6272 if (found_key.offset < ctx->pos) 6273 continue; 6274 if (found_key.offset > private->last_index) 6275 break; 6276 if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) 6277 continue; 6278 di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item); 6279 name_len = btrfs_dir_name_len(leaf, di); 6280 if ((total_len + sizeof(struct dir_entry) + name_len) >= 6281 PAGE_SIZE) { 6282 btrfs_release_path(path); 6283 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6284 if (ret) 6285 goto nopos; 6286 addr = private->filldir_buf; 6287 entries = 0; 6288 total_len = 0; 6289 goto again; 6290 } 6291 6292 ftype = btrfs_dir_flags_to_ftype(btrfs_dir_flags(leaf, di)); 6293 entry = addr; 6294 name_ptr = (char *)(entry + 1); 6295 read_extent_buffer(leaf, name_ptr, 6296 (unsigned long)(di + 1), name_len); 6297 put_unaligned(name_len, &entry->name_len); 6298 put_unaligned(fs_ftype_to_dtype(ftype), &entry->type); 6299 btrfs_dir_item_key_to_cpu(leaf, di, &location); 6300 put_unaligned(location.objectid, &entry->ino); 6301 put_unaligned(found_key.offset, &entry->offset); 6302 entries++; 6303 addr += sizeof(struct dir_entry) + name_len; 6304 total_len += sizeof(struct dir_entry) + name_len; 6305 } 6306 /* Catch error encountered during iteration */ 6307 if (ret < 0) 6308 goto err; 6309 6310 btrfs_release_path(path); 6311 6312 ret = btrfs_filldir(private->filldir_buf, entries, ctx); 6313 if (ret) 6314 goto nopos; 6315 6316 if (btrfs_readdir_delayed_dir_index(ctx, &ins_list)) 6317 goto nopos; 6318 6319 /* 6320 * Stop new entries from being returned after we return the last 6321 * entry. 6322 * 6323 * New directory entries are assigned a strictly increasing 6324 * offset. This means that new entries created during readdir 6325 * are *guaranteed* to be seen in the future by that readdir. 6326 * This has broken buggy programs which operate on names as 6327 * they're returned by readdir. Until we reuse freed offsets 6328 * we have this hack to stop new entries from being returned 6329 * under the assumption that they'll never reach this huge 6330 * offset. 6331 * 6332 * This is being careful not to overflow 32bit loff_t unless the 6333 * last entry requires it because doing so has broken 32bit apps 6334 * in the past. 6335 */ 6336 if (ctx->pos >= INT_MAX) 6337 ctx->pos = LLONG_MAX; 6338 else 6339 ctx->pos = INT_MAX; 6340nopos: 6341 ret = 0; 6342err: 6343 if (put) 6344 btrfs_readdir_put_delayed_items(BTRFS_I(inode), &ins_list, &del_list); 6345 return ret; 6346} 6347 6348/* 6349 * This is somewhat expensive, updating the tree every time the 6350 * inode changes. But, it is most likely to find the inode in cache. 6351 * FIXME, needs more benchmarking...there are no reasons other than performance 6352 * to keep or drop this code. 6353 */ 6354static int btrfs_dirty_inode(struct btrfs_inode *inode) 6355{ 6356 struct btrfs_root *root = inode->root; 6357 struct btrfs_fs_info *fs_info = root->fs_info; 6358 struct btrfs_trans_handle *trans; 6359 int ret; 6360 6361 if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags)) 6362 return 0; 6363 6364 trans = btrfs_join_transaction(root); 6365 if (IS_ERR(trans)) 6366 return PTR_ERR(trans); 6367 6368 ret = btrfs_update_inode(trans, inode); 6369 if (ret == -ENOSPC || ret == -EDQUOT) { 6370 /* whoops, lets try again with the full transaction */ 6371 btrfs_end_transaction(trans); 6372 trans = btrfs_start_transaction(root, 1); 6373 if (IS_ERR(trans)) 6374 return PTR_ERR(trans); 6375 6376 ret = btrfs_update_inode(trans, inode); 6377 } 6378 btrfs_end_transaction(trans); 6379 if (inode->delayed_node) 6380 btrfs_balance_delayed_items(fs_info); 6381 6382 return ret; 6383} 6384 6385/* 6386 * We need our own ->update_time so that we can return error on ENOSPC for 6387 * updating the inode in the case of file write and mmap writes. 6388 */ 6389static int btrfs_update_time(struct inode *inode, int flags) 6390{ 6391 struct btrfs_root *root = BTRFS_I(inode)->root; 6392 bool dirty; 6393 6394 if (btrfs_root_readonly(root)) 6395 return -EROFS; 6396 6397 dirty = inode_update_timestamps(inode, flags); 6398 return dirty ? btrfs_dirty_inode(BTRFS_I(inode)) : 0; 6399} 6400 6401/* 6402 * helper to find a free sequence number in a given directory. This current 6403 * code is very simple, later versions will do smarter things in the btree 6404 */ 6405int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) 6406{ 6407 int ret = 0; 6408 6409 if (dir->index_cnt == (u64)-1) { 6410 ret = btrfs_inode_delayed_dir_index_count(dir); 6411 if (ret) { 6412 ret = btrfs_set_inode_index_count(dir); 6413 if (ret) 6414 return ret; 6415 } 6416 } 6417 6418 *index = dir->index_cnt; 6419 dir->index_cnt++; 6420 6421 return ret; 6422} 6423 6424static int btrfs_insert_inode_locked(struct inode *inode) 6425{ 6426 struct btrfs_iget_args args; 6427 6428 args.ino = btrfs_ino(BTRFS_I(inode)); 6429 args.root = BTRFS_I(inode)->root; 6430 6431 return insert_inode_locked4(inode, 6432 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), 6433 btrfs_find_actor, &args); 6434} 6435 6436int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args, 6437 unsigned int *trans_num_items) 6438{ 6439 struct inode *dir = args->dir; 6440 struct inode *inode = args->inode; 6441 int ret; 6442 6443 if (!args->orphan) { 6444 ret = fscrypt_setup_filename(dir, &args->dentry->d_name, 0, 6445 &args->fname); 6446 if (ret) 6447 return ret; 6448 } 6449 6450 ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl); 6451 if (ret) { 6452 fscrypt_free_filename(&args->fname); 6453 return ret; 6454 } 6455 6456 /* 1 to add inode item */ 6457 *trans_num_items = 1; 6458 /* 1 to add compression property */ 6459 if (BTRFS_I(dir)->prop_compress) 6460 (*trans_num_items)++; 6461 /* 1 to add default ACL xattr */ 6462 if (args->default_acl) 6463 (*trans_num_items)++; 6464 /* 1 to add access ACL xattr */ 6465 if (args->acl) 6466 (*trans_num_items)++; 6467#ifdef CONFIG_SECURITY 6468 /* 1 to add LSM xattr */ 6469 if (dir->i_security) 6470 (*trans_num_items)++; 6471#endif 6472 if (args->orphan) { 6473 /* 1 to add orphan item */ 6474 (*trans_num_items)++; 6475 } else { 6476 /* 6477 * 1 to add dir item 6478 * 1 to add dir index 6479 * 1 to update parent inode item 6480 * 6481 * No need for 1 unit for the inode ref item because it is 6482 * inserted in a batch together with the inode item at 6483 * btrfs_create_new_inode(). 6484 */ 6485 *trans_num_items += 3; 6486 } 6487 return 0; 6488} 6489 6490void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args) 6491{ 6492 posix_acl_release(args->acl); 6493 posix_acl_release(args->default_acl); 6494 fscrypt_free_filename(&args->fname); 6495} 6496 6497/* 6498 * Inherit flags from the parent inode. 6499 * 6500 * Currently only the compression flags and the cow flags are inherited. 6501 */ 6502static void btrfs_inherit_iflags(struct btrfs_inode *inode, struct btrfs_inode *dir) 6503{ 6504 unsigned int flags; 6505 6506 flags = dir->flags; 6507 6508 if (flags & BTRFS_INODE_NOCOMPRESS) { 6509 inode->flags &= ~BTRFS_INODE_COMPRESS; 6510 inode->flags |= BTRFS_INODE_NOCOMPRESS; 6511 } else if (flags & BTRFS_INODE_COMPRESS) { 6512 inode->flags &= ~BTRFS_INODE_NOCOMPRESS; 6513 inode->flags |= BTRFS_INODE_COMPRESS; 6514 } 6515 6516 if (flags & BTRFS_INODE_NODATACOW) { 6517 inode->flags |= BTRFS_INODE_NODATACOW; 6518 if (S_ISREG(inode->vfs_inode.i_mode)) 6519 inode->flags |= BTRFS_INODE_NODATASUM; 6520 } 6521 6522 btrfs_sync_inode_flags_to_i_flags(inode); 6523} 6524 6525int btrfs_create_new_inode(struct btrfs_trans_handle *trans, 6526 struct btrfs_new_inode_args *args) 6527{ 6528 struct timespec64 ts; 6529 struct inode *dir = args->dir; 6530 struct inode *inode = args->inode; 6531 const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name; 6532 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); 6533 struct btrfs_root *root; 6534 struct btrfs_inode_item *inode_item; 6535 struct btrfs_path *path; 6536 u64 objectid; 6537 struct btrfs_inode_ref *ref; 6538 struct btrfs_key key[2]; 6539 u32 sizes[2]; 6540 struct btrfs_item_batch batch; 6541 unsigned long ptr; 6542 int ret; 6543 bool xa_reserved = false; 6544 6545 if (!args->orphan && !args->subvol) { 6546 /* 6547 * Before anything else, check if we can add the name to the 6548 * parent directory. We want to avoid a dir item overflow in 6549 * case we have an existing dir item due to existing name 6550 * hash collisions. We do this check here before we call 6551 * btrfs_add_link() down below so that we can avoid a 6552 * transaction abort (which could be exploited by malicious 6553 * users). 6554 * 6555 * For subvolumes we already do this in btrfs_mksubvol(). 6556 */ 6557 ret = btrfs_check_dir_item_collision(BTRFS_I(dir)->root, 6558 btrfs_ino(BTRFS_I(dir)), 6559 name); 6560 if (ret < 0) 6561 return ret; 6562 } 6563 6564 path = btrfs_alloc_path(); 6565 if (!path) 6566 return -ENOMEM; 6567 6568 if (!args->subvol) 6569 BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root); 6570 root = BTRFS_I(inode)->root; 6571 6572 ret = btrfs_init_file_extent_tree(BTRFS_I(inode)); 6573 if (ret) 6574 goto out; 6575 6576 ret = btrfs_get_free_objectid(root, &objectid); 6577 if (ret) 6578 goto out; 6579 btrfs_set_inode_number(BTRFS_I(inode), objectid); 6580 6581 ret = xa_reserve(&root->inodes, objectid, GFP_NOFS); 6582 if (ret) 6583 goto out; 6584 xa_reserved = true; 6585 6586 if (args->orphan) { 6587 /* 6588 * O_TMPFILE, set link count to 0, so that after this point, we 6589 * fill in an inode item with the correct link count. 6590 */ 6591 set_nlink(inode, 0); 6592 } else { 6593 trace_btrfs_inode_request(dir); 6594 6595 ret = btrfs_set_inode_index(BTRFS_I(dir), &BTRFS_I(inode)->dir_index); 6596 if (ret) 6597 goto out; 6598 } 6599 6600 if (S_ISDIR(inode->i_mode)) 6601 BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX; 6602 6603 BTRFS_I(inode)->generation = trans->transid; 6604 inode->i_generation = BTRFS_I(inode)->generation; 6605 6606 /* 6607 * We don't have any capability xattrs set here yet, shortcut any 6608 * queries for the xattrs here. If we add them later via the inode 6609 * security init path or any other path this flag will be cleared. 6610 */ 6611 set_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags); 6612 6613 /* 6614 * Subvolumes don't inherit flags from their parent directory. 6615 * Originally this was probably by accident, but we probably can't 6616 * change it now without compatibility issues. 6617 */ 6618 if (!args->subvol) 6619 btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir)); 6620 6621 btrfs_set_inode_mapping_order(BTRFS_I(inode)); 6622 if (S_ISREG(inode->i_mode)) { 6623 if (btrfs_test_opt(fs_info, NODATASUM)) 6624 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; 6625 if (btrfs_test_opt(fs_info, NODATACOW)) 6626 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | 6627 BTRFS_INODE_NODATASUM; 6628 btrfs_update_inode_mapping_flags(BTRFS_I(inode)); 6629 } 6630 6631 ret = btrfs_insert_inode_locked(inode); 6632 if (ret < 0) { 6633 if (!args->orphan) 6634 BTRFS_I(dir)->index_cnt--; 6635 goto out; 6636 } 6637 6638 /* 6639 * We could have gotten an inode number from somebody who was fsynced 6640 * and then removed in this same transaction, so let's just set full 6641 * sync since it will be a full sync anyway and this will blow away the 6642 * old info in the log. 6643 */ 6644 btrfs_set_inode_full_sync(BTRFS_I(inode)); 6645 6646 key[0].objectid = objectid; 6647 key[0].type = BTRFS_INODE_ITEM_KEY; 6648 key[0].offset = 0; 6649 6650 sizes[0] = sizeof(struct btrfs_inode_item); 6651 6652 if (!args->orphan) { 6653 /* 6654 * Start new inodes with an inode_ref. This is slightly more 6655 * efficient for small numbers of hard links since they will 6656 * be packed into one item. Extended refs will kick in if we 6657 * add more hard links than can fit in the ref item. 6658 */ 6659 key[1].objectid = objectid; 6660 key[1].type = BTRFS_INODE_REF_KEY; 6661 if (args->subvol) { 6662 key[1].offset = objectid; 6663 sizes[1] = 2 + sizeof(*ref); 6664 } else { 6665 key[1].offset = btrfs_ino(BTRFS_I(dir)); 6666 sizes[1] = name->len + sizeof(*ref); 6667 } 6668 } 6669 6670 batch.keys = &key[0]; 6671 batch.data_sizes = &sizes[0]; 6672 batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]); 6673 batch.nr = args->orphan ? 1 : 2; 6674 ret = btrfs_insert_empty_items(trans, root, path, &batch); 6675 if (unlikely(ret != 0)) { 6676 btrfs_abort_transaction(trans, ret); 6677 goto discard; 6678 } 6679 6680 ts = simple_inode_init_ts(inode); 6681 BTRFS_I(inode)->i_otime_sec = ts.tv_sec; 6682 BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec; 6683 6684 /* 6685 * We're going to fill the inode item now, so at this point the inode 6686 * must be fully initialized. 6687 */ 6688 6689 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], 6690 struct btrfs_inode_item); 6691 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, 6692 sizeof(*inode_item)); 6693 fill_inode_item(trans, path->nodes[0], inode_item, inode); 6694 6695 if (!args->orphan) { 6696 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, 6697 struct btrfs_inode_ref); 6698 ptr = (unsigned long)(ref + 1); 6699 if (args->subvol) { 6700 btrfs_set_inode_ref_name_len(path->nodes[0], ref, 2); 6701 btrfs_set_inode_ref_index(path->nodes[0], ref, 0); 6702 write_extent_buffer(path->nodes[0], "..", ptr, 2); 6703 } else { 6704 btrfs_set_inode_ref_name_len(path->nodes[0], ref, 6705 name->len); 6706 btrfs_set_inode_ref_index(path->nodes[0], ref, 6707 BTRFS_I(inode)->dir_index); 6708 write_extent_buffer(path->nodes[0], name->name, ptr, 6709 name->len); 6710 } 6711 } 6712 6713 /* 6714 * We don't need the path anymore, plus inheriting properties, adding 6715 * ACLs, security xattrs, orphan item or adding the link, will result in 6716 * allocating yet another path. So just free our path. 6717 */ 6718 btrfs_free_path(path); 6719 path = NULL; 6720 6721 if (args->subvol) { 6722 struct btrfs_inode *parent; 6723 6724 /* 6725 * Subvolumes inherit properties from their parent subvolume, 6726 * not the directory they were created in. 6727 */ 6728 parent = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, BTRFS_I(dir)->root); 6729 if (IS_ERR(parent)) { 6730 ret = PTR_ERR(parent); 6731 } else { 6732 ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode), 6733 parent); 6734 iput(&parent->vfs_inode); 6735 } 6736 } else { 6737 ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode), 6738 BTRFS_I(dir)); 6739 } 6740 if (ret) { 6741 btrfs_err(fs_info, 6742 "error inheriting props for ino %llu (root %llu): %d", 6743 btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ret); 6744 } 6745 6746 /* 6747 * Subvolumes don't inherit ACLs or get passed to the LSM. This is 6748 * probably a bug. 6749 */ 6750 if (!args->subvol) { 6751 ret = btrfs_init_inode_security(trans, args); 6752 if (unlikely(ret)) { 6753 btrfs_abort_transaction(trans, ret); 6754 goto discard; 6755 } 6756 } 6757 6758 ret = btrfs_add_inode_to_root(BTRFS_I(inode), false); 6759 if (WARN_ON(ret)) { 6760 /* Shouldn't happen, we used xa_reserve() before. */ 6761 btrfs_abort_transaction(trans, ret); 6762 goto discard; 6763 } 6764 6765 trace_btrfs_inode_new(inode); 6766 btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); 6767 6768 btrfs_update_root_times(trans, root); 6769 6770 if (args->orphan) { 6771 ret = btrfs_orphan_add(trans, BTRFS_I(inode)); 6772 if (unlikely(ret)) { 6773 btrfs_abort_transaction(trans, ret); 6774 goto discard; 6775 } 6776 } else { 6777 ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name, 6778 0, BTRFS_I(inode)->dir_index); 6779 if (unlikely(ret)) { 6780 btrfs_abort_transaction(trans, ret); 6781 goto discard; 6782 } 6783 } 6784 6785 return 0; 6786 6787discard: 6788 /* 6789 * discard_new_inode() calls iput(), but the caller owns the reference 6790 * to the inode. 6791 */ 6792 ihold(inode); 6793 discard_new_inode(inode); 6794out: 6795 if (xa_reserved) 6796 xa_release(&root->inodes, objectid); 6797 6798 btrfs_free_path(path); 6799 return ret; 6800} 6801 6802/* 6803 * utility function to add 'inode' into 'parent_inode' with 6804 * a give name and a given sequence number. 6805 * if 'add_backref' is true, also insert a backref from the 6806 * inode to the parent directory. 6807 */ 6808int btrfs_add_link(struct btrfs_trans_handle *trans, 6809 struct btrfs_inode *parent_inode, struct btrfs_inode *inode, 6810 const struct fscrypt_str *name, bool add_backref, u64 index) 6811{ 6812 int ret = 0; 6813 struct btrfs_key key; 6814 struct btrfs_root *root = parent_inode->root; 6815 u64 ino = btrfs_ino(inode); 6816 u64 parent_ino = btrfs_ino(parent_inode); 6817 6818 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6819 memcpy(&key, &inode->root->root_key, sizeof(key)); 6820 } else { 6821 key.objectid = ino; 6822 key.type = BTRFS_INODE_ITEM_KEY; 6823 key.offset = 0; 6824 } 6825 6826 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6827 ret = btrfs_add_root_ref(trans, key.objectid, 6828 btrfs_root_id(root), parent_ino, 6829 index, name); 6830 } else if (add_backref) { 6831 ret = btrfs_insert_inode_ref(trans, root, name, 6832 ino, parent_ino, index); 6833 } 6834 6835 /* Nothing to clean up yet */ 6836 if (ret) 6837 return ret; 6838 6839 ret = btrfs_insert_dir_item(trans, name, parent_inode, &key, 6840 btrfs_inode_type(inode), index); 6841 if (ret == -EEXIST || ret == -EOVERFLOW) 6842 goto fail_dir_item; 6843 else if (unlikely(ret)) { 6844 btrfs_abort_transaction(trans, ret); 6845 return ret; 6846 } 6847 6848 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + 6849 name->len * 2); 6850 inode_inc_iversion(&parent_inode->vfs_inode); 6851 update_time_after_link_or_unlink(parent_inode); 6852 6853 ret = btrfs_update_inode(trans, parent_inode); 6854 if (ret) 6855 btrfs_abort_transaction(trans, ret); 6856 return ret; 6857 6858fail_dir_item: 6859 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { 6860 u64 local_index; 6861 int ret2; 6862 6863 ret2 = btrfs_del_root_ref(trans, key.objectid, btrfs_root_id(root), 6864 parent_ino, &local_index, name); 6865 if (ret2) 6866 btrfs_abort_transaction(trans, ret2); 6867 } else if (add_backref) { 6868 int ret2; 6869 6870 ret2 = btrfs_del_inode_ref(trans, root, name, ino, parent_ino, NULL); 6871 if (ret2) 6872 btrfs_abort_transaction(trans, ret2); 6873 } 6874 6875 /* Return the original error code */ 6876 return ret; 6877} 6878 6879static int btrfs_create_common(struct inode *dir, struct dentry *dentry, 6880 struct inode *inode) 6881{ 6882 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); 6883 struct btrfs_root *root = BTRFS_I(dir)->root; 6884 struct btrfs_new_inode_args new_inode_args = { 6885 .dir = dir, 6886 .dentry = dentry, 6887 .inode = inode, 6888 }; 6889 unsigned int trans_num_items; 6890 struct btrfs_trans_handle *trans; 6891 int ret; 6892 6893 ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); 6894 if (ret) 6895 goto out_inode; 6896 6897 trans = btrfs_start_transaction(root, trans_num_items); 6898 if (IS_ERR(trans)) { 6899 ret = PTR_ERR(trans); 6900 goto out_new_inode_args; 6901 } 6902 6903 ret = btrfs_create_new_inode(trans, &new_inode_args); 6904 if (!ret) { 6905 if (S_ISDIR(inode->i_mode)) 6906 inode->i_opflags |= IOP_FASTPERM_MAY_EXEC; 6907 d_instantiate_new(dentry, inode); 6908 } 6909 6910 btrfs_end_transaction(trans); 6911 btrfs_btree_balance_dirty(fs_info); 6912out_new_inode_args: 6913 btrfs_new_inode_args_destroy(&new_inode_args); 6914out_inode: 6915 if (ret) 6916 iput(inode); 6917 return ret; 6918} 6919 6920static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir, 6921 struct dentry *dentry, umode_t mode, dev_t rdev) 6922{ 6923 struct inode *inode; 6924 6925 inode = new_inode(dir->i_sb); 6926 if (!inode) 6927 return -ENOMEM; 6928 inode_init_owner(idmap, inode, dir, mode); 6929 inode->i_op = &btrfs_special_inode_operations; 6930 init_special_inode(inode, inode->i_mode, rdev); 6931 return btrfs_create_common(dir, dentry, inode); 6932} 6933 6934static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir, 6935 struct dentry *dentry, umode_t mode, bool excl) 6936{ 6937 struct inode *inode; 6938 6939 inode = new_inode(dir->i_sb); 6940 if (!inode) 6941 return -ENOMEM; 6942 inode_init_owner(idmap, inode, dir, mode); 6943 inode->i_fop = &btrfs_file_operations; 6944 inode->i_op = &btrfs_file_inode_operations; 6945 inode->i_mapping->a_ops = &btrfs_aops; 6946 return btrfs_create_common(dir, dentry, inode); 6947} 6948 6949static int btrfs_link(struct dentry *old_dentry, struct inode *dir, 6950 struct dentry *dentry) 6951{ 6952 struct btrfs_trans_handle *trans = NULL; 6953 struct btrfs_root *root = BTRFS_I(dir)->root; 6954 struct inode *inode = d_inode(old_dentry); 6955 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); 6956 struct fscrypt_name fname; 6957 u64 index; 6958 int ret; 6959 6960 /* do not allow sys_link's with other subvols of the same device */ 6961 if (btrfs_root_id(root) != btrfs_root_id(BTRFS_I(inode)->root)) 6962 return -EXDEV; 6963 6964 if (inode->i_nlink >= BTRFS_LINK_MAX) 6965 return -EMLINK; 6966 6967 ret = fscrypt_setup_filename(dir, &dentry->d_name, 0, &fname); 6968 if (ret) 6969 goto fail; 6970 6971 ret = btrfs_set_inode_index(BTRFS_I(dir), &index); 6972 if (ret) 6973 goto fail; 6974 6975 /* 6976 * 2 items for inode and inode ref 6977 * 2 items for dir items 6978 * 1 item for parent inode 6979 * 1 item for orphan item deletion if O_TMPFILE 6980 */ 6981 trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6); 6982 if (IS_ERR(trans)) { 6983 ret = PTR_ERR(trans); 6984 trans = NULL; 6985 goto fail; 6986 } 6987 6988 /* There are several dir indexes for this inode, clear the cache. */ 6989 BTRFS_I(inode)->dir_index = 0ULL; 6990 inode_inc_iversion(inode); 6991 inode_set_ctime_current(inode); 6992 6993 ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), 6994 &fname.disk_name, 1, index); 6995 if (ret) 6996 goto fail; 6997 6998 /* Link added now we update the inode item with the new link count. */ 6999 inc_nlink(inode); 7000 ret = btrfs_update_inode(trans, BTRFS_I(inode)); 7001 if (unlikely(ret)) { 7002 btrfs_abort_transaction(trans, ret); 7003 goto fail; 7004 } 7005 7006 if (inode->i_nlink == 1) { 7007 /* 7008 * If the new hard link count is 1, it's a file created with the 7009 * open(2) O_TMPFILE flag. 7010 */ 7011 ret = btrfs_orphan_del(trans, BTRFS_I(inode)); 7012 if (unlikely(ret)) { 7013 btrfs_abort_transaction(trans, ret); 7014 goto fail; 7015 } 7016 } 7017 7018 /* Grab reference for the new dentry passed to d_instantiate(). */ 7019 ihold(inode); 7020 d_instantiate(dentry, inode); 7021 btrfs_log_new_name(trans, old_dentry, NULL, 0, dentry->d_parent); 7022 7023fail: 7024 fscrypt_free_filename(&fname); 7025 if (trans) 7026 btrfs_end_transaction(trans); 7027 btrfs_btree_balance_dirty(fs_info); 7028 return ret; 7029} 7030 7031static struct dentry *btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, 7032 struct dentry *dentry, umode_t mode) 7033{ 7034 struct inode *inode; 7035 7036 inode = new_inode(dir->i_sb); 7037 if (!inode) 7038 return ERR_PTR(-ENOMEM); 7039 inode_init_owner(idmap, inode, dir, S_IFDIR | mode); 7040 inode->i_op = &btrfs_dir_inode_operations; 7041 inode->i_fop = &btrfs_dir_file_operations; 7042 return ERR_PTR(btrfs_create_common(dir, dentry, inode)); 7043} 7044 7045static noinline int uncompress_inline(struct btrfs_path *path, 7046 struct folio *folio, 7047 struct btrfs_file_extent_item *item) 7048{ 7049 int ret; 7050 struct extent_buffer *leaf = path->nodes[0]; 7051 const u32 blocksize = leaf->fs_info->sectorsize; 7052 char *tmp; 7053 size_t max_size; 7054 unsigned long inline_size; 7055 unsigned long ptr; 7056 int compress_type; 7057 7058 compress_type = btrfs_file_extent_compression(leaf, item); 7059 max_size = btrfs_file_extent_ram_bytes(leaf, item); 7060 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]); 7061 tmp = kmalloc(inline_size, GFP_NOFS); 7062 if (!tmp) 7063 return -ENOMEM; 7064 ptr = btrfs_file_extent_inline_start(item); 7065 7066 read_extent_buffer(leaf, tmp, ptr, inline_size); 7067 7068 max_size = min_t(unsigned long, blocksize, max_size); 7069 ret = btrfs_decompress(compress_type, tmp, folio, 0, inline_size, 7070 max_size); 7071 7072 /* 7073 * decompression code contains a memset to fill in any space between the end 7074 * of the uncompressed data and the end of max_size in case the decompressed 7075 * data ends up shorter than ram_bytes. That doesn't cover the hole between 7076 * the end of an inline extent and the beginning of the next block, so we 7077 * cover that region here. 7078 */ 7079 7080 if (max_size < blocksize) 7081 folio_zero_range(folio, max_size, blocksize - max_size); 7082 kfree(tmp); 7083 return ret; 7084} 7085 7086static int read_inline_extent(struct btrfs_path *path, struct folio *folio) 7087{ 7088 const u32 blocksize = path->nodes[0]->fs_info->sectorsize; 7089 struct btrfs_file_extent_item *fi; 7090 void *kaddr; 7091 size_t copy_size; 7092 7093 if (!folio || folio_test_uptodate(folio)) 7094 return 0; 7095 7096 ASSERT(folio_pos(folio) == 0); 7097 7098 fi = btrfs_item_ptr(path->nodes[0], path->slots[0], 7099 struct btrfs_file_extent_item); 7100 if (btrfs_file_extent_compression(path->nodes[0], fi) != BTRFS_COMPRESS_NONE) 7101 return uncompress_inline(path, folio, fi); 7102 7103 copy_size = min_t(u64, blocksize, 7104 btrfs_file_extent_ram_bytes(path->nodes[0], fi)); 7105 kaddr = kmap_local_folio(folio, 0); 7106 read_extent_buffer(path->nodes[0], kaddr, 7107 btrfs_file_extent_inline_start(fi), copy_size); 7108 kunmap_local(kaddr); 7109 if (copy_size < blocksize) 7110 folio_zero_range(folio, copy_size, blocksize - copy_size); 7111 return 0; 7112} 7113 7114/* 7115 * Lookup the first extent overlapping a range in a file. 7116 * 7117 * @inode: file to search in 7118 * @page: page to read extent data into if the extent is inline 7119 * @start: file offset 7120 * @len: length of range starting at @start 7121 * 7122 * Return the first &struct extent_map which overlaps the given range, reading 7123 * it from the B-tree and caching it if necessary. Note that there may be more 7124 * extents which overlap the given range after the returned extent_map. 7125 * 7126 * If @page is not NULL and the extent is inline, this also reads the extent 7127 * data directly into the page and marks the extent up to date in the io_tree. 7128 * 7129 * Return: ERR_PTR on error, non-NULL extent_map on success. 7130 */ 7131struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, 7132 struct folio *folio, u64 start, u64 len) 7133{ 7134 struct btrfs_fs_info *fs_info = inode->root->fs_info; 7135 int ret = 0; 7136 u64 extent_start = 0; 7137 u64 extent_end = 0; 7138 u64 objectid = btrfs_ino(inode); 7139 int extent_type = -1; 7140 struct btrfs_path *path = NULL; 7141 struct btrfs_root *root = inode->root; 7142 struct btrfs_file_extent_item *item; 7143 struct extent_buffer *leaf; 7144 struct btrfs_key found_key; 7145 struct extent_map *em = NULL; 7146 struct extent_map_tree *em_tree = &inode->extent_tree; 7147 7148 read_lock(&em_tree->lock); 7149 em = btrfs_lookup_extent_mapping(em_tree, start, len); 7150 read_unlock(&em_tree->lock); 7151 7152 if (em) { 7153 if (em->start > start || em->start + em->len <= start) 7154 btrfs_free_extent_map(em); 7155 else if (em->disk_bytenr == EXTENT_MAP_INLINE && folio) 7156 btrfs_free_extent_map(em); 7157 else 7158 goto out; 7159 } 7160 em = btrfs_alloc_extent_map(); 7161 if (!em) { 7162 ret = -ENOMEM; 7163 goto out; 7164 } 7165 em->start = EXTENT_MAP_HOLE; 7166 em->disk_bytenr = EXTENT_MAP_HOLE; 7167 em->len = (u64)-1; 7168 7169 path = btrfs_alloc_path(); 7170 if (!path) { 7171 ret = -ENOMEM; 7172 goto out; 7173 } 7174 7175 /* Chances are we'll be called again, so go ahead and do readahead */ 7176 path->reada = READA_FORWARD; 7177 7178 /* 7179 * The same explanation in load_free_space_cache applies here as well, 7180 * we only read when we're loading the free space cache, and at that 7181 * point the commit_root has everything we need. 7182 */ 7183 if (btrfs_is_free_space_inode(inode)) { 7184 path->search_commit_root = true; 7185 path->skip_locking = true; 7186 } 7187 7188 ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); 7189 if (ret < 0) { 7190 goto out; 7191 } else if (ret > 0) { 7192 if (path->slots[0] == 0) 7193 goto not_found; 7194 path->slots[0]--; 7195 ret = 0; 7196 } 7197 7198 leaf = path->nodes[0]; 7199 item = btrfs_item_ptr(leaf, path->slots[0], 7200 struct btrfs_file_extent_item); 7201 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 7202 if (found_key.objectid != objectid || 7203 found_key.type != BTRFS_EXTENT_DATA_KEY) { 7204 /* 7205 * If we backup past the first extent we want to move forward 7206 * and see if there is an extent in front of us, otherwise we'll 7207 * say there is a hole for our whole search range which can 7208 * cause problems. 7209 */ 7210 extent_end = start; 7211 goto next; 7212 } 7213 7214 extent_type = btrfs_file_extent_type(leaf, item); 7215 extent_start = found_key.offset; 7216 extent_end = btrfs_file_extent_end(path); 7217 if (extent_type == BTRFS_FILE_EXTENT_REG || 7218 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 7219 /* Only regular file could have regular/prealloc extent */ 7220 if (unlikely(!S_ISREG(inode->vfs_inode.i_mode))) { 7221 ret = -EUCLEAN; 7222 btrfs_crit(fs_info, 7223 "regular/prealloc extent found for non-regular inode %llu", 7224 btrfs_ino(inode)); 7225 goto out; 7226 } 7227 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, 7228 extent_start); 7229 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 7230 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, 7231 path->slots[0], 7232 extent_start); 7233 } 7234next: 7235 if (start >= extent_end) { 7236 path->slots[0]++; 7237 if (path->slots[0] >= btrfs_header_nritems(leaf)) { 7238 ret = btrfs_next_leaf(root, path); 7239 if (ret < 0) 7240 goto out; 7241 else if (ret > 0) 7242 goto not_found; 7243 7244 leaf = path->nodes[0]; 7245 } 7246 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); 7247 if (found_key.objectid != objectid || 7248 found_key.type != BTRFS_EXTENT_DATA_KEY) 7249 goto not_found; 7250 if (start + len <= found_key.offset) 7251 goto not_found; 7252 if (start > found_key.offset) 7253 goto next; 7254 7255 /* New extent overlaps with existing one */ 7256 em->start = start; 7257 em->len = found_key.offset - start; 7258 em->disk_bytenr = EXTENT_MAP_HOLE; 7259 goto insert; 7260 } 7261 7262 btrfs_extent_item_to_extent_map(inode, path, item, em); 7263 7264 if (extent_type == BTRFS_FILE_EXTENT_REG || 7265 extent_type == BTRFS_FILE_EXTENT_PREALLOC) { 7266 goto insert; 7267 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { 7268 /* 7269 * Inline extent can only exist at file offset 0. This is 7270 * ensured by tree-checker and inline extent creation path. 7271 * Thus all members representing file offsets should be zero. 7272 */ 7273 ASSERT(extent_start == 0); 7274 ASSERT(em->start == 0); 7275 7276 /* 7277 * btrfs_extent_item_to_extent_map() should have properly 7278 * initialized em members already. 7279 * 7280 * Other members are not utilized for inline extents. 7281 */ 7282 ASSERT(em->disk_bytenr == EXTENT_MAP_INLINE); 7283 ASSERT(em->len == fs_info->sectorsize); 7284 7285 ret = read_inline_extent(path, folio); 7286 if (ret < 0) 7287 goto out; 7288 goto insert; 7289 } 7290not_found: 7291 em->start = start; 7292 em->len = len; 7293 em->disk_bytenr = EXTENT_MAP_HOLE; 7294insert: 7295 ret = 0; 7296 btrfs_release_path(path); 7297 if (unlikely(em->start > start || btrfs_extent_map_end(em) <= start)) { 7298 btrfs_err(fs_info, 7299 "bad extent! em: [%llu %llu] passed [%llu %llu]", 7300 em->start, em->len, start, len); 7301 ret = -EIO; 7302 goto out; 7303 } 7304 7305 write_lock(&em_tree->lock); 7306 ret = btrfs_add_extent_mapping(inode, &em, start, len); 7307 write_unlock(&em_tree->lock); 7308out: 7309 btrfs_free_path(path); 7310 7311 trace_btrfs_get_extent(root, inode, em); 7312 7313 if (ret) { 7314 btrfs_free_extent_map(em); 7315 return ERR_PTR(ret); 7316 } 7317 return em; 7318} 7319 7320static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr) 7321{ 7322 struct btrfs_block_group *block_group; 7323 bool readonly = false; 7324 7325 block_group = btrfs_lookup_block_group(fs_info, bytenr); 7326 if (!block_group || block_group->ro) 7327 readonly = true; 7328 if (block_group) 7329 btrfs_put_block_group(block_group); 7330 return readonly; 7331} 7332 7333/* 7334 * Check if we can do nocow write into the range [@offset, @offset + @len) 7335 * 7336 * @offset: File offset 7337 * @len: The length to write, will be updated to the nocow writeable 7338 * range 7339 * @orig_start: (optional) Return the original file offset of the file extent 7340 * @orig_len: (optional) Return the original on-disk length of the file extent 7341 * @ram_bytes: (optional) Return the ram_bytes of the file extent 7342 * 7343 * Return: 7344 * >0 and update @len if we can do nocow write 7345 * 0 if we can't do nocow write 7346 * <0 if error happened 7347 * 7348 * NOTE: This only checks the file extents, caller is responsible to wait for 7349 * any ordered extents. 7350 */ 7351noinline int can_nocow_extent(struct btrfs_inode *inode, u64 offset, u64 *len, 7352 struct btrfs_file_extent *file_extent, 7353 bool nowait) 7354{ 7355 struct btrfs_root *root = inode->root; 7356 struct btrfs_fs_info *fs_info = root->fs_info; 7357 struct can_nocow_file_extent_args nocow_args = { 0 }; 7358 BTRFS_PATH_AUTO_FREE(path); 7359 int ret; 7360 struct extent_buffer *leaf; 7361 struct extent_io_tree *io_tree = &inode->io_tree; 7362 struct btrfs_file_extent_item *fi; 7363 struct btrfs_key key; 7364 int found_type; 7365 7366 path = btrfs_alloc_path(); 7367 if (!path) 7368 return -ENOMEM; 7369 path->nowait = nowait; 7370 7371 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), 7372 offset, 0); 7373 if (ret < 0) 7374 return ret; 7375 7376 if (ret == 1) { 7377 if (path->slots[0] == 0) { 7378 /* Can't find the item, must COW. */ 7379 return 0; 7380 } 7381 path->slots[0]--; 7382 } 7383 ret = 0; 7384 leaf = path->nodes[0]; 7385 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); 7386 if (key.objectid != btrfs_ino(inode) || 7387 key.type != BTRFS_EXTENT_DATA_KEY) { 7388 /* Not our file or wrong item type, must COW. */ 7389 return 0; 7390 } 7391 7392 if (key.offset > offset) { 7393 /* Wrong offset, must COW. */ 7394 return 0; 7395 } 7396 7397 if (btrfs_file_extent_end(path) <= offset) 7398 return 0; 7399 7400 fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 7401 found_type = btrfs_file_extent_type(leaf, fi); 7402 7403 nocow_args.start = offset; 7404 nocow_args.end = offset + *len - 1; 7405 nocow_args.free_path = true; 7406 7407 ret = can_nocow_file_extent(path, &key, inode, &nocow_args); 7408 /* can_nocow_file_extent() has freed the path. */ 7409 path = NULL; 7410 7411 if (ret != 1) { 7412 /* Treat errors as not being able to NOCOW. */ 7413 return 0; 7414 } 7415 7416 if (btrfs_extent_readonly(fs_info, 7417 nocow_args.file_extent.disk_bytenr + 7418 nocow_args.file_extent.offset)) 7419 return 0; 7420 7421 if (!(inode->flags & BTRFS_INODE_NODATACOW) && 7422 found_type == BTRFS_FILE_EXTENT_PREALLOC) { 7423 u64 range_end; 7424 7425 range_end = round_up(offset + nocow_args.file_extent.num_bytes, 7426 root->fs_info->sectorsize) - 1; 7427 ret = btrfs_test_range_bit_exists(io_tree, offset, range_end, 7428 EXTENT_DELALLOC); 7429 if (ret) 7430 return -EAGAIN; 7431 } 7432 7433 if (file_extent) 7434 memcpy(file_extent, &nocow_args.file_extent, sizeof(*file_extent)); 7435 7436 *len = nocow_args.file_extent.num_bytes; 7437 7438 return 1; 7439} 7440 7441/* The callers of this must take lock_extent() */ 7442struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start, 7443 const struct btrfs_file_extent *file_extent, 7444 int type) 7445{ 7446 struct extent_map *em; 7447 int ret; 7448 7449 /* 7450 * Note the missing NOCOW type. 7451 * 7452 * For pure NOCOW writes, we should not create an io extent map, but 7453 * just reusing the existing one. 7454 * Only PREALLOC writes (NOCOW write into preallocated range) can 7455 * create an io extent map. 7456 */ 7457 ASSERT(type == BTRFS_ORDERED_PREALLOC || 7458 type == BTRFS_ORDERED_COMPRESSED || 7459 type == BTRFS_ORDERED_REGULAR); 7460 7461 switch (type) { 7462 case BTRFS_ORDERED_PREALLOC: 7463 /* We're only referring part of a larger preallocated extent. */ 7464 ASSERT(file_extent->num_bytes <= file_extent->ram_bytes); 7465 break; 7466 case BTRFS_ORDERED_REGULAR: 7467 /* COW results a new extent matching our file extent size. */ 7468 ASSERT(file_extent->disk_num_bytes == file_extent->num_bytes); 7469 ASSERT(file_extent->ram_bytes == file_extent->num_bytes); 7470 7471 /* Since it's a new extent, we should not have any offset. */ 7472 ASSERT(file_extent->offset == 0); 7473 break; 7474 case BTRFS_ORDERED_COMPRESSED: 7475 /* Must be compressed. */ 7476 ASSERT(file_extent->compression != BTRFS_COMPRESS_NONE); 7477 7478 /* 7479 * Encoded write can make us to refer to part of the 7480 * uncompressed extent. 7481 */ 7482 ASSERT(file_extent->num_bytes <= file_extent->ram_bytes); 7483 break; 7484 } 7485 7486 em = btrfs_alloc_extent_map(); 7487 if (!em) 7488 return ERR_PTR(-ENOMEM); 7489 7490 em->start = start; 7491 em->len = file_extent->num_bytes; 7492 em->disk_bytenr = file_extent->disk_bytenr; 7493 em->disk_num_bytes = file_extent->disk_num_bytes; 7494 em->ram_bytes = file_extent->ram_bytes; 7495 em->generation = -1; 7496 em->offset = file_extent->offset; 7497 em->flags |= EXTENT_FLAG_PINNED; 7498 if (type == BTRFS_ORDERED_COMPRESSED) 7499 btrfs_extent_map_set_compression(em, file_extent->compression); 7500 7501 ret = btrfs_replace_extent_map_range(inode, em, true); 7502 if (ret) { 7503 btrfs_free_extent_map(em); 7504 return ERR_PTR(ret); 7505 } 7506 7507 /* em got 2 refs now, callers needs to do btrfs_free_extent_map once. */ 7508 return em; 7509} 7510 7511/* 7512 * For release_folio() and invalidate_folio() we have a race window where 7513 * folio_end_writeback() is called but the subpage spinlock is not yet released. 7514 * If we continue to release/invalidate the page, we could cause use-after-free 7515 * for subpage spinlock. So this function is to spin and wait for subpage 7516 * spinlock. 7517 */ 7518static void wait_subpage_spinlock(struct folio *folio) 7519{ 7520 struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); 7521 struct btrfs_folio_state *bfs; 7522 7523 if (!btrfs_is_subpage(fs_info, folio)) 7524 return; 7525 7526 ASSERT(folio_test_private(folio) && folio_get_private(folio)); 7527 bfs = folio_get_private(folio); 7528 7529 /* 7530 * This may look insane as we just acquire the spinlock and release it, 7531 * without doing anything. But we just want to make sure no one is 7532 * still holding the subpage spinlock. 7533 * And since the page is not dirty nor writeback, and we have page 7534 * locked, the only possible way to hold a spinlock is from the endio 7535 * function to clear page writeback. 7536 * 7537 * Here we just acquire the spinlock so that all existing callers 7538 * should exit and we're safe to release/invalidate the page. 7539 */ 7540 spin_lock_irq(&bfs->lock); 7541 spin_unlock_irq(&bfs->lock); 7542} 7543 7544static int btrfs_launder_folio(struct folio *folio) 7545{ 7546 return btrfs_qgroup_free_data(folio_to_inode(folio), NULL, folio_pos(folio), 7547 folio_size(folio), NULL); 7548} 7549 7550static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags) 7551{ 7552 if (try_release_extent_mapping(folio, gfp_flags)) { 7553 wait_subpage_spinlock(folio); 7554 clear_folio_extent_mapped(folio); 7555 return true; 7556 } 7557 return false; 7558} 7559 7560static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags) 7561{ 7562 if (folio_test_writeback(folio) || folio_test_dirty(folio)) 7563 return false; 7564 return __btrfs_release_folio(folio, gfp_flags); 7565} 7566 7567#ifdef CONFIG_MIGRATION 7568static int btrfs_migrate_folio(struct address_space *mapping, 7569 struct folio *dst, struct folio *src, 7570 enum migrate_mode mode) 7571{ 7572 int ret = filemap_migrate_folio(mapping, dst, src, mode); 7573 7574 if (ret) 7575 return ret; 7576 7577 if (folio_test_ordered(src)) { 7578 folio_clear_ordered(src); 7579 folio_set_ordered(dst); 7580 } 7581 7582 return 0; 7583} 7584#else 7585#define btrfs_migrate_folio NULL 7586#endif 7587 7588static void btrfs_invalidate_folio(struct folio *folio, size_t offset, 7589 size_t length) 7590{ 7591 struct btrfs_inode *inode = folio_to_inode(folio); 7592 struct btrfs_fs_info *fs_info = inode->root->fs_info; 7593 struct extent_io_tree *tree = &inode->io_tree; 7594 struct extent_state *cached_state = NULL; 7595 u64 page_start = folio_pos(folio); 7596 u64 page_end = page_start + folio_size(folio) - 1; 7597 u64 cur; 7598 int inode_evicting = inode_state_read_once(&inode->vfs_inode) & I_FREEING; 7599 7600 /* 7601 * We have folio locked so no new ordered extent can be created on this 7602 * page, nor bio can be submitted for this folio. 7603 * 7604 * But already submitted bio can still be finished on this folio. 7605 * Furthermore, endio function won't skip folio which has Ordered 7606 * already cleared, so it's possible for endio and 7607 * invalidate_folio to do the same ordered extent accounting twice 7608 * on one folio. 7609 * 7610 * So here we wait for any submitted bios to finish, so that we won't 7611 * do double ordered extent accounting on the same folio. 7612 */ 7613 folio_wait_writeback(folio); 7614 wait_subpage_spinlock(folio); 7615 7616 /* 7617 * For subpage case, we have call sites like 7618 * btrfs_punch_hole_lock_range() which passes range not aligned to 7619 * sectorsize. 7620 * If the range doesn't cover the full folio, we don't need to and 7621 * shouldn't clear page extent mapped, as folio->private can still 7622 * record subpage dirty bits for other part of the range. 7623 * 7624 * For cases that invalidate the full folio even the range doesn't 7625 * cover the full folio, like invalidating the last folio, we're 7626 * still safe to wait for ordered extent to finish. 7627 */ 7628 if (!(offset == 0 && length == folio_size(folio))) { 7629 btrfs_release_folio(folio, GFP_NOFS); 7630 return; 7631 } 7632 7633 if (!inode_evicting) 7634 btrfs_lock_extent(tree, page_start, page_end, &cached_state); 7635 7636 cur = page_start; 7637 while (cur < page_end) { 7638 struct btrfs_ordered_extent *ordered; 7639 u64 range_end; 7640 u32 range_len; 7641 u32 extra_flags = 0; 7642 7643 ordered = btrfs_lookup_first_ordered_range(inode, cur, 7644 page_end + 1 - cur); 7645 if (!ordered) { 7646 range_end = page_end; 7647 /* 7648 * No ordered extent covering this range, we are safe 7649 * to delete all extent states in the range. 7650 */ 7651 extra_flags = EXTENT_CLEAR_ALL_BITS; 7652 goto next; 7653 } 7654 if (ordered->file_offset > cur) { 7655 /* 7656 * There is a range between [cur, oe->file_offset) not 7657 * covered by any ordered extent. 7658 * We are safe to delete all extent states, and handle 7659 * the ordered extent in the next iteration. 7660 */ 7661 range_end = ordered->file_offset - 1; 7662 extra_flags = EXTENT_CLEAR_ALL_BITS; 7663 goto next; 7664 } 7665 7666 range_end = min(ordered->file_offset + ordered->num_bytes - 1, 7667 page_end); 7668 ASSERT(range_end + 1 - cur < U32_MAX); 7669 range_len = range_end + 1 - cur; 7670 if (!btrfs_folio_test_ordered(fs_info, folio, cur, range_len)) { 7671 /* 7672 * If Ordered is cleared, it means endio has 7673 * already been executed for the range. 7674 * We can't delete the extent states as 7675 * btrfs_finish_ordered_io() may still use some of them. 7676 */ 7677 goto next; 7678 } 7679 btrfs_folio_clear_ordered(fs_info, folio, cur, range_len); 7680 7681 /* 7682 * IO on this page will never be started, so we need to account 7683 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW 7684 * here, must leave that up for the ordered extent completion. 7685 * 7686 * This will also unlock the range for incoming 7687 * btrfs_finish_ordered_io(). 7688 */ 7689 if (!inode_evicting) 7690 btrfs_clear_extent_bit(tree, cur, range_end, 7691 EXTENT_DELALLOC | 7692 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | 7693 EXTENT_DEFRAG, &cached_state); 7694 7695 spin_lock(&inode->ordered_tree_lock); 7696 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); 7697 ordered->truncated_len = min(ordered->truncated_len, 7698 cur - ordered->file_offset); 7699 spin_unlock(&inode->ordered_tree_lock); 7700 7701 /* 7702 * If the ordered extent has finished, we're safe to delete all 7703 * the extent states of the range, otherwise 7704 * btrfs_finish_ordered_io() will get executed by endio for 7705 * other pages, so we can't delete extent states. 7706 */ 7707 if (btrfs_dec_test_ordered_pending(inode, &ordered, 7708 cur, range_end + 1 - cur)) { 7709 btrfs_finish_ordered_io(ordered); 7710 /* 7711 * The ordered extent has finished, now we're again 7712 * safe to delete all extent states of the range. 7713 */ 7714 extra_flags = EXTENT_CLEAR_ALL_BITS; 7715 } 7716next: 7717 if (ordered) 7718 btrfs_put_ordered_extent(ordered); 7719 /* 7720 * Qgroup reserved space handler 7721 * Sector(s) here will be either: 7722 * 7723 * 1) Already written to disk or bio already finished 7724 * Then its QGROUP_RESERVED bit in io_tree is already cleared. 7725 * Qgroup will be handled by its qgroup_record then. 7726 * btrfs_qgroup_free_data() call will do nothing here. 7727 * 7728 * 2) Not written to disk yet 7729 * Then btrfs_qgroup_free_data() call will clear the 7730 * QGROUP_RESERVED bit of its io_tree, and free the qgroup 7731 * reserved data space. 7732 * Since the IO will never happen for this page. 7733 */ 7734 btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur, NULL); 7735 if (!inode_evicting) 7736 btrfs_clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED | 7737 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | 7738 EXTENT_DEFRAG | extra_flags, 7739 &cached_state); 7740 cur = range_end + 1; 7741 } 7742 /* 7743 * We have iterated through all ordered extents of the page, the page 7744 * should not have Ordered anymore, or the above iteration 7745 * did something wrong. 7746 */ 7747 ASSERT(!folio_test_ordered(folio)); 7748 btrfs_folio_clear_checked(fs_info, folio, folio_pos(folio), folio_size(folio)); 7749 if (!inode_evicting) 7750 __btrfs_release_folio(folio, GFP_NOFS); 7751 clear_folio_extent_mapped(folio); 7752} 7753 7754static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback) 7755{ 7756 struct btrfs_truncate_control control = { 7757 .inode = inode, 7758 .ino = btrfs_ino(inode), 7759 .min_type = BTRFS_EXTENT_DATA_KEY, 7760 .clear_extent_range = true, 7761 .new_size = inode->vfs_inode.i_size, 7762 }; 7763 struct btrfs_root *root = inode->root; 7764 struct btrfs_fs_info *fs_info = root->fs_info; 7765 struct btrfs_block_rsv rsv; 7766 int ret; 7767 struct btrfs_trans_handle *trans; 7768 const u64 min_size = btrfs_calc_metadata_size(fs_info, 1); 7769 const u64 lock_start = round_down(inode->vfs_inode.i_size, fs_info->sectorsize); 7770 const u64 i_size_up = round_up(inode->vfs_inode.i_size, fs_info->sectorsize); 7771 7772 /* Our inode is locked and the i_size can't be changed concurrently. */ 7773 btrfs_assert_inode_locked(inode); 7774 7775 if (!skip_writeback) { 7776 ret = btrfs_wait_ordered_range(inode, lock_start, (u64)-1); 7777 if (ret) 7778 return ret; 7779 } 7780 7781 /* 7782 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of 7783 * things going on here: 7784 * 7785 * 1) We need to reserve space to update our inode. 7786 * 7787 * 2) We need to have something to cache all the space that is going to 7788 * be free'd up by the truncate operation, but also have some slack 7789 * space reserved in case it uses space during the truncate (thank you 7790 * very much snapshotting). 7791 * 7792 * And we need these to be separate. The fact is we can use a lot of 7793 * space doing the truncate, and we have no earthly idea how much space 7794 * we will use, so we need the truncate reservation to be separate so it 7795 * doesn't end up using space reserved for updating the inode. We also 7796 * need to be able to stop the transaction and start a new one, which 7797 * means we need to be able to update the inode several times, and we 7798 * have no idea of knowing how many times that will be, so we can't just 7799 * reserve 1 item for the entirety of the operation, so that has to be 7800 * done separately as well. 7801 * 7802 * So that leaves us with 7803 * 7804 * 1) rsv - for the truncate reservation, which we will steal from the 7805 * transaction reservation. 7806 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for 7807 * updating the inode. 7808 */ 7809 btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); 7810 rsv.size = min_size; 7811 rsv.failfast = true; 7812 7813 /* 7814 * 1 for the truncate slack space 7815 * 1 for updating the inode. 7816 */ 7817 trans = btrfs_start_transaction(root, 2); 7818 if (IS_ERR(trans)) { 7819 ret = PTR_ERR(trans); 7820 goto out; 7821 } 7822 7823 /* Migrate the slack space for the truncate to our reserve */ 7824 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, &rsv, 7825 min_size, false); 7826 /* 7827 * We have reserved 2 metadata units when we started the transaction and 7828 * min_size matches 1 unit, so this should never fail, but if it does, 7829 * it's not critical we just fail truncation. 7830 */ 7831 if (WARN_ON(ret)) { 7832 btrfs_end_transaction(trans); 7833 goto out; 7834 } 7835 7836 trans->block_rsv = &rsv; 7837 7838 while (1) { 7839 struct extent_state *cached_state = NULL; 7840 7841 btrfs_lock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state); 7842 /* 7843 * We want to drop from the next block forward in case this new 7844 * size is not block aligned since we will be keeping the last 7845 * block of the extent just the way it is. 7846 */ 7847 btrfs_drop_extent_map_range(inode, i_size_up, (u64)-1, false); 7848 7849 ret = btrfs_truncate_inode_items(trans, root, &control); 7850 7851 inode_sub_bytes(&inode->vfs_inode, control.sub_bytes); 7852 btrfs_inode_safe_disk_i_size_write(inode, control.last_size); 7853 7854 btrfs_unlock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state); 7855 7856 trans->block_rsv = &fs_info->trans_block_rsv; 7857 if (ret != -ENOSPC && ret != -EAGAIN) 7858 break; 7859 7860 ret = btrfs_update_inode(trans, inode); 7861 if (ret) 7862 break; 7863 7864 btrfs_end_transaction(trans); 7865 btrfs_btree_balance_dirty(fs_info); 7866 7867 trans = btrfs_start_transaction(root, 2); 7868 if (IS_ERR(trans)) { 7869 ret = PTR_ERR(trans); 7870 trans = NULL; 7871 break; 7872 } 7873 7874 btrfs_block_rsv_release(fs_info, &rsv, -1, NULL); 7875 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, 7876 &rsv, min_size, false); 7877 /* 7878 * We have reserved 2 metadata units when we started the 7879 * transaction and min_size matches 1 unit, so this should never 7880 * fail, but if it does, it's not critical we just fail truncation. 7881 */ 7882 if (WARN_ON(ret)) 7883 break; 7884 7885 trans->block_rsv = &rsv; 7886 } 7887 7888 /* 7889 * We can't call btrfs_truncate_block inside a trans handle as we could 7890 * deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we 7891 * know we've truncated everything except the last little bit, and can 7892 * do btrfs_truncate_block and then update the disk_i_size. 7893 */ 7894 if (ret == BTRFS_NEED_TRUNCATE_BLOCK) { 7895 btrfs_end_transaction(trans); 7896 btrfs_btree_balance_dirty(fs_info); 7897 7898 ret = btrfs_truncate_block(inode, inode->vfs_inode.i_size, 7899 inode->vfs_inode.i_size, (u64)-1); 7900 if (ret) 7901 goto out; 7902 trans = btrfs_start_transaction(root, 1); 7903 if (IS_ERR(trans)) { 7904 ret = PTR_ERR(trans); 7905 goto out; 7906 } 7907 btrfs_inode_safe_disk_i_size_write(inode, 0); 7908 } 7909 7910 if (trans) { 7911 int ret2; 7912 7913 trans->block_rsv = &fs_info->trans_block_rsv; 7914 ret2 = btrfs_update_inode(trans, inode); 7915 if (ret2 && !ret) 7916 ret = ret2; 7917 7918 ret2 = btrfs_end_transaction(trans); 7919 if (ret2 && !ret) 7920 ret = ret2; 7921 btrfs_btree_balance_dirty(fs_info); 7922 } 7923out: 7924 btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL); 7925 /* 7926 * So if we truncate and then write and fsync we normally would just 7927 * write the extents that changed, which is a problem if we need to 7928 * first truncate that entire inode. So set this flag so we write out 7929 * all of the extents in the inode to the sync log so we're completely 7930 * safe. 7931 * 7932 * If no extents were dropped or trimmed we don't need to force the next 7933 * fsync to truncate all the inode's items from the log and re-log them 7934 * all. This means the truncate operation did not change the file size, 7935 * or changed it to a smaller size but there was only an implicit hole 7936 * between the old i_size and the new i_size, and there were no prealloc 7937 * extents beyond i_size to drop. 7938 */ 7939 if (control.extents_found > 0) 7940 btrfs_set_inode_full_sync(inode); 7941 7942 return ret; 7943} 7944 7945struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap, 7946 struct inode *dir) 7947{ 7948 struct inode *inode; 7949 7950 inode = new_inode(dir->i_sb); 7951 if (inode) { 7952 /* 7953 * Subvolumes don't inherit the sgid bit or the parent's gid if 7954 * the parent's sgid bit is set. This is probably a bug. 7955 */ 7956 inode_init_owner(idmap, inode, NULL, 7957 S_IFDIR | (~current_umask() & S_IRWXUGO)); 7958 inode->i_op = &btrfs_dir_inode_operations; 7959 inode->i_fop = &btrfs_dir_file_operations; 7960 } 7961 return inode; 7962} 7963 7964struct inode *btrfs_alloc_inode(struct super_block *sb) 7965{ 7966 struct btrfs_fs_info *fs_info = btrfs_sb(sb); 7967 struct btrfs_inode *ei; 7968 struct inode *inode; 7969 7970 ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL); 7971 if (!ei) 7972 return NULL; 7973 7974 ei->root = NULL; 7975 ei->generation = 0; 7976 ei->last_trans = 0; 7977 ei->last_sub_trans = 0; 7978 ei->logged_trans = 0; 7979 ei->delalloc_bytes = 0; 7980 /* new_delalloc_bytes and last_dir_index_offset are in a union. */ 7981 ei->new_delalloc_bytes = 0; 7982 ei->defrag_bytes = 0; 7983 ei->disk_i_size = 0; 7984 ei->flags = 0; 7985 ei->ro_flags = 0; 7986 /* 7987 * ->index_cnt will be properly initialized later when creating a new 7988 * inode (btrfs_create_new_inode()) or when reading an existing inode 7989 * from disk (btrfs_read_locked_inode()). 7990 */ 7991 ei->csum_bytes = 0; 7992 ei->dir_index = 0; 7993 ei->last_unlink_trans = 0; 7994 ei->last_reflink_trans = 0; 7995 ei->last_log_commit = 0; 7996 7997 spin_lock_init(&ei->lock); 7998 ei->outstanding_extents = 0; 7999 if (sb->s_magic != BTRFS_TEST_MAGIC) 8000 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, 8001 BTRFS_BLOCK_RSV_DELALLOC); 8002 ei->runtime_flags = 0; 8003 ei->prop_compress = BTRFS_COMPRESS_NONE; 8004 ei->defrag_compress = BTRFS_COMPRESS_NONE; 8005 8006 ei->delayed_node = NULL; 8007 8008 ei->i_otime_sec = 0; 8009 ei->i_otime_nsec = 0; 8010 8011 inode = &ei->vfs_inode; 8012 btrfs_extent_map_tree_init(&ei->extent_tree); 8013 8014 /* This io tree sets the valid inode. */ 8015 btrfs_extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO); 8016 ei->io_tree.inode = ei; 8017 8018 ei->file_extent_tree = NULL; 8019 8020 mutex_init(&ei->log_mutex); 8021 spin_lock_init(&ei->ordered_tree_lock); 8022 ei->ordered_tree = RB_ROOT; 8023 ei->ordered_tree_last = NULL; 8024 INIT_LIST_HEAD(&ei->delalloc_inodes); 8025 INIT_LIST_HEAD(&ei->delayed_iput); 8026 init_rwsem(&ei->i_mmap_lock); 8027 8028 return inode; 8029} 8030 8031#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS 8032void btrfs_test_destroy_inode(struct inode *inode) 8033{ 8034 btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); 8035 kfree(BTRFS_I(inode)->file_extent_tree); 8036 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8037} 8038#endif 8039 8040void btrfs_free_inode(struct inode *inode) 8041{ 8042 kfree(BTRFS_I(inode)->file_extent_tree); 8043 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); 8044} 8045 8046void btrfs_destroy_inode(struct inode *vfs_inode) 8047{ 8048 struct btrfs_ordered_extent *ordered; 8049 struct btrfs_inode *inode = BTRFS_I(vfs_inode); 8050 struct btrfs_root *root = inode->root; 8051 bool freespace_inode; 8052 8053 WARN_ON(!hlist_empty(&vfs_inode->i_dentry)); 8054 WARN_ON(vfs_inode->i_data.nrpages); 8055 WARN_ON(inode->block_rsv.reserved); 8056 WARN_ON(inode->block_rsv.size); 8057 WARN_ON(inode->outstanding_extents); 8058 if (!S_ISDIR(vfs_inode->i_mode)) { 8059 WARN_ON(inode->delalloc_bytes); 8060 WARN_ON(inode->new_delalloc_bytes); 8061 WARN_ON(inode->csum_bytes); 8062 } 8063 if (!root || !btrfs_is_data_reloc_root(root)) 8064 WARN_ON(inode->defrag_bytes); 8065 8066 /* 8067 * This can happen where we create an inode, but somebody else also 8068 * created the same inode and we need to destroy the one we already 8069 * created. 8070 */ 8071 if (!root) 8072 return; 8073 8074 /* 8075 * If this is a free space inode do not take the ordered extents lockdep 8076 * map. 8077 */ 8078 freespace_inode = btrfs_is_free_space_inode(inode); 8079 8080 while (1) { 8081 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); 8082 if (!ordered) 8083 break; 8084 else { 8085 btrfs_err(root->fs_info, 8086 "found ordered extent %llu %llu on inode cleanup", 8087 ordered->file_offset, ordered->num_bytes); 8088 8089 if (!freespace_inode) 8090 btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent); 8091 8092 btrfs_remove_ordered_extent(inode, ordered); 8093 btrfs_put_ordered_extent(ordered); 8094 btrfs_put_ordered_extent(ordered); 8095 } 8096 } 8097 btrfs_qgroup_check_reserved_leak(inode); 8098 btrfs_del_inode_from_root(inode); 8099 btrfs_drop_extent_map_range(inode, 0, (u64)-1, false); 8100 btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1); 8101 btrfs_put_root(inode->root); 8102} 8103 8104int btrfs_drop_inode(struct inode *inode) 8105{ 8106 struct btrfs_root *root = BTRFS_I(inode)->root; 8107 8108 if (root == NULL) 8109 return 1; 8110 8111 /* the snap/subvol tree is on deleting */ 8112 if (btrfs_root_refs(&root->root_item) == 0) 8113 return 1; 8114 else 8115 return inode_generic_drop(inode); 8116} 8117 8118static void init_once(void *foo) 8119{ 8120 struct btrfs_inode *ei = foo; 8121 8122 inode_init_once(&ei->vfs_inode); 8123#ifdef CONFIG_FS_VERITY 8124 ei->i_verity_info = NULL; 8125#endif 8126} 8127 8128void __cold btrfs_destroy_cachep(void) 8129{ 8130 /* 8131 * Make sure all delayed rcu free inodes are flushed before we 8132 * destroy cache. 8133 */ 8134 rcu_barrier(); 8135 kmem_cache_destroy(btrfs_inode_cachep); 8136} 8137 8138int __init btrfs_init_cachep(void) 8139{ 8140 btrfs_inode_cachep = kmem_cache_create("btrfs_inode", 8141 sizeof(struct btrfs_inode), 0, 8142 SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, 8143 init_once); 8144 if (!btrfs_inode_cachep) 8145 return -ENOMEM; 8146 8147 return 0; 8148} 8149 8150static int btrfs_getattr(struct mnt_idmap *idmap, 8151 const struct path *path, struct kstat *stat, 8152 u32 request_mask, unsigned int flags) 8153{ 8154 u64 delalloc_bytes; 8155 u64 inode_bytes; 8156 struct inode *inode = d_inode(path->dentry); 8157 u32 blocksize = btrfs_sb(inode->i_sb)->sectorsize; 8158 u32 bi_flags = BTRFS_I(inode)->flags; 8159 u32 bi_ro_flags = BTRFS_I(inode)->ro_flags; 8160 8161 stat->result_mask |= STATX_BTIME; 8162 stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec; 8163 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec; 8164 if (bi_flags & BTRFS_INODE_APPEND) 8165 stat->attributes |= STATX_ATTR_APPEND; 8166 if (bi_flags & BTRFS_INODE_COMPRESS) 8167 stat->attributes |= STATX_ATTR_COMPRESSED; 8168 if (bi_flags & BTRFS_INODE_IMMUTABLE) 8169 stat->attributes |= STATX_ATTR_IMMUTABLE; 8170 if (bi_flags & BTRFS_INODE_NODUMP) 8171 stat->attributes |= STATX_ATTR_NODUMP; 8172 if (bi_ro_flags & BTRFS_INODE_RO_VERITY) 8173 stat->attributes |= STATX_ATTR_VERITY; 8174 8175 stat->attributes_mask |= (STATX_ATTR_APPEND | 8176 STATX_ATTR_COMPRESSED | 8177 STATX_ATTR_IMMUTABLE | 8178 STATX_ATTR_NODUMP); 8179 8180 generic_fillattr(idmap, request_mask, inode, stat); 8181 stat->dev = BTRFS_I(inode)->root->anon_dev; 8182 8183 stat->subvol = btrfs_root_id(BTRFS_I(inode)->root); 8184 stat->result_mask |= STATX_SUBVOL; 8185 8186 spin_lock(&BTRFS_I(inode)->lock); 8187 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; 8188 inode_bytes = inode_get_bytes(inode); 8189 spin_unlock(&BTRFS_I(inode)->lock); 8190 stat->blocks = (ALIGN(inode_bytes, blocksize) + 8191 ALIGN(delalloc_bytes, blocksize)) >> SECTOR_SHIFT; 8192 return 0; 8193} 8194 8195static int btrfs_rename_exchange(struct inode *old_dir, 8196 struct dentry *old_dentry, 8197 struct inode *new_dir, 8198 struct dentry *new_dentry) 8199{ 8200 struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir); 8201 struct btrfs_trans_handle *trans; 8202 unsigned int trans_num_items; 8203 struct btrfs_root *root = BTRFS_I(old_dir)->root; 8204 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 8205 struct inode *new_inode = new_dentry->d_inode; 8206 struct inode *old_inode = old_dentry->d_inode; 8207 struct btrfs_rename_ctx old_rename_ctx; 8208 struct btrfs_rename_ctx new_rename_ctx; 8209 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 8210 u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); 8211 u64 old_idx = 0; 8212 u64 new_idx = 0; 8213 int ret; 8214 int ret2; 8215 bool need_abort = false; 8216 bool logs_pinned = false; 8217 struct fscrypt_name old_fname, new_fname; 8218 struct fscrypt_str *old_name, *new_name; 8219 8220 /* 8221 * For non-subvolumes allow exchange only within one subvolume, in the 8222 * same inode namespace. Two subvolumes (represented as directory) can 8223 * be exchanged as they're a logical link and have a fixed inode number. 8224 */ 8225 if (root != dest && 8226 (old_ino != BTRFS_FIRST_FREE_OBJECTID || 8227 new_ino != BTRFS_FIRST_FREE_OBJECTID)) 8228 return -EXDEV; 8229 8230 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname); 8231 if (ret) 8232 return ret; 8233 8234 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname); 8235 if (ret) { 8236 fscrypt_free_filename(&old_fname); 8237 return ret; 8238 } 8239 8240 old_name = &old_fname.disk_name; 8241 new_name = &new_fname.disk_name; 8242 8243 /* close the race window with snapshot create/destroy ioctl */ 8244 if (old_ino == BTRFS_FIRST_FREE_OBJECTID || 8245 new_ino == BTRFS_FIRST_FREE_OBJECTID) 8246 down_read(&fs_info->subvol_sem); 8247 8248 /* 8249 * For each inode: 8250 * 1 to remove old dir item 8251 * 1 to remove old dir index 8252 * 1 to add new dir item 8253 * 1 to add new dir index 8254 * 1 to update parent inode 8255 * 8256 * If the parents are the same, we only need to account for one 8257 */ 8258 trans_num_items = (old_dir == new_dir ? 9 : 10); 8259 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8260 /* 8261 * 1 to remove old root ref 8262 * 1 to remove old root backref 8263 * 1 to add new root ref 8264 * 1 to add new root backref 8265 */ 8266 trans_num_items += 4; 8267 } else { 8268 /* 8269 * 1 to update inode item 8270 * 1 to remove old inode ref 8271 * 1 to add new inode ref 8272 */ 8273 trans_num_items += 3; 8274 } 8275 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) 8276 trans_num_items += 4; 8277 else 8278 trans_num_items += 3; 8279 trans = btrfs_start_transaction(root, trans_num_items); 8280 if (IS_ERR(trans)) { 8281 ret = PTR_ERR(trans); 8282 goto out_notrans; 8283 } 8284 8285 if (dest != root) { 8286 ret = btrfs_record_root_in_trans(trans, dest); 8287 if (ret) 8288 goto out_fail; 8289 } 8290 8291 /* 8292 * We need to find a free sequence number both in the source and 8293 * in the destination directory for the exchange. 8294 */ 8295 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); 8296 if (ret) 8297 goto out_fail; 8298 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); 8299 if (ret) 8300 goto out_fail; 8301 8302 BTRFS_I(old_inode)->dir_index = 0ULL; 8303 BTRFS_I(new_inode)->dir_index = 0ULL; 8304 8305 /* Reference for the source. */ 8306 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8307 /* force full log commit if subvolume involved. */ 8308 btrfs_set_log_full_commit(trans); 8309 } else { 8310 ret = btrfs_insert_inode_ref(trans, dest, new_name, old_ino, 8311 btrfs_ino(BTRFS_I(new_dir)), 8312 old_idx); 8313 if (ret) 8314 goto out_fail; 8315 need_abort = true; 8316 } 8317 8318 /* And now for the dest. */ 8319 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 8320 /* force full log commit if subvolume involved. */ 8321 btrfs_set_log_full_commit(trans); 8322 } else { 8323 ret = btrfs_insert_inode_ref(trans, root, old_name, new_ino, 8324 btrfs_ino(BTRFS_I(old_dir)), 8325 new_idx); 8326 if (ret) { 8327 if (unlikely(need_abort)) 8328 btrfs_abort_transaction(trans, ret); 8329 goto out_fail; 8330 } 8331 } 8332 8333 /* Update inode version and ctime/mtime. */ 8334 inode_inc_iversion(old_dir); 8335 inode_inc_iversion(new_dir); 8336 inode_inc_iversion(old_inode); 8337 inode_inc_iversion(new_inode); 8338 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); 8339 8340 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && 8341 new_ino != BTRFS_FIRST_FREE_OBJECTID) { 8342 /* 8343 * If we are renaming in the same directory (and it's not for 8344 * root entries) pin the log early to prevent any concurrent 8345 * task from logging the directory after we removed the old 8346 * entries and before we add the new entries, otherwise that 8347 * task can sync a log without any entry for the inodes we are 8348 * renaming and therefore replaying that log, if a power failure 8349 * happens after syncing the log, would result in deleting the 8350 * inodes. 8351 * 8352 * If the rename affects two different directories, we want to 8353 * make sure the that there's no log commit that contains 8354 * updates for only one of the directories but not for the 8355 * other. 8356 * 8357 * If we are renaming an entry for a root, we don't care about 8358 * log updates since we called btrfs_set_log_full_commit(). 8359 */ 8360 btrfs_pin_log_trans(root); 8361 btrfs_pin_log_trans(dest); 8362 logs_pinned = true; 8363 } 8364 8365 if (old_dentry->d_parent != new_dentry->d_parent) { 8366 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 8367 BTRFS_I(old_inode), true); 8368 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), 8369 BTRFS_I(new_inode), true); 8370 } 8371 8372 /* src is a subvolume */ 8373 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8374 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry); 8375 if (unlikely(ret)) { 8376 btrfs_abort_transaction(trans, ret); 8377 goto out_fail; 8378 } 8379 } else { /* src is an inode */ 8380 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir), 8381 BTRFS_I(old_dentry->d_inode), 8382 old_name, &old_rename_ctx); 8383 if (unlikely(ret)) { 8384 btrfs_abort_transaction(trans, ret); 8385 goto out_fail; 8386 } 8387 ret = btrfs_update_inode(trans, BTRFS_I(old_inode)); 8388 if (unlikely(ret)) { 8389 btrfs_abort_transaction(trans, ret); 8390 goto out_fail; 8391 } 8392 } 8393 8394 /* dest is a subvolume */ 8395 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { 8396 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry); 8397 if (unlikely(ret)) { 8398 btrfs_abort_transaction(trans, ret); 8399 goto out_fail; 8400 } 8401 } else { /* dest is an inode */ 8402 ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir), 8403 BTRFS_I(new_dentry->d_inode), 8404 new_name, &new_rename_ctx); 8405 if (unlikely(ret)) { 8406 btrfs_abort_transaction(trans, ret); 8407 goto out_fail; 8408 } 8409 ret = btrfs_update_inode(trans, BTRFS_I(new_inode)); 8410 if (unlikely(ret)) { 8411 btrfs_abort_transaction(trans, ret); 8412 goto out_fail; 8413 } 8414 } 8415 8416 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 8417 new_name, 0, old_idx); 8418 if (unlikely(ret)) { 8419 btrfs_abort_transaction(trans, ret); 8420 goto out_fail; 8421 } 8422 8423 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), 8424 old_name, 0, new_idx); 8425 if (unlikely(ret)) { 8426 btrfs_abort_transaction(trans, ret); 8427 goto out_fail; 8428 } 8429 8430 if (old_inode->i_nlink == 1) 8431 BTRFS_I(old_inode)->dir_index = old_idx; 8432 if (new_inode->i_nlink == 1) 8433 BTRFS_I(new_inode)->dir_index = new_idx; 8434 8435 /* 8436 * Do the log updates for all inodes. 8437 * 8438 * If either entry is for a root we don't need to update the logs since 8439 * we've called btrfs_set_log_full_commit() before. 8440 */ 8441 if (logs_pinned) { 8442 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), 8443 old_rename_ctx.index, new_dentry->d_parent); 8444 btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir), 8445 new_rename_ctx.index, old_dentry->d_parent); 8446 } 8447 8448out_fail: 8449 if (logs_pinned) { 8450 btrfs_end_log_trans(root); 8451 btrfs_end_log_trans(dest); 8452 } 8453 ret2 = btrfs_end_transaction(trans); 8454 ret = ret ? ret : ret2; 8455out_notrans: 8456 if (new_ino == BTRFS_FIRST_FREE_OBJECTID || 8457 old_ino == BTRFS_FIRST_FREE_OBJECTID) 8458 up_read(&fs_info->subvol_sem); 8459 8460 fscrypt_free_filename(&new_fname); 8461 fscrypt_free_filename(&old_fname); 8462 return ret; 8463} 8464 8465static struct inode *new_whiteout_inode(struct mnt_idmap *idmap, 8466 struct inode *dir) 8467{ 8468 struct inode *inode; 8469 8470 inode = new_inode(dir->i_sb); 8471 if (inode) { 8472 inode_init_owner(idmap, inode, dir, 8473 S_IFCHR | WHITEOUT_MODE); 8474 inode->i_op = &btrfs_special_inode_operations; 8475 init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); 8476 } 8477 return inode; 8478} 8479 8480static int btrfs_rename(struct mnt_idmap *idmap, 8481 struct inode *old_dir, struct dentry *old_dentry, 8482 struct inode *new_dir, struct dentry *new_dentry, 8483 unsigned int flags) 8484{ 8485 struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir); 8486 struct btrfs_new_inode_args whiteout_args = { 8487 .dir = old_dir, 8488 .dentry = old_dentry, 8489 }; 8490 struct btrfs_trans_handle *trans; 8491 unsigned int trans_num_items; 8492 struct btrfs_root *root = BTRFS_I(old_dir)->root; 8493 struct btrfs_root *dest = BTRFS_I(new_dir)->root; 8494 struct inode *new_inode = d_inode(new_dentry); 8495 struct inode *old_inode = d_inode(old_dentry); 8496 struct btrfs_rename_ctx rename_ctx; 8497 u64 index = 0; 8498 int ret; 8499 int ret2; 8500 u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); 8501 struct fscrypt_name old_fname, new_fname; 8502 bool logs_pinned = false; 8503 8504 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) 8505 return -EPERM; 8506 8507 /* we only allow rename subvolume link between subvolumes */ 8508 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) 8509 return -EXDEV; 8510 8511 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || 8512 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) 8513 return -ENOTEMPTY; 8514 8515 if (S_ISDIR(old_inode->i_mode) && new_inode && 8516 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) 8517 return -ENOTEMPTY; 8518 8519 ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname); 8520 if (ret) 8521 return ret; 8522 8523 ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname); 8524 if (ret) { 8525 fscrypt_free_filename(&old_fname); 8526 return ret; 8527 } 8528 8529 /* check for collisions, even if the name isn't there */ 8530 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, &new_fname.disk_name); 8531 if (ret) { 8532 if (ret == -EEXIST) { 8533 /* we shouldn't get 8534 * eexist without a new_inode */ 8535 if (WARN_ON(!new_inode)) { 8536 goto out_fscrypt_names; 8537 } 8538 } else { 8539 /* maybe -EOVERFLOW */ 8540 goto out_fscrypt_names; 8541 } 8542 } 8543 ret = 0; 8544 8545 /* 8546 * we're using rename to replace one file with another. Start IO on it 8547 * now so we don't add too much work to the end of the transaction 8548 */ 8549 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) 8550 filemap_flush(old_inode->i_mapping); 8551 8552 if (flags & RENAME_WHITEOUT) { 8553 whiteout_args.inode = new_whiteout_inode(idmap, old_dir); 8554 if (!whiteout_args.inode) { 8555 ret = -ENOMEM; 8556 goto out_fscrypt_names; 8557 } 8558 ret = btrfs_new_inode_prepare(&whiteout_args, &trans_num_items); 8559 if (ret) 8560 goto out_whiteout_inode; 8561 } else { 8562 /* 1 to update the old parent inode. */ 8563 trans_num_items = 1; 8564 } 8565 8566 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { 8567 /* Close the race window with snapshot create/destroy ioctl */ 8568 down_read(&fs_info->subvol_sem); 8569 /* 8570 * 1 to remove old root ref 8571 * 1 to remove old root backref 8572 * 1 to add new root ref 8573 * 1 to add new root backref 8574 */ 8575 trans_num_items += 4; 8576 } else { 8577 /* 8578 * 1 to update inode 8579 * 1 to remove old inode ref 8580 * 1 to add new inode ref 8581 */ 8582 trans_num_items += 3; 8583 } 8584 /* 8585 * 1 to remove old dir item 8586 * 1 to remove old dir index 8587 * 1 to add new dir item 8588 * 1 to add new dir index 8589 */ 8590 trans_num_items += 4; 8591 /* 1 to update new parent inode if it's not the same as the old parent */ 8592 if (new_dir != old_dir) 8593 trans_num_items++; 8594 if (new_inode) { 8595 /* 8596 * 1 to update inode 8597 * 1 to remove inode ref 8598 * 1 to remove dir item 8599 * 1 to remove dir index 8600 * 1 to possibly add orphan item 8601 */ 8602 trans_num_items += 5; 8603 } 8604 trans = btrfs_start_transaction(root, trans_num_items); 8605 if (IS_ERR(trans)) { 8606 ret = PTR_ERR(trans); 8607 goto out_notrans; 8608 } 8609 8610 if (dest != root) { 8611 ret = btrfs_record_root_in_trans(trans, dest); 8612 if (ret) 8613 goto out_fail; 8614 } 8615 8616 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); 8617 if (ret) 8618 goto out_fail; 8619 8620 BTRFS_I(old_inode)->dir_index = 0ULL; 8621 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 8622 /* force full log commit if subvolume involved. */ 8623 btrfs_set_log_full_commit(trans); 8624 } else { 8625 ret = btrfs_insert_inode_ref(trans, dest, &new_fname.disk_name, 8626 old_ino, btrfs_ino(BTRFS_I(new_dir)), 8627 index); 8628 if (ret) 8629 goto out_fail; 8630 } 8631 8632 inode_inc_iversion(old_dir); 8633 inode_inc_iversion(new_dir); 8634 inode_inc_iversion(old_inode); 8635 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); 8636 8637 if (old_ino != BTRFS_FIRST_FREE_OBJECTID) { 8638 /* 8639 * If we are renaming in the same directory (and it's not a 8640 * root entry) pin the log to prevent any concurrent task from 8641 * logging the directory after we removed the old entry and 8642 * before we add the new entry, otherwise that task can sync 8643 * a log without any entry for the inode we are renaming and 8644 * therefore replaying that log, if a power failure happens 8645 * after syncing the log, would result in deleting the inode. 8646 * 8647 * If the rename affects two different directories, we want to 8648 * make sure the that there's no log commit that contains 8649 * updates for only one of the directories but not for the 8650 * other. 8651 * 8652 * If we are renaming an entry for a root, we don't care about 8653 * log updates since we called btrfs_set_log_full_commit(). 8654 */ 8655 btrfs_pin_log_trans(root); 8656 btrfs_pin_log_trans(dest); 8657 logs_pinned = true; 8658 } 8659 8660 if (old_dentry->d_parent != new_dentry->d_parent) 8661 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), 8662 BTRFS_I(old_inode), true); 8663 8664 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { 8665 ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry); 8666 if (unlikely(ret)) { 8667 btrfs_abort_transaction(trans, ret); 8668 goto out_fail; 8669 } 8670 } else { 8671 ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir), 8672 BTRFS_I(d_inode(old_dentry)), 8673 &old_fname.disk_name, &rename_ctx); 8674 if (unlikely(ret)) { 8675 btrfs_abort_transaction(trans, ret); 8676 goto out_fail; 8677 } 8678 ret = btrfs_update_inode(trans, BTRFS_I(old_inode)); 8679 if (unlikely(ret)) { 8680 btrfs_abort_transaction(trans, ret); 8681 goto out_fail; 8682 } 8683 } 8684 8685 if (new_inode) { 8686 inode_inc_iversion(new_inode); 8687 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == 8688 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { 8689 ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry); 8690 if (unlikely(ret)) { 8691 btrfs_abort_transaction(trans, ret); 8692 goto out_fail; 8693 } 8694 BUG_ON(new_inode->i_nlink == 0); 8695 } else { 8696 ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir), 8697 BTRFS_I(d_inode(new_dentry)), 8698 &new_fname.disk_name); 8699 if (unlikely(ret)) { 8700 btrfs_abort_transaction(trans, ret); 8701 goto out_fail; 8702 } 8703 } 8704 if (new_inode->i_nlink == 0) { 8705 ret = btrfs_orphan_add(trans, 8706 BTRFS_I(d_inode(new_dentry))); 8707 if (unlikely(ret)) { 8708 btrfs_abort_transaction(trans, ret); 8709 goto out_fail; 8710 } 8711 } 8712 } 8713 8714 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), 8715 &new_fname.disk_name, 0, index); 8716 if (unlikely(ret)) { 8717 btrfs_abort_transaction(trans, ret); 8718 goto out_fail; 8719 } 8720 8721 if (old_inode->i_nlink == 1) 8722 BTRFS_I(old_inode)->dir_index = index; 8723 8724 if (logs_pinned) 8725 btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), 8726 rename_ctx.index, new_dentry->d_parent); 8727 8728 if (flags & RENAME_WHITEOUT) { 8729 ret = btrfs_create_new_inode(trans, &whiteout_args); 8730 if (unlikely(ret)) { 8731 btrfs_abort_transaction(trans, ret); 8732 goto out_fail; 8733 } else { 8734 unlock_new_inode(whiteout_args.inode); 8735 iput(whiteout_args.inode); 8736 whiteout_args.inode = NULL; 8737 } 8738 } 8739out_fail: 8740 if (logs_pinned) { 8741 btrfs_end_log_trans(root); 8742 btrfs_end_log_trans(dest); 8743 } 8744 ret2 = btrfs_end_transaction(trans); 8745 ret = ret ? ret : ret2; 8746out_notrans: 8747 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) 8748 up_read(&fs_info->subvol_sem); 8749 if (flags & RENAME_WHITEOUT) 8750 btrfs_new_inode_args_destroy(&whiteout_args); 8751out_whiteout_inode: 8752 if (flags & RENAME_WHITEOUT) 8753 iput(whiteout_args.inode); 8754out_fscrypt_names: 8755 fscrypt_free_filename(&old_fname); 8756 fscrypt_free_filename(&new_fname); 8757 return ret; 8758} 8759 8760static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir, 8761 struct dentry *old_dentry, struct inode *new_dir, 8762 struct dentry *new_dentry, unsigned int flags) 8763{ 8764 int ret; 8765 8766 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 8767 return -EINVAL; 8768 8769 if (flags & RENAME_EXCHANGE) 8770 ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir, 8771 new_dentry); 8772 else 8773 ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir, 8774 new_dentry, flags); 8775 8776 btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info); 8777 8778 return ret; 8779} 8780 8781struct btrfs_delalloc_work { 8782 struct inode *inode; 8783 struct completion completion; 8784 struct list_head list; 8785 struct btrfs_work work; 8786}; 8787 8788static void btrfs_run_delalloc_work(struct btrfs_work *work) 8789{ 8790 struct btrfs_delalloc_work *delalloc_work; 8791 struct inode *inode; 8792 8793 delalloc_work = container_of(work, struct btrfs_delalloc_work, 8794 work); 8795 inode = delalloc_work->inode; 8796 filemap_flush(inode->i_mapping); 8797 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, 8798 &BTRFS_I(inode)->runtime_flags)) 8799 filemap_flush(inode->i_mapping); 8800 8801 iput(inode); 8802 complete(&delalloc_work->completion); 8803} 8804 8805static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode) 8806{ 8807 struct btrfs_delalloc_work *work; 8808 8809 work = kmalloc(sizeof(*work), GFP_NOFS); 8810 if (!work) 8811 return NULL; 8812 8813 init_completion(&work->completion); 8814 INIT_LIST_HEAD(&work->list); 8815 work->inode = inode; 8816 btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL); 8817 8818 return work; 8819} 8820 8821/* 8822 * some fairly slow code that needs optimization. This walks the list 8823 * of all the inodes with pending delalloc and forces them to disk. 8824 */ 8825static int start_delalloc_inodes(struct btrfs_root *root, long *nr_to_write, 8826 bool snapshot, bool in_reclaim_context) 8827{ 8828 struct btrfs_delalloc_work *work, *next; 8829 LIST_HEAD(works); 8830 LIST_HEAD(splice); 8831 int ret = 0; 8832 8833 mutex_lock(&root->delalloc_mutex); 8834 spin_lock(&root->delalloc_lock); 8835 list_splice_init(&root->delalloc_inodes, &splice); 8836 while (!list_empty(&splice)) { 8837 struct btrfs_inode *inode; 8838 struct inode *tmp_inode; 8839 8840 inode = list_first_entry(&splice, struct btrfs_inode, delalloc_inodes); 8841 8842 list_move_tail(&inode->delalloc_inodes, &root->delalloc_inodes); 8843 8844 if (in_reclaim_context && 8845 test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags)) 8846 continue; 8847 8848 tmp_inode = igrab(&inode->vfs_inode); 8849 if (!tmp_inode) { 8850 cond_resched_lock(&root->delalloc_lock); 8851 continue; 8852 } 8853 spin_unlock(&root->delalloc_lock); 8854 8855 if (snapshot) 8856 set_bit(BTRFS_INODE_SNAPSHOT_FLUSH, &inode->runtime_flags); 8857 if (nr_to_write == NULL) { 8858 work = btrfs_alloc_delalloc_work(tmp_inode); 8859 if (!work) { 8860 iput(tmp_inode); 8861 ret = -ENOMEM; 8862 goto out; 8863 } 8864 list_add_tail(&work->list, &works); 8865 btrfs_queue_work(root->fs_info->flush_workers, 8866 &work->work); 8867 } else { 8868 ret = filemap_flush_nr(tmp_inode->i_mapping, 8869 nr_to_write); 8870 btrfs_add_delayed_iput(inode); 8871 8872 if (ret || *nr_to_write <= 0) 8873 goto out; 8874 } 8875 cond_resched(); 8876 spin_lock(&root->delalloc_lock); 8877 } 8878 spin_unlock(&root->delalloc_lock); 8879 8880out: 8881 list_for_each_entry_safe(work, next, &works, list) { 8882 list_del_init(&work->list); 8883 wait_for_completion(&work->completion); 8884 kfree(work); 8885 } 8886 8887 if (!list_empty(&splice)) { 8888 spin_lock(&root->delalloc_lock); 8889 list_splice_tail(&splice, &root->delalloc_inodes); 8890 spin_unlock(&root->delalloc_lock); 8891 } 8892 mutex_unlock(&root->delalloc_mutex); 8893 return ret; 8894} 8895 8896int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context) 8897{ 8898 struct btrfs_fs_info *fs_info = root->fs_info; 8899 8900 if (BTRFS_FS_ERROR(fs_info)) 8901 return -EROFS; 8902 return start_delalloc_inodes(root, NULL, true, in_reclaim_context); 8903} 8904 8905int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, 8906 bool in_reclaim_context) 8907{ 8908 long *nr_to_write = nr == LONG_MAX ? NULL : &nr; 8909 struct btrfs_root *root; 8910 LIST_HEAD(splice); 8911 int ret; 8912 8913 if (BTRFS_FS_ERROR(fs_info)) 8914 return -EROFS; 8915 8916 mutex_lock(&fs_info->delalloc_root_mutex); 8917 spin_lock(&fs_info->delalloc_root_lock); 8918 list_splice_init(&fs_info->delalloc_roots, &splice); 8919 while (!list_empty(&splice)) { 8920 root = list_first_entry(&splice, struct btrfs_root, 8921 delalloc_root); 8922 root = btrfs_grab_root(root); 8923 BUG_ON(!root); 8924 list_move_tail(&root->delalloc_root, 8925 &fs_info->delalloc_roots); 8926 spin_unlock(&fs_info->delalloc_root_lock); 8927 8928 ret = start_delalloc_inodes(root, nr_to_write, false, 8929 in_reclaim_context); 8930 btrfs_put_root(root); 8931 if (ret < 0 || nr <= 0) 8932 goto out; 8933 spin_lock(&fs_info->delalloc_root_lock); 8934 } 8935 spin_unlock(&fs_info->delalloc_root_lock); 8936 8937 ret = 0; 8938out: 8939 if (!list_empty(&splice)) { 8940 spin_lock(&fs_info->delalloc_root_lock); 8941 list_splice_tail(&splice, &fs_info->delalloc_roots); 8942 spin_unlock(&fs_info->delalloc_root_lock); 8943 } 8944 mutex_unlock(&fs_info->delalloc_root_mutex); 8945 return ret; 8946} 8947 8948static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir, 8949 struct dentry *dentry, const char *symname) 8950{ 8951 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); 8952 struct btrfs_trans_handle *trans; 8953 struct btrfs_root *root = BTRFS_I(dir)->root; 8954 struct btrfs_path *path; 8955 struct btrfs_key key; 8956 struct inode *inode; 8957 struct btrfs_new_inode_args new_inode_args = { 8958 .dir = dir, 8959 .dentry = dentry, 8960 }; 8961 unsigned int trans_num_items; 8962 int ret; 8963 int name_len; 8964 int datasize; 8965 unsigned long ptr; 8966 struct btrfs_file_extent_item *ei; 8967 struct extent_buffer *leaf; 8968 8969 name_len = strlen(symname); 8970 /* 8971 * Symlinks utilize uncompressed inline extent data, which should not 8972 * reach block size. 8973 */ 8974 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || 8975 name_len >= fs_info->sectorsize) 8976 return -ENAMETOOLONG; 8977 8978 inode = new_inode(dir->i_sb); 8979 if (!inode) 8980 return -ENOMEM; 8981 inode_init_owner(idmap, inode, dir, S_IFLNK | S_IRWXUGO); 8982 inode->i_op = &btrfs_symlink_inode_operations; 8983 inode_nohighmem(inode); 8984 inode->i_mapping->a_ops = &btrfs_aops; 8985 btrfs_i_size_write(BTRFS_I(inode), name_len); 8986 inode_set_bytes(inode, name_len); 8987 8988 new_inode_args.inode = inode; 8989 ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); 8990 if (ret) 8991 goto out_inode; 8992 /* 1 additional item for the inline extent */ 8993 trans_num_items++; 8994 8995 trans = btrfs_start_transaction(root, trans_num_items); 8996 if (IS_ERR(trans)) { 8997 ret = PTR_ERR(trans); 8998 goto out_new_inode_args; 8999 } 9000 9001 ret = btrfs_create_new_inode(trans, &new_inode_args); 9002 if (ret) 9003 goto out; 9004 9005 path = btrfs_alloc_path(); 9006 if (unlikely(!path)) { 9007 ret = -ENOMEM; 9008 btrfs_abort_transaction(trans, ret); 9009 discard_new_inode(inode); 9010 inode = NULL; 9011 goto out; 9012 } 9013 key.objectid = btrfs_ino(BTRFS_I(inode)); 9014 key.type = BTRFS_EXTENT_DATA_KEY; 9015 key.offset = 0; 9016 datasize = btrfs_file_extent_calc_inline_size(name_len); 9017 ret = btrfs_insert_empty_item(trans, root, path, &key, datasize); 9018 if (unlikely(ret)) { 9019 btrfs_abort_transaction(trans, ret); 9020 btrfs_free_path(path); 9021 discard_new_inode(inode); 9022 inode = NULL; 9023 goto out; 9024 } 9025 leaf = path->nodes[0]; 9026 ei = btrfs_item_ptr(leaf, path->slots[0], 9027 struct btrfs_file_extent_item); 9028 btrfs_set_file_extent_generation(leaf, ei, trans->transid); 9029 btrfs_set_file_extent_type(leaf, ei, 9030 BTRFS_FILE_EXTENT_INLINE); 9031 btrfs_set_file_extent_encryption(leaf, ei, 0); 9032 btrfs_set_file_extent_compression(leaf, ei, 0); 9033 btrfs_set_file_extent_other_encoding(leaf, ei, 0); 9034 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); 9035 9036 ptr = btrfs_file_extent_inline_start(ei); 9037 write_extent_buffer(leaf, symname, ptr, name_len); 9038 btrfs_free_path(path); 9039 9040 d_instantiate_new(dentry, inode); 9041 ret = 0; 9042out: 9043 btrfs_end_transaction(trans); 9044 btrfs_btree_balance_dirty(fs_info); 9045out_new_inode_args: 9046 btrfs_new_inode_args_destroy(&new_inode_args); 9047out_inode: 9048 if (ret) 9049 iput(inode); 9050 return ret; 9051} 9052 9053static struct btrfs_trans_handle *insert_prealloc_file_extent( 9054 struct btrfs_trans_handle *trans_in, 9055 struct btrfs_inode *inode, 9056 struct btrfs_key *ins, 9057 u64 file_offset) 9058{ 9059 struct btrfs_file_extent_item stack_fi; 9060 struct btrfs_replace_extent_info extent_info; 9061 struct btrfs_trans_handle *trans = trans_in; 9062 struct btrfs_path *path; 9063 u64 start = ins->objectid; 9064 u64 len = ins->offset; 9065 u64 qgroup_released = 0; 9066 int ret; 9067 9068 memset(&stack_fi, 0, sizeof(stack_fi)); 9069 9070 btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC); 9071 btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start); 9072 btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len); 9073 btrfs_set_stack_file_extent_num_bytes(&stack_fi, len); 9074 btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len); 9075 btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE); 9076 /* Encryption and other encoding is reserved and all 0 */ 9077 9078 ret = btrfs_qgroup_release_data(inode, file_offset, len, &qgroup_released); 9079 if (ret < 0) 9080 return ERR_PTR(ret); 9081 9082 if (trans) { 9083 ret = insert_reserved_file_extent(trans, inode, 9084 file_offset, &stack_fi, 9085 true, qgroup_released); 9086 if (ret) 9087 goto free_qgroup; 9088 return trans; 9089 } 9090 9091 extent_info.disk_offset = start; 9092 extent_info.disk_len = len; 9093 extent_info.data_offset = 0; 9094 extent_info.data_len = len; 9095 extent_info.file_offset = file_offset; 9096 extent_info.extent_buf = (char *)&stack_fi; 9097 extent_info.is_new_extent = true; 9098 extent_info.update_times = true; 9099 extent_info.qgroup_reserved = qgroup_released; 9100 extent_info.insertions = 0; 9101 9102 path = btrfs_alloc_path(); 9103 if (!path) { 9104 ret = -ENOMEM; 9105 goto free_qgroup; 9106 } 9107 9108 ret = btrfs_replace_file_extents(inode, path, file_offset, 9109 file_offset + len - 1, &extent_info, 9110 &trans); 9111 btrfs_free_path(path); 9112 if (ret) 9113 goto free_qgroup; 9114 return trans; 9115 9116free_qgroup: 9117 /* 9118 * We have released qgroup data range at the beginning of the function, 9119 * and normally qgroup_released bytes will be freed when committing 9120 * transaction. 9121 * But if we error out early, we have to free what we have released 9122 * or we leak qgroup data reservation. 9123 */ 9124 btrfs_qgroup_free_refroot(inode->root->fs_info, 9125 btrfs_root_id(inode->root), qgroup_released, 9126 BTRFS_QGROUP_RSV_DATA); 9127 return ERR_PTR(ret); 9128} 9129 9130static int __btrfs_prealloc_file_range(struct inode *inode, int mode, 9131 u64 start, u64 num_bytes, u64 min_size, 9132 loff_t actual_len, u64 *alloc_hint, 9133 struct btrfs_trans_handle *trans) 9134{ 9135 struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); 9136 struct extent_map *em; 9137 struct btrfs_root *root = BTRFS_I(inode)->root; 9138 struct btrfs_key ins; 9139 u64 cur_offset = start; 9140 u64 clear_offset = start; 9141 u64 i_size; 9142 u64 cur_bytes; 9143 u64 last_alloc = (u64)-1; 9144 int ret = 0; 9145 bool own_trans = true; 9146 u64 end = start + num_bytes - 1; 9147 9148 if (trans) 9149 own_trans = false; 9150 while (num_bytes > 0) { 9151 cur_bytes = min_t(u64, num_bytes, SZ_256M); 9152 cur_bytes = max(cur_bytes, min_size); 9153 /* 9154 * If we are severely fragmented we could end up with really 9155 * small allocations, so if the allocator is returning small 9156 * chunks lets make its job easier by only searching for those 9157 * sized chunks. 9158 */ 9159 cur_bytes = min(cur_bytes, last_alloc); 9160 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, 9161 min_size, 0, *alloc_hint, &ins, true, false); 9162 if (ret) 9163 break; 9164 9165 /* 9166 * We've reserved this space, and thus converted it from 9167 * ->bytes_may_use to ->bytes_reserved. Any error that happens 9168 * from here on out we will only need to clear our reservation 9169 * for the remaining unreserved area, so advance our 9170 * clear_offset by our extent size. 9171 */ 9172 clear_offset += ins.offset; 9173 9174 last_alloc = ins.offset; 9175 trans = insert_prealloc_file_extent(trans, BTRFS_I(inode), 9176 &ins, cur_offset); 9177 /* 9178 * Now that we inserted the prealloc extent we can finally 9179 * decrement the number of reservations in the block group. 9180 * If we did it before, we could race with relocation and have 9181 * relocation miss the reserved extent, making it fail later. 9182 */ 9183 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 9184 if (IS_ERR(trans)) { 9185 ret = PTR_ERR(trans); 9186 btrfs_free_reserved_extent(fs_info, ins.objectid, 9187 ins.offset, false); 9188 break; 9189 } 9190 9191 em = btrfs_alloc_extent_map(); 9192 if (!em) { 9193 btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset, 9194 cur_offset + ins.offset - 1, false); 9195 btrfs_set_inode_full_sync(BTRFS_I(inode)); 9196 goto next; 9197 } 9198 9199 em->start = cur_offset; 9200 em->len = ins.offset; 9201 em->disk_bytenr = ins.objectid; 9202 em->offset = 0; 9203 em->disk_num_bytes = ins.offset; 9204 em->ram_bytes = ins.offset; 9205 em->flags |= EXTENT_FLAG_PREALLOC; 9206 em->generation = trans->transid; 9207 9208 ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true); 9209 btrfs_free_extent_map(em); 9210next: 9211 num_bytes -= ins.offset; 9212 cur_offset += ins.offset; 9213 *alloc_hint = ins.objectid + ins.offset; 9214 9215 inode_inc_iversion(inode); 9216 inode_set_ctime_current(inode); 9217 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; 9218 if (!(mode & FALLOC_FL_KEEP_SIZE) && 9219 (actual_len > inode->i_size) && 9220 (cur_offset > inode->i_size)) { 9221 if (cur_offset > actual_len) 9222 i_size = actual_len; 9223 else 9224 i_size = cur_offset; 9225 i_size_write(inode, i_size); 9226 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); 9227 } 9228 9229 ret = btrfs_update_inode(trans, BTRFS_I(inode)); 9230 9231 if (unlikely(ret)) { 9232 btrfs_abort_transaction(trans, ret); 9233 if (own_trans) 9234 btrfs_end_transaction(trans); 9235 break; 9236 } 9237 9238 if (own_trans) { 9239 btrfs_end_transaction(trans); 9240 trans = NULL; 9241 } 9242 } 9243 if (clear_offset < end) 9244 btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset, 9245 end - clear_offset + 1); 9246 return ret; 9247} 9248 9249int btrfs_prealloc_file_range(struct inode *inode, int mode, 9250 u64 start, u64 num_bytes, u64 min_size, 9251 loff_t actual_len, u64 *alloc_hint) 9252{ 9253 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9254 min_size, actual_len, alloc_hint, 9255 NULL); 9256} 9257 9258int btrfs_prealloc_file_range_trans(struct inode *inode, 9259 struct btrfs_trans_handle *trans, int mode, 9260 u64 start, u64 num_bytes, u64 min_size, 9261 loff_t actual_len, u64 *alloc_hint) 9262{ 9263 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, 9264 min_size, actual_len, alloc_hint, trans); 9265} 9266 9267/* 9268 * NOTE: in case you are adding MAY_EXEC check for directories: 9269 * we are marking them with IOP_FASTPERM_MAY_EXEC, allowing path lookup to 9270 * elide calls here. 9271 */ 9272static int btrfs_permission(struct mnt_idmap *idmap, 9273 struct inode *inode, int mask) 9274{ 9275 struct btrfs_root *root = BTRFS_I(inode)->root; 9276 umode_t mode = inode->i_mode; 9277 9278 if (mask & MAY_WRITE && 9279 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { 9280 if (btrfs_root_readonly(root)) 9281 return -EROFS; 9282 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) 9283 return -EACCES; 9284 } 9285 return generic_permission(idmap, inode, mask); 9286} 9287 9288static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, 9289 struct file *file, umode_t mode) 9290{ 9291 struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); 9292 struct btrfs_trans_handle *trans; 9293 struct btrfs_root *root = BTRFS_I(dir)->root; 9294 struct inode *inode; 9295 struct btrfs_new_inode_args new_inode_args = { 9296 .dir = dir, 9297 .dentry = file->f_path.dentry, 9298 .orphan = true, 9299 }; 9300 unsigned int trans_num_items; 9301 int ret; 9302 9303 inode = new_inode(dir->i_sb); 9304 if (!inode) 9305 return -ENOMEM; 9306 inode_init_owner(idmap, inode, dir, mode); 9307 inode->i_fop = &btrfs_file_operations; 9308 inode->i_op = &btrfs_file_inode_operations; 9309 inode->i_mapping->a_ops = &btrfs_aops; 9310 9311 new_inode_args.inode = inode; 9312 ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); 9313 if (ret) 9314 goto out_inode; 9315 9316 trans = btrfs_start_transaction(root, trans_num_items); 9317 if (IS_ERR(trans)) { 9318 ret = PTR_ERR(trans); 9319 goto out_new_inode_args; 9320 } 9321 9322 ret = btrfs_create_new_inode(trans, &new_inode_args); 9323 9324 /* 9325 * We set number of links to 0 in btrfs_create_new_inode(), and here we 9326 * set it to 1 because d_tmpfile() will issue a warning if the count is 9327 * 0, through: 9328 * 9329 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() 9330 */ 9331 set_nlink(inode, 1); 9332 9333 if (!ret) { 9334 d_tmpfile(file, inode); 9335 unlock_new_inode(inode); 9336 mark_inode_dirty(inode); 9337 } 9338 9339 btrfs_end_transaction(trans); 9340 btrfs_btree_balance_dirty(fs_info); 9341out_new_inode_args: 9342 btrfs_new_inode_args_destroy(&new_inode_args); 9343out_inode: 9344 if (ret) 9345 iput(inode); 9346 return finish_open_simple(file, ret); 9347} 9348 9349int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info, 9350 int compress_type) 9351{ 9352 switch (compress_type) { 9353 case BTRFS_COMPRESS_NONE: 9354 return BTRFS_ENCODED_IO_COMPRESSION_NONE; 9355 case BTRFS_COMPRESS_ZLIB: 9356 return BTRFS_ENCODED_IO_COMPRESSION_ZLIB; 9357 case BTRFS_COMPRESS_LZO: 9358 /* 9359 * The LZO format depends on the sector size. 64K is the maximum 9360 * sector size that we support. 9361 */ 9362 if (fs_info->sectorsize < SZ_4K || fs_info->sectorsize > SZ_64K) 9363 return -EINVAL; 9364 return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 9365 (fs_info->sectorsize_bits - 12); 9366 case BTRFS_COMPRESS_ZSTD: 9367 return BTRFS_ENCODED_IO_COMPRESSION_ZSTD; 9368 default: 9369 return -EUCLEAN; 9370 } 9371} 9372 9373static ssize_t btrfs_encoded_read_inline( 9374 struct kiocb *iocb, 9375 struct iov_iter *iter, u64 start, 9376 u64 lockend, 9377 struct extent_state **cached_state, 9378 u64 extent_start, size_t count, 9379 struct btrfs_ioctl_encoded_io_args *encoded, 9380 bool *unlocked) 9381{ 9382 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 9383 struct btrfs_root *root = inode->root; 9384 struct btrfs_fs_info *fs_info = root->fs_info; 9385 struct extent_io_tree *io_tree = &inode->io_tree; 9386 BTRFS_PATH_AUTO_FREE(path); 9387 struct extent_buffer *leaf; 9388 struct btrfs_file_extent_item *item; 9389 u64 ram_bytes; 9390 unsigned long ptr; 9391 void *tmp; 9392 ssize_t ret; 9393 const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); 9394 9395 path = btrfs_alloc_path(); 9396 if (!path) 9397 return -ENOMEM; 9398 9399 path->nowait = nowait; 9400 9401 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), 9402 extent_start, 0); 9403 if (ret) { 9404 if (unlikely(ret > 0)) { 9405 /* The extent item disappeared? */ 9406 return -EIO; 9407 } 9408 return ret; 9409 } 9410 leaf = path->nodes[0]; 9411 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 9412 9413 ram_bytes = btrfs_file_extent_ram_bytes(leaf, item); 9414 ptr = btrfs_file_extent_inline_start(item); 9415 9416 encoded->len = min_t(u64, extent_start + ram_bytes, 9417 inode->vfs_inode.i_size) - iocb->ki_pos; 9418 ret = btrfs_encoded_io_compression_from_extent(fs_info, 9419 btrfs_file_extent_compression(leaf, item)); 9420 if (ret < 0) 9421 return ret; 9422 encoded->compression = ret; 9423 if (encoded->compression) { 9424 size_t inline_size; 9425 9426 inline_size = btrfs_file_extent_inline_item_len(leaf, 9427 path->slots[0]); 9428 if (inline_size > count) 9429 return -ENOBUFS; 9430 9431 count = inline_size; 9432 encoded->unencoded_len = ram_bytes; 9433 encoded->unencoded_offset = iocb->ki_pos - extent_start; 9434 } else { 9435 count = min_t(u64, count, encoded->len); 9436 encoded->len = count; 9437 encoded->unencoded_len = count; 9438 ptr += iocb->ki_pos - extent_start; 9439 } 9440 9441 tmp = kmalloc(count, GFP_NOFS); 9442 if (!tmp) 9443 return -ENOMEM; 9444 9445 read_extent_buffer(leaf, tmp, ptr, count); 9446 btrfs_release_path(path); 9447 btrfs_unlock_extent(io_tree, start, lockend, cached_state); 9448 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9449 *unlocked = true; 9450 9451 ret = copy_to_iter(tmp, count, iter); 9452 if (ret != count) 9453 ret = -EFAULT; 9454 kfree(tmp); 9455 9456 return ret; 9457} 9458 9459struct btrfs_encoded_read_private { 9460 struct completion *sync_reads; 9461 void *uring_ctx; 9462 refcount_t pending_refs; 9463 blk_status_t status; 9464}; 9465 9466static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) 9467{ 9468 struct btrfs_encoded_read_private *priv = bbio->private; 9469 9470 if (bbio->bio.bi_status) { 9471 /* 9472 * The memory barrier implied by the refcount_dec_and_test() here 9473 * pairs with the memory barrier implied by the refcount_dec_and_test() 9474 * in btrfs_encoded_read_regular_fill_pages() to ensure that 9475 * this write is observed before the load of status in 9476 * btrfs_encoded_read_regular_fill_pages(). 9477 */ 9478 WRITE_ONCE(priv->status, bbio->bio.bi_status); 9479 } 9480 if (refcount_dec_and_test(&priv->pending_refs)) { 9481 int err = blk_status_to_errno(READ_ONCE(priv->status)); 9482 9483 if (priv->uring_ctx) { 9484 btrfs_uring_read_extent_endio(priv->uring_ctx, err); 9485 kfree(priv); 9486 } else { 9487 complete(priv->sync_reads); 9488 } 9489 } 9490 bio_put(&bbio->bio); 9491} 9492 9493int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, 9494 u64 disk_bytenr, u64 disk_io_size, 9495 struct page **pages, void *uring_ctx) 9496{ 9497 struct btrfs_encoded_read_private *priv, sync_priv; 9498 struct completion sync_reads; 9499 unsigned long i = 0; 9500 struct btrfs_bio *bbio; 9501 int ret; 9502 9503 /* 9504 * Fast path for synchronous reads which completes in this call, io_uring 9505 * needs longer time span. 9506 */ 9507 if (uring_ctx) { 9508 priv = kmalloc(sizeof(struct btrfs_encoded_read_private), GFP_NOFS); 9509 if (!priv) 9510 return -ENOMEM; 9511 } else { 9512 priv = &sync_priv; 9513 init_completion(&sync_reads); 9514 priv->sync_reads = &sync_reads; 9515 } 9516 9517 refcount_set(&priv->pending_refs, 1); 9518 priv->status = 0; 9519 priv->uring_ctx = uring_ctx; 9520 9521 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, 9522 btrfs_encoded_read_endio, priv); 9523 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; 9524 9525 do { 9526 size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE); 9527 9528 if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) { 9529 refcount_inc(&priv->pending_refs); 9530 btrfs_submit_bbio(bbio, 0); 9531 9532 bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, 9533 btrfs_encoded_read_endio, priv); 9534 bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; 9535 continue; 9536 } 9537 9538 i++; 9539 disk_bytenr += bytes; 9540 disk_io_size -= bytes; 9541 } while (disk_io_size); 9542 9543 refcount_inc(&priv->pending_refs); 9544 btrfs_submit_bbio(bbio, 0); 9545 9546 if (uring_ctx) { 9547 if (refcount_dec_and_test(&priv->pending_refs)) { 9548 ret = blk_status_to_errno(READ_ONCE(priv->status)); 9549 btrfs_uring_read_extent_endio(uring_ctx, ret); 9550 kfree(priv); 9551 return ret; 9552 } 9553 9554 return -EIOCBQUEUED; 9555 } else { 9556 if (!refcount_dec_and_test(&priv->pending_refs)) 9557 wait_for_completion_io(&sync_reads); 9558 /* See btrfs_encoded_read_endio() for ordering. */ 9559 return blk_status_to_errno(READ_ONCE(priv->status)); 9560 } 9561} 9562 9563ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, struct iov_iter *iter, 9564 u64 start, u64 lockend, 9565 struct extent_state **cached_state, 9566 u64 disk_bytenr, u64 disk_io_size, 9567 size_t count, bool compressed, bool *unlocked) 9568{ 9569 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 9570 struct extent_io_tree *io_tree = &inode->io_tree; 9571 struct page **pages; 9572 unsigned long nr_pages, i; 9573 u64 cur; 9574 size_t page_offset; 9575 ssize_t ret; 9576 9577 nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE); 9578 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); 9579 if (!pages) 9580 return -ENOMEM; 9581 ret = btrfs_alloc_page_array(nr_pages, pages, false); 9582 if (ret) { 9583 ret = -ENOMEM; 9584 goto out; 9585 } 9586 9587 ret = btrfs_encoded_read_regular_fill_pages(inode, disk_bytenr, 9588 disk_io_size, pages, NULL); 9589 if (ret) 9590 goto out; 9591 9592 btrfs_unlock_extent(io_tree, start, lockend, cached_state); 9593 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9594 *unlocked = true; 9595 9596 if (compressed) { 9597 i = 0; 9598 page_offset = 0; 9599 } else { 9600 i = (iocb->ki_pos - start) >> PAGE_SHIFT; 9601 page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1); 9602 } 9603 cur = 0; 9604 while (cur < count) { 9605 size_t bytes = min_t(size_t, count - cur, 9606 PAGE_SIZE - page_offset); 9607 9608 if (copy_page_to_iter(pages[i], page_offset, bytes, 9609 iter) != bytes) { 9610 ret = -EFAULT; 9611 goto out; 9612 } 9613 i++; 9614 cur += bytes; 9615 page_offset = 0; 9616 } 9617 ret = count; 9618out: 9619 for (i = 0; i < nr_pages; i++) { 9620 if (pages[i]) 9621 __free_page(pages[i]); 9622 } 9623 kfree(pages); 9624 return ret; 9625} 9626 9627ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, 9628 struct btrfs_ioctl_encoded_io_args *encoded, 9629 struct extent_state **cached_state, 9630 u64 *disk_bytenr, u64 *disk_io_size) 9631{ 9632 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 9633 struct btrfs_fs_info *fs_info = inode->root->fs_info; 9634 struct extent_io_tree *io_tree = &inode->io_tree; 9635 ssize_t ret; 9636 size_t count = iov_iter_count(iter); 9637 u64 start, lockend; 9638 struct extent_map *em; 9639 const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); 9640 bool unlocked = false; 9641 9642 file_accessed(iocb->ki_filp); 9643 9644 ret = btrfs_inode_lock(inode, 9645 BTRFS_ILOCK_SHARED | (nowait ? BTRFS_ILOCK_TRY : 0)); 9646 if (ret) 9647 return ret; 9648 9649 if (iocb->ki_pos >= inode->vfs_inode.i_size) { 9650 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9651 return 0; 9652 } 9653 start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize); 9654 /* 9655 * We don't know how long the extent containing iocb->ki_pos is, but if 9656 * it's compressed we know that it won't be longer than this. 9657 */ 9658 lockend = start + BTRFS_MAX_UNCOMPRESSED - 1; 9659 9660 if (nowait) { 9661 struct btrfs_ordered_extent *ordered; 9662 9663 if (filemap_range_needs_writeback(inode->vfs_inode.i_mapping, 9664 start, lockend)) { 9665 ret = -EAGAIN; 9666 goto out_unlock_inode; 9667 } 9668 9669 if (!btrfs_try_lock_extent(io_tree, start, lockend, cached_state)) { 9670 ret = -EAGAIN; 9671 goto out_unlock_inode; 9672 } 9673 9674 ordered = btrfs_lookup_ordered_range(inode, start, 9675 lockend - start + 1); 9676 if (ordered) { 9677 btrfs_put_ordered_extent(ordered); 9678 btrfs_unlock_extent(io_tree, start, lockend, cached_state); 9679 ret = -EAGAIN; 9680 goto out_unlock_inode; 9681 } 9682 } else { 9683 for (;;) { 9684 struct btrfs_ordered_extent *ordered; 9685 9686 ret = btrfs_wait_ordered_range(inode, start, 9687 lockend - start + 1); 9688 if (ret) 9689 goto out_unlock_inode; 9690 9691 btrfs_lock_extent(io_tree, start, lockend, cached_state); 9692 ordered = btrfs_lookup_ordered_range(inode, start, 9693 lockend - start + 1); 9694 if (!ordered) 9695 break; 9696 btrfs_put_ordered_extent(ordered); 9697 btrfs_unlock_extent(io_tree, start, lockend, cached_state); 9698 cond_resched(); 9699 } 9700 } 9701 9702 em = btrfs_get_extent(inode, NULL, start, lockend - start + 1); 9703 if (IS_ERR(em)) { 9704 ret = PTR_ERR(em); 9705 goto out_unlock_extent; 9706 } 9707 9708 if (em->disk_bytenr == EXTENT_MAP_INLINE) { 9709 u64 extent_start = em->start; 9710 9711 /* 9712 * For inline extents we get everything we need out of the 9713 * extent item. 9714 */ 9715 btrfs_free_extent_map(em); 9716 em = NULL; 9717 ret = btrfs_encoded_read_inline(iocb, iter, start, lockend, 9718 cached_state, extent_start, 9719 count, encoded, &unlocked); 9720 goto out_unlock_extent; 9721 } 9722 9723 /* 9724 * We only want to return up to EOF even if the extent extends beyond 9725 * that. 9726 */ 9727 encoded->len = min_t(u64, btrfs_extent_map_end(em), 9728 inode->vfs_inode.i_size) - iocb->ki_pos; 9729 if (em->disk_bytenr == EXTENT_MAP_HOLE || 9730 (em->flags & EXTENT_FLAG_PREALLOC)) { 9731 *disk_bytenr = EXTENT_MAP_HOLE; 9732 count = min_t(u64, count, encoded->len); 9733 encoded->len = count; 9734 encoded->unencoded_len = count; 9735 } else if (btrfs_extent_map_is_compressed(em)) { 9736 *disk_bytenr = em->disk_bytenr; 9737 /* 9738 * Bail if the buffer isn't large enough to return the whole 9739 * compressed extent. 9740 */ 9741 if (em->disk_num_bytes > count) { 9742 ret = -ENOBUFS; 9743 goto out_em; 9744 } 9745 *disk_io_size = em->disk_num_bytes; 9746 count = em->disk_num_bytes; 9747 encoded->unencoded_len = em->ram_bytes; 9748 encoded->unencoded_offset = iocb->ki_pos - (em->start - em->offset); 9749 ret = btrfs_encoded_io_compression_from_extent(fs_info, 9750 btrfs_extent_map_compression(em)); 9751 if (ret < 0) 9752 goto out_em; 9753 encoded->compression = ret; 9754 } else { 9755 *disk_bytenr = btrfs_extent_map_block_start(em) + (start - em->start); 9756 if (encoded->len > count) 9757 encoded->len = count; 9758 /* 9759 * Don't read beyond what we locked. This also limits the page 9760 * allocations that we'll do. 9761 */ 9762 *disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start; 9763 count = start + *disk_io_size - iocb->ki_pos; 9764 encoded->len = count; 9765 encoded->unencoded_len = count; 9766 *disk_io_size = ALIGN(*disk_io_size, fs_info->sectorsize); 9767 } 9768 btrfs_free_extent_map(em); 9769 em = NULL; 9770 9771 if (*disk_bytenr == EXTENT_MAP_HOLE) { 9772 btrfs_unlock_extent(io_tree, start, lockend, cached_state); 9773 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9774 unlocked = true; 9775 ret = iov_iter_zero(count, iter); 9776 if (ret != count) 9777 ret = -EFAULT; 9778 } else { 9779 ret = -EIOCBQUEUED; 9780 goto out_unlock_extent; 9781 } 9782 9783out_em: 9784 btrfs_free_extent_map(em); 9785out_unlock_extent: 9786 /* Leave inode and extent locked if we need to do a read. */ 9787 if (!unlocked && ret != -EIOCBQUEUED) 9788 btrfs_unlock_extent(io_tree, start, lockend, cached_state); 9789out_unlock_inode: 9790 if (!unlocked && ret != -EIOCBQUEUED) 9791 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); 9792 return ret; 9793} 9794 9795ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, 9796 const struct btrfs_ioctl_encoded_io_args *encoded) 9797{ 9798 struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); 9799 struct btrfs_root *root = inode->root; 9800 struct btrfs_fs_info *fs_info = root->fs_info; 9801 struct extent_io_tree *io_tree = &inode->io_tree; 9802 struct extent_changeset *data_reserved = NULL; 9803 struct extent_state *cached_state = NULL; 9804 struct btrfs_ordered_extent *ordered; 9805 struct btrfs_file_extent file_extent; 9806 int compression; 9807 size_t orig_count; 9808 u64 start, end; 9809 u64 num_bytes, ram_bytes, disk_num_bytes; 9810 unsigned long nr_folios, i; 9811 struct folio **folios; 9812 struct btrfs_key ins; 9813 bool extent_reserved = false; 9814 struct extent_map *em; 9815 ssize_t ret; 9816 9817 switch (encoded->compression) { 9818 case BTRFS_ENCODED_IO_COMPRESSION_ZLIB: 9819 compression = BTRFS_COMPRESS_ZLIB; 9820 break; 9821 case BTRFS_ENCODED_IO_COMPRESSION_ZSTD: 9822 compression = BTRFS_COMPRESS_ZSTD; 9823 break; 9824 case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K: 9825 case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K: 9826 case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K: 9827 case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K: 9828 case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K: 9829 /* The sector size must match for LZO. */ 9830 if (encoded->compression - 9831 BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 != 9832 fs_info->sectorsize_bits) 9833 return -EINVAL; 9834 compression = BTRFS_COMPRESS_LZO; 9835 break; 9836 default: 9837 return -EINVAL; 9838 } 9839 if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE) 9840 return -EINVAL; 9841 9842 /* 9843 * Compressed extents should always have checksums, so error out if we 9844 * have a NOCOW file or inode was created while mounted with NODATASUM. 9845 */ 9846 if (inode->flags & BTRFS_INODE_NODATASUM) 9847 return -EINVAL; 9848 9849 orig_count = iov_iter_count(from); 9850 9851 /* The extent size must be sane. */ 9852 if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED || 9853 orig_count > BTRFS_MAX_COMPRESSED || orig_count == 0) 9854 return -EINVAL; 9855 9856 /* 9857 * The compressed data must be smaller than the decompressed data. 9858 * 9859 * It's of course possible for data to compress to larger or the same 9860 * size, but the buffered I/O path falls back to no compression for such 9861 * data, and we don't want to break any assumptions by creating these 9862 * extents. 9863 * 9864 * Note that this is less strict than the current check we have that the 9865 * compressed data must be at least one sector smaller than the 9866 * decompressed data. We only want to enforce the weaker requirement 9867 * from old kernels that it is at least one byte smaller. 9868 */ 9869 if (orig_count >= encoded->unencoded_len) 9870 return -EINVAL; 9871 9872 /* The extent must start on a sector boundary. */ 9873 start = iocb->ki_pos; 9874 if (!IS_ALIGNED(start, fs_info->sectorsize)) 9875 return -EINVAL; 9876 9877 /* 9878 * The extent must end on a sector boundary. However, we allow a write 9879 * which ends at or extends i_size to have an unaligned length; we round 9880 * up the extent size and set i_size to the unaligned end. 9881 */ 9882 if (start + encoded->len < inode->vfs_inode.i_size && 9883 !IS_ALIGNED(start + encoded->len, fs_info->sectorsize)) 9884 return -EINVAL; 9885 9886 /* Finally, the offset in the unencoded data must be sector-aligned. */ 9887 if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize)) 9888 return -EINVAL; 9889 9890 num_bytes = ALIGN(encoded->len, fs_info->sectorsize); 9891 ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize); 9892 end = start + num_bytes - 1; 9893 9894 /* 9895 * If the extent cannot be inline, the compressed data on disk must be 9896 * sector-aligned. For convenience, we extend it with zeroes if it 9897 * isn't. 9898 */ 9899 disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize); 9900 nr_folios = DIV_ROUND_UP(disk_num_bytes, PAGE_SIZE); 9901 folios = kvcalloc(nr_folios, sizeof(struct folio *), GFP_KERNEL_ACCOUNT); 9902 if (!folios) 9903 return -ENOMEM; 9904 for (i = 0; i < nr_folios; i++) { 9905 size_t bytes = min_t(size_t, PAGE_SIZE, iov_iter_count(from)); 9906 char *kaddr; 9907 9908 folios[i] = folio_alloc(GFP_KERNEL_ACCOUNT, 0); 9909 if (!folios[i]) { 9910 ret = -ENOMEM; 9911 goto out_folios; 9912 } 9913 kaddr = kmap_local_folio(folios[i], 0); 9914 if (copy_from_iter(kaddr, bytes, from) != bytes) { 9915 kunmap_local(kaddr); 9916 ret = -EFAULT; 9917 goto out_folios; 9918 } 9919 if (bytes < PAGE_SIZE) 9920 memset(kaddr + bytes, 0, PAGE_SIZE - bytes); 9921 kunmap_local(kaddr); 9922 } 9923 9924 for (;;) { 9925 ret = btrfs_wait_ordered_range(inode, start, num_bytes); 9926 if (ret) 9927 goto out_folios; 9928 ret = invalidate_inode_pages2_range(inode->vfs_inode.i_mapping, 9929 start >> PAGE_SHIFT, 9930 end >> PAGE_SHIFT); 9931 if (ret) 9932 goto out_folios; 9933 btrfs_lock_extent(io_tree, start, end, &cached_state); 9934 ordered = btrfs_lookup_ordered_range(inode, start, num_bytes); 9935 if (!ordered && 9936 !filemap_range_has_page(inode->vfs_inode.i_mapping, start, end)) 9937 break; 9938 if (ordered) 9939 btrfs_put_ordered_extent(ordered); 9940 btrfs_unlock_extent(io_tree, start, end, &cached_state); 9941 cond_resched(); 9942 } 9943 9944 /* 9945 * We don't use the higher-level delalloc space functions because our 9946 * num_bytes and disk_num_bytes are different. 9947 */ 9948 ret = btrfs_alloc_data_chunk_ondemand(inode, disk_num_bytes); 9949 if (ret) 9950 goto out_unlock; 9951 ret = btrfs_qgroup_reserve_data(inode, &data_reserved, start, num_bytes); 9952 if (ret) 9953 goto out_free_data_space; 9954 ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes, 9955 false); 9956 if (ret) 9957 goto out_qgroup_free_data; 9958 9959 /* Try an inline extent first. */ 9960 if (encoded->unencoded_len == encoded->len && 9961 encoded->unencoded_offset == 0 && 9962 can_cow_file_range_inline(inode, start, encoded->len, orig_count)) { 9963 ret = __cow_file_range_inline(inode, encoded->len, 9964 orig_count, compression, folios[0], 9965 true); 9966 if (ret <= 0) { 9967 if (ret == 0) 9968 ret = orig_count; 9969 goto out_delalloc_release; 9970 } 9971 } 9972 9973 ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes, 9974 disk_num_bytes, 0, 0, &ins, true, true); 9975 if (ret) 9976 goto out_delalloc_release; 9977 extent_reserved = true; 9978 9979 file_extent.disk_bytenr = ins.objectid; 9980 file_extent.disk_num_bytes = ins.offset; 9981 file_extent.num_bytes = num_bytes; 9982 file_extent.ram_bytes = ram_bytes; 9983 file_extent.offset = encoded->unencoded_offset; 9984 file_extent.compression = compression; 9985 em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED); 9986 if (IS_ERR(em)) { 9987 ret = PTR_ERR(em); 9988 goto out_free_reserved; 9989 } 9990 btrfs_free_extent_map(em); 9991 9992 ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent, 9993 (1U << BTRFS_ORDERED_ENCODED) | 9994 (1U << BTRFS_ORDERED_COMPRESSED)); 9995 if (IS_ERR(ordered)) { 9996 btrfs_drop_extent_map_range(inode, start, end, false); 9997 ret = PTR_ERR(ordered); 9998 goto out_free_reserved; 9999 } 10000 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10001 10002 if (start + encoded->len > inode->vfs_inode.i_size) 10003 i_size_write(&inode->vfs_inode, start + encoded->len); 10004 10005 btrfs_unlock_extent(io_tree, start, end, &cached_state); 10006 10007 btrfs_delalloc_release_extents(inode, num_bytes); 10008 10009 btrfs_submit_compressed_write(ordered, folios, nr_folios, 0, false); 10010 ret = orig_count; 10011 goto out; 10012 10013out_free_reserved: 10014 btrfs_dec_block_group_reservations(fs_info, ins.objectid); 10015 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true); 10016out_delalloc_release: 10017 btrfs_delalloc_release_extents(inode, num_bytes); 10018 btrfs_delalloc_release_metadata(inode, disk_num_bytes, ret < 0); 10019out_qgroup_free_data: 10020 if (ret < 0) 10021 btrfs_qgroup_free_data(inode, data_reserved, start, num_bytes, NULL); 10022out_free_data_space: 10023 /* 10024 * If btrfs_reserve_extent() succeeded, then we already decremented 10025 * bytes_may_use. 10026 */ 10027 if (!extent_reserved) 10028 btrfs_free_reserved_data_space_noquota(inode, disk_num_bytes); 10029out_unlock: 10030 btrfs_unlock_extent(io_tree, start, end, &cached_state); 10031out_folios: 10032 for (i = 0; i < nr_folios; i++) { 10033 if (folios[i]) 10034 folio_put(folios[i]); 10035 } 10036 kvfree(folios); 10037out: 10038 if (ret >= 0) 10039 iocb->ki_pos += encoded->len; 10040 return ret; 10041} 10042 10043#ifdef CONFIG_SWAP 10044/* 10045 * Add an entry indicating a block group or device which is pinned by a 10046 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a 10047 * negative errno on failure. 10048 */ 10049static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, 10050 bool is_block_group) 10051{ 10052 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10053 struct btrfs_swapfile_pin *sp, *entry; 10054 struct rb_node **p; 10055 struct rb_node *parent = NULL; 10056 10057 sp = kmalloc(sizeof(*sp), GFP_NOFS); 10058 if (!sp) 10059 return -ENOMEM; 10060 sp->ptr = ptr; 10061 sp->inode = inode; 10062 sp->is_block_group = is_block_group; 10063 sp->bg_extent_count = 1; 10064 10065 spin_lock(&fs_info->swapfile_pins_lock); 10066 p = &fs_info->swapfile_pins.rb_node; 10067 while (*p) { 10068 parent = *p; 10069 entry = rb_entry(parent, struct btrfs_swapfile_pin, node); 10070 if (sp->ptr < entry->ptr || 10071 (sp->ptr == entry->ptr && sp->inode < entry->inode)) { 10072 p = &(*p)->rb_left; 10073 } else if (sp->ptr > entry->ptr || 10074 (sp->ptr == entry->ptr && sp->inode > entry->inode)) { 10075 p = &(*p)->rb_right; 10076 } else { 10077 if (is_block_group) 10078 entry->bg_extent_count++; 10079 spin_unlock(&fs_info->swapfile_pins_lock); 10080 kfree(sp); 10081 return 1; 10082 } 10083 } 10084 rb_link_node(&sp->node, parent, p); 10085 rb_insert_color(&sp->node, &fs_info->swapfile_pins); 10086 spin_unlock(&fs_info->swapfile_pins_lock); 10087 return 0; 10088} 10089 10090/* Free all of the entries pinned by this swapfile. */ 10091static void btrfs_free_swapfile_pins(struct inode *inode) 10092{ 10093 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 10094 struct btrfs_swapfile_pin *sp; 10095 struct rb_node *node, *next; 10096 10097 spin_lock(&fs_info->swapfile_pins_lock); 10098 node = rb_first(&fs_info->swapfile_pins); 10099 while (node) { 10100 next = rb_next(node); 10101 sp = rb_entry(node, struct btrfs_swapfile_pin, node); 10102 if (sp->inode == inode) { 10103 rb_erase(&sp->node, &fs_info->swapfile_pins); 10104 if (sp->is_block_group) { 10105 btrfs_dec_block_group_swap_extents(sp->ptr, 10106 sp->bg_extent_count); 10107 btrfs_put_block_group(sp->ptr); 10108 } 10109 kfree(sp); 10110 } 10111 node = next; 10112 } 10113 spin_unlock(&fs_info->swapfile_pins_lock); 10114} 10115 10116struct btrfs_swap_info { 10117 u64 start; 10118 u64 block_start; 10119 u64 block_len; 10120 u64 lowest_ppage; 10121 u64 highest_ppage; 10122 unsigned long nr_pages; 10123 int nr_extents; 10124}; 10125 10126static int btrfs_add_swap_extent(struct swap_info_struct *sis, 10127 struct btrfs_swap_info *bsi) 10128{ 10129 unsigned long nr_pages; 10130 unsigned long max_pages; 10131 u64 first_ppage, first_ppage_reported, next_ppage; 10132 int ret; 10133 10134 /* 10135 * Our swapfile may have had its size extended after the swap header was 10136 * written. In that case activating the swapfile should not go beyond 10137 * the max size set in the swap header. 10138 */ 10139 if (bsi->nr_pages >= sis->max) 10140 return 0; 10141 10142 max_pages = sis->max - bsi->nr_pages; 10143 first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT; 10144 next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT; 10145 10146 if (first_ppage >= next_ppage) 10147 return 0; 10148 nr_pages = next_ppage - first_ppage; 10149 nr_pages = min(nr_pages, max_pages); 10150 10151 first_ppage_reported = first_ppage; 10152 if (bsi->start == 0) 10153 first_ppage_reported++; 10154 if (bsi->lowest_ppage > first_ppage_reported) 10155 bsi->lowest_ppage = first_ppage_reported; 10156 if (bsi->highest_ppage < (next_ppage - 1)) 10157 bsi->highest_ppage = next_ppage - 1; 10158 10159 ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage); 10160 if (ret < 0) 10161 return ret; 10162 bsi->nr_extents += ret; 10163 bsi->nr_pages += nr_pages; 10164 return 0; 10165} 10166 10167static void btrfs_swap_deactivate(struct file *file) 10168{ 10169 struct inode *inode = file_inode(file); 10170 10171 btrfs_free_swapfile_pins(inode); 10172 atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles); 10173} 10174 10175static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10176 sector_t *span) 10177{ 10178 struct inode *inode = file_inode(file); 10179 struct btrfs_root *root = BTRFS_I(inode)->root; 10180 struct btrfs_fs_info *fs_info = root->fs_info; 10181 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; 10182 struct extent_state *cached_state = NULL; 10183 struct btrfs_chunk_map *map = NULL; 10184 struct btrfs_device *device = NULL; 10185 struct btrfs_swap_info bsi = { 10186 .lowest_ppage = (sector_t)-1ULL, 10187 }; 10188 struct btrfs_backref_share_check_ctx *backref_ctx = NULL; 10189 struct btrfs_path *path = NULL; 10190 int ret = 0; 10191 u64 isize; 10192 u64 prev_extent_end = 0; 10193 10194 /* 10195 * Acquire the inode's mmap lock to prevent races with memory mapped 10196 * writes, as they could happen after we flush delalloc below and before 10197 * we lock the extent range further below. The inode was already locked 10198 * up in the call chain. 10199 */ 10200 btrfs_assert_inode_locked(BTRFS_I(inode)); 10201 down_write(&BTRFS_I(inode)->i_mmap_lock); 10202 10203 /* 10204 * If the swap file was just created, make sure delalloc is done. If the 10205 * file changes again after this, the user is doing something stupid and 10206 * we don't really care. 10207 */ 10208 ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1); 10209 if (ret) 10210 goto out_unlock_mmap; 10211 10212 /* 10213 * The inode is locked, so these flags won't change after we check them. 10214 */ 10215 if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) { 10216 btrfs_warn(fs_info, "swapfile must not be compressed"); 10217 ret = -EINVAL; 10218 goto out_unlock_mmap; 10219 } 10220 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) { 10221 btrfs_warn(fs_info, "swapfile must not be copy-on-write"); 10222 ret = -EINVAL; 10223 goto out_unlock_mmap; 10224 } 10225 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { 10226 btrfs_warn(fs_info, "swapfile must not be checksummed"); 10227 ret = -EINVAL; 10228 goto out_unlock_mmap; 10229 } 10230 10231 path = btrfs_alloc_path(); 10232 backref_ctx = btrfs_alloc_backref_share_check_ctx(); 10233 if (!path || !backref_ctx) { 10234 ret = -ENOMEM; 10235 goto out_unlock_mmap; 10236 } 10237 10238 /* 10239 * Balance or device remove/replace/resize can move stuff around from 10240 * under us. The exclop protection makes sure they aren't running/won't 10241 * run concurrently while we are mapping the swap extents, and 10242 * fs_info->swapfile_pins prevents them from running while the swap 10243 * file is active and moving the extents. Note that this also prevents 10244 * a concurrent device add which isn't actually necessary, but it's not 10245 * really worth the trouble to allow it. 10246 */ 10247 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) { 10248 btrfs_warn(fs_info, 10249 "cannot activate swapfile while exclusive operation is running"); 10250 ret = -EBUSY; 10251 goto out_unlock_mmap; 10252 } 10253 10254 /* 10255 * Prevent snapshot creation while we are activating the swap file. 10256 * We do not want to race with snapshot creation. If snapshot creation 10257 * already started before we bumped nr_swapfiles from 0 to 1 and 10258 * completes before the first write into the swap file after it is 10259 * activated, than that write would fallback to COW. 10260 */ 10261 if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) { 10262 btrfs_exclop_finish(fs_info); 10263 btrfs_warn(fs_info, 10264 "cannot activate swapfile because snapshot creation is in progress"); 10265 ret = -EINVAL; 10266 goto out_unlock_mmap; 10267 } 10268 /* 10269 * Snapshots can create extents which require COW even if NODATACOW is 10270 * set. We use this counter to prevent snapshots. We must increment it 10271 * before walking the extents because we don't want a concurrent 10272 * snapshot to run after we've already checked the extents. 10273 * 10274 * It is possible that subvolume is marked for deletion but still not 10275 * removed yet. To prevent this race, we check the root status before 10276 * activating the swapfile. 10277 */ 10278 spin_lock(&root->root_item_lock); 10279 if (btrfs_root_dead(root)) { 10280 spin_unlock(&root->root_item_lock); 10281 10282 btrfs_drew_write_unlock(&root->snapshot_lock); 10283 btrfs_exclop_finish(fs_info); 10284 btrfs_warn(fs_info, 10285 "cannot activate swapfile because subvolume %llu is being deleted", 10286 btrfs_root_id(root)); 10287 ret = -EPERM; 10288 goto out_unlock_mmap; 10289 } 10290 atomic_inc(&root->nr_swapfiles); 10291 spin_unlock(&root->root_item_lock); 10292 10293 isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); 10294 10295 btrfs_lock_extent(io_tree, 0, isize - 1, &cached_state); 10296 while (prev_extent_end < isize) { 10297 struct btrfs_key key; 10298 struct extent_buffer *leaf; 10299 struct btrfs_file_extent_item *ei; 10300 struct btrfs_block_group *bg; 10301 u64 logical_block_start; 10302 u64 physical_block_start; 10303 u64 extent_gen; 10304 u64 disk_bytenr; 10305 u64 len; 10306 10307 key.objectid = btrfs_ino(BTRFS_I(inode)); 10308 key.type = BTRFS_EXTENT_DATA_KEY; 10309 key.offset = prev_extent_end; 10310 10311 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 10312 if (ret < 0) 10313 goto out; 10314 10315 /* 10316 * If key not found it means we have an implicit hole (NO_HOLES 10317 * is enabled). 10318 */ 10319 if (ret > 0) { 10320 btrfs_warn(fs_info, "swapfile must not have holes"); 10321 ret = -EINVAL; 10322 goto out; 10323 } 10324 10325 leaf = path->nodes[0]; 10326 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); 10327 10328 if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) { 10329 /* 10330 * It's unlikely we'll ever actually find ourselves 10331 * here, as a file small enough to fit inline won't be 10332 * big enough to store more than the swap header, but in 10333 * case something changes in the future, let's catch it 10334 * here rather than later. 10335 */ 10336 btrfs_warn(fs_info, "swapfile must not be inline"); 10337 ret = -EINVAL; 10338 goto out; 10339 } 10340 10341 if (btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) { 10342 btrfs_warn(fs_info, "swapfile must not be compressed"); 10343 ret = -EINVAL; 10344 goto out; 10345 } 10346 10347 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); 10348 if (disk_bytenr == 0) { 10349 btrfs_warn(fs_info, "swapfile must not have holes"); 10350 ret = -EINVAL; 10351 goto out; 10352 } 10353 10354 logical_block_start = disk_bytenr + btrfs_file_extent_offset(leaf, ei); 10355 extent_gen = btrfs_file_extent_generation(leaf, ei); 10356 prev_extent_end = btrfs_file_extent_end(path); 10357 10358 if (prev_extent_end > isize) 10359 len = isize - key.offset; 10360 else 10361 len = btrfs_file_extent_num_bytes(leaf, ei); 10362 10363 backref_ctx->curr_leaf_bytenr = leaf->start; 10364 10365 /* 10366 * Don't need the path anymore, release to avoid deadlocks when 10367 * calling btrfs_is_data_extent_shared() because when joining a 10368 * transaction it can block waiting for the current one's commit 10369 * which in turn may be trying to lock the same leaf to flush 10370 * delayed items for example. 10371 */ 10372 btrfs_release_path(path); 10373 10374 ret = btrfs_is_data_extent_shared(BTRFS_I(inode), disk_bytenr, 10375 extent_gen, backref_ctx); 10376 if (ret < 0) { 10377 goto out; 10378 } else if (ret > 0) { 10379 btrfs_warn(fs_info, 10380 "swapfile must not be copy-on-write"); 10381 ret = -EINVAL; 10382 goto out; 10383 } 10384 10385 map = btrfs_get_chunk_map(fs_info, logical_block_start, len); 10386 if (IS_ERR(map)) { 10387 ret = PTR_ERR(map); 10388 goto out; 10389 } 10390 10391 if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 10392 btrfs_warn(fs_info, 10393 "swapfile must have single data profile"); 10394 ret = -EINVAL; 10395 goto out; 10396 } 10397 10398 if (device == NULL) { 10399 device = map->stripes[0].dev; 10400 ret = btrfs_add_swapfile_pin(inode, device, false); 10401 if (ret == 1) 10402 ret = 0; 10403 else if (ret) 10404 goto out; 10405 } else if (device != map->stripes[0].dev) { 10406 btrfs_warn(fs_info, "swapfile must be on one device"); 10407 ret = -EINVAL; 10408 goto out; 10409 } 10410 10411 physical_block_start = (map->stripes[0].physical + 10412 (logical_block_start - map->start)); 10413 btrfs_free_chunk_map(map); 10414 map = NULL; 10415 10416 bg = btrfs_lookup_block_group(fs_info, logical_block_start); 10417 if (!bg) { 10418 btrfs_warn(fs_info, 10419 "could not find block group containing swapfile"); 10420 ret = -EINVAL; 10421 goto out; 10422 } 10423 10424 if (!btrfs_inc_block_group_swap_extents(bg)) { 10425 btrfs_warn(fs_info, 10426 "block group for swapfile at %llu is read-only%s", 10427 bg->start, 10428 atomic_read(&fs_info->scrubs_running) ? 10429 " (scrub running)" : ""); 10430 btrfs_put_block_group(bg); 10431 ret = -EINVAL; 10432 goto out; 10433 } 10434 10435 ret = btrfs_add_swapfile_pin(inode, bg, true); 10436 if (ret) { 10437 btrfs_put_block_group(bg); 10438 if (ret == 1) 10439 ret = 0; 10440 else 10441 goto out; 10442 } 10443 10444 if (bsi.block_len && 10445 bsi.block_start + bsi.block_len == physical_block_start) { 10446 bsi.block_len += len; 10447 } else { 10448 if (bsi.block_len) { 10449 ret = btrfs_add_swap_extent(sis, &bsi); 10450 if (ret) 10451 goto out; 10452 } 10453 bsi.start = key.offset; 10454 bsi.block_start = physical_block_start; 10455 bsi.block_len = len; 10456 } 10457 10458 if (fatal_signal_pending(current)) { 10459 ret = -EINTR; 10460 goto out; 10461 } 10462 10463 cond_resched(); 10464 } 10465 10466 if (bsi.block_len) 10467 ret = btrfs_add_swap_extent(sis, &bsi); 10468 10469out: 10470 if (!IS_ERR_OR_NULL(map)) 10471 btrfs_free_chunk_map(map); 10472 10473 btrfs_unlock_extent(io_tree, 0, isize - 1, &cached_state); 10474 10475 if (ret) 10476 btrfs_swap_deactivate(file); 10477 10478 btrfs_drew_write_unlock(&root->snapshot_lock); 10479 10480 btrfs_exclop_finish(fs_info); 10481 10482out_unlock_mmap: 10483 up_write(&BTRFS_I(inode)->i_mmap_lock); 10484 btrfs_free_backref_share_ctx(backref_ctx); 10485 btrfs_free_path(path); 10486 if (ret) 10487 return ret; 10488 10489 if (device) 10490 sis->bdev = device->bdev; 10491 *span = bsi.highest_ppage - bsi.lowest_ppage + 1; 10492 sis->max = bsi.nr_pages; 10493 sis->pages = bsi.nr_pages - 1; 10494 return bsi.nr_extents; 10495} 10496#else 10497static void btrfs_swap_deactivate(struct file *file) 10498{ 10499} 10500 10501static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, 10502 sector_t *span) 10503{ 10504 return -EOPNOTSUPP; 10505} 10506#endif 10507 10508/* 10509 * Update the number of bytes used in the VFS' inode. When we replace extents in 10510 * a range (clone, dedupe, fallocate's zero range), we must update the number of 10511 * bytes used by the inode in an atomic manner, so that concurrent stat(2) calls 10512 * always get a correct value. 10513 */ 10514void btrfs_update_inode_bytes(struct btrfs_inode *inode, 10515 const u64 add_bytes, 10516 const u64 del_bytes) 10517{ 10518 if (add_bytes == del_bytes) 10519 return; 10520 10521 spin_lock(&inode->lock); 10522 if (del_bytes > 0) 10523 inode_sub_bytes(&inode->vfs_inode, del_bytes); 10524 if (add_bytes > 0) 10525 inode_add_bytes(&inode->vfs_inode, add_bytes); 10526 spin_unlock(&inode->lock); 10527} 10528 10529/* 10530 * Verify that there are no ordered extents for a given file range. 10531 * 10532 * @inode: The target inode. 10533 * @start: Start offset of the file range, should be sector size aligned. 10534 * @end: End offset (inclusive) of the file range, its value +1 should be 10535 * sector size aligned. 10536 * 10537 * This should typically be used for cases where we locked an inode's VFS lock in 10538 * exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode, 10539 * we have flushed all delalloc in the range, we have waited for all ordered 10540 * extents in the range to complete and finally we have locked the file range in 10541 * the inode's io_tree. 10542 */ 10543void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end) 10544{ 10545 struct btrfs_root *root = inode->root; 10546 struct btrfs_ordered_extent *ordered; 10547 10548 if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) 10549 return; 10550 10551 ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start); 10552 if (ordered) { 10553 btrfs_err(root->fs_info, 10554"found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])", 10555 start, end, btrfs_ino(inode), btrfs_root_id(root), 10556 ordered->file_offset, 10557 ordered->file_offset + ordered->num_bytes - 1); 10558 btrfs_put_ordered_extent(ordered); 10559 } 10560 10561 ASSERT(ordered == NULL); 10562} 10563 10564/* 10565 * Find the first inode with a minimum number. 10566 * 10567 * @root: The root to search for. 10568 * @min_ino: The minimum inode number. 10569 * 10570 * Find the first inode in the @root with a number >= @min_ino and return it. 10571 * Returns NULL if no such inode found. 10572 */ 10573struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino) 10574{ 10575 struct btrfs_inode *inode; 10576 unsigned long from = min_ino; 10577 10578 xa_lock(&root->inodes); 10579 while (true) { 10580 inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT); 10581 if (!inode) 10582 break; 10583 if (igrab(&inode->vfs_inode)) 10584 break; 10585 10586 from = btrfs_ino(inode) + 1; 10587 cond_resched_lock(&root->inodes.xa_lock); 10588 } 10589 xa_unlock(&root->inodes); 10590 10591 return inode; 10592} 10593 10594static const struct inode_operations btrfs_dir_inode_operations = { 10595 .getattr = btrfs_getattr, 10596 .lookup = btrfs_lookup, 10597 .create = btrfs_create, 10598 .unlink = btrfs_unlink, 10599 .link = btrfs_link, 10600 .mkdir = btrfs_mkdir, 10601 .rmdir = btrfs_rmdir, 10602 .rename = btrfs_rename2, 10603 .symlink = btrfs_symlink, 10604 .setattr = btrfs_setattr, 10605 .mknod = btrfs_mknod, 10606 .listxattr = btrfs_listxattr, 10607 .permission = btrfs_permission, 10608 .get_inode_acl = btrfs_get_acl, 10609 .set_acl = btrfs_set_acl, 10610 .update_time = btrfs_update_time, 10611 .tmpfile = btrfs_tmpfile, 10612 .fileattr_get = btrfs_fileattr_get, 10613 .fileattr_set = btrfs_fileattr_set, 10614}; 10615 10616static const struct file_operations btrfs_dir_file_operations = { 10617 .llseek = btrfs_dir_llseek, 10618 .read = generic_read_dir, 10619 .iterate_shared = btrfs_real_readdir, 10620 .open = btrfs_opendir, 10621 .unlocked_ioctl = btrfs_ioctl, 10622#ifdef CONFIG_COMPAT 10623 .compat_ioctl = btrfs_compat_ioctl, 10624#endif 10625 .release = btrfs_release_file, 10626 .fsync = btrfs_sync_file, 10627}; 10628 10629/* 10630 * btrfs doesn't support the bmap operation because swapfiles 10631 * use bmap to make a mapping of extents in the file. They assume 10632 * these extents won't change over the life of the file and they 10633 * use the bmap result to do IO directly to the drive. 10634 * 10635 * the btrfs bmap call would return logical addresses that aren't 10636 * suitable for IO and they also will change frequently as COW 10637 * operations happen. So, swapfile + btrfs == corruption. 10638 * 10639 * For now we're avoiding this by dropping bmap. 10640 */ 10641static const struct address_space_operations btrfs_aops = { 10642 .read_folio = btrfs_read_folio, 10643 .writepages = btrfs_writepages, 10644 .readahead = btrfs_readahead, 10645 .invalidate_folio = btrfs_invalidate_folio, 10646 .launder_folio = btrfs_launder_folio, 10647 .release_folio = btrfs_release_folio, 10648 .migrate_folio = btrfs_migrate_folio, 10649 .dirty_folio = filemap_dirty_folio, 10650 .error_remove_folio = generic_error_remove_folio, 10651 .swap_activate = btrfs_swap_activate, 10652 .swap_deactivate = btrfs_swap_deactivate, 10653}; 10654 10655static const struct inode_operations btrfs_file_inode_operations = { 10656 .getattr = btrfs_getattr, 10657 .setattr = btrfs_setattr, 10658 .listxattr = btrfs_listxattr, 10659 .permission = btrfs_permission, 10660 .fiemap = btrfs_fiemap, 10661 .get_inode_acl = btrfs_get_acl, 10662 .set_acl = btrfs_set_acl, 10663 .update_time = btrfs_update_time, 10664 .fileattr_get = btrfs_fileattr_get, 10665 .fileattr_set = btrfs_fileattr_set, 10666}; 10667static const struct inode_operations btrfs_special_inode_operations = { 10668 .getattr = btrfs_getattr, 10669 .setattr = btrfs_setattr, 10670 .permission = btrfs_permission, 10671 .listxattr = btrfs_listxattr, 10672 .get_inode_acl = btrfs_get_acl, 10673 .set_acl = btrfs_set_acl, 10674 .update_time = btrfs_update_time, 10675}; 10676static const struct inode_operations btrfs_symlink_inode_operations = { 10677 .get_link = page_get_link, 10678 .getattr = btrfs_getattr, 10679 .setattr = btrfs_setattr, 10680 .permission = btrfs_permission, 10681 .listxattr = btrfs_listxattr, 10682 .update_time = btrfs_update_time, 10683}; 10684 10685const struct dentry_operations btrfs_dentry_operations = { 10686 .d_delete = btrfs_dentry_delete, 10687};
~/
Read-only file preview.