FILE
data.c
File preview with syntax highlighting.
linux-6.19.14/fs/f2fs/data.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * fs/f2fs/data.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8#include <linux/fs.h> 9#include <linux/f2fs_fs.h> 10#include <linux/sched/mm.h> 11#include <linux/mpage.h> 12#include <linux/writeback.h> 13#include <linux/pagevec.h> 14#include <linux/blkdev.h> 15#include <linux/bio.h> 16#include <linux/blk-crypto.h> 17#include <linux/swap.h> 18#include <linux/prefetch.h> 19#include <linux/uio.h> 20#include <linux/sched/signal.h> 21#include <linux/fiemap.h> 22#include <linux/iomap.h> 23 24#include "f2fs.h" 25#include "node.h" 26#include "segment.h" 27#include "iostat.h" 28#include <trace/events/f2fs.h> 29 30#define NUM_PREALLOC_POST_READ_CTXS 128 31 32static struct kmem_cache *bio_post_read_ctx_cache; 33static struct kmem_cache *bio_entry_slab; 34static mempool_t *bio_post_read_ctx_pool; 35static struct bio_set f2fs_bioset; 36 37#define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE 38 39int __init f2fs_init_bioset(void) 40{ 41 return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, 42 0, BIOSET_NEED_BVECS); 43} 44 45void f2fs_destroy_bioset(void) 46{ 47 bioset_exit(&f2fs_bioset); 48} 49 50bool f2fs_is_cp_guaranteed(const struct folio *folio) 51{ 52 struct address_space *mapping = folio->mapping; 53 struct inode *inode; 54 struct f2fs_sb_info *sbi; 55 56 if (fscrypt_is_bounce_folio(folio)) 57 return folio_test_f2fs_gcing(fscrypt_pagecache_folio(folio)); 58 59 inode = mapping->host; 60 sbi = F2FS_I_SB(inode); 61 62 if (inode->i_ino == F2FS_META_INO(sbi) || 63 inode->i_ino == F2FS_NODE_INO(sbi) || 64 S_ISDIR(inode->i_mode)) 65 return true; 66 67 if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || 68 folio_test_f2fs_gcing(folio)) 69 return true; 70 return false; 71} 72 73static enum count_type __read_io_type(struct folio *folio) 74{ 75 struct address_space *mapping = folio->mapping; 76 77 if (mapping) { 78 struct inode *inode = mapping->host; 79 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 80 81 if (inode->i_ino == F2FS_META_INO(sbi)) 82 return F2FS_RD_META; 83 84 if (inode->i_ino == F2FS_NODE_INO(sbi)) 85 return F2FS_RD_NODE; 86 } 87 return F2FS_RD_DATA; 88} 89 90/* postprocessing steps for read bios */ 91enum bio_post_read_step { 92#ifdef CONFIG_FS_ENCRYPTION 93 STEP_DECRYPT = BIT(0), 94#else 95 STEP_DECRYPT = 0, /* compile out the decryption-related code */ 96#endif 97#ifdef CONFIG_F2FS_FS_COMPRESSION 98 STEP_DECOMPRESS = BIT(1), 99#else 100 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */ 101#endif 102#ifdef CONFIG_FS_VERITY 103 STEP_VERITY = BIT(2), 104#else 105 STEP_VERITY = 0, /* compile out the verity-related code */ 106#endif 107}; 108 109struct bio_post_read_ctx { 110 struct bio *bio; 111 struct f2fs_sb_info *sbi; 112 struct work_struct work; 113 unsigned int enabled_steps; 114 /* 115 * decompression_attempted keeps track of whether 116 * f2fs_end_read_compressed_page() has been called on the pages in the 117 * bio that belong to a compressed cluster yet. 118 */ 119 bool decompression_attempted; 120 block_t fs_blkaddr; 121}; 122 123/* 124 * Update and unlock a bio's pages, and free the bio. 125 * 126 * This marks pages up-to-date only if there was no error in the bio (I/O error, 127 * decryption error, or verity error), as indicated by bio->bi_status. 128 * 129 * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) 130 * aren't marked up-to-date here, as decompression is done on a per-compression- 131 * cluster basis rather than a per-bio basis. Instead, we only must do two 132 * things for each compressed page here: call f2fs_end_read_compressed_page() 133 * with failed=true if an error occurred before it would have normally gotten 134 * called (i.e., I/O error or decryption error, but *not* verity error), and 135 * release the bio's reference to the decompress_io_ctx of the page's cluster. 136 */ 137static void f2fs_finish_read_bio(struct bio *bio, bool in_task) 138{ 139 struct folio_iter fi; 140 struct bio_post_read_ctx *ctx = bio->bi_private; 141 142 bio_for_each_folio_all(fi, bio) { 143 struct folio *folio = fi.folio; 144 145 if (f2fs_is_compressed_page(folio)) { 146 if (ctx && !ctx->decompression_attempted) 147 f2fs_end_read_compressed_page(folio, true, 0, 148 in_task); 149 f2fs_put_folio_dic(folio, in_task); 150 continue; 151 } 152 153 dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); 154 155 if (F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && 156 f2fs_sanity_check_node_footer(F2FS_F_SB(folio), 157 folio, folio->index, NODE_TYPE_REGULAR, true)) 158 bio->bi_status = BLK_STS_IOERR; 159 160 folio_end_read(folio, bio->bi_status == BLK_STS_OK); 161 } 162 163 if (ctx) 164 mempool_free(ctx, bio_post_read_ctx_pool); 165 bio_put(bio); 166} 167 168static void f2fs_verify_bio(struct work_struct *work) 169{ 170 struct bio_post_read_ctx *ctx = 171 container_of(work, struct bio_post_read_ctx, work); 172 struct bio *bio = ctx->bio; 173 bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); 174 175 /* 176 * fsverity_verify_bio() may call readahead() again, and while verity 177 * will be disabled for this, decryption and/or decompression may still 178 * be needed, resulting in another bio_post_read_ctx being allocated. 179 * So to prevent deadlocks we need to release the current ctx to the 180 * mempool first. This assumes that verity is the last post-read step. 181 */ 182 mempool_free(ctx, bio_post_read_ctx_pool); 183 bio->bi_private = NULL; 184 185 /* 186 * Verify the bio's pages with fs-verity. Exclude compressed pages, 187 * as those were handled separately by f2fs_end_read_compressed_page(). 188 */ 189 if (may_have_compressed_pages) { 190 struct folio_iter fi; 191 192 bio_for_each_folio_all(fi, bio) { 193 struct folio *folio = fi.folio; 194 195 if (!f2fs_is_compressed_page(folio) && 196 !fsverity_verify_page(&folio->page)) { 197 bio->bi_status = BLK_STS_IOERR; 198 break; 199 } 200 } 201 } else { 202 fsverity_verify_bio(bio); 203 } 204 205 f2fs_finish_read_bio(bio, true); 206} 207 208/* 209 * If the bio's data needs to be verified with fs-verity, then enqueue the 210 * verity work for the bio. Otherwise finish the bio now. 211 * 212 * Note that to avoid deadlocks, the verity work can't be done on the 213 * decryption/decompression workqueue. This is because verifying the data pages 214 * can involve reading verity metadata pages from the file, and these verity 215 * metadata pages may be encrypted and/or compressed. 216 */ 217static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) 218{ 219 struct bio_post_read_ctx *ctx = bio->bi_private; 220 221 if (ctx && (ctx->enabled_steps & STEP_VERITY)) { 222 INIT_WORK(&ctx->work, f2fs_verify_bio); 223 fsverity_enqueue_verify_work(&ctx->work); 224 } else { 225 f2fs_finish_read_bio(bio, in_task); 226 } 227} 228 229/* 230 * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last 231 * remaining page was read by @ctx->bio. 232 * 233 * Note that a bio may span clusters (even a mix of compressed and uncompressed 234 * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates 235 * that the bio includes at least one compressed page. The actual decompression 236 * is done on a per-cluster basis, not a per-bio basis. 237 */ 238static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, 239 bool in_task) 240{ 241 struct folio_iter fi; 242 bool all_compressed = true; 243 block_t blkaddr = ctx->fs_blkaddr; 244 245 bio_for_each_folio_all(fi, ctx->bio) { 246 struct folio *folio = fi.folio; 247 248 if (f2fs_is_compressed_page(folio)) 249 f2fs_end_read_compressed_page(folio, false, blkaddr, 250 in_task); 251 else 252 all_compressed = false; 253 254 blkaddr++; 255 } 256 257 ctx->decompression_attempted = true; 258 259 /* 260 * Optimization: if all the bio's pages are compressed, then scheduling 261 * the per-bio verity work is unnecessary, as verity will be fully 262 * handled at the compression cluster level. 263 */ 264 if (all_compressed) 265 ctx->enabled_steps &= ~STEP_VERITY; 266} 267 268static void f2fs_post_read_work(struct work_struct *work) 269{ 270 struct bio_post_read_ctx *ctx = 271 container_of(work, struct bio_post_read_ctx, work); 272 struct bio *bio = ctx->bio; 273 274 if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { 275 f2fs_finish_read_bio(bio, true); 276 return; 277 } 278 279 if (ctx->enabled_steps & STEP_DECOMPRESS) 280 f2fs_handle_step_decompress(ctx, true); 281 282 f2fs_verify_and_finish_bio(bio, true); 283} 284 285static void f2fs_read_end_io(struct bio *bio) 286{ 287 struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio)); 288 struct bio_post_read_ctx *ctx; 289 bool intask = in_task() && !irqs_disabled(); 290 291 iostat_update_and_unbind_ctx(bio); 292 ctx = bio->bi_private; 293 294 if (time_to_inject(sbi, FAULT_READ_IO)) 295 bio->bi_status = BLK_STS_IOERR; 296 297 if (bio->bi_status != BLK_STS_OK) { 298 f2fs_finish_read_bio(bio, intask); 299 return; 300 } 301 302 if (ctx) { 303 unsigned int enabled_steps = ctx->enabled_steps & 304 (STEP_DECRYPT | STEP_DECOMPRESS); 305 306 /* 307 * If we have only decompression step between decompression and 308 * decrypt, we don't need post processing for this. 309 */ 310 if (enabled_steps == STEP_DECOMPRESS && 311 !f2fs_low_mem_mode(sbi)) { 312 f2fs_handle_step_decompress(ctx, intask); 313 } else if (enabled_steps) { 314 INIT_WORK(&ctx->work, f2fs_post_read_work); 315 queue_work(ctx->sbi->post_read_wq, &ctx->work); 316 return; 317 } 318 } 319 320 f2fs_verify_and_finish_bio(bio, intask); 321} 322 323static void f2fs_write_end_io(struct bio *bio) 324{ 325 struct f2fs_sb_info *sbi; 326 struct folio_iter fi; 327 328 iostat_update_and_unbind_ctx(bio); 329 sbi = bio->bi_private; 330 331 if (time_to_inject(sbi, FAULT_WRITE_IO)) 332 bio->bi_status = BLK_STS_IOERR; 333 334 bio_for_each_folio_all(fi, bio) { 335 struct folio *folio = fi.folio; 336 enum count_type type; 337 338 if (fscrypt_is_bounce_folio(folio)) { 339 struct folio *io_folio = folio; 340 341 folio = fscrypt_pagecache_folio(io_folio); 342 fscrypt_free_bounce_page(&io_folio->page); 343 } 344 345#ifdef CONFIG_F2FS_FS_COMPRESSION 346 if (f2fs_is_compressed_page(folio)) { 347 f2fs_compress_write_end_io(bio, folio); 348 continue; 349 } 350#endif 351 352 type = WB_DATA_TYPE(folio, false); 353 354 if (unlikely(bio->bi_status != BLK_STS_OK)) { 355 mapping_set_error(folio->mapping, -EIO); 356 if (type == F2FS_WB_CP_DATA) 357 f2fs_stop_checkpoint(sbi, true, 358 STOP_CP_REASON_WRITE_FAIL); 359 } 360 361 if (is_node_folio(folio)) { 362 f2fs_sanity_check_node_footer(sbi, folio, 363 folio->index, NODE_TYPE_REGULAR, true); 364 f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); 365 } 366 367 dec_page_count(sbi, type); 368 369 /* 370 * we should access sbi before folio_end_writeback() to 371 * avoid racing w/ kill_f2fs_super() 372 */ 373 if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && 374 wq_has_sleeper(&sbi->cp_wait)) 375 wake_up(&sbi->cp_wait); 376 377 if (f2fs_in_warm_node_list(sbi, folio)) 378 f2fs_del_fsync_node_entry(sbi, folio); 379 folio_clear_f2fs_gcing(folio); 380 folio_end_writeback(folio); 381 } 382 383 bio_put(bio); 384} 385 386#ifdef CONFIG_BLK_DEV_ZONED 387static void f2fs_zone_write_end_io(struct bio *bio) 388{ 389 struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; 390 391 bio->bi_private = io->bi_private; 392 complete(&io->zone_wait); 393 f2fs_write_end_io(bio); 394} 395#endif 396 397struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, 398 block_t blk_addr, sector_t *sector) 399{ 400 struct block_device *bdev = sbi->sb->s_bdev; 401 int i; 402 403 if (f2fs_is_multi_device(sbi)) { 404 for (i = 0; i < sbi->s_ndevs; i++) { 405 if (FDEV(i).start_blk <= blk_addr && 406 FDEV(i).end_blk >= blk_addr) { 407 blk_addr -= FDEV(i).start_blk; 408 bdev = FDEV(i).bdev; 409 break; 410 } 411 } 412 } 413 414 if (sector) 415 *sector = SECTOR_FROM_BLOCK(blk_addr); 416 return bdev; 417} 418 419int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) 420{ 421 int i; 422 423 if (!f2fs_is_multi_device(sbi)) 424 return 0; 425 426 for (i = 0; i < sbi->s_ndevs; i++) 427 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) 428 return i; 429 return 0; 430} 431 432static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) 433{ 434 unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); 435 unsigned int fua_flag, meta_flag, io_flag; 436 blk_opf_t op_flags = 0; 437 438 if (fio->op != REQ_OP_WRITE) 439 return 0; 440 if (fio->type == DATA) 441 io_flag = fio->sbi->data_io_flag; 442 else if (fio->type == NODE) 443 io_flag = fio->sbi->node_io_flag; 444 else 445 return 0; 446 447 fua_flag = io_flag & temp_mask; 448 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; 449 450 /* 451 * data/node io flag bits per temp: 452 * REQ_META | REQ_FUA | 453 * 5 | 4 | 3 | 2 | 1 | 0 | 454 * Cold | Warm | Hot | Cold | Warm | Hot | 455 */ 456 if (BIT(fio->temp) & meta_flag) 457 op_flags |= REQ_META; 458 if (BIT(fio->temp) & fua_flag) 459 op_flags |= REQ_FUA; 460 461 if (fio->type == DATA && 462 F2FS_I(fio->folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE) 463 op_flags |= REQ_PRIO; 464 465 return op_flags; 466} 467 468static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) 469{ 470 struct f2fs_sb_info *sbi = fio->sbi; 471 struct block_device *bdev; 472 sector_t sector; 473 struct bio *bio; 474 475 bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); 476 bio = bio_alloc_bioset(bdev, npages, 477 fio->op | fio->op_flags | f2fs_io_flags(fio), 478 GFP_NOIO, &f2fs_bioset); 479 bio->bi_iter.bi_sector = sector; 480 if (is_read_io(fio->op)) { 481 bio->bi_end_io = f2fs_read_end_io; 482 bio->bi_private = NULL; 483 } else { 484 bio->bi_end_io = f2fs_write_end_io; 485 bio->bi_private = sbi; 486 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, 487 fio->type, fio->temp); 488 } 489 iostat_alloc_and_bind_ctx(sbi, bio, NULL); 490 491 if (fio->io_wbc) 492 wbc_init_bio(fio->io_wbc, bio); 493 494 return bio; 495} 496 497static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, 498 pgoff_t first_idx, 499 const struct f2fs_io_info *fio, 500 gfp_t gfp_mask) 501{ 502 /* 503 * The f2fs garbage collector sets ->encrypted_page when it wants to 504 * read/write raw data without encryption. 505 */ 506 if (!fio || !fio->encrypted_page) 507 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); 508} 509 510static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, 511 pgoff_t next_idx, 512 const struct f2fs_io_info *fio) 513{ 514 /* 515 * The f2fs garbage collector sets ->encrypted_page when it wants to 516 * read/write raw data without encryption. 517 */ 518 if (fio && fio->encrypted_page) 519 return !bio_has_crypt_ctx(bio); 520 521 return fscrypt_mergeable_bio(bio, inode, next_idx); 522} 523 524void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, 525 enum page_type type) 526{ 527 WARN_ON_ONCE(!is_read_io(bio_op(bio))); 528 trace_f2fs_submit_read_bio(sbi->sb, type, bio); 529 530 iostat_update_submit_ctx(bio, type); 531 submit_bio(bio); 532} 533 534static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, 535 enum page_type type) 536{ 537 WARN_ON_ONCE(is_read_io(bio_op(bio))); 538 trace_f2fs_submit_write_bio(sbi->sb, type, bio); 539 iostat_update_submit_ctx(bio, type); 540 submit_bio(bio); 541} 542 543static void __submit_merged_bio(struct f2fs_bio_info *io) 544{ 545 struct f2fs_io_info *fio = &io->fio; 546 547 if (!io->bio) 548 return; 549 550 if (is_read_io(fio->op)) { 551 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 552 f2fs_submit_read_bio(io->sbi, io->bio, fio->type); 553 } else { 554 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 555 f2fs_submit_write_bio(io->sbi, io->bio, fio->type); 556 } 557 io->bio = NULL; 558} 559 560static bool __has_merged_page(struct bio *bio, struct inode *inode, 561 struct folio *folio, nid_t ino) 562{ 563 struct folio_iter fi; 564 565 if (!bio) 566 return false; 567 568 if (!inode && !folio && !ino) 569 return true; 570 571 bio_for_each_folio_all(fi, bio) { 572 struct folio *target = fi.folio; 573 574 if (fscrypt_is_bounce_folio(target)) { 575 target = fscrypt_pagecache_folio(target); 576 if (IS_ERR(target)) 577 continue; 578 } 579 if (f2fs_is_compressed_page(target)) { 580 target = f2fs_compress_control_folio(target); 581 if (IS_ERR(target)) 582 continue; 583 } 584 585 if (inode && inode == target->mapping->host) 586 return true; 587 if (folio && folio == target) 588 return true; 589 if (ino && ino == ino_of_node(target)) 590 return true; 591 } 592 593 return false; 594} 595 596int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) 597{ 598 int i; 599 600 for (i = 0; i < NR_PAGE_TYPE; i++) { 601 int n = (i == META) ? 1 : NR_TEMP_TYPE; 602 int j; 603 604 sbi->write_io[i] = f2fs_kmalloc(sbi, 605 array_size(n, sizeof(struct f2fs_bio_info)), 606 GFP_KERNEL); 607 if (!sbi->write_io[i]) 608 return -ENOMEM; 609 610 for (j = HOT; j < n; j++) { 611 struct f2fs_bio_info *io = &sbi->write_io[i][j]; 612 613 init_f2fs_rwsem(&io->io_rwsem); 614 io->sbi = sbi; 615 io->bio = NULL; 616 io->last_block_in_bio = 0; 617 spin_lock_init(&io->io_lock); 618 INIT_LIST_HEAD(&io->io_list); 619 INIT_LIST_HEAD(&io->bio_list); 620 init_f2fs_rwsem(&io->bio_list_lock); 621#ifdef CONFIG_BLK_DEV_ZONED 622 init_completion(&io->zone_wait); 623 io->zone_pending_bio = NULL; 624 io->bi_private = NULL; 625#endif 626 } 627 } 628 629 return 0; 630} 631 632static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, 633 enum page_type type, enum temp_type temp) 634{ 635 enum page_type btype = PAGE_TYPE_OF_BIO(type); 636 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 637 638 f2fs_down_write(&io->io_rwsem); 639 640 if (!io->bio) 641 goto unlock_out; 642 643 /* change META to META_FLUSH in the checkpoint procedure */ 644 if (type >= META_FLUSH) { 645 io->fio.type = META_FLUSH; 646 io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC; 647 if (!test_opt(sbi, NOBARRIER)) 648 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; 649 } 650 __submit_merged_bio(io); 651unlock_out: 652 f2fs_up_write(&io->io_rwsem); 653} 654 655static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, 656 struct inode *inode, struct folio *folio, 657 nid_t ino, enum page_type type, bool force) 658{ 659 enum temp_type temp; 660 bool ret = true; 661 662 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 663 if (!force) { 664 enum page_type btype = PAGE_TYPE_OF_BIO(type); 665 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 666 667 f2fs_down_read(&io->io_rwsem); 668 ret = __has_merged_page(io->bio, inode, folio, ino); 669 f2fs_up_read(&io->io_rwsem); 670 } 671 if (ret) 672 __f2fs_submit_merged_write(sbi, type, temp); 673 674 /* TODO: use HOT temp only for meta pages now. */ 675 if (type >= META) 676 break; 677 } 678} 679 680void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) 681{ 682 __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true); 683} 684 685void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, 686 struct inode *inode, struct folio *folio, 687 nid_t ino, enum page_type type) 688{ 689 __submit_merged_write_cond(sbi, inode, folio, ino, type, false); 690} 691 692void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) 693{ 694 f2fs_submit_merged_write(sbi, DATA); 695 f2fs_submit_merged_write(sbi, NODE); 696 f2fs_submit_merged_write(sbi, META); 697} 698 699/* 700 * Fill the locked page with data located in the block address. 701 * A caller needs to unlock the page on failure. 702 */ 703int f2fs_submit_page_bio(struct f2fs_io_info *fio) 704{ 705 struct bio *bio; 706 struct folio *fio_folio = fio->folio; 707 struct folio *data_folio = fio->encrypted_page ? 708 page_folio(fio->encrypted_page) : fio_folio; 709 710 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 711 fio->is_por ? META_POR : (__is_meta_io(fio) ? 712 META_GENERIC : DATA_GENERIC_ENHANCE))) 713 return -EFSCORRUPTED; 714 715 trace_f2fs_submit_folio_bio(data_folio, fio); 716 717 /* Allocate a new bio */ 718 bio = __bio_alloc(fio, 1); 719 720 f2fs_set_bio_crypt_ctx(bio, fio_folio->mapping->host, 721 fio_folio->index, fio, GFP_NOIO); 722 bio_add_folio_nofail(bio, data_folio, folio_size(data_folio), 0); 723 724 if (fio->io_wbc && !is_read_io(fio->op)) 725 wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE); 726 727 inc_page_count(fio->sbi, is_read_io(fio->op) ? 728 __read_io_type(data_folio) : WB_DATA_TYPE(fio->folio, false)); 729 730 if (is_read_io(bio_op(bio))) 731 f2fs_submit_read_bio(fio->sbi, bio, fio->type); 732 else 733 f2fs_submit_write_bio(fio->sbi, bio, fio->type); 734 return 0; 735} 736 737static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 738 block_t last_blkaddr, block_t cur_blkaddr) 739{ 740 if (unlikely(sbi->max_io_bytes && 741 bio->bi_iter.bi_size >= sbi->max_io_bytes)) 742 return false; 743 if (last_blkaddr + 1 != cur_blkaddr) 744 return false; 745 return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); 746} 747 748static bool io_type_is_mergeable(struct f2fs_bio_info *io, 749 struct f2fs_io_info *fio) 750{ 751 blk_opf_t mask = ~(REQ_PREFLUSH | REQ_FUA); 752 753 if (io->fio.op != fio->op) 754 return false; 755 return (io->fio.op_flags & mask) == (fio->op_flags & mask); 756} 757 758static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 759 struct f2fs_bio_info *io, 760 struct f2fs_io_info *fio, 761 block_t last_blkaddr, 762 block_t cur_blkaddr) 763{ 764 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) 765 return false; 766 return io_type_is_mergeable(io, fio); 767} 768 769static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, 770 struct folio *folio, enum temp_type temp) 771{ 772 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 773 struct bio_entry *be; 774 775 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL); 776 be->bio = bio; 777 bio_get(bio); 778 779 bio_add_folio_nofail(bio, folio, folio_size(folio), 0); 780 781 f2fs_down_write(&io->bio_list_lock); 782 list_add_tail(&be->list, &io->bio_list); 783 f2fs_up_write(&io->bio_list_lock); 784} 785 786static void del_bio_entry(struct bio_entry *be) 787{ 788 list_del(&be->list); 789 kmem_cache_free(bio_entry_slab, be); 790} 791 792static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, 793 struct folio *folio) 794{ 795 struct folio *fio_folio = fio->folio; 796 struct f2fs_sb_info *sbi = fio->sbi; 797 enum temp_type temp; 798 bool found = false; 799 int ret = -EAGAIN; 800 801 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 802 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 803 struct list_head *head = &io->bio_list; 804 struct bio_entry *be; 805 806 f2fs_down_write(&io->bio_list_lock); 807 list_for_each_entry(be, head, list) { 808 if (be->bio != *bio) 809 continue; 810 811 found = true; 812 813 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio, 814 *fio->last_block, 815 fio->new_blkaddr)); 816 if (f2fs_crypt_mergeable_bio(*bio, 817 fio_folio->mapping->host, 818 fio_folio->index, fio) && 819 bio_add_folio(*bio, folio, folio_size(folio), 0)) { 820 ret = 0; 821 break; 822 } 823 824 /* page can't be merged into bio; submit the bio */ 825 del_bio_entry(be); 826 f2fs_submit_write_bio(sbi, *bio, DATA); 827 break; 828 } 829 f2fs_up_write(&io->bio_list_lock); 830 } 831 832 if (ret) { 833 bio_put(*bio); 834 *bio = NULL; 835 } 836 837 return ret; 838} 839 840void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, 841 struct bio **bio, struct folio *folio) 842{ 843 enum temp_type temp; 844 bool found = false; 845 struct bio *target = bio ? *bio : NULL; 846 847 f2fs_bug_on(sbi, !target && !folio); 848 849 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 850 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 851 struct list_head *head = &io->bio_list; 852 struct bio_entry *be; 853 854 if (list_empty(head)) 855 continue; 856 857 f2fs_down_read(&io->bio_list_lock); 858 list_for_each_entry(be, head, list) { 859 if (target) 860 found = (target == be->bio); 861 else 862 found = __has_merged_page(be->bio, NULL, 863 folio, 0); 864 if (found) 865 break; 866 } 867 f2fs_up_read(&io->bio_list_lock); 868 869 if (!found) 870 continue; 871 872 found = false; 873 874 f2fs_down_write(&io->bio_list_lock); 875 list_for_each_entry(be, head, list) { 876 if (target) 877 found = (target == be->bio); 878 else 879 found = __has_merged_page(be->bio, NULL, 880 folio, 0); 881 if (found) { 882 target = be->bio; 883 del_bio_entry(be); 884 break; 885 } 886 } 887 f2fs_up_write(&io->bio_list_lock); 888 } 889 890 if (found) 891 f2fs_submit_write_bio(sbi, target, DATA); 892 if (bio && *bio) { 893 bio_put(*bio); 894 *bio = NULL; 895 } 896} 897 898int f2fs_merge_page_bio(struct f2fs_io_info *fio) 899{ 900 struct bio *bio = *fio->bio; 901 struct folio *data_folio = fio->encrypted_page ? 902 page_folio(fio->encrypted_page) : fio->folio; 903 struct folio *folio = fio->folio; 904 905 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 906 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) 907 return -EFSCORRUPTED; 908 909 trace_f2fs_submit_folio_bio(data_folio, fio); 910 911 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, 912 fio->new_blkaddr)) 913 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); 914alloc_new: 915 if (!bio) { 916 bio = __bio_alloc(fio, BIO_MAX_VECS); 917 f2fs_set_bio_crypt_ctx(bio, folio->mapping->host, 918 folio->index, fio, GFP_NOIO); 919 920 add_bio_entry(fio->sbi, bio, data_folio, fio->temp); 921 } else { 922 if (add_ipu_page(fio, &bio, data_folio)) 923 goto alloc_new; 924 } 925 926 if (fio->io_wbc) 927 wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio)); 928 929 inc_page_count(fio->sbi, WB_DATA_TYPE(folio, false)); 930 931 *fio->last_block = fio->new_blkaddr; 932 *fio->bio = bio; 933 934 return 0; 935} 936 937#ifdef CONFIG_BLK_DEV_ZONED 938static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr) 939{ 940 struct block_device *bdev = sbi->sb->s_bdev; 941 int devi = 0; 942 943 if (f2fs_is_multi_device(sbi)) { 944 devi = f2fs_target_device_index(sbi, blkaddr); 945 if (blkaddr < FDEV(devi).start_blk || 946 blkaddr > FDEV(devi).end_blk) { 947 f2fs_err(sbi, "Invalid block %x", blkaddr); 948 return false; 949 } 950 blkaddr -= FDEV(devi).start_blk; 951 bdev = FDEV(devi).bdev; 952 } 953 return bdev_is_zoned(bdev) && 954 f2fs_blkz_is_seq(sbi, devi, blkaddr) && 955 (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); 956} 957#endif 958 959void f2fs_submit_page_write(struct f2fs_io_info *fio) 960{ 961 struct f2fs_sb_info *sbi = fio->sbi; 962 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 963 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; 964 struct folio *bio_folio; 965 enum count_type type; 966 967 f2fs_bug_on(sbi, is_read_io(fio->op)); 968 969 f2fs_down_write(&io->io_rwsem); 970next: 971#ifdef CONFIG_BLK_DEV_ZONED 972 if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { 973 wait_for_completion_io(&io->zone_wait); 974 bio_put(io->zone_pending_bio); 975 io->zone_pending_bio = NULL; 976 io->bi_private = NULL; 977 } 978#endif 979 980 if (fio->in_list) { 981 spin_lock(&io->io_lock); 982 if (list_empty(&io->io_list)) { 983 spin_unlock(&io->io_lock); 984 goto out; 985 } 986 fio = list_first_entry(&io->io_list, 987 struct f2fs_io_info, list); 988 list_del(&fio->list); 989 spin_unlock(&io->io_lock); 990 } 991 992 verify_fio_blkaddr(fio); 993 994 if (fio->encrypted_page) 995 bio_folio = page_folio(fio->encrypted_page); 996 else if (fio->compressed_page) 997 bio_folio = page_folio(fio->compressed_page); 998 else 999 bio_folio = fio->folio; 1000 1001 /* set submitted = true as a return value */ 1002 fio->submitted = 1; 1003 1004 type = WB_DATA_TYPE(bio_folio, fio->compressed_page); 1005 inc_page_count(sbi, type); 1006 1007 if (io->bio && 1008 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, 1009 fio->new_blkaddr) || 1010 !f2fs_crypt_mergeable_bio(io->bio, fio_inode(fio), 1011 bio_folio->index, fio))) 1012 __submit_merged_bio(io); 1013alloc_new: 1014 if (io->bio == NULL) { 1015 io->bio = __bio_alloc(fio, BIO_MAX_VECS); 1016 f2fs_set_bio_crypt_ctx(io->bio, fio_inode(fio), 1017 bio_folio->index, fio, GFP_NOIO); 1018 io->fio = *fio; 1019 } 1020 1021 if (!bio_add_folio(io->bio, bio_folio, folio_size(bio_folio), 0)) { 1022 __submit_merged_bio(io); 1023 goto alloc_new; 1024 } 1025 1026 if (fio->io_wbc) 1027 wbc_account_cgroup_owner(fio->io_wbc, fio->folio, 1028 folio_size(fio->folio)); 1029 1030 io->last_block_in_bio = fio->new_blkaddr; 1031 1032 trace_f2fs_submit_folio_write(fio->folio, fio); 1033#ifdef CONFIG_BLK_DEV_ZONED 1034 if (f2fs_sb_has_blkzoned(sbi) && btype < META && 1035 is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { 1036 bio_get(io->bio); 1037 reinit_completion(&io->zone_wait); 1038 io->bi_private = io->bio->bi_private; 1039 io->bio->bi_private = io; 1040 io->bio->bi_end_io = f2fs_zone_write_end_io; 1041 io->zone_pending_bio = io->bio; 1042 __submit_merged_bio(io); 1043 } 1044#endif 1045 if (fio->in_list) 1046 goto next; 1047out: 1048 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || 1049 !f2fs_is_checkpoint_ready(sbi)) 1050 __submit_merged_bio(io); 1051 f2fs_up_write(&io->io_rwsem); 1052} 1053 1054static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, 1055 unsigned nr_pages, blk_opf_t op_flag, 1056 pgoff_t first_idx, bool for_write) 1057{ 1058 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1059 struct bio *bio; 1060 struct bio_post_read_ctx *ctx = NULL; 1061 unsigned int post_read_steps = 0; 1062 sector_t sector; 1063 struct block_device *bdev = f2fs_target_device(sbi, blkaddr, §or); 1064 1065 bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages), 1066 REQ_OP_READ | op_flag, 1067 for_write ? GFP_NOIO : GFP_KERNEL, &f2fs_bioset); 1068 bio->bi_iter.bi_sector = sector; 1069 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); 1070 bio->bi_end_io = f2fs_read_end_io; 1071 1072 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 1073 post_read_steps |= STEP_DECRYPT; 1074 1075 if (f2fs_need_verity(inode, first_idx)) 1076 post_read_steps |= STEP_VERITY; 1077 1078 /* 1079 * STEP_DECOMPRESS is handled specially, since a compressed file might 1080 * contain both compressed and uncompressed clusters. We'll allocate a 1081 * bio_post_read_ctx if the file is compressed, but the caller is 1082 * responsible for enabling STEP_DECOMPRESS if it's actually needed. 1083 */ 1084 1085 if (post_read_steps || f2fs_compressed_file(inode)) { 1086 /* Due to the mempool, this never fails. */ 1087 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); 1088 ctx->bio = bio; 1089 ctx->sbi = sbi; 1090 ctx->enabled_steps = post_read_steps; 1091 ctx->fs_blkaddr = blkaddr; 1092 ctx->decompression_attempted = false; 1093 bio->bi_private = ctx; 1094 } 1095 iostat_alloc_and_bind_ctx(sbi, bio, ctx); 1096 1097 return bio; 1098} 1099 1100/* This can handle encryption stuffs */ 1101static void f2fs_submit_page_read(struct inode *inode, struct folio *folio, 1102 block_t blkaddr, blk_opf_t op_flags, 1103 bool for_write) 1104{ 1105 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1106 struct bio *bio; 1107 1108 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, 1109 folio->index, for_write); 1110 1111 /* wait for GCed page writeback via META_MAPPING */ 1112 f2fs_wait_on_block_writeback(inode, blkaddr); 1113 1114 if (!bio_add_folio(bio, folio, PAGE_SIZE, 0)) 1115 f2fs_bug_on(sbi, 1); 1116 1117 inc_page_count(sbi, F2FS_RD_DATA); 1118 f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); 1119 f2fs_submit_read_bio(sbi, bio, DATA); 1120} 1121 1122static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1123{ 1124 __le32 *addr = get_dnode_addr(dn->inode, dn->node_folio); 1125 1126 dn->data_blkaddr = blkaddr; 1127 addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 1128} 1129 1130/* 1131 * Lock ordering for the change of data block address: 1132 * ->data_page 1133 * ->node_folio 1134 * update block addresses in the node page 1135 */ 1136void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1137{ 1138 f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true); 1139 __set_data_blkaddr(dn, blkaddr); 1140 if (folio_mark_dirty(dn->node_folio)) 1141 dn->node_changed = true; 1142} 1143 1144void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1145{ 1146 f2fs_set_data_blkaddr(dn, blkaddr); 1147 f2fs_update_read_extent_cache(dn); 1148} 1149 1150/* dn->ofs_in_node will be returned with up-to-date last block pointer */ 1151int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) 1152{ 1153 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1154 int err; 1155 1156 if (!count) 1157 return 0; 1158 1159 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1160 return -EPERM; 1161 err = inc_valid_block_count(sbi, dn->inode, &count, true); 1162 if (unlikely(err)) 1163 return err; 1164 1165 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, 1166 dn->ofs_in_node, count); 1167 1168 f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true); 1169 1170 for (; count > 0; dn->ofs_in_node++) { 1171 block_t blkaddr = f2fs_data_blkaddr(dn); 1172 1173 if (blkaddr == NULL_ADDR) { 1174 __set_data_blkaddr(dn, NEW_ADDR); 1175 count--; 1176 } 1177 } 1178 1179 if (folio_mark_dirty(dn->node_folio)) 1180 dn->node_changed = true; 1181 return 0; 1182} 1183 1184/* Should keep dn->ofs_in_node unchanged */ 1185int f2fs_reserve_new_block(struct dnode_of_data *dn) 1186{ 1187 unsigned int ofs_in_node = dn->ofs_in_node; 1188 int ret; 1189 1190 ret = f2fs_reserve_new_blocks(dn, 1); 1191 dn->ofs_in_node = ofs_in_node; 1192 return ret; 1193} 1194 1195int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 1196{ 1197 bool need_put = dn->inode_folio ? false : true; 1198 int err; 1199 1200 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); 1201 if (err) 1202 return err; 1203 1204 if (dn->data_blkaddr == NULL_ADDR) 1205 err = f2fs_reserve_new_block(dn); 1206 if (err || need_put) 1207 f2fs_put_dnode(dn); 1208 return err; 1209} 1210 1211struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index, 1212 blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs) 1213{ 1214 struct address_space *mapping = inode->i_mapping; 1215 struct dnode_of_data dn; 1216 struct folio *folio; 1217 int err; 1218 1219 folio = f2fs_grab_cache_folio(mapping, index, for_write); 1220 if (IS_ERR(folio)) 1221 return folio; 1222 1223 if (f2fs_lookup_read_extent_cache_block(inode, index, 1224 &dn.data_blkaddr)) { 1225 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, 1226 DATA_GENERIC_ENHANCE_READ)) { 1227 err = -EFSCORRUPTED; 1228 goto put_err; 1229 } 1230 goto got_it; 1231 } 1232 1233 set_new_dnode(&dn, inode, NULL, NULL, 0); 1234 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 1235 if (err) { 1236 if (err == -ENOENT && next_pgofs) 1237 *next_pgofs = f2fs_get_next_page_offset(&dn, index); 1238 goto put_err; 1239 } 1240 f2fs_put_dnode(&dn); 1241 1242 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 1243 err = -ENOENT; 1244 if (next_pgofs) 1245 *next_pgofs = index + 1; 1246 goto put_err; 1247 } 1248 if (dn.data_blkaddr != NEW_ADDR && 1249 !f2fs_is_valid_blkaddr(F2FS_I_SB(inode), 1250 dn.data_blkaddr, 1251 DATA_GENERIC_ENHANCE)) { 1252 err = -EFSCORRUPTED; 1253 goto put_err; 1254 } 1255got_it: 1256 if (folio_test_uptodate(folio)) { 1257 folio_unlock(folio); 1258 return folio; 1259 } 1260 1261 /* 1262 * A new dentry page is allocated but not able to be written, since its 1263 * new inode page couldn't be allocated due to -ENOSPC. 1264 * In such the case, its blkaddr can be remained as NEW_ADDR. 1265 * see, f2fs_add_link -> f2fs_get_new_data_folio -> 1266 * f2fs_init_inode_metadata. 1267 */ 1268 if (dn.data_blkaddr == NEW_ADDR) { 1269 folio_zero_segment(folio, 0, folio_size(folio)); 1270 if (!folio_test_uptodate(folio)) 1271 folio_mark_uptodate(folio); 1272 folio_unlock(folio); 1273 return folio; 1274 } 1275 1276 f2fs_submit_page_read(inode, folio, dn.data_blkaddr, 1277 op_flags, for_write); 1278 return folio; 1279 1280put_err: 1281 f2fs_folio_put(folio, true); 1282 return ERR_PTR(err); 1283} 1284 1285struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index, 1286 pgoff_t *next_pgofs) 1287{ 1288 struct address_space *mapping = inode->i_mapping; 1289 struct folio *folio; 1290 1291 folio = f2fs_filemap_get_folio(mapping, index, FGP_ACCESSED, 0); 1292 if (IS_ERR(folio)) 1293 goto read; 1294 if (folio_test_uptodate(folio)) 1295 return folio; 1296 f2fs_folio_put(folio, false); 1297 1298read: 1299 folio = f2fs_get_read_data_folio(inode, index, 0, false, next_pgofs); 1300 if (IS_ERR(folio)) 1301 return folio; 1302 1303 if (folio_test_uptodate(folio)) 1304 return folio; 1305 1306 folio_wait_locked(folio); 1307 if (unlikely(!folio_test_uptodate(folio))) { 1308 f2fs_folio_put(folio, false); 1309 return ERR_PTR(-EIO); 1310 } 1311 return folio; 1312} 1313 1314/* 1315 * If it tries to access a hole, return an error. 1316 * Because, the callers, functions in dir.c and GC, should be able to know 1317 * whether this page exists or not. 1318 */ 1319struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index, 1320 bool for_write) 1321{ 1322 struct address_space *mapping = inode->i_mapping; 1323 struct folio *folio; 1324 1325 folio = f2fs_get_read_data_folio(inode, index, 0, for_write, NULL); 1326 if (IS_ERR(folio)) 1327 return folio; 1328 1329 /* wait for read completion */ 1330 folio_lock(folio); 1331 if (unlikely(folio->mapping != mapping || !folio_test_uptodate(folio))) { 1332 f2fs_folio_put(folio, true); 1333 return ERR_PTR(-EIO); 1334 } 1335 return folio; 1336} 1337 1338/* 1339 * Caller ensures that this data page is never allocated. 1340 * A new zero-filled data page is allocated in the page cache. 1341 * 1342 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 1343 * f2fs_unlock_op(). 1344 * Note that, ifolio is set only by make_empty_dir, and if any error occur, 1345 * ifolio should be released by this function. 1346 */ 1347struct folio *f2fs_get_new_data_folio(struct inode *inode, 1348 struct folio *ifolio, pgoff_t index, bool new_i_size) 1349{ 1350 struct address_space *mapping = inode->i_mapping; 1351 struct folio *folio; 1352 struct dnode_of_data dn; 1353 int err; 1354 1355 folio = f2fs_grab_cache_folio(mapping, index, true); 1356 if (IS_ERR(folio)) { 1357 /* 1358 * before exiting, we should make sure ifolio will be released 1359 * if any error occur. 1360 */ 1361 f2fs_folio_put(ifolio, true); 1362 return ERR_PTR(-ENOMEM); 1363 } 1364 1365 set_new_dnode(&dn, inode, ifolio, NULL, 0); 1366 err = f2fs_reserve_block(&dn, index); 1367 if (err) { 1368 f2fs_folio_put(folio, true); 1369 return ERR_PTR(err); 1370 } 1371 if (!ifolio) 1372 f2fs_put_dnode(&dn); 1373 1374 if (folio_test_uptodate(folio)) 1375 goto got_it; 1376 1377 if (dn.data_blkaddr == NEW_ADDR) { 1378 folio_zero_segment(folio, 0, folio_size(folio)); 1379 if (!folio_test_uptodate(folio)) 1380 folio_mark_uptodate(folio); 1381 } else { 1382 f2fs_folio_put(folio, true); 1383 1384 /* if ifolio exists, blkaddr should be NEW_ADDR */ 1385 f2fs_bug_on(F2FS_I_SB(inode), ifolio); 1386 folio = f2fs_get_lock_data_folio(inode, index, true); 1387 if (IS_ERR(folio)) 1388 return folio; 1389 } 1390got_it: 1391 if (new_i_size && i_size_read(inode) < 1392 ((loff_t)(index + 1) << PAGE_SHIFT)) 1393 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); 1394 return folio; 1395} 1396 1397static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) 1398{ 1399 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1400 struct f2fs_summary sum; 1401 struct node_info ni; 1402 block_t old_blkaddr; 1403 blkcnt_t count = 1; 1404 int err; 1405 1406 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1407 return -EPERM; 1408 1409 err = f2fs_get_node_info(sbi, dn->nid, &ni, false); 1410 if (err) 1411 return err; 1412 1413 dn->data_blkaddr = f2fs_data_blkaddr(dn); 1414 if (dn->data_blkaddr == NULL_ADDR) { 1415 err = inc_valid_block_count(sbi, dn->inode, &count, true); 1416 if (unlikely(err)) 1417 return err; 1418 } 1419 1420 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1421 old_blkaddr = dn->data_blkaddr; 1422 err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, 1423 &dn->data_blkaddr, &sum, seg_type, NULL); 1424 if (err) 1425 return err; 1426 1427 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 1428 f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); 1429 1430 f2fs_update_data_blkaddr(dn, dn->data_blkaddr); 1431 return 0; 1432} 1433 1434static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) 1435{ 1436 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1437 f2fs_down_read(&sbi->node_change); 1438 else 1439 f2fs_lock_op(sbi); 1440} 1441 1442static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) 1443{ 1444 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1445 f2fs_up_read(&sbi->node_change); 1446 else 1447 f2fs_unlock_op(sbi); 1448} 1449 1450int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) 1451{ 1452 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1453 int err = 0; 1454 1455 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1456 if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, 1457 &dn->data_blkaddr)) 1458 err = f2fs_reserve_block(dn, index); 1459 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1460 1461 return err; 1462} 1463 1464static int f2fs_map_no_dnode(struct inode *inode, 1465 struct f2fs_map_blocks *map, struct dnode_of_data *dn, 1466 pgoff_t pgoff) 1467{ 1468 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1469 1470 /* 1471 * There is one exceptional case that read_node_page() may return 1472 * -ENOENT due to filesystem has been shutdown or cp_error, return 1473 * -EIO in that case. 1474 */ 1475 if (map->m_may_create && 1476 (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi))) 1477 return -EIO; 1478 1479 if (map->m_next_pgofs) 1480 *map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff); 1481 if (map->m_next_extent) 1482 *map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff); 1483 return 0; 1484} 1485 1486static bool f2fs_map_blocks_cached(struct inode *inode, 1487 struct f2fs_map_blocks *map, int flag) 1488{ 1489 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1490 unsigned int maxblocks = map->m_len; 1491 pgoff_t pgoff = (pgoff_t)map->m_lblk; 1492 struct extent_info ei = {}; 1493 1494 if (!f2fs_lookup_read_extent_cache(inode, pgoff, &ei)) 1495 return false; 1496 1497 map->m_pblk = ei.blk + pgoff - ei.fofs; 1498 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff); 1499 map->m_flags = F2FS_MAP_MAPPED; 1500 if (map->m_next_extent) 1501 *map->m_next_extent = pgoff + map->m_len; 1502 1503 /* for hardware encryption, but to avoid potential issue in future */ 1504 if (flag == F2FS_GET_BLOCK_DIO) 1505 f2fs_wait_on_block_writeback_range(inode, 1506 map->m_pblk, map->m_len); 1507 1508 if (f2fs_allow_multi_device_dio(sbi, flag)) { 1509 int bidx = f2fs_target_device_index(sbi, map->m_pblk); 1510 struct f2fs_dev_info *dev = &sbi->devs[bidx]; 1511 1512 map->m_bdev = dev->bdev; 1513 map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); 1514 map->m_pblk -= dev->start_blk; 1515 } else { 1516 map->m_bdev = inode->i_sb->s_bdev; 1517 } 1518 return true; 1519} 1520 1521static bool map_is_mergeable(struct f2fs_sb_info *sbi, 1522 struct f2fs_map_blocks *map, 1523 block_t blkaddr, int flag, int bidx, 1524 int ofs) 1525{ 1526 if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) 1527 return false; 1528 if (map->m_pblk != NEW_ADDR && blkaddr == (map->m_pblk + ofs)) 1529 return true; 1530 if (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) 1531 return true; 1532 if (flag == F2FS_GET_BLOCK_PRE_DIO) 1533 return true; 1534 if (flag == F2FS_GET_BLOCK_DIO && 1535 map->m_pblk == NULL_ADDR && blkaddr == NULL_ADDR) 1536 return true; 1537 return false; 1538} 1539 1540/* 1541 * f2fs_map_blocks() tries to find or build mapping relationship which 1542 * maps continuous logical blocks to physical blocks, and return such 1543 * info via f2fs_map_blocks structure. 1544 */ 1545int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) 1546{ 1547 unsigned int maxblocks = map->m_len; 1548 struct dnode_of_data dn; 1549 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1550 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; 1551 pgoff_t pgofs, end_offset, end; 1552 int err = 0, ofs = 1; 1553 unsigned int ofs_in_node, last_ofs_in_node; 1554 blkcnt_t prealloc; 1555 block_t blkaddr; 1556 unsigned int start_pgofs; 1557 int bidx = 0; 1558 bool is_hole; 1559 bool lfs_dio_write; 1560 1561 if (!maxblocks) 1562 return 0; 1563 1564 lfs_dio_write = (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && 1565 map->m_may_create); 1566 1567 if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) 1568 goto out; 1569 1570 map->m_bdev = inode->i_sb->s_bdev; 1571 map->m_multidev_dio = 1572 f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag); 1573 1574 map->m_len = 0; 1575 map->m_flags = 0; 1576 1577 /* it only supports block size == page size */ 1578 pgofs = (pgoff_t)map->m_lblk; 1579 end = pgofs + maxblocks; 1580 1581 if (flag == F2FS_GET_BLOCK_PRECACHE) 1582 mode = LOOKUP_NODE_RA; 1583 1584next_dnode: 1585 if (map->m_may_create) { 1586 if (f2fs_lfs_mode(sbi)) 1587 f2fs_balance_fs(sbi, true); 1588 f2fs_map_lock(sbi, flag); 1589 } 1590 1591 /* When reading holes, we need its node page */ 1592 set_new_dnode(&dn, inode, NULL, NULL, 0); 1593 err = f2fs_get_dnode_of_data(&dn, pgofs, mode); 1594 if (err) { 1595 if (flag == F2FS_GET_BLOCK_BMAP) 1596 map->m_pblk = 0; 1597 if (err == -ENOENT) 1598 err = f2fs_map_no_dnode(inode, map, &dn, pgofs); 1599 goto unlock_out; 1600 } 1601 1602 start_pgofs = pgofs; 1603 prealloc = 0; 1604 last_ofs_in_node = ofs_in_node = dn.ofs_in_node; 1605 end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); 1606 1607next_block: 1608 blkaddr = f2fs_data_blkaddr(&dn); 1609 is_hole = !__is_valid_data_blkaddr(blkaddr); 1610 if (!is_hole && 1611 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { 1612 err = -EFSCORRUPTED; 1613 goto sync_out; 1614 } 1615 1616 /* use out-place-update for direct IO under LFS mode */ 1617 if (map->m_may_create && (is_hole || 1618 (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && 1619 !f2fs_is_pinned_file(inode) && map->m_last_pblk != blkaddr))) { 1620 if (unlikely(f2fs_cp_error(sbi))) { 1621 err = -EIO; 1622 goto sync_out; 1623 } 1624 1625 switch (flag) { 1626 case F2FS_GET_BLOCK_PRE_AIO: 1627 if (blkaddr == NULL_ADDR) { 1628 prealloc++; 1629 last_ofs_in_node = dn.ofs_in_node; 1630 } 1631 break; 1632 case F2FS_GET_BLOCK_PRE_DIO: 1633 case F2FS_GET_BLOCK_DIO: 1634 err = __allocate_data_block(&dn, map->m_seg_type); 1635 if (err) 1636 goto sync_out; 1637 if (flag == F2FS_GET_BLOCK_PRE_DIO) 1638 file_need_truncate(inode); 1639 set_inode_flag(inode, FI_APPEND_WRITE); 1640 break; 1641 default: 1642 WARN_ON_ONCE(1); 1643 err = -EIO; 1644 goto sync_out; 1645 } 1646 1647 blkaddr = dn.data_blkaddr; 1648 if (is_hole) 1649 map->m_flags |= F2FS_MAP_NEW; 1650 } else if (is_hole) { 1651 if (f2fs_compressed_file(inode) && 1652 f2fs_sanity_check_cluster(&dn)) { 1653 err = -EFSCORRUPTED; 1654 f2fs_handle_error(sbi, 1655 ERROR_CORRUPTED_CLUSTER); 1656 goto sync_out; 1657 } 1658 1659 switch (flag) { 1660 case F2FS_GET_BLOCK_PRECACHE: 1661 goto sync_out; 1662 case F2FS_GET_BLOCK_BMAP: 1663 map->m_pblk = 0; 1664 goto sync_out; 1665 case F2FS_GET_BLOCK_FIEMAP: 1666 if (blkaddr == NULL_ADDR) { 1667 if (map->m_next_pgofs) 1668 *map->m_next_pgofs = pgofs + 1; 1669 goto sync_out; 1670 } 1671 break; 1672 case F2FS_GET_BLOCK_DIO: 1673 if (map->m_next_pgofs) 1674 *map->m_next_pgofs = pgofs + 1; 1675 break; 1676 default: 1677 /* for defragment case */ 1678 if (map->m_next_pgofs) 1679 *map->m_next_pgofs = pgofs + 1; 1680 goto sync_out; 1681 } 1682 } 1683 1684 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1685 goto skip; 1686 1687 if (map->m_multidev_dio) 1688 bidx = f2fs_target_device_index(sbi, blkaddr); 1689 1690 if (map->m_len == 0) { 1691 /* reserved delalloc block should be mapped for fiemap. */ 1692 if (blkaddr == NEW_ADDR) 1693 map->m_flags |= F2FS_MAP_DELALLOC; 1694 /* DIO READ and hole case, should not map the blocks. */ 1695 if (!(flag == F2FS_GET_BLOCK_DIO && is_hole && !map->m_may_create)) 1696 map->m_flags |= F2FS_MAP_MAPPED; 1697 1698 map->m_pblk = blkaddr; 1699 map->m_len = 1; 1700 1701 if (map->m_multidev_dio) 1702 map->m_bdev = FDEV(bidx).bdev; 1703 1704 if (lfs_dio_write) 1705 map->m_last_pblk = NULL_ADDR; 1706 } else if (map_is_mergeable(sbi, map, blkaddr, flag, bidx, ofs)) { 1707 ofs++; 1708 map->m_len++; 1709 } else { 1710 if (lfs_dio_write && !f2fs_is_pinned_file(inode)) 1711 map->m_last_pblk = blkaddr; 1712 goto sync_out; 1713 } 1714 1715skip: 1716 dn.ofs_in_node++; 1717 pgofs++; 1718 1719 /* preallocate blocks in batch for one dnode page */ 1720 if (flag == F2FS_GET_BLOCK_PRE_AIO && 1721 (pgofs == end || dn.ofs_in_node == end_offset)) { 1722 1723 dn.ofs_in_node = ofs_in_node; 1724 err = f2fs_reserve_new_blocks(&dn, prealloc); 1725 if (err) 1726 goto sync_out; 1727 1728 map->m_len += dn.ofs_in_node - ofs_in_node; 1729 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { 1730 err = -ENOSPC; 1731 goto sync_out; 1732 } 1733 dn.ofs_in_node = end_offset; 1734 } 1735 1736 if (pgofs >= end) 1737 goto sync_out; 1738 else if (dn.ofs_in_node < end_offset) 1739 goto next_block; 1740 1741 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1742 if (map->m_flags & F2FS_MAP_MAPPED) { 1743 unsigned int ofs = start_pgofs - map->m_lblk; 1744 1745 f2fs_update_read_extent_cache_range(&dn, 1746 start_pgofs, map->m_pblk + ofs, 1747 map->m_len - ofs); 1748 } 1749 } 1750 1751 f2fs_put_dnode(&dn); 1752 1753 if (map->m_may_create) { 1754 f2fs_map_unlock(sbi, flag); 1755 f2fs_balance_fs(sbi, dn.node_changed); 1756 } 1757 goto next_dnode; 1758 1759sync_out: 1760 1761 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) { 1762 /* 1763 * for hardware encryption, but to avoid potential issue 1764 * in future 1765 */ 1766 f2fs_wait_on_block_writeback_range(inode, 1767 map->m_pblk, map->m_len); 1768 1769 if (map->m_multidev_dio) { 1770 block_t blk_addr = map->m_pblk; 1771 1772 bidx = f2fs_target_device_index(sbi, map->m_pblk); 1773 1774 map->m_bdev = FDEV(bidx).bdev; 1775 map->m_pblk -= FDEV(bidx).start_blk; 1776 1777 if (map->m_may_create) 1778 f2fs_update_device_state(sbi, inode->i_ino, 1779 blk_addr, map->m_len); 1780 1781 f2fs_bug_on(sbi, blk_addr + map->m_len > 1782 FDEV(bidx).end_blk + 1); 1783 } 1784 } 1785 1786 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1787 if (map->m_flags & F2FS_MAP_MAPPED) { 1788 unsigned int ofs = start_pgofs - map->m_lblk; 1789 1790 if (map->m_len > ofs) 1791 f2fs_update_read_extent_cache_range(&dn, 1792 start_pgofs, map->m_pblk + ofs, 1793 map->m_len - ofs); 1794 } 1795 if (map->m_next_extent) 1796 *map->m_next_extent = is_hole ? pgofs + 1 : pgofs; 1797 } 1798 f2fs_put_dnode(&dn); 1799unlock_out: 1800 if (map->m_may_create) { 1801 f2fs_map_unlock(sbi, flag); 1802 f2fs_balance_fs(sbi, dn.node_changed); 1803 } 1804out: 1805 trace_f2fs_map_blocks(inode, map, flag, err); 1806 return err; 1807} 1808 1809static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, 1810 bool check_first) 1811{ 1812 struct f2fs_map_blocks map; 1813 block_t last_lblk; 1814 int err; 1815 1816 if (pos + len > i_size_read(inode)) 1817 return false; 1818 1819 map.m_lblk = F2FS_BYTES_TO_BLK(pos); 1820 map.m_next_pgofs = NULL; 1821 map.m_next_extent = NULL; 1822 map.m_seg_type = NO_CHECK_TYPE; 1823 map.m_may_create = false; 1824 last_lblk = F2FS_BLK_ALIGN(pos + len); 1825 1826 while (map.m_lblk < last_lblk) { 1827 map.m_len = last_lblk - map.m_lblk; 1828 err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); 1829 if (err || map.m_len == 0) 1830 return false; 1831 map.m_lblk += map.m_len; 1832 if (check_first) 1833 break; 1834 } 1835 return true; 1836} 1837 1838bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) 1839{ 1840 return __f2fs_overwrite_io(inode, pos, len, false); 1841} 1842 1843static int f2fs_xattr_fiemap(struct inode *inode, 1844 struct fiemap_extent_info *fieinfo) 1845{ 1846 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1847 struct node_info ni; 1848 __u64 phys = 0, len; 1849 __u32 flags; 1850 nid_t xnid = F2FS_I(inode)->i_xattr_nid; 1851 int err = 0; 1852 1853 if (f2fs_has_inline_xattr(inode)) { 1854 int offset; 1855 struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), 1856 inode->i_ino, false); 1857 1858 if (IS_ERR(folio)) 1859 return PTR_ERR(folio); 1860 1861 err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); 1862 if (err) { 1863 f2fs_folio_put(folio, true); 1864 return err; 1865 } 1866 1867 phys = F2FS_BLK_TO_BYTES(ni.blk_addr); 1868 offset = offsetof(struct f2fs_inode, i_addr) + 1869 sizeof(__le32) * (DEF_ADDRS_PER_INODE - 1870 get_inline_xattr_addrs(inode)); 1871 1872 phys += offset; 1873 len = inline_xattr_size(inode); 1874 1875 f2fs_folio_put(folio, true); 1876 1877 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; 1878 1879 if (!xnid) 1880 flags |= FIEMAP_EXTENT_LAST; 1881 1882 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1883 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1884 if (err) 1885 return err; 1886 } 1887 1888 if (xnid) { 1889 struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), 1890 xnid, false); 1891 1892 if (IS_ERR(folio)) 1893 return PTR_ERR(folio); 1894 1895 err = f2fs_get_node_info(sbi, xnid, &ni, false); 1896 if (err) { 1897 f2fs_folio_put(folio, true); 1898 return err; 1899 } 1900 1901 phys = F2FS_BLK_TO_BYTES(ni.blk_addr); 1902 len = inode->i_sb->s_blocksize; 1903 1904 f2fs_folio_put(folio, true); 1905 1906 flags = FIEMAP_EXTENT_LAST; 1907 } 1908 1909 if (phys) { 1910 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1911 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1912 } 1913 1914 return (err < 0 ? err : 0); 1915} 1916 1917int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 1918 u64 start, u64 len) 1919{ 1920 struct f2fs_map_blocks map; 1921 sector_t start_blk, last_blk, blk_len, max_len; 1922 pgoff_t next_pgofs; 1923 u64 logical = 0, phys = 0, size = 0; 1924 u32 flags = 0; 1925 int ret = 0; 1926 bool compr_cluster = false, compr_appended; 1927 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 1928 unsigned int count_in_cluster = 0; 1929 loff_t maxbytes; 1930 1931 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { 1932 ret = f2fs_precache_extents(inode); 1933 if (ret) 1934 return ret; 1935 } 1936 1937 ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR); 1938 if (ret) 1939 return ret; 1940 1941 inode_lock_shared(inode); 1942 1943 maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); 1944 if (start > maxbytes) { 1945 ret = -EFBIG; 1946 goto out; 1947 } 1948 1949 if (len > maxbytes || (maxbytes - len) < start) 1950 len = maxbytes - start; 1951 1952 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { 1953 ret = f2fs_xattr_fiemap(inode, fieinfo); 1954 goto out; 1955 } 1956 1957 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) { 1958 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 1959 if (ret != -EAGAIN) 1960 goto out; 1961 } 1962 1963 start_blk = F2FS_BYTES_TO_BLK(start); 1964 last_blk = F2FS_BYTES_TO_BLK(start + len - 1); 1965 blk_len = last_blk - start_blk + 1; 1966 max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk; 1967 1968next: 1969 memset(&map, 0, sizeof(map)); 1970 map.m_lblk = start_blk; 1971 map.m_len = blk_len; 1972 map.m_next_pgofs = &next_pgofs; 1973 map.m_seg_type = NO_CHECK_TYPE; 1974 1975 if (compr_cluster) { 1976 map.m_lblk += 1; 1977 map.m_len = cluster_size - count_in_cluster; 1978 } 1979 1980 ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 1981 if (ret) 1982 goto out; 1983 1984 /* HOLE */ 1985 if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) { 1986 start_blk = next_pgofs; 1987 1988 if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes) 1989 goto prep_next; 1990 1991 flags |= FIEMAP_EXTENT_LAST; 1992 } 1993 1994 /* 1995 * current extent may cross boundary of inquiry, increase len to 1996 * requery. 1997 */ 1998 if (!compr_cluster && (map.m_flags & F2FS_MAP_MAPPED) && 1999 map.m_lblk + map.m_len - 1 == last_blk && 2000 blk_len != max_len) { 2001 blk_len = max_len; 2002 goto next; 2003 } 2004 2005 compr_appended = false; 2006 /* In a case of compressed cluster, append this to the last extent */ 2007 if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) || 2008 !(map.m_flags & F2FS_MAP_FLAGS))) { 2009 compr_appended = true; 2010 goto skip_fill; 2011 } 2012 2013 if (size) { 2014 flags |= FIEMAP_EXTENT_MERGED; 2015 if (IS_ENCRYPTED(inode)) 2016 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 2017 2018 ret = fiemap_fill_next_extent(fieinfo, logical, 2019 phys, size, flags); 2020 trace_f2fs_fiemap(inode, logical, phys, size, flags, ret); 2021 if (ret) 2022 goto out; 2023 size = 0; 2024 } 2025 2026 if (start_blk > last_blk) 2027 goto out; 2028 2029skip_fill: 2030 if (map.m_pblk == COMPRESS_ADDR) { 2031 compr_cluster = true; 2032 count_in_cluster = 1; 2033 } else if (compr_appended) { 2034 unsigned int appended_blks = cluster_size - 2035 count_in_cluster + 1; 2036 size += F2FS_BLK_TO_BYTES(appended_blks); 2037 start_blk += appended_blks; 2038 compr_cluster = false; 2039 } else { 2040 logical = F2FS_BLK_TO_BYTES(start_blk); 2041 phys = __is_valid_data_blkaddr(map.m_pblk) ? 2042 F2FS_BLK_TO_BYTES(map.m_pblk) : 0; 2043 size = F2FS_BLK_TO_BYTES(map.m_len); 2044 flags = 0; 2045 2046 if (compr_cluster) { 2047 flags = FIEMAP_EXTENT_ENCODED; 2048 count_in_cluster += map.m_len; 2049 if (count_in_cluster == cluster_size) { 2050 compr_cluster = false; 2051 size += F2FS_BLKSIZE; 2052 } 2053 } else if (map.m_flags & F2FS_MAP_DELALLOC) { 2054 flags = FIEMAP_EXTENT_UNWRITTEN; 2055 } 2056 2057 start_blk += F2FS_BYTES_TO_BLK(size); 2058 } 2059 2060prep_next: 2061 cond_resched(); 2062 if (fatal_signal_pending(current)) 2063 ret = -EINTR; 2064 else 2065 goto next; 2066out: 2067 if (ret == 1) 2068 ret = 0; 2069 2070 inode_unlock_shared(inode); 2071 return ret; 2072} 2073 2074static inline loff_t f2fs_readpage_limit(struct inode *inode) 2075{ 2076 if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) 2077 return F2FS_BLK_TO_BYTES(max_file_blocks(inode)); 2078 2079 return i_size_read(inode); 2080} 2081 2082static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac) 2083{ 2084 return rac ? REQ_RAHEAD : 0; 2085} 2086 2087static int f2fs_read_single_page(struct inode *inode, struct folio *folio, 2088 unsigned nr_pages, 2089 struct f2fs_map_blocks *map, 2090 struct bio **bio_ret, 2091 sector_t *last_block_in_bio, 2092 struct readahead_control *rac) 2093{ 2094 struct bio *bio = *bio_ret; 2095 const unsigned int blocksize = F2FS_BLKSIZE; 2096 sector_t block_in_file; 2097 sector_t last_block; 2098 sector_t last_block_in_file; 2099 sector_t block_nr; 2100 pgoff_t index = folio->index; 2101 int ret = 0; 2102 2103 block_in_file = (sector_t)index; 2104 last_block = block_in_file + nr_pages; 2105 last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + 2106 blocksize - 1); 2107 if (last_block > last_block_in_file) 2108 last_block = last_block_in_file; 2109 2110 /* just zeroing out page which is beyond EOF */ 2111 if (block_in_file >= last_block) 2112 goto zero_out; 2113 /* 2114 * Map blocks using the previous result first. 2115 */ 2116 if ((map->m_flags & F2FS_MAP_MAPPED) && 2117 block_in_file > map->m_lblk && 2118 block_in_file < (map->m_lblk + map->m_len)) 2119 goto got_it; 2120 2121 /* 2122 * Then do more f2fs_map_blocks() calls until we are 2123 * done with this page. 2124 */ 2125 map->m_lblk = block_in_file; 2126 map->m_len = last_block - block_in_file; 2127 2128 ret = f2fs_map_blocks(inode, map, F2FS_GET_BLOCK_DEFAULT); 2129 if (ret) 2130 goto out; 2131got_it: 2132 if ((map->m_flags & F2FS_MAP_MAPPED)) { 2133 block_nr = map->m_pblk + block_in_file - map->m_lblk; 2134 folio_set_mappedtodisk(folio); 2135 2136 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, 2137 DATA_GENERIC_ENHANCE_READ)) { 2138 ret = -EFSCORRUPTED; 2139 goto out; 2140 } 2141 } else { 2142zero_out: 2143 folio_zero_segment(folio, 0, folio_size(folio)); 2144 if (f2fs_need_verity(inode, index) && 2145 !fsverity_verify_folio(folio)) { 2146 ret = -EIO; 2147 goto out; 2148 } 2149 if (!folio_test_uptodate(folio)) 2150 folio_mark_uptodate(folio); 2151 folio_unlock(folio); 2152 goto out; 2153 } 2154 2155 /* 2156 * This page will go to BIO. Do we need to send this 2157 * BIO off first? 2158 */ 2159 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, 2160 *last_block_in_bio, block_nr) || 2161 !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { 2162submit_and_realloc: 2163 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2164 bio = NULL; 2165 } 2166 if (bio == NULL) 2167 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, 2168 f2fs_ra_op_flags(rac), index, 2169 false); 2170 2171 /* 2172 * If the page is under writeback, we need to wait for 2173 * its completion to see the correct decrypted data. 2174 */ 2175 f2fs_wait_on_block_writeback(inode, block_nr); 2176 2177 if (!bio_add_folio(bio, folio, blocksize, 0)) 2178 goto submit_and_realloc; 2179 2180 inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); 2181 f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, 2182 F2FS_BLKSIZE); 2183 *last_block_in_bio = block_nr; 2184out: 2185 *bio_ret = bio; 2186 return ret; 2187} 2188 2189#ifdef CONFIG_F2FS_FS_COMPRESSION 2190int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, 2191 unsigned nr_pages, sector_t *last_block_in_bio, 2192 struct readahead_control *rac, bool for_write) 2193{ 2194 struct dnode_of_data dn; 2195 struct inode *inode = cc->inode; 2196 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2197 struct bio *bio = *bio_ret; 2198 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; 2199 sector_t last_block_in_file; 2200 const unsigned int blocksize = F2FS_BLKSIZE; 2201 struct decompress_io_ctx *dic = NULL; 2202 struct extent_info ei = {}; 2203 bool from_dnode = true; 2204 int i; 2205 int ret = 0; 2206 2207 if (unlikely(f2fs_cp_error(sbi))) { 2208 ret = -EIO; 2209 from_dnode = false; 2210 goto out_put_dnode; 2211 } 2212 2213 f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); 2214 2215 last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + 2216 blocksize - 1); 2217 2218 /* get rid of pages beyond EOF */ 2219 for (i = 0; i < cc->cluster_size; i++) { 2220 struct page *page = cc->rpages[i]; 2221 struct folio *folio; 2222 2223 if (!page) 2224 continue; 2225 2226 folio = page_folio(page); 2227 if ((sector_t)folio->index >= last_block_in_file) { 2228 folio_zero_segment(folio, 0, folio_size(folio)); 2229 if (!folio_test_uptodate(folio)) 2230 folio_mark_uptodate(folio); 2231 } else if (!folio_test_uptodate(folio)) { 2232 continue; 2233 } 2234 folio_unlock(folio); 2235 if (for_write) 2236 folio_put(folio); 2237 cc->rpages[i] = NULL; 2238 cc->nr_rpages--; 2239 } 2240 2241 /* we are done since all pages are beyond EOF */ 2242 if (f2fs_cluster_is_empty(cc)) 2243 goto out; 2244 2245 if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei)) 2246 from_dnode = false; 2247 2248 if (!from_dnode) 2249 goto skip_reading_dnode; 2250 2251 set_new_dnode(&dn, inode, NULL, NULL, 0); 2252 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 2253 if (ret) 2254 goto out; 2255 2256 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR); 2257 2258skip_reading_dnode: 2259 for (i = 1; i < cc->cluster_size; i++) { 2260 block_t blkaddr; 2261 2262 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio, 2263 dn.ofs_in_node + i) : 2264 ei.blk + i - 1; 2265 2266 if (!__is_valid_data_blkaddr(blkaddr)) 2267 break; 2268 2269 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) { 2270 ret = -EFAULT; 2271 goto out_put_dnode; 2272 } 2273 cc->nr_cpages++; 2274 2275 if (!from_dnode && i >= ei.c_len) 2276 break; 2277 } 2278 2279 /* nothing to decompress */ 2280 if (cc->nr_cpages == 0) { 2281 ret = 0; 2282 goto out_put_dnode; 2283 } 2284 2285 dic = f2fs_alloc_dic(cc); 2286 if (IS_ERR(dic)) { 2287 ret = PTR_ERR(dic); 2288 goto out_put_dnode; 2289 } 2290 2291 for (i = 0; i < cc->nr_cpages; i++) { 2292 struct folio *folio = page_folio(dic->cpages[i]); 2293 block_t blkaddr; 2294 struct bio_post_read_ctx *ctx; 2295 2296 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio, 2297 dn.ofs_in_node + i + 1) : 2298 ei.blk + i; 2299 2300 f2fs_wait_on_block_writeback(inode, blkaddr); 2301 2302 if (f2fs_load_compressed_folio(sbi, folio, blkaddr)) { 2303 if (atomic_dec_and_test(&dic->remaining_pages)) { 2304 f2fs_decompress_cluster(dic, true); 2305 break; 2306 } 2307 continue; 2308 } 2309 2310 if (bio && (!page_is_mergeable(sbi, bio, 2311 *last_block_in_bio, blkaddr) || 2312 !f2fs_crypt_mergeable_bio(bio, inode, folio->index, NULL))) { 2313submit_and_realloc: 2314 f2fs_submit_read_bio(sbi, bio, DATA); 2315 bio = NULL; 2316 } 2317 2318 if (!bio) 2319 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages - i, 2320 f2fs_ra_op_flags(rac), 2321 folio->index, for_write); 2322 2323 if (!bio_add_folio(bio, folio, blocksize, 0)) 2324 goto submit_and_realloc; 2325 2326 ctx = get_post_read_ctx(bio); 2327 ctx->enabled_steps |= STEP_DECOMPRESS; 2328 refcount_inc(&dic->refcnt); 2329 2330 inc_page_count(sbi, F2FS_RD_DATA); 2331 f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); 2332 *last_block_in_bio = blkaddr; 2333 } 2334 2335 if (from_dnode) 2336 f2fs_put_dnode(&dn); 2337 2338 *bio_ret = bio; 2339 return 0; 2340 2341out_put_dnode: 2342 if (from_dnode) 2343 f2fs_put_dnode(&dn); 2344out: 2345 for (i = 0; i < cc->cluster_size; i++) { 2346 if (cc->rpages[i]) { 2347 ClearPageUptodate(cc->rpages[i]); 2348 unlock_page(cc->rpages[i]); 2349 } 2350 } 2351 *bio_ret = bio; 2352 return ret; 2353} 2354#endif 2355 2356/* 2357 * This function was originally taken from fs/mpage.c, and customized for f2fs. 2358 * Major change was from block_size == page_size in f2fs by default. 2359 */ 2360static int f2fs_mpage_readpages(struct inode *inode, 2361 struct readahead_control *rac, struct folio *folio) 2362{ 2363 struct bio *bio = NULL; 2364 sector_t last_block_in_bio = 0; 2365 struct f2fs_map_blocks map; 2366#ifdef CONFIG_F2FS_FS_COMPRESSION 2367 struct compress_ctx cc = { 2368 .inode = inode, 2369 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2370 .cluster_size = F2FS_I(inode)->i_cluster_size, 2371 .cluster_idx = NULL_CLUSTER, 2372 .rpages = NULL, 2373 .cpages = NULL, 2374 .nr_rpages = 0, 2375 .nr_cpages = 0, 2376 }; 2377 pgoff_t nc_cluster_idx = NULL_CLUSTER; 2378 pgoff_t index; 2379#endif 2380 unsigned nr_pages = rac ? readahead_count(rac) : 1; 2381 unsigned max_nr_pages = nr_pages; 2382 int ret = 0; 2383 2384#ifdef CONFIG_F2FS_FS_COMPRESSION 2385 if (f2fs_compressed_file(inode)) { 2386 index = rac ? readahead_index(rac) : folio->index; 2387 max_nr_pages = round_up(index + nr_pages, cc.cluster_size) - 2388 round_down(index, cc.cluster_size); 2389 } 2390#endif 2391 2392 map.m_pblk = 0; 2393 map.m_lblk = 0; 2394 map.m_len = 0; 2395 map.m_flags = 0; 2396 map.m_next_pgofs = NULL; 2397 map.m_next_extent = NULL; 2398 map.m_seg_type = NO_CHECK_TYPE; 2399 map.m_may_create = false; 2400 2401 for (; nr_pages; nr_pages--) { 2402 if (rac) { 2403 folio = readahead_folio(rac); 2404 prefetchw(&folio->flags); 2405 } 2406 2407#ifdef CONFIG_F2FS_FS_COMPRESSION 2408 index = folio->index; 2409 2410 if (!f2fs_compressed_file(inode)) 2411 goto read_single_page; 2412 2413 /* there are remained compressed pages, submit them */ 2414 if (!f2fs_cluster_can_merge_page(&cc, index)) { 2415 ret = f2fs_read_multi_pages(&cc, &bio, 2416 max_nr_pages, 2417 &last_block_in_bio, 2418 rac, false); 2419 f2fs_destroy_compress_ctx(&cc, false); 2420 if (ret) 2421 goto set_error_page; 2422 } 2423 if (cc.cluster_idx == NULL_CLUSTER) { 2424 if (nc_cluster_idx == index >> cc.log_cluster_size) 2425 goto read_single_page; 2426 2427 ret = f2fs_is_compressed_cluster(inode, index); 2428 if (ret < 0) 2429 goto set_error_page; 2430 else if (!ret) { 2431 nc_cluster_idx = 2432 index >> cc.log_cluster_size; 2433 goto read_single_page; 2434 } 2435 2436 nc_cluster_idx = NULL_CLUSTER; 2437 } 2438 ret = f2fs_init_compress_ctx(&cc); 2439 if (ret) 2440 goto set_error_page; 2441 2442 f2fs_compress_ctx_add_page(&cc, folio); 2443 2444 goto next_page; 2445read_single_page: 2446#endif 2447 2448 ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map, 2449 &bio, &last_block_in_bio, rac); 2450 if (ret) { 2451#ifdef CONFIG_F2FS_FS_COMPRESSION 2452set_error_page: 2453#endif 2454 folio_zero_segment(folio, 0, folio_size(folio)); 2455 folio_unlock(folio); 2456 } 2457#ifdef CONFIG_F2FS_FS_COMPRESSION 2458next_page: 2459#endif 2460 2461#ifdef CONFIG_F2FS_FS_COMPRESSION 2462 if (f2fs_compressed_file(inode)) { 2463 /* last page */ 2464 if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) { 2465 ret = f2fs_read_multi_pages(&cc, &bio, 2466 max_nr_pages, 2467 &last_block_in_bio, 2468 rac, false); 2469 f2fs_destroy_compress_ctx(&cc, false); 2470 } 2471 } 2472#endif 2473 } 2474 if (bio) 2475 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2476 return ret; 2477} 2478 2479static int f2fs_read_data_folio(struct file *file, struct folio *folio) 2480{ 2481 struct inode *inode = folio->mapping->host; 2482 int ret = -EAGAIN; 2483 2484 trace_f2fs_readpage(folio, DATA); 2485 2486 if (!f2fs_is_compress_backend_ready(inode)) { 2487 folio_unlock(folio); 2488 return -EOPNOTSUPP; 2489 } 2490 2491 /* If the file has inline data, try to read it directly */ 2492 if (f2fs_has_inline_data(inode)) 2493 ret = f2fs_read_inline_data(inode, folio); 2494 if (ret == -EAGAIN) 2495 ret = f2fs_mpage_readpages(inode, NULL, folio); 2496 return ret; 2497} 2498 2499static void f2fs_readahead(struct readahead_control *rac) 2500{ 2501 struct inode *inode = rac->mapping->host; 2502 2503 trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac)); 2504 2505 if (!f2fs_is_compress_backend_ready(inode)) 2506 return; 2507 2508 /* If the file has inline data, skip readahead */ 2509 if (f2fs_has_inline_data(inode)) 2510 return; 2511 2512 f2fs_mpage_readpages(inode, rac, NULL); 2513} 2514 2515int f2fs_encrypt_one_page(struct f2fs_io_info *fio) 2516{ 2517 struct inode *inode = fio_inode(fio); 2518 struct folio *mfolio; 2519 struct page *page; 2520 gfp_t gfp_flags = GFP_NOFS; 2521 2522 if (!f2fs_encrypted_file(inode)) 2523 return 0; 2524 2525 page = fio->compressed_page ? fio->compressed_page : fio->page; 2526 2527 if (fscrypt_inode_uses_inline_crypto(inode)) 2528 return 0; 2529 2530retry_encrypt: 2531 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page_folio(page), 2532 PAGE_SIZE, 0, gfp_flags); 2533 if (IS_ERR(fio->encrypted_page)) { 2534 /* flush pending IOs and wait for a while in the ENOMEM case */ 2535 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { 2536 f2fs_flush_merged_writes(fio->sbi); 2537 memalloc_retry_wait(GFP_NOFS); 2538 gfp_flags |= __GFP_NOFAIL; 2539 goto retry_encrypt; 2540 } 2541 return PTR_ERR(fio->encrypted_page); 2542 } 2543 2544 mfolio = filemap_lock_folio(META_MAPPING(fio->sbi), fio->old_blkaddr); 2545 if (!IS_ERR(mfolio)) { 2546 if (folio_test_uptodate(mfolio)) 2547 memcpy(folio_address(mfolio), 2548 page_address(fio->encrypted_page), PAGE_SIZE); 2549 f2fs_folio_put(mfolio, true); 2550 } 2551 return 0; 2552} 2553 2554static inline bool check_inplace_update_policy(struct inode *inode, 2555 struct f2fs_io_info *fio) 2556{ 2557 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2558 2559 if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) && 2560 is_inode_flag_set(inode, FI_OPU_WRITE)) 2561 return false; 2562 if (IS_F2FS_IPU_FORCE(sbi)) 2563 return true; 2564 if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi)) 2565 return true; 2566 if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util) 2567 return true; 2568 if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) && 2569 utilization(sbi) > SM_I(sbi)->min_ipu_util) 2570 return true; 2571 2572 /* 2573 * IPU for rewrite async pages 2574 */ 2575 if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && 2576 !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) 2577 return true; 2578 2579 /* this is only set during fdatasync */ 2580 if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, FI_NEED_IPU)) 2581 return true; 2582 2583 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2584 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2585 return true; 2586 2587 return false; 2588} 2589 2590bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) 2591{ 2592 /* swap file is migrating in aligned write mode */ 2593 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 2594 return false; 2595 2596 if (f2fs_is_pinned_file(inode)) 2597 return true; 2598 2599 /* if this is cold file, we should overwrite to avoid fragmentation */ 2600 if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE)) 2601 return true; 2602 2603 return check_inplace_update_policy(inode, fio); 2604} 2605 2606bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) 2607{ 2608 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2609 2610 /* The below cases were checked when setting it. */ 2611 if (f2fs_is_pinned_file(inode)) 2612 return false; 2613 if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 2614 return true; 2615 if (f2fs_lfs_mode(sbi)) 2616 return true; 2617 if (S_ISDIR(inode->i_mode)) 2618 return true; 2619 if (IS_NOQUOTA(inode)) 2620 return true; 2621 if (f2fs_used_in_atomic_write(inode)) 2622 return true; 2623 /* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */ 2624 if (f2fs_compressed_file(inode) && 2625 F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER && 2626 is_inode_flag_set(inode, FI_ENABLE_COMPRESS)) 2627 return true; 2628 2629 /* swap file is migrating in aligned write mode */ 2630 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 2631 return true; 2632 2633 if (is_inode_flag_set(inode, FI_OPU_WRITE)) 2634 return true; 2635 2636 if (fio) { 2637 if (page_private_gcing(fio->page)) 2638 return true; 2639 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2640 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2641 return true; 2642 } 2643 return false; 2644} 2645 2646static inline bool need_inplace_update(struct f2fs_io_info *fio) 2647{ 2648 struct inode *inode = fio_inode(fio); 2649 2650 if (f2fs_should_update_outplace(inode, fio)) 2651 return false; 2652 2653 return f2fs_should_update_inplace(inode, fio); 2654} 2655 2656int f2fs_do_write_data_page(struct f2fs_io_info *fio) 2657{ 2658 struct folio *folio = fio->folio; 2659 struct inode *inode = folio->mapping->host; 2660 struct dnode_of_data dn; 2661 struct node_info ni; 2662 bool ipu_force = false; 2663 bool atomic_commit; 2664 int err = 0; 2665 2666 /* Use COW inode to make dnode_of_data for atomic write */ 2667 atomic_commit = f2fs_is_atomic_file(inode) && 2668 folio_test_f2fs_atomic(folio); 2669 if (atomic_commit) 2670 set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0); 2671 else 2672 set_new_dnode(&dn, inode, NULL, NULL, 0); 2673 2674 if (need_inplace_update(fio) && 2675 f2fs_lookup_read_extent_cache_block(inode, folio->index, 2676 &fio->old_blkaddr)) { 2677 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2678 DATA_GENERIC_ENHANCE)) 2679 return -EFSCORRUPTED; 2680 2681 ipu_force = true; 2682 fio->need_lock = LOCK_DONE; 2683 goto got_it; 2684 } 2685 2686 /* Deadlock due to between page->lock and f2fs_lock_op */ 2687 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) 2688 return -EAGAIN; 2689 2690 err = f2fs_get_dnode_of_data(&dn, folio->index, LOOKUP_NODE); 2691 if (err) 2692 goto out; 2693 2694 fio->old_blkaddr = dn.data_blkaddr; 2695 2696 /* This page is already truncated */ 2697 if (fio->old_blkaddr == NULL_ADDR) { 2698 folio_clear_uptodate(folio); 2699 folio_clear_f2fs_gcing(folio); 2700 goto out_writepage; 2701 } 2702got_it: 2703 if (__is_valid_data_blkaddr(fio->old_blkaddr) && 2704 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2705 DATA_GENERIC_ENHANCE)) { 2706 err = -EFSCORRUPTED; 2707 goto out_writepage; 2708 } 2709 2710 /* wait for GCed page writeback via META_MAPPING */ 2711 if (fio->meta_gc) 2712 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); 2713 2714 /* 2715 * If current allocation needs SSR, 2716 * it had better in-place writes for updated data. 2717 */ 2718 if (ipu_force || 2719 (__is_valid_data_blkaddr(fio->old_blkaddr) && 2720 need_inplace_update(fio))) { 2721 err = f2fs_encrypt_one_page(fio); 2722 if (err) 2723 goto out_writepage; 2724 2725 folio_start_writeback(folio); 2726 f2fs_put_dnode(&dn); 2727 if (fio->need_lock == LOCK_REQ) 2728 f2fs_unlock_op(fio->sbi); 2729 err = f2fs_inplace_write_data(fio); 2730 if (err) { 2731 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 2732 fscrypt_finalize_bounce_page(&fio->encrypted_page); 2733 folio_end_writeback(folio); 2734 } else { 2735 set_inode_flag(inode, FI_UPDATE_WRITE); 2736 } 2737 trace_f2fs_do_write_data_page(folio, IPU); 2738 return err; 2739 } 2740 2741 if (fio->need_lock == LOCK_RETRY) { 2742 if (!f2fs_trylock_op(fio->sbi)) { 2743 err = -EAGAIN; 2744 goto out_writepage; 2745 } 2746 fio->need_lock = LOCK_REQ; 2747 } 2748 2749 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); 2750 if (err) 2751 goto out_writepage; 2752 2753 fio->version = ni.version; 2754 2755 err = f2fs_encrypt_one_page(fio); 2756 if (err) 2757 goto out_writepage; 2758 2759 folio_start_writeback(folio); 2760 2761 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) 2762 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); 2763 2764 /* LFS mode write path */ 2765 f2fs_outplace_write_data(&dn, fio); 2766 trace_f2fs_do_write_data_page(folio, OPU); 2767 set_inode_flag(inode, FI_APPEND_WRITE); 2768 if (atomic_commit) 2769 folio_clear_f2fs_atomic(folio); 2770out_writepage: 2771 f2fs_put_dnode(&dn); 2772out: 2773 if (fio->need_lock == LOCK_REQ) 2774 f2fs_unlock_op(fio->sbi); 2775 return err; 2776} 2777 2778int f2fs_write_single_data_page(struct folio *folio, int *submitted, 2779 struct bio **bio, 2780 sector_t *last_block, 2781 struct writeback_control *wbc, 2782 enum iostat_type io_type, 2783 int compr_blocks, 2784 bool allow_balance) 2785{ 2786 struct inode *inode = folio->mapping->host; 2787 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2788 loff_t i_size = i_size_read(inode); 2789 const pgoff_t end_index = ((unsigned long long)i_size) 2790 >> PAGE_SHIFT; 2791 loff_t psize = (loff_t)(folio->index + 1) << PAGE_SHIFT; 2792 unsigned offset = 0; 2793 bool need_balance_fs = false; 2794 bool quota_inode = IS_NOQUOTA(inode); 2795 int err = 0; 2796 struct f2fs_io_info fio = { 2797 .sbi = sbi, 2798 .ino = inode->i_ino, 2799 .type = DATA, 2800 .op = REQ_OP_WRITE, 2801 .op_flags = wbc_to_write_flags(wbc), 2802 .old_blkaddr = NULL_ADDR, 2803 .folio = folio, 2804 .encrypted_page = NULL, 2805 .submitted = 0, 2806 .compr_blocks = compr_blocks, 2807 .need_lock = compr_blocks ? LOCK_DONE : LOCK_RETRY, 2808 .meta_gc = f2fs_meta_inode_gc_required(inode) ? 1 : 0, 2809 .io_type = io_type, 2810 .io_wbc = wbc, 2811 .bio = bio, 2812 .last_block = last_block, 2813 }; 2814 2815 trace_f2fs_writepage(folio, DATA); 2816 2817 /* we should bypass data pages to proceed the kworker jobs */ 2818 if (unlikely(f2fs_cp_error(sbi))) { 2819 mapping_set_error(folio->mapping, -EIO); 2820 /* 2821 * don't drop any dirty dentry pages for keeping lastest 2822 * directory structure. 2823 */ 2824 if (S_ISDIR(inode->i_mode) && 2825 !is_sbi_flag_set(sbi, SBI_IS_CLOSE)) 2826 goto redirty_out; 2827 2828 /* keep data pages in remount-ro mode */ 2829 if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) 2830 goto redirty_out; 2831 goto out; 2832 } 2833 2834 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 2835 goto redirty_out; 2836 2837 if (folio->index < end_index || 2838 f2fs_verity_in_progress(inode) || 2839 compr_blocks) 2840 goto write; 2841 2842 /* 2843 * If the offset is out-of-range of file size, 2844 * this page does not have to be written to disk. 2845 */ 2846 offset = i_size & (PAGE_SIZE - 1); 2847 if ((folio->index >= end_index + 1) || !offset) 2848 goto out; 2849 2850 folio_zero_segment(folio, offset, folio_size(folio)); 2851write: 2852 /* Dentry/quota blocks are controlled by checkpoint */ 2853 if (S_ISDIR(inode->i_mode) || quota_inode) { 2854 /* 2855 * We need to wait for node_write to avoid block allocation during 2856 * checkpoint. This can only happen to quota writes which can cause 2857 * the below discard race condition. 2858 */ 2859 if (quota_inode) 2860 f2fs_down_read(&sbi->node_write); 2861 2862 fio.need_lock = LOCK_DONE; 2863 err = f2fs_do_write_data_page(&fio); 2864 2865 if (quota_inode) 2866 f2fs_up_read(&sbi->node_write); 2867 2868 goto done; 2869 } 2870 2871 need_balance_fs = true; 2872 err = -EAGAIN; 2873 if (f2fs_has_inline_data(inode)) { 2874 err = f2fs_write_inline_data(inode, folio); 2875 if (!err) 2876 goto out; 2877 } 2878 2879 if (err == -EAGAIN) { 2880 err = f2fs_do_write_data_page(&fio); 2881 if (err == -EAGAIN) { 2882 f2fs_bug_on(sbi, compr_blocks); 2883 fio.need_lock = LOCK_REQ; 2884 err = f2fs_do_write_data_page(&fio); 2885 } 2886 } 2887 2888 if (err) { 2889 file_set_keep_isize(inode); 2890 } else { 2891 spin_lock(&F2FS_I(inode)->i_size_lock); 2892 if (F2FS_I(inode)->last_disk_size < psize) 2893 F2FS_I(inode)->last_disk_size = psize; 2894 spin_unlock(&F2FS_I(inode)->i_size_lock); 2895 } 2896 2897done: 2898 if (err && err != -ENOENT) 2899 goto redirty_out; 2900 2901out: 2902 inode_dec_dirty_pages(inode); 2903 if (err) { 2904 folio_clear_uptodate(folio); 2905 folio_clear_f2fs_gcing(folio); 2906 } 2907 folio_unlock(folio); 2908 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && 2909 !F2FS_I(inode)->wb_task && allow_balance) 2910 f2fs_balance_fs(sbi, need_balance_fs); 2911 2912 if (unlikely(f2fs_cp_error(sbi))) { 2913 f2fs_submit_merged_write(sbi, DATA); 2914 if (bio && *bio) 2915 f2fs_submit_merged_ipu_write(sbi, bio, NULL); 2916 submitted = NULL; 2917 } 2918 2919 if (submitted) 2920 *submitted = fio.submitted; 2921 2922 return 0; 2923 2924redirty_out: 2925 folio_redirty_for_writepage(wbc, folio); 2926 /* 2927 * pageout() in MM translates EAGAIN, so calls handle_write_error() 2928 * -> mapping_set_error() -> set_bit(AS_EIO, ...). 2929 * file_write_and_wait_range() will see EIO error, which is critical 2930 * to return value of fsync() followed by atomic_write failure to user. 2931 */ 2932 folio_unlock(folio); 2933 if (!err) 2934 return 1; 2935 return err; 2936} 2937 2938/* 2939 * This function was copied from write_cache_pages from mm/page-writeback.c. 2940 * The major change is making write step of cold data page separately from 2941 * warm/hot data page. 2942 */ 2943static int f2fs_write_cache_pages(struct address_space *mapping, 2944 struct writeback_control *wbc, 2945 enum iostat_type io_type) 2946{ 2947 int ret = 0; 2948 int done = 0, retry = 0; 2949 struct page *pages_local[F2FS_ONSTACK_PAGES]; 2950 struct page **pages = pages_local; 2951 struct folio_batch fbatch; 2952 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 2953 struct bio *bio = NULL; 2954 sector_t last_block; 2955#ifdef CONFIG_F2FS_FS_COMPRESSION 2956 struct inode *inode = mapping->host; 2957 struct compress_ctx cc = { 2958 .inode = inode, 2959 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2960 .cluster_size = F2FS_I(inode)->i_cluster_size, 2961 .cluster_idx = NULL_CLUSTER, 2962 .rpages = NULL, 2963 .nr_rpages = 0, 2964 .cpages = NULL, 2965 .valid_nr_cpages = 0, 2966 .rbuf = NULL, 2967 .cbuf = NULL, 2968 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size, 2969 .private = NULL, 2970 }; 2971#endif 2972 int nr_folios, p, idx; 2973 int nr_pages; 2974 unsigned int max_pages = F2FS_ONSTACK_PAGES; 2975 pgoff_t index; 2976 pgoff_t end; /* Inclusive */ 2977 pgoff_t done_index; 2978 int range_whole = 0; 2979 xa_mark_t tag; 2980 int nwritten = 0; 2981 int submitted = 0; 2982 int i; 2983 2984#ifdef CONFIG_F2FS_FS_COMPRESSION 2985 if (f2fs_compressed_file(inode) && 2986 1 << cc.log_cluster_size > F2FS_ONSTACK_PAGES) { 2987 pages = f2fs_kzalloc(sbi, sizeof(struct page *) << 2988 cc.log_cluster_size, GFP_NOFS | __GFP_NOFAIL); 2989 max_pages = 1 << cc.log_cluster_size; 2990 } 2991#endif 2992 2993 folio_batch_init(&fbatch); 2994 2995 if (get_dirty_pages(mapping->host) <= 2996 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) 2997 set_inode_flag(mapping->host, FI_HOT_DATA); 2998 else 2999 clear_inode_flag(mapping->host, FI_HOT_DATA); 3000 3001 if (wbc->range_cyclic) { 3002 index = mapping->writeback_index; /* prev offset */ 3003 end = -1; 3004 } else { 3005 index = wbc->range_start >> PAGE_SHIFT; 3006 end = wbc->range_end >> PAGE_SHIFT; 3007 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 3008 range_whole = 1; 3009 } 3010 tag = wbc_to_tag(wbc); 3011retry: 3012 retry = 0; 3013 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 3014 tag_pages_for_writeback(mapping, index, end); 3015 done_index = index; 3016 while (!done && !retry && (index <= end)) { 3017 nr_pages = 0; 3018again: 3019 nr_folios = filemap_get_folios_tag(mapping, &index, end, 3020 tag, &fbatch); 3021 if (nr_folios == 0) { 3022 if (nr_pages) 3023 goto write; 3024 break; 3025 } 3026 3027 for (i = 0; i < nr_folios; i++) { 3028 struct folio *folio = fbatch.folios[i]; 3029 3030 idx = 0; 3031 p = folio_nr_pages(folio); 3032add_more: 3033 pages[nr_pages] = folio_page(folio, idx); 3034 folio_get(folio); 3035 if (++nr_pages == max_pages) { 3036 index = folio->index + idx + 1; 3037 folio_batch_release(&fbatch); 3038 goto write; 3039 } 3040 if (++idx < p) 3041 goto add_more; 3042 } 3043 folio_batch_release(&fbatch); 3044 goto again; 3045write: 3046 for (i = 0; i < nr_pages; i++) { 3047 struct page *page = pages[i]; 3048 struct folio *folio = page_folio(page); 3049 bool need_readd; 3050readd: 3051 need_readd = false; 3052#ifdef CONFIG_F2FS_FS_COMPRESSION 3053 if (f2fs_compressed_file(inode)) { 3054 void *fsdata = NULL; 3055 struct page *pagep; 3056 int ret2; 3057 3058 ret = f2fs_init_compress_ctx(&cc); 3059 if (ret) { 3060 done = 1; 3061 break; 3062 } 3063 3064 if (!f2fs_cluster_can_merge_page(&cc, 3065 folio->index)) { 3066 ret = f2fs_write_multi_pages(&cc, 3067 &submitted, wbc, io_type); 3068 if (!ret) 3069 need_readd = true; 3070 goto result; 3071 } 3072 3073 if (unlikely(f2fs_cp_error(sbi))) 3074 goto lock_folio; 3075 3076 if (!f2fs_cluster_is_empty(&cc)) 3077 goto lock_folio; 3078 3079 if (f2fs_all_cluster_page_ready(&cc, 3080 pages, i, nr_pages, true)) 3081 goto lock_folio; 3082 3083 ret2 = f2fs_prepare_compress_overwrite( 3084 inode, &pagep, 3085 folio->index, &fsdata); 3086 if (ret2 < 0) { 3087 ret = ret2; 3088 done = 1; 3089 break; 3090 } else if (ret2 && 3091 (!f2fs_compress_write_end(inode, 3092 fsdata, folio->index, 1) || 3093 !f2fs_all_cluster_page_ready(&cc, 3094 pages, i, nr_pages, 3095 false))) { 3096 retry = 1; 3097 break; 3098 } 3099 } 3100#endif 3101 /* give a priority to WB_SYNC threads */ 3102 if (atomic_read(&sbi->wb_sync_req[DATA]) && 3103 wbc->sync_mode == WB_SYNC_NONE) { 3104 done = 1; 3105 break; 3106 } 3107#ifdef CONFIG_F2FS_FS_COMPRESSION 3108lock_folio: 3109#endif 3110 done_index = folio->index; 3111retry_write: 3112 folio_lock(folio); 3113 3114 if (unlikely(folio->mapping != mapping)) { 3115continue_unlock: 3116 folio_unlock(folio); 3117 continue; 3118 } 3119 3120 if (!folio_test_dirty(folio)) { 3121 /* someone wrote it for us */ 3122 goto continue_unlock; 3123 } 3124 3125 if (folio_test_writeback(folio)) { 3126 if (wbc->sync_mode == WB_SYNC_NONE) 3127 goto continue_unlock; 3128 f2fs_folio_wait_writeback(folio, DATA, true, true); 3129 } 3130 3131 if (!folio_clear_dirty_for_io(folio)) 3132 goto continue_unlock; 3133 3134#ifdef CONFIG_F2FS_FS_COMPRESSION 3135 if (f2fs_compressed_file(inode)) { 3136 folio_get(folio); 3137 f2fs_compress_ctx_add_page(&cc, folio); 3138 continue; 3139 } 3140#endif 3141 submitted = 0; 3142 ret = f2fs_write_single_data_page(folio, 3143 &submitted, &bio, &last_block, 3144 wbc, io_type, 0, true); 3145#ifdef CONFIG_F2FS_FS_COMPRESSION 3146result: 3147#endif 3148 nwritten += submitted; 3149 wbc->nr_to_write -= submitted; 3150 3151 if (unlikely(ret)) { 3152 /* 3153 * keep nr_to_write, since vfs uses this to 3154 * get # of written pages. 3155 */ 3156 if (ret == 1) { 3157 ret = 0; 3158 goto next; 3159 } else if (ret == -EAGAIN) { 3160 ret = 0; 3161 if (wbc->sync_mode == WB_SYNC_ALL) { 3162 f2fs_schedule_timeout( 3163 DEFAULT_SCHEDULE_TIMEOUT); 3164 goto retry_write; 3165 } 3166 goto next; 3167 } 3168 done_index = folio_next_index(folio); 3169 done = 1; 3170 break; 3171 } 3172 3173 if (wbc->nr_to_write <= 0 && 3174 wbc->sync_mode == WB_SYNC_NONE) { 3175 done = 1; 3176 break; 3177 } 3178next: 3179 if (need_readd) 3180 goto readd; 3181 } 3182 release_pages(pages, nr_pages); 3183 cond_resched(); 3184 } 3185#ifdef CONFIG_F2FS_FS_COMPRESSION 3186 /* flush remained pages in compress cluster */ 3187 if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) { 3188 ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type); 3189 nwritten += submitted; 3190 wbc->nr_to_write -= submitted; 3191 if (ret) { 3192 done = 1; 3193 retry = 0; 3194 } 3195 } 3196 if (f2fs_compressed_file(inode)) 3197 f2fs_destroy_compress_ctx(&cc, false); 3198#endif 3199 if (retry) { 3200 index = 0; 3201 end = -1; 3202 goto retry; 3203 } 3204 if (wbc->range_cyclic && !done) 3205 done_index = 0; 3206 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 3207 mapping->writeback_index = done_index; 3208 3209 if (nwritten) 3210 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, 3211 NULL, 0, DATA); 3212 /* submit cached bio of IPU write */ 3213 if (bio) 3214 f2fs_submit_merged_ipu_write(sbi, &bio, NULL); 3215 3216#ifdef CONFIG_F2FS_FS_COMPRESSION 3217 if (pages != pages_local) 3218 kfree(pages); 3219#endif 3220 3221 return ret; 3222} 3223 3224static inline bool __should_serialize_io(struct inode *inode, 3225 struct writeback_control *wbc) 3226{ 3227 /* to avoid deadlock in path of data flush */ 3228 if (F2FS_I(inode)->wb_task) 3229 return false; 3230 3231 if (!S_ISREG(inode->i_mode)) 3232 return false; 3233 if (IS_NOQUOTA(inode)) 3234 return false; 3235 3236 if (f2fs_need_compress_data(inode)) 3237 return true; 3238 if (wbc->sync_mode != WB_SYNC_ALL) 3239 return true; 3240 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) 3241 return true; 3242 return false; 3243} 3244 3245static inline void account_writeback(struct inode *inode, bool inc) 3246{ 3247 if (!f2fs_sb_has_compression(F2FS_I_SB(inode))) 3248 return; 3249 3250 f2fs_down_read(&F2FS_I(inode)->i_sem); 3251 if (inc) 3252 atomic_inc(&F2FS_I(inode)->writeback); 3253 else 3254 atomic_dec(&F2FS_I(inode)->writeback); 3255 f2fs_up_read(&F2FS_I(inode)->i_sem); 3256} 3257 3258static int __f2fs_write_data_pages(struct address_space *mapping, 3259 struct writeback_control *wbc, 3260 enum iostat_type io_type) 3261{ 3262 struct inode *inode = mapping->host; 3263 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3264 struct blk_plug plug; 3265 int ret; 3266 bool locked = false; 3267 3268 /* skip writing if there is no dirty page in this inode */ 3269 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 3270 return 0; 3271 3272 /* during POR, we don't need to trigger writepage at all. */ 3273 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 3274 goto skip_write; 3275 3276 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && 3277 wbc->sync_mode == WB_SYNC_NONE && 3278 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 3279 f2fs_available_free_memory(sbi, DIRTY_DENTS)) 3280 goto skip_write; 3281 3282 /* skip writing in file defragment preparing stage */ 3283 if (is_inode_flag_set(inode, FI_SKIP_WRITES)) 3284 goto skip_write; 3285 3286 trace_f2fs_writepages(mapping->host, wbc, DATA); 3287 3288 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ 3289 if (wbc->sync_mode == WB_SYNC_ALL) 3290 atomic_inc(&sbi->wb_sync_req[DATA]); 3291 else if (atomic_read(&sbi->wb_sync_req[DATA])) { 3292 /* to avoid potential deadlock */ 3293 if (current->plug) 3294 blk_finish_plug(current->plug); 3295 goto skip_write; 3296 } 3297 3298 if (__should_serialize_io(inode, wbc)) { 3299 mutex_lock(&sbi->writepages); 3300 locked = true; 3301 } 3302 3303 account_writeback(inode, true); 3304 3305 blk_start_plug(&plug); 3306 ret = f2fs_write_cache_pages(mapping, wbc, io_type); 3307 blk_finish_plug(&plug); 3308 3309 account_writeback(inode, false); 3310 3311 if (locked) 3312 mutex_unlock(&sbi->writepages); 3313 3314 if (wbc->sync_mode == WB_SYNC_ALL) 3315 atomic_dec(&sbi->wb_sync_req[DATA]); 3316 /* 3317 * if some pages were truncated, we cannot guarantee its mapping->host 3318 * to detect pending bios. 3319 */ 3320 3321 f2fs_remove_dirty_inode(inode); 3322 return ret; 3323 3324skip_write: 3325 wbc->pages_skipped += get_dirty_pages(inode); 3326 trace_f2fs_writepages(mapping->host, wbc, DATA); 3327 return 0; 3328} 3329 3330static int f2fs_write_data_pages(struct address_space *mapping, 3331 struct writeback_control *wbc) 3332{ 3333 struct inode *inode = mapping->host; 3334 3335 return __f2fs_write_data_pages(mapping, wbc, 3336 F2FS_I(inode)->cp_task == current ? 3337 FS_CP_DATA_IO : FS_DATA_IO); 3338} 3339 3340void f2fs_write_failed(struct inode *inode, loff_t to) 3341{ 3342 loff_t i_size = i_size_read(inode); 3343 3344 if (IS_NOQUOTA(inode)) 3345 return; 3346 3347 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */ 3348 if (to > i_size && !f2fs_verity_in_progress(inode)) { 3349 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3350 filemap_invalidate_lock(inode->i_mapping); 3351 3352 truncate_pagecache(inode, i_size); 3353 f2fs_truncate_blocks(inode, i_size, true); 3354 3355 filemap_invalidate_unlock(inode->i_mapping); 3356 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3357 } 3358} 3359 3360static int prepare_write_begin(struct f2fs_sb_info *sbi, 3361 struct folio *folio, loff_t pos, unsigned int len, 3362 block_t *blk_addr, bool *node_changed) 3363{ 3364 struct inode *inode = folio->mapping->host; 3365 pgoff_t index = folio->index; 3366 struct dnode_of_data dn; 3367 struct folio *ifolio; 3368 bool locked = false; 3369 int flag = F2FS_GET_BLOCK_PRE_AIO; 3370 int err = 0; 3371 3372 /* 3373 * If a whole page is being written and we already preallocated all the 3374 * blocks, then there is no need to get a block address now. 3375 */ 3376 if (len == PAGE_SIZE && is_inode_flag_set(inode, FI_PREALLOCATED_ALL)) 3377 return 0; 3378 3379 /* f2fs_lock_op avoids race between write CP and convert_inline_page */ 3380 if (f2fs_has_inline_data(inode)) { 3381 if (pos + len > MAX_INLINE_DATA(inode)) 3382 flag = F2FS_GET_BLOCK_DEFAULT; 3383 f2fs_map_lock(sbi, flag); 3384 locked = true; 3385 } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { 3386 f2fs_map_lock(sbi, flag); 3387 locked = true; 3388 } 3389 3390restart: 3391 /* check inline_data */ 3392 ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); 3393 if (IS_ERR(ifolio)) { 3394 err = PTR_ERR(ifolio); 3395 goto unlock_out; 3396 } 3397 3398 set_new_dnode(&dn, inode, ifolio, ifolio, 0); 3399 3400 if (f2fs_has_inline_data(inode)) { 3401 if (pos + len <= MAX_INLINE_DATA(inode)) { 3402 f2fs_do_read_inline_data(folio, ifolio); 3403 set_inode_flag(inode, FI_DATA_EXIST); 3404 if (inode->i_nlink) 3405 folio_set_f2fs_inline(ifolio); 3406 goto out; 3407 } 3408 err = f2fs_convert_inline_folio(&dn, folio); 3409 if (err || dn.data_blkaddr != NULL_ADDR) 3410 goto out; 3411 } 3412 3413 if (!f2fs_lookup_read_extent_cache_block(inode, index, 3414 &dn.data_blkaddr)) { 3415 if (IS_DEVICE_ALIASING(inode)) { 3416 err = -ENODATA; 3417 goto out; 3418 } 3419 3420 if (locked) { 3421 err = f2fs_reserve_block(&dn, index); 3422 goto out; 3423 } 3424 3425 /* hole case */ 3426 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3427 if (!err && dn.data_blkaddr != NULL_ADDR) 3428 goto out; 3429 f2fs_put_dnode(&dn); 3430 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3431 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); 3432 locked = true; 3433 goto restart; 3434 } 3435out: 3436 if (!err) { 3437 /* convert_inline_page can make node_changed */ 3438 *blk_addr = dn.data_blkaddr; 3439 *node_changed = dn.node_changed; 3440 } 3441 f2fs_put_dnode(&dn); 3442unlock_out: 3443 if (locked) 3444 f2fs_map_unlock(sbi, flag); 3445 return err; 3446} 3447 3448static int __find_data_block(struct inode *inode, pgoff_t index, 3449 block_t *blk_addr) 3450{ 3451 struct dnode_of_data dn; 3452 struct folio *ifolio; 3453 int err = 0; 3454 3455 ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); 3456 if (IS_ERR(ifolio)) 3457 return PTR_ERR(ifolio); 3458 3459 set_new_dnode(&dn, inode, ifolio, ifolio, 0); 3460 3461 if (!f2fs_lookup_read_extent_cache_block(inode, index, 3462 &dn.data_blkaddr)) { 3463 /* hole case */ 3464 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3465 if (err) { 3466 dn.data_blkaddr = NULL_ADDR; 3467 err = 0; 3468 } 3469 } 3470 *blk_addr = dn.data_blkaddr; 3471 f2fs_put_dnode(&dn); 3472 return err; 3473} 3474 3475static int __reserve_data_block(struct inode *inode, pgoff_t index, 3476 block_t *blk_addr, bool *node_changed) 3477{ 3478 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3479 struct dnode_of_data dn; 3480 struct folio *ifolio; 3481 int err = 0; 3482 3483 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3484 3485 ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); 3486 if (IS_ERR(ifolio)) { 3487 err = PTR_ERR(ifolio); 3488 goto unlock_out; 3489 } 3490 set_new_dnode(&dn, inode, ifolio, ifolio, 0); 3491 3492 if (!f2fs_lookup_read_extent_cache_block(dn.inode, index, 3493 &dn.data_blkaddr)) 3494 err = f2fs_reserve_block(&dn, index); 3495 3496 *blk_addr = dn.data_blkaddr; 3497 *node_changed = dn.node_changed; 3498 f2fs_put_dnode(&dn); 3499 3500unlock_out: 3501 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3502 return err; 3503} 3504 3505static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, 3506 struct folio *folio, loff_t pos, unsigned int len, 3507 block_t *blk_addr, bool *node_changed, bool *use_cow) 3508{ 3509 struct inode *inode = folio->mapping->host; 3510 struct inode *cow_inode = F2FS_I(inode)->cow_inode; 3511 pgoff_t index = folio->index; 3512 int err = 0; 3513 block_t ori_blk_addr = NULL_ADDR; 3514 3515 /* If pos is beyond the end of file, reserve a new block in COW inode */ 3516 if ((pos & PAGE_MASK) >= i_size_read(inode)) 3517 goto reserve_block; 3518 3519 /* Look for the block in COW inode first */ 3520 err = __find_data_block(cow_inode, index, blk_addr); 3521 if (err) { 3522 return err; 3523 } else if (*blk_addr != NULL_ADDR) { 3524 *use_cow = true; 3525 return 0; 3526 } 3527 3528 if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE)) 3529 goto reserve_block; 3530 3531 /* Look for the block in the original inode */ 3532 err = __find_data_block(inode, index, &ori_blk_addr); 3533 if (err) 3534 return err; 3535 3536reserve_block: 3537 /* Finally, we should reserve a new block in COW inode for the update */ 3538 err = __reserve_data_block(cow_inode, index, blk_addr, node_changed); 3539 if (err) 3540 return err; 3541 inc_atomic_write_cnt(inode); 3542 3543 if (ori_blk_addr != NULL_ADDR) 3544 *blk_addr = ori_blk_addr; 3545 return 0; 3546} 3547 3548static int f2fs_write_begin(const struct kiocb *iocb, 3549 struct address_space *mapping, 3550 loff_t pos, unsigned len, struct folio **foliop, 3551 void **fsdata) 3552{ 3553 struct inode *inode = mapping->host; 3554 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3555 struct folio *folio; 3556 pgoff_t index = pos >> PAGE_SHIFT; 3557 bool need_balance = false; 3558 bool use_cow = false; 3559 block_t blkaddr = NULL_ADDR; 3560 int err = 0; 3561 3562 trace_f2fs_write_begin(inode, pos, len); 3563 3564 if (!f2fs_is_checkpoint_ready(sbi)) { 3565 err = -ENOSPC; 3566 goto fail; 3567 } 3568 3569 /* 3570 * We should check this at this moment to avoid deadlock on inode page 3571 * and #0 page. The locking rule for inline_data conversion should be: 3572 * folio_lock(folio #0) -> folio_lock(inode_page) 3573 */ 3574 if (index != 0) { 3575 err = f2fs_convert_inline_inode(inode); 3576 if (err) 3577 goto fail; 3578 } 3579 3580#ifdef CONFIG_F2FS_FS_COMPRESSION 3581 if (f2fs_compressed_file(inode)) { 3582 int ret; 3583 struct page *page; 3584 3585 *fsdata = NULL; 3586 3587 if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode))) 3588 goto repeat; 3589 3590 ret = f2fs_prepare_compress_overwrite(inode, &page, 3591 index, fsdata); 3592 if (ret < 0) { 3593 err = ret; 3594 goto fail; 3595 } else if (ret) { 3596 *foliop = page_folio(page); 3597 return 0; 3598 } 3599 } 3600#endif 3601 3602repeat: 3603 /* 3604 * Do not use FGP_STABLE to avoid deadlock. 3605 * Will wait that below with our IO control. 3606 */ 3607 folio = f2fs_filemap_get_folio(mapping, index, 3608 FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_NOFS, 3609 mapping_gfp_mask(mapping)); 3610 if (IS_ERR(folio)) { 3611 err = PTR_ERR(folio); 3612 goto fail; 3613 } 3614 3615 /* TODO: cluster can be compressed due to race with .writepage */ 3616 3617 *foliop = folio; 3618 3619 if (f2fs_is_atomic_file(inode)) 3620 err = prepare_atomic_write_begin(sbi, folio, pos, len, 3621 &blkaddr, &need_balance, &use_cow); 3622 else 3623 err = prepare_write_begin(sbi, folio, pos, len, 3624 &blkaddr, &need_balance); 3625 if (err) 3626 goto put_folio; 3627 3628 if (need_balance && !IS_NOQUOTA(inode) && 3629 has_not_enough_free_secs(sbi, 0, 0)) { 3630 folio_unlock(folio); 3631 f2fs_balance_fs(sbi, true); 3632 folio_lock(folio); 3633 if (folio->mapping != mapping) { 3634 /* The folio got truncated from under us */ 3635 folio_unlock(folio); 3636 folio_put(folio); 3637 goto repeat; 3638 } 3639 } 3640 3641 f2fs_folio_wait_writeback(folio, DATA, false, true); 3642 3643 if (len == folio_size(folio) || folio_test_uptodate(folio)) 3644 return 0; 3645 3646 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) && 3647 !f2fs_verity_in_progress(inode)) { 3648 folio_zero_segment(folio, len, folio_size(folio)); 3649 return 0; 3650 } 3651 3652 if (blkaddr == NEW_ADDR) { 3653 folio_zero_segment(folio, 0, folio_size(folio)); 3654 folio_mark_uptodate(folio); 3655 } else { 3656 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 3657 DATA_GENERIC_ENHANCE_READ)) { 3658 err = -EFSCORRUPTED; 3659 goto put_folio; 3660 } 3661 f2fs_submit_page_read(use_cow ? 3662 F2FS_I(inode)->cow_inode : inode, 3663 folio, blkaddr, 0, true); 3664 3665 folio_lock(folio); 3666 if (unlikely(folio->mapping != mapping)) { 3667 folio_unlock(folio); 3668 folio_put(folio); 3669 goto repeat; 3670 } 3671 if (unlikely(!folio_test_uptodate(folio))) { 3672 err = -EIO; 3673 goto put_folio; 3674 } 3675 } 3676 return 0; 3677 3678put_folio: 3679 f2fs_folio_put(folio, true); 3680fail: 3681 f2fs_write_failed(inode, pos + len); 3682 return err; 3683} 3684 3685static int f2fs_write_end(const struct kiocb *iocb, 3686 struct address_space *mapping, 3687 loff_t pos, unsigned len, unsigned copied, 3688 struct folio *folio, void *fsdata) 3689{ 3690 struct inode *inode = folio->mapping->host; 3691 3692 trace_f2fs_write_end(inode, pos, len, copied); 3693 3694 /* 3695 * This should be come from len == PAGE_SIZE, and we expect copied 3696 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and 3697 * let generic_perform_write() try to copy data again through copied=0. 3698 */ 3699 if (!folio_test_uptodate(folio)) { 3700 if (unlikely(copied != len)) 3701 copied = 0; 3702 else 3703 folio_mark_uptodate(folio); 3704 } 3705 3706#ifdef CONFIG_F2FS_FS_COMPRESSION 3707 /* overwrite compressed file */ 3708 if (f2fs_compressed_file(inode) && fsdata) { 3709 f2fs_compress_write_end(inode, fsdata, folio->index, copied); 3710 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3711 3712 if (pos + copied > i_size_read(inode) && 3713 !f2fs_verity_in_progress(inode)) 3714 f2fs_i_size_write(inode, pos + copied); 3715 return copied; 3716 } 3717#endif 3718 3719 if (!copied) 3720 goto unlock_out; 3721 3722 folio_mark_dirty(folio); 3723 3724 if (f2fs_is_atomic_file(inode)) 3725 folio_set_f2fs_atomic(folio); 3726 3727 if (pos + copied > i_size_read(inode) && 3728 !f2fs_verity_in_progress(inode)) { 3729 f2fs_i_size_write(inode, pos + copied); 3730 if (f2fs_is_atomic_file(inode)) 3731 f2fs_i_size_write(F2FS_I(inode)->cow_inode, 3732 pos + copied); 3733 } 3734unlock_out: 3735 f2fs_folio_put(folio, true); 3736 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3737 return copied; 3738} 3739 3740void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length) 3741{ 3742 struct inode *inode = folio->mapping->host; 3743 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3744 3745 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 3746 (offset || length != folio_size(folio))) 3747 return; 3748 3749 if (folio_test_dirty(folio)) { 3750 if (inode->i_ino == F2FS_META_INO(sbi)) { 3751 dec_page_count(sbi, F2FS_DIRTY_META); 3752 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { 3753 dec_page_count(sbi, F2FS_DIRTY_NODES); 3754 } else { 3755 inode_dec_dirty_pages(inode); 3756 f2fs_remove_dirty_inode(inode); 3757 } 3758 } 3759 folio_detach_private(folio); 3760} 3761 3762bool f2fs_release_folio(struct folio *folio, gfp_t wait) 3763{ 3764 /* If this is dirty folio, keep private data */ 3765 if (folio_test_dirty(folio)) 3766 return false; 3767 3768 folio_detach_private(folio); 3769 return true; 3770} 3771 3772static bool f2fs_dirty_data_folio(struct address_space *mapping, 3773 struct folio *folio) 3774{ 3775 struct inode *inode = mapping->host; 3776 3777 trace_f2fs_set_page_dirty(folio, DATA); 3778 3779 if (!folio_test_uptodate(folio)) 3780 folio_mark_uptodate(folio); 3781 BUG_ON(folio_test_swapcache(folio)); 3782 3783 if (filemap_dirty_folio(mapping, folio)) { 3784 f2fs_update_dirty_folio(inode, folio); 3785 return true; 3786 } 3787 return false; 3788} 3789 3790 3791static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) 3792{ 3793#ifdef CONFIG_F2FS_FS_COMPRESSION 3794 struct dnode_of_data dn; 3795 sector_t start_idx, blknr = 0; 3796 int ret; 3797 3798 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size); 3799 3800 set_new_dnode(&dn, inode, NULL, NULL, 0); 3801 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 3802 if (ret) 3803 return 0; 3804 3805 if (dn.data_blkaddr != COMPRESS_ADDR) { 3806 dn.ofs_in_node += block - start_idx; 3807 blknr = f2fs_data_blkaddr(&dn); 3808 if (!__is_valid_data_blkaddr(blknr)) 3809 blknr = 0; 3810 } 3811 3812 f2fs_put_dnode(&dn); 3813 return blknr; 3814#else 3815 return 0; 3816#endif 3817} 3818 3819 3820static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 3821{ 3822 struct inode *inode = mapping->host; 3823 sector_t blknr = 0; 3824 3825 if (f2fs_has_inline_data(inode)) 3826 goto out; 3827 3828 /* make sure allocating whole blocks */ 3829 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 3830 filemap_write_and_wait(mapping); 3831 3832 /* Block number less than F2FS MAX BLOCKS */ 3833 if (unlikely(block >= max_file_blocks(inode))) 3834 goto out; 3835 3836 if (f2fs_compressed_file(inode)) { 3837 blknr = f2fs_bmap_compress(inode, block); 3838 } else { 3839 struct f2fs_map_blocks map; 3840 3841 memset(&map, 0, sizeof(map)); 3842 map.m_lblk = block; 3843 map.m_len = 1; 3844 map.m_next_pgofs = NULL; 3845 map.m_seg_type = NO_CHECK_TYPE; 3846 3847 if (!f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_BMAP)) 3848 blknr = map.m_pblk; 3849 } 3850out: 3851 trace_f2fs_bmap(inode, block, blknr); 3852 return blknr; 3853} 3854 3855#ifdef CONFIG_SWAP 3856static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, 3857 unsigned int blkcnt) 3858{ 3859 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3860 unsigned int blkofs; 3861 unsigned int blk_per_sec = BLKS_PER_SEC(sbi); 3862 unsigned int end_blk = start_blk + blkcnt - 1; 3863 unsigned int secidx = start_blk / blk_per_sec; 3864 unsigned int end_sec; 3865 int ret = 0; 3866 3867 if (!blkcnt) 3868 return 0; 3869 end_sec = end_blk / blk_per_sec; 3870 3871 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3872 filemap_invalidate_lock(inode->i_mapping); 3873 3874 set_inode_flag(inode, FI_ALIGNED_WRITE); 3875 set_inode_flag(inode, FI_OPU_WRITE); 3876 3877 for (; secidx <= end_sec; secidx++) { 3878 unsigned int blkofs_end = secidx == end_sec ? 3879 end_blk % blk_per_sec : blk_per_sec - 1; 3880 3881 f2fs_down_write(&sbi->pin_sem); 3882 3883 ret = f2fs_allocate_pinning_section(sbi); 3884 if (ret) { 3885 f2fs_up_write(&sbi->pin_sem); 3886 break; 3887 } 3888 3889 set_inode_flag(inode, FI_SKIP_WRITES); 3890 3891 for (blkofs = 0; blkofs <= blkofs_end; blkofs++) { 3892 struct folio *folio; 3893 unsigned int blkidx = secidx * blk_per_sec + blkofs; 3894 3895 folio = f2fs_get_lock_data_folio(inode, blkidx, true); 3896 if (IS_ERR(folio)) { 3897 f2fs_up_write(&sbi->pin_sem); 3898 ret = PTR_ERR(folio); 3899 goto done; 3900 } 3901 3902 folio_mark_dirty(folio); 3903 f2fs_folio_put(folio, true); 3904 } 3905 3906 clear_inode_flag(inode, FI_SKIP_WRITES); 3907 3908 ret = filemap_fdatawrite(inode->i_mapping); 3909 3910 f2fs_up_write(&sbi->pin_sem); 3911 3912 if (ret) 3913 break; 3914 } 3915 3916done: 3917 clear_inode_flag(inode, FI_SKIP_WRITES); 3918 clear_inode_flag(inode, FI_OPU_WRITE); 3919 clear_inode_flag(inode, FI_ALIGNED_WRITE); 3920 3921 filemap_invalidate_unlock(inode->i_mapping); 3922 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3923 3924 return ret; 3925} 3926 3927static int check_swap_activate(struct swap_info_struct *sis, 3928 struct file *swap_file, sector_t *span) 3929{ 3930 struct address_space *mapping = swap_file->f_mapping; 3931 struct inode *inode = mapping->host; 3932 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3933 block_t cur_lblock; 3934 block_t last_lblock; 3935 block_t pblock; 3936 block_t lowest_pblock = -1; 3937 block_t highest_pblock = 0; 3938 int nr_extents = 0; 3939 unsigned int nr_pblocks; 3940 unsigned int blks_per_sec = BLKS_PER_SEC(sbi); 3941 unsigned int not_aligned = 0; 3942 int ret = 0; 3943 3944 /* 3945 * Map all the blocks into the extent list. This code doesn't try 3946 * to be very smart. 3947 */ 3948 cur_lblock = 0; 3949 last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode)); 3950 3951 while (cur_lblock < last_lblock && cur_lblock < sis->max) { 3952 struct f2fs_map_blocks map; 3953 bool last_extent = false; 3954retry: 3955 cond_resched(); 3956 3957 memset(&map, 0, sizeof(map)); 3958 map.m_lblk = cur_lblock; 3959 map.m_len = last_lblock - cur_lblock; 3960 map.m_next_pgofs = NULL; 3961 map.m_next_extent = NULL; 3962 map.m_seg_type = NO_CHECK_TYPE; 3963 map.m_may_create = false; 3964 3965 ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 3966 if (ret) 3967 goto out; 3968 3969 /* hole */ 3970 if (!(map.m_flags & F2FS_MAP_FLAGS)) { 3971 f2fs_err(sbi, "Swapfile has holes"); 3972 ret = -EINVAL; 3973 goto out; 3974 } 3975 3976 pblock = map.m_pblk; 3977 nr_pblocks = map.m_len; 3978 3979 if (!last_extent && 3980 ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || 3981 nr_pblocks % blks_per_sec || 3982 f2fs_is_sequential_zone_area(sbi, pblock))) { 3983 not_aligned++; 3984 3985 nr_pblocks = roundup(nr_pblocks, blks_per_sec); 3986 if (cur_lblock + nr_pblocks > sis->max) 3987 nr_pblocks -= blks_per_sec; 3988 3989 /* this extent is last one */ 3990 if (!nr_pblocks) { 3991 nr_pblocks = last_lblock - cur_lblock; 3992 last_extent = true; 3993 } 3994 3995 ret = f2fs_migrate_blocks(inode, cur_lblock, 3996 nr_pblocks); 3997 if (ret) { 3998 if (ret == -ENOENT) 3999 ret = -EINVAL; 4000 goto out; 4001 } 4002 4003 /* lookup block mapping info after block migration */ 4004 goto retry; 4005 } 4006 4007 if (cur_lblock + nr_pblocks >= sis->max) 4008 nr_pblocks = sis->max - cur_lblock; 4009 4010 if (cur_lblock) { /* exclude the header page */ 4011 if (pblock < lowest_pblock) 4012 lowest_pblock = pblock; 4013 if (pblock + nr_pblocks - 1 > highest_pblock) 4014 highest_pblock = pblock + nr_pblocks - 1; 4015 } 4016 4017 /* 4018 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 4019 */ 4020 ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock); 4021 if (ret < 0) 4022 goto out; 4023 nr_extents += ret; 4024 cur_lblock += nr_pblocks; 4025 } 4026 ret = nr_extents; 4027 *span = 1 + highest_pblock - lowest_pblock; 4028 if (cur_lblock == 0) 4029 cur_lblock = 1; /* force Empty message */ 4030 sis->max = cur_lblock; 4031 sis->pages = cur_lblock - 1; 4032out: 4033 if (not_aligned) 4034 f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)", 4035 not_aligned, blks_per_sec * F2FS_BLKSIZE); 4036 return ret; 4037} 4038 4039static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4040 sector_t *span) 4041{ 4042 struct inode *inode = file_inode(file); 4043 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4044 int ret; 4045 4046 if (!S_ISREG(inode->i_mode)) 4047 return -EINVAL; 4048 4049 if (f2fs_readonly(sbi->sb)) 4050 return -EROFS; 4051 4052 if (f2fs_lfs_mode(sbi) && !f2fs_sb_has_blkzoned(sbi)) { 4053 f2fs_err(sbi, "Swapfile not supported in LFS mode"); 4054 return -EINVAL; 4055 } 4056 4057 ret = f2fs_convert_inline_inode(inode); 4058 if (ret) 4059 return ret; 4060 4061 if (!f2fs_disable_compressed_file(inode)) 4062 return -EINVAL; 4063 4064 ret = filemap_fdatawrite(inode->i_mapping); 4065 if (ret < 0) 4066 return ret; 4067 4068 f2fs_precache_extents(inode); 4069 4070 ret = check_swap_activate(sis, file, span); 4071 if (ret < 0) 4072 return ret; 4073 4074 stat_inc_swapfile_inode(inode); 4075 set_inode_flag(inode, FI_PIN_FILE); 4076 f2fs_update_time(sbi, REQ_TIME); 4077 return ret; 4078} 4079 4080static void f2fs_swap_deactivate(struct file *file) 4081{ 4082 struct inode *inode = file_inode(file); 4083 4084 stat_dec_swapfile_inode(inode); 4085 clear_inode_flag(inode, FI_PIN_FILE); 4086} 4087#else 4088static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4089 sector_t *span) 4090{ 4091 return -EOPNOTSUPP; 4092} 4093 4094static void f2fs_swap_deactivate(struct file *file) 4095{ 4096} 4097#endif 4098 4099const struct address_space_operations f2fs_dblock_aops = { 4100 .read_folio = f2fs_read_data_folio, 4101 .readahead = f2fs_readahead, 4102 .writepages = f2fs_write_data_pages, 4103 .write_begin = f2fs_write_begin, 4104 .write_end = f2fs_write_end, 4105 .dirty_folio = f2fs_dirty_data_folio, 4106 .migrate_folio = filemap_migrate_folio, 4107 .invalidate_folio = f2fs_invalidate_folio, 4108 .release_folio = f2fs_release_folio, 4109 .bmap = f2fs_bmap, 4110 .swap_activate = f2fs_swap_activate, 4111 .swap_deactivate = f2fs_swap_deactivate, 4112}; 4113 4114void f2fs_clear_page_cache_dirty_tag(struct folio *folio) 4115{ 4116 struct address_space *mapping = folio->mapping; 4117 unsigned long flags; 4118 4119 xa_lock_irqsave(&mapping->i_pages, flags); 4120 __xa_clear_mark(&mapping->i_pages, folio->index, 4121 PAGECACHE_TAG_DIRTY); 4122 xa_unlock_irqrestore(&mapping->i_pages, flags); 4123} 4124 4125int __init f2fs_init_post_read_processing(void) 4126{ 4127 bio_post_read_ctx_cache = 4128 kmem_cache_create("f2fs_bio_post_read_ctx", 4129 sizeof(struct bio_post_read_ctx), 0, 0, NULL); 4130 if (!bio_post_read_ctx_cache) 4131 goto fail; 4132 bio_post_read_ctx_pool = 4133 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, 4134 bio_post_read_ctx_cache); 4135 if (!bio_post_read_ctx_pool) 4136 goto fail_free_cache; 4137 return 0; 4138 4139fail_free_cache: 4140 kmem_cache_destroy(bio_post_read_ctx_cache); 4141fail: 4142 return -ENOMEM; 4143} 4144 4145void f2fs_destroy_post_read_processing(void) 4146{ 4147 mempool_destroy(bio_post_read_ctx_pool); 4148 kmem_cache_destroy(bio_post_read_ctx_cache); 4149} 4150 4151int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi) 4152{ 4153 if (!f2fs_sb_has_encrypt(sbi) && 4154 !f2fs_sb_has_verity(sbi) && 4155 !f2fs_sb_has_compression(sbi)) 4156 return 0; 4157 4158 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq", 4159 WQ_UNBOUND | WQ_HIGHPRI, 4160 num_online_cpus()); 4161 return sbi->post_read_wq ? 0 : -ENOMEM; 4162} 4163 4164void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi) 4165{ 4166 if (sbi->post_read_wq) 4167 destroy_workqueue(sbi->post_read_wq); 4168} 4169 4170int __init f2fs_init_bio_entry_cache(void) 4171{ 4172 bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab", 4173 sizeof(struct bio_entry)); 4174 return bio_entry_slab ? 0 : -ENOMEM; 4175} 4176 4177void f2fs_destroy_bio_entry_cache(void) 4178{ 4179 kmem_cache_destroy(bio_entry_slab); 4180} 4181 4182static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, 4183 unsigned int flags, struct iomap *iomap, 4184 struct iomap *srcmap) 4185{ 4186 struct f2fs_map_blocks map = { NULL, }; 4187 pgoff_t next_pgofs = 0; 4188 int err; 4189 4190 map.m_lblk = F2FS_BYTES_TO_BLK(offset); 4191 map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1; 4192 map.m_next_pgofs = &next_pgofs; 4193 map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), 4194 inode->i_write_hint); 4195 if (flags & IOMAP_WRITE && iomap->private) { 4196 map.m_last_pblk = (unsigned long)iomap->private; 4197 iomap->private = NULL; 4198 } 4199 4200 /* 4201 * If the blocks being overwritten are already allocated, 4202 * f2fs_map_lock and f2fs_balance_fs are not necessary. 4203 */ 4204 if ((flags & IOMAP_WRITE) && 4205 !__f2fs_overwrite_io(inode, offset, length, true)) 4206 map.m_may_create = true; 4207 4208 err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); 4209 if (err) 4210 return err; 4211 4212 iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk); 4213 4214 /* 4215 * When inline encryption is enabled, sometimes I/O to an encrypted file 4216 * has to be broken up to guarantee DUN contiguity. Handle this by 4217 * limiting the length of the mapping returned. 4218 */ 4219 map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); 4220 4221 /* 4222 * We should never see delalloc or compressed extents here based on 4223 * prior flushing and checks. 4224 */ 4225 if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) 4226 return -EINVAL; 4227 4228 if (map.m_flags & F2FS_MAP_MAPPED) { 4229 if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) 4230 return -EINVAL; 4231 4232 iomap->length = F2FS_BLK_TO_BYTES(map.m_len); 4233 iomap->type = IOMAP_MAPPED; 4234 iomap->flags |= IOMAP_F_MERGED; 4235 iomap->bdev = map.m_bdev; 4236 iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk); 4237 4238 if (flags & IOMAP_WRITE && map.m_last_pblk) 4239 iomap->private = (void *)map.m_last_pblk; 4240 } else { 4241 if (flags & IOMAP_WRITE) 4242 return -ENOTBLK; 4243 4244 if (map.m_pblk == NULL_ADDR) { 4245 iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) - 4246 iomap->offset; 4247 iomap->type = IOMAP_HOLE; 4248 } else if (map.m_pblk == NEW_ADDR) { 4249 iomap->length = F2FS_BLK_TO_BYTES(map.m_len); 4250 iomap->type = IOMAP_UNWRITTEN; 4251 } else { 4252 f2fs_bug_on(F2FS_I_SB(inode), 1); 4253 } 4254 iomap->addr = IOMAP_NULL_ADDR; 4255 } 4256 4257 if (map.m_flags & F2FS_MAP_NEW) 4258 iomap->flags |= IOMAP_F_NEW; 4259 if ((inode_state_read_once(inode) & I_DIRTY_DATASYNC) || 4260 offset + length > i_size_read(inode)) 4261 iomap->flags |= IOMAP_F_DIRTY; 4262 4263 return 0; 4264} 4265 4266const struct iomap_ops f2fs_iomap_ops = { 4267 .iomap_begin = f2fs_iomap_begin, 4268};
~/
Read-only file preview.