FILE
filemap.c
File preview with syntax highlighting.
linux-6.19.14/mm/filemap.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * linux/mm/filemap.c 4 * 5 * Copyright (C) 1994-1999 Linus Torvalds 6 */ 7 8/* 9 * This file handles the generic file mmap semantics used by 10 * most "normal" filesystems (but you don't /have/ to use this: 11 * the NFS filesystem used to do this differently, for example) 12 */ 13#include <linux/export.h> 14#include <linux/compiler.h> 15#include <linux/dax.h> 16#include <linux/fs.h> 17#include <linux/sched/signal.h> 18#include <linux/uaccess.h> 19#include <linux/capability.h> 20#include <linux/kernel_stat.h> 21#include <linux/gfp.h> 22#include <linux/mm.h> 23#include <linux/swap.h> 24#include <linux/leafops.h> 25#include <linux/syscalls.h> 26#include <linux/mman.h> 27#include <linux/pagemap.h> 28#include <linux/file.h> 29#include <linux/uio.h> 30#include <linux/error-injection.h> 31#include <linux/hash.h> 32#include <linux/writeback.h> 33#include <linux/backing-dev.h> 34#include <linux/pagevec.h> 35#include <linux/security.h> 36#include <linux/cpuset.h> 37#include <linux/hugetlb.h> 38#include <linux/memcontrol.h> 39#include <linux/shmem_fs.h> 40#include <linux/rmap.h> 41#include <linux/delayacct.h> 42#include <linux/psi.h> 43#include <linux/ramfs.h> 44#include <linux/page_idle.h> 45#include <linux/migrate.h> 46#include <linux/pipe_fs_i.h> 47#include <linux/splice.h> 48#include <linux/rcupdate_wait.h> 49#include <linux/sched/mm.h> 50#include <linux/sysctl.h> 51#include <linux/pgalloc.h> 52 53#include <asm/tlbflush.h> 54#include "internal.h" 55 56#define CREATE_TRACE_POINTS 57#include <trace/events/filemap.h> 58 59/* 60 * FIXME: remove all knowledge of the buffer layer from the core VM 61 */ 62#include <linux/buffer_head.h> /* for try_to_free_buffers */ 63 64#include <asm/mman.h> 65 66#include "swap.h" 67 68/* 69 * Shared mappings implemented 30.11.1994. It's not fully working yet, 70 * though. 71 * 72 * Shared mappings now work. 15.8.1995 Bruno. 73 * 74 * finished 'unifying' the page and buffer cache and SMP-threaded the 75 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 76 * 77 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 78 */ 79 80/* 81 * Lock ordering: 82 * 83 * ->i_mmap_rwsem (truncate_pagecache) 84 * ->private_lock (__free_pte->block_dirty_folio) 85 * ->swap_lock (exclusive_swap_page, others) 86 * ->i_pages lock 87 * 88 * ->i_rwsem 89 * ->invalidate_lock (acquired by fs in truncate path) 90 * ->i_mmap_rwsem (truncate->unmap_mapping_range) 91 * 92 * ->mmap_lock 93 * ->i_mmap_rwsem 94 * ->page_table_lock or pte_lock (various, mainly in memory.c) 95 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) 96 * 97 * ->mmap_lock 98 * ->invalidate_lock (filemap_fault) 99 * ->lock_page (filemap_fault, access_process_vm) 100 * 101 * ->i_rwsem (generic_perform_write) 102 * ->mmap_lock (fault_in_readable->do_page_fault) 103 * 104 * bdi->wb.list_lock 105 * sb_lock (fs/fs-writeback.c) 106 * ->i_pages lock (__sync_single_inode) 107 * 108 * ->i_mmap_rwsem 109 * ->anon_vma.lock (vma_merge) 110 * 111 * ->anon_vma.lock 112 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 113 * 114 * ->page_table_lock or pte_lock 115 * ->swap_lock (try_to_unmap_one) 116 * ->private_lock (try_to_unmap_one) 117 * ->i_pages lock (try_to_unmap_one) 118 * ->lruvec->lru_lock (follow_page_mask->mark_page_accessed) 119 * ->lruvec->lru_lock (check_pte_range->folio_isolate_lru) 120 * ->private_lock (folio_remove_rmap_pte->set_page_dirty) 121 * ->i_pages lock (folio_remove_rmap_pte->set_page_dirty) 122 * bdi.wb->list_lock (folio_remove_rmap_pte->set_page_dirty) 123 * ->inode->i_lock (folio_remove_rmap_pte->set_page_dirty) 124 * bdi.wb->list_lock (zap_pte_range->set_page_dirty) 125 * ->inode->i_lock (zap_pte_range->set_page_dirty) 126 * ->private_lock (zap_pte_range->block_dirty_folio) 127 */ 128 129static void page_cache_delete(struct address_space *mapping, 130 struct folio *folio, void *shadow) 131{ 132 XA_STATE(xas, &mapping->i_pages, folio->index); 133 long nr = 1; 134 135 mapping_set_update(&xas, mapping); 136 137 xas_set_order(&xas, folio->index, folio_order(folio)); 138 nr = folio_nr_pages(folio); 139 140 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 141 142 xas_store(&xas, shadow); 143 xas_init_marks(&xas); 144 145 folio->mapping = NULL; 146 /* Leave folio->index set: truncation lookup relies upon it */ 147 mapping->nrpages -= nr; 148} 149 150static void filemap_unaccount_folio(struct address_space *mapping, 151 struct folio *folio) 152{ 153 long nr; 154 155 VM_BUG_ON_FOLIO(folio_mapped(folio), folio); 156 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { 157 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n", 158 current->comm, folio_pfn(folio)); 159 dump_page(&folio->page, "still mapped when deleted"); 160 dump_stack(); 161 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 162 163 if (mapping_exiting(mapping) && !folio_test_large(folio)) { 164 int mapcount = folio_mapcount(folio); 165 166 if (folio_ref_count(folio) >= mapcount + 2) { 167 /* 168 * All vmas have already been torn down, so it's 169 * a good bet that actually the page is unmapped 170 * and we'd rather not leak it: if we're wrong, 171 * another bad page check should catch it later. 172 */ 173 atomic_set(&folio->_mapcount, -1); 174 folio_ref_sub(folio, mapcount); 175 } 176 } 177 } 178 179 /* hugetlb folios do not participate in page cache accounting. */ 180 if (folio_test_hugetlb(folio)) 181 return; 182 183 nr = folio_nr_pages(folio); 184 185 lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 186 if (folio_test_swapbacked(folio)) { 187 lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); 188 if (folio_test_pmd_mappable(folio)) 189 lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); 190 } else if (folio_test_pmd_mappable(folio)) { 191 lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); 192 filemap_nr_thps_dec(mapping); 193 } 194 if (test_bit(AS_KERNEL_FILE, &folio->mapping->flags)) 195 mod_node_page_state(folio_pgdat(folio), 196 NR_KERNEL_FILE_PAGES, -nr); 197 198 /* 199 * At this point folio must be either written or cleaned by 200 * truncate. Dirty folio here signals a bug and loss of 201 * unwritten data - on ordinary filesystems. 202 * 203 * But it's harmless on in-memory filesystems like tmpfs; and can 204 * occur when a driver which did get_user_pages() sets page dirty 205 * before putting it, while the inode is being finally evicted. 206 * 207 * Below fixes dirty accounting after removing the folio entirely 208 * but leaves the dirty flag set: it has no effect for truncated 209 * folio and anyway will be cleared before returning folio to 210 * buddy allocator. 211 */ 212 if (WARN_ON_ONCE(folio_test_dirty(folio) && 213 mapping_can_writeback(mapping))) 214 folio_account_cleaned(folio, inode_to_wb(mapping->host)); 215} 216 217/* 218 * Delete a page from the page cache and free it. Caller has to make 219 * sure the page is locked and that nobody else uses it - or that usage 220 * is safe. The caller must hold the i_pages lock. 221 */ 222void __filemap_remove_folio(struct folio *folio, void *shadow) 223{ 224 struct address_space *mapping = folio->mapping; 225 226 trace_mm_filemap_delete_from_page_cache(folio); 227 filemap_unaccount_folio(mapping, folio); 228 page_cache_delete(mapping, folio, shadow); 229} 230 231static void filemap_free_folio(const struct address_space *mapping, 232 struct folio *folio) 233{ 234 void (*free_folio)(struct folio *); 235 236 free_folio = mapping->a_ops->free_folio; 237 if (free_folio) 238 free_folio(folio); 239 240 folio_put_refs(folio, folio_nr_pages(folio)); 241} 242 243/** 244 * filemap_remove_folio - Remove folio from page cache. 245 * @folio: The folio. 246 * 247 * This must be called only on folios that are locked and have been 248 * verified to be in the page cache. It will never put the folio into 249 * the free list because the caller has a reference on the page. 250 */ 251void filemap_remove_folio(struct folio *folio) 252{ 253 struct address_space *mapping = folio->mapping; 254 255 BUG_ON(!folio_test_locked(folio)); 256 spin_lock(&mapping->host->i_lock); 257 xa_lock_irq(&mapping->i_pages); 258 __filemap_remove_folio(folio, NULL); 259 xa_unlock_irq(&mapping->i_pages); 260 if (mapping_shrinkable(mapping)) 261 inode_lru_list_add(mapping->host); 262 spin_unlock(&mapping->host->i_lock); 263 264 filemap_free_folio(mapping, folio); 265} 266 267/* 268 * page_cache_delete_batch - delete several folios from page cache 269 * @mapping: the mapping to which folios belong 270 * @fbatch: batch of folios to delete 271 * 272 * The function walks over mapping->i_pages and removes folios passed in 273 * @fbatch from the mapping. The function expects @fbatch to be sorted 274 * by page index and is optimised for it to be dense. 275 * It tolerates holes in @fbatch (mapping entries at those indices are not 276 * modified). 277 * 278 * The function expects the i_pages lock to be held. 279 */ 280static void page_cache_delete_batch(struct address_space *mapping, 281 struct folio_batch *fbatch) 282{ 283 XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); 284 long total_pages = 0; 285 int i = 0; 286 struct folio *folio; 287 288 mapping_set_update(&xas, mapping); 289 xas_for_each(&xas, folio, ULONG_MAX) { 290 if (i >= folio_batch_count(fbatch)) 291 break; 292 293 /* A swap/dax/shadow entry got inserted? Skip it. */ 294 if (xa_is_value(folio)) 295 continue; 296 /* 297 * A page got inserted in our range? Skip it. We have our 298 * pages locked so they are protected from being removed. 299 * If we see a page whose index is higher than ours, it 300 * means our page has been removed, which shouldn't be 301 * possible because we're holding the PageLock. 302 */ 303 if (folio != fbatch->folios[i]) { 304 VM_BUG_ON_FOLIO(folio->index > 305 fbatch->folios[i]->index, folio); 306 continue; 307 } 308 309 WARN_ON_ONCE(!folio_test_locked(folio)); 310 311 folio->mapping = NULL; 312 /* Leave folio->index set: truncation lookup relies on it */ 313 314 i++; 315 xas_store(&xas, NULL); 316 total_pages += folio_nr_pages(folio); 317 } 318 mapping->nrpages -= total_pages; 319} 320 321void delete_from_page_cache_batch(struct address_space *mapping, 322 struct folio_batch *fbatch) 323{ 324 int i; 325 326 if (!folio_batch_count(fbatch)) 327 return; 328 329 spin_lock(&mapping->host->i_lock); 330 xa_lock_irq(&mapping->i_pages); 331 for (i = 0; i < folio_batch_count(fbatch); i++) { 332 struct folio *folio = fbatch->folios[i]; 333 334 trace_mm_filemap_delete_from_page_cache(folio); 335 filemap_unaccount_folio(mapping, folio); 336 } 337 page_cache_delete_batch(mapping, fbatch); 338 xa_unlock_irq(&mapping->i_pages); 339 if (mapping_shrinkable(mapping)) 340 inode_lru_list_add(mapping->host); 341 spin_unlock(&mapping->host->i_lock); 342 343 for (i = 0; i < folio_batch_count(fbatch); i++) 344 filemap_free_folio(mapping, fbatch->folios[i]); 345} 346 347int filemap_check_errors(struct address_space *mapping) 348{ 349 int ret = 0; 350 /* Check for outstanding write errors */ 351 if (test_bit(AS_ENOSPC, &mapping->flags) && 352 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 353 ret = -ENOSPC; 354 if (test_bit(AS_EIO, &mapping->flags) && 355 test_and_clear_bit(AS_EIO, &mapping->flags)) 356 ret = -EIO; 357 return ret; 358} 359EXPORT_SYMBOL(filemap_check_errors); 360 361static int filemap_check_and_keep_errors(struct address_space *mapping) 362{ 363 /* Check for outstanding write errors */ 364 if (test_bit(AS_EIO, &mapping->flags)) 365 return -EIO; 366 if (test_bit(AS_ENOSPC, &mapping->flags)) 367 return -ENOSPC; 368 return 0; 369} 370 371static int filemap_writeback(struct address_space *mapping, loff_t start, 372 loff_t end, enum writeback_sync_modes sync_mode, 373 long *nr_to_write) 374{ 375 struct writeback_control wbc = { 376 .sync_mode = sync_mode, 377 .nr_to_write = nr_to_write ? *nr_to_write : LONG_MAX, 378 .range_start = start, 379 .range_end = end, 380 }; 381 int ret; 382 383 if (!mapping_can_writeback(mapping) || 384 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 385 return 0; 386 387 wbc_attach_fdatawrite_inode(&wbc, mapping->host); 388 ret = do_writepages(mapping, &wbc); 389 wbc_detach_inode(&wbc); 390 391 if (!ret && nr_to_write) 392 *nr_to_write = wbc.nr_to_write; 393 return ret; 394} 395 396/** 397 * filemap_fdatawrite_range - start writeback on mapping dirty pages in range 398 * @mapping: address space structure to write 399 * @start: offset in bytes where the range starts 400 * @end: offset in bytes where the range ends (inclusive) 401 * 402 * Start writeback against all of a mapping's dirty pages that lie 403 * within the byte offsets <start, end> inclusive. 404 * 405 * This is a data integrity operation that waits upon dirty or in writeback 406 * pages. 407 * 408 * Return: %0 on success, negative error code otherwise. 409 */ 410int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 411 loff_t end) 412{ 413 return filemap_writeback(mapping, start, end, WB_SYNC_ALL, NULL); 414} 415EXPORT_SYMBOL(filemap_fdatawrite_range); 416 417int filemap_fdatawrite(struct address_space *mapping) 418{ 419 return filemap_fdatawrite_range(mapping, 0, LLONG_MAX); 420} 421EXPORT_SYMBOL(filemap_fdatawrite); 422 423/** 424 * filemap_flush_range - start writeback on a range 425 * @mapping: target address_space 426 * @start: index to start writeback on 427 * @end: last (inclusive) index for writeback 428 * 429 * This is a non-integrity writeback helper, to start writing back folios 430 * for the indicated range. 431 * 432 * Return: %0 on success, negative error code otherwise. 433 */ 434int filemap_flush_range(struct address_space *mapping, loff_t start, 435 loff_t end) 436{ 437 return filemap_writeback(mapping, start, end, WB_SYNC_NONE, NULL); 438} 439EXPORT_SYMBOL_GPL(filemap_flush_range); 440 441/** 442 * filemap_flush - mostly a non-blocking flush 443 * @mapping: target address_space 444 * 445 * This is a mostly non-blocking flush. Not suitable for data-integrity 446 * purposes - I/O may not be started against all dirty pages. 447 * 448 * Return: %0 on success, negative error code otherwise. 449 */ 450int filemap_flush(struct address_space *mapping) 451{ 452 return filemap_flush_range(mapping, 0, LLONG_MAX); 453} 454EXPORT_SYMBOL(filemap_flush); 455 456/* 457 * Start writeback on @nr_to_write pages from @mapping. No one but the existing 458 * btrfs caller should be using this. Talk to linux-mm if you think adding a 459 * new caller is a good idea. 460 */ 461int filemap_flush_nr(struct address_space *mapping, long *nr_to_write) 462{ 463 return filemap_writeback(mapping, 0, LLONG_MAX, WB_SYNC_NONE, 464 nr_to_write); 465} 466EXPORT_SYMBOL_FOR_MODULES(filemap_flush_nr, "btrfs"); 467 468/** 469 * filemap_range_has_page - check if a page exists in range. 470 * @mapping: address space within which to check 471 * @start_byte: offset in bytes where the range starts 472 * @end_byte: offset in bytes where the range ends (inclusive) 473 * 474 * Find at least one page in the range supplied, usually used to check if 475 * direct writing in this range will trigger a writeback. 476 * 477 * Return: %true if at least one page exists in the specified range, 478 * %false otherwise. 479 */ 480bool filemap_range_has_page(struct address_space *mapping, 481 loff_t start_byte, loff_t end_byte) 482{ 483 struct folio *folio; 484 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 485 pgoff_t max = end_byte >> PAGE_SHIFT; 486 487 if (end_byte < start_byte) 488 return false; 489 490 rcu_read_lock(); 491 for (;;) { 492 folio = xas_find(&xas, max); 493 if (xas_retry(&xas, folio)) 494 continue; 495 /* Shadow entries don't count */ 496 if (xa_is_value(folio)) 497 continue; 498 /* 499 * We don't need to try to pin this page; we're about to 500 * release the RCU lock anyway. It is enough to know that 501 * there was a page here recently. 502 */ 503 break; 504 } 505 rcu_read_unlock(); 506 507 return folio != NULL; 508} 509EXPORT_SYMBOL(filemap_range_has_page); 510 511static void __filemap_fdatawait_range(struct address_space *mapping, 512 loff_t start_byte, loff_t end_byte) 513{ 514 pgoff_t index = start_byte >> PAGE_SHIFT; 515 pgoff_t end = end_byte >> PAGE_SHIFT; 516 struct folio_batch fbatch; 517 unsigned nr_folios; 518 519 folio_batch_init(&fbatch); 520 521 while (index <= end) { 522 unsigned i; 523 524 nr_folios = filemap_get_folios_tag(mapping, &index, end, 525 PAGECACHE_TAG_WRITEBACK, &fbatch); 526 527 if (!nr_folios) 528 break; 529 530 for (i = 0; i < nr_folios; i++) { 531 struct folio *folio = fbatch.folios[i]; 532 533 folio_wait_writeback(folio); 534 } 535 folio_batch_release(&fbatch); 536 cond_resched(); 537 } 538} 539 540/** 541 * filemap_fdatawait_range - wait for writeback to complete 542 * @mapping: address space structure to wait for 543 * @start_byte: offset in bytes where the range starts 544 * @end_byte: offset in bytes where the range ends (inclusive) 545 * 546 * Walk the list of under-writeback pages of the given address space 547 * in the given range and wait for all of them. Check error status of 548 * the address space and return it. 549 * 550 * Since the error status of the address space is cleared by this function, 551 * callers are responsible for checking the return value and handling and/or 552 * reporting the error. 553 * 554 * Return: error status of the address space. 555 */ 556int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 557 loff_t end_byte) 558{ 559 __filemap_fdatawait_range(mapping, start_byte, end_byte); 560 return filemap_check_errors(mapping); 561} 562EXPORT_SYMBOL(filemap_fdatawait_range); 563 564/** 565 * filemap_fdatawait_range_keep_errors - wait for writeback to complete 566 * @mapping: address space structure to wait for 567 * @start_byte: offset in bytes where the range starts 568 * @end_byte: offset in bytes where the range ends (inclusive) 569 * 570 * Walk the list of under-writeback pages of the given address space in the 571 * given range and wait for all of them. Unlike filemap_fdatawait_range(), 572 * this function does not clear error status of the address space. 573 * 574 * Use this function if callers don't handle errors themselves. Expected 575 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 576 * fsfreeze(8) 577 */ 578int filemap_fdatawait_range_keep_errors(struct address_space *mapping, 579 loff_t start_byte, loff_t end_byte) 580{ 581 __filemap_fdatawait_range(mapping, start_byte, end_byte); 582 return filemap_check_and_keep_errors(mapping); 583} 584EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors); 585 586/** 587 * file_fdatawait_range - wait for writeback to complete 588 * @file: file pointing to address space structure to wait for 589 * @start_byte: offset in bytes where the range starts 590 * @end_byte: offset in bytes where the range ends (inclusive) 591 * 592 * Walk the list of under-writeback pages of the address space that file 593 * refers to, in the given range and wait for all of them. Check error 594 * status of the address space vs. the file->f_wb_err cursor and return it. 595 * 596 * Since the error status of the file is advanced by this function, 597 * callers are responsible for checking the return value and handling and/or 598 * reporting the error. 599 * 600 * Return: error status of the address space vs. the file->f_wb_err cursor. 601 */ 602int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte) 603{ 604 struct address_space *mapping = file->f_mapping; 605 606 __filemap_fdatawait_range(mapping, start_byte, end_byte); 607 return file_check_and_advance_wb_err(file); 608} 609EXPORT_SYMBOL(file_fdatawait_range); 610 611/** 612 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors 613 * @mapping: address space structure to wait for 614 * 615 * Walk the list of under-writeback pages of the given address space 616 * and wait for all of them. Unlike filemap_fdatawait(), this function 617 * does not clear error status of the address space. 618 * 619 * Use this function if callers don't handle errors themselves. Expected 620 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 621 * fsfreeze(8) 622 * 623 * Return: error status of the address space. 624 */ 625int filemap_fdatawait_keep_errors(struct address_space *mapping) 626{ 627 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); 628 return filemap_check_and_keep_errors(mapping); 629} 630EXPORT_SYMBOL(filemap_fdatawait_keep_errors); 631 632/* Returns true if writeback might be needed or already in progress. */ 633static bool mapping_needs_writeback(struct address_space *mapping) 634{ 635 return mapping->nrpages; 636} 637 638bool filemap_range_has_writeback(struct address_space *mapping, 639 loff_t start_byte, loff_t end_byte) 640{ 641 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 642 pgoff_t max = end_byte >> PAGE_SHIFT; 643 struct folio *folio; 644 645 if (end_byte < start_byte) 646 return false; 647 648 rcu_read_lock(); 649 xas_for_each(&xas, folio, max) { 650 if (xas_retry(&xas, folio)) 651 continue; 652 if (xa_is_value(folio)) 653 continue; 654 if (folio_test_dirty(folio) || folio_test_locked(folio) || 655 folio_test_writeback(folio)) 656 break; 657 } 658 rcu_read_unlock(); 659 return folio != NULL; 660} 661EXPORT_SYMBOL_GPL(filemap_range_has_writeback); 662 663/** 664 * filemap_write_and_wait_range - write out & wait on a file range 665 * @mapping: the address_space for the pages 666 * @lstart: offset in bytes where the range starts 667 * @lend: offset in bytes where the range ends (inclusive) 668 * 669 * Write out and wait upon file offsets lstart->lend, inclusive. 670 * 671 * Note that @lend is inclusive (describes the last byte to be written) so 672 * that this function can be used to write to the very end-of-file (end = -1). 673 * 674 * Return: error status of the address space. 675 */ 676int filemap_write_and_wait_range(struct address_space *mapping, 677 loff_t lstart, loff_t lend) 678{ 679 int err = 0, err2; 680 681 if (lend < lstart) 682 return 0; 683 684 if (mapping_needs_writeback(mapping)) { 685 err = filemap_fdatawrite_range(mapping, lstart, lend); 686 /* 687 * Even if the above returned error, the pages may be 688 * written partially (e.g. -ENOSPC), so we wait for it. 689 * But the -EIO is special case, it may indicate the worst 690 * thing (e.g. bug) happened, so we avoid waiting for it. 691 */ 692 if (err != -EIO) 693 __filemap_fdatawait_range(mapping, lstart, lend); 694 } 695 err2 = filemap_check_errors(mapping); 696 if (!err) 697 err = err2; 698 return err; 699} 700EXPORT_SYMBOL(filemap_write_and_wait_range); 701 702void __filemap_set_wb_err(struct address_space *mapping, int err) 703{ 704 errseq_t eseq = errseq_set(&mapping->wb_err, err); 705 706 trace_filemap_set_wb_err(mapping, eseq); 707} 708EXPORT_SYMBOL(__filemap_set_wb_err); 709 710/** 711 * file_check_and_advance_wb_err - report wb error (if any) that was previously 712 * and advance wb_err to current one 713 * @file: struct file on which the error is being reported 714 * 715 * When userland calls fsync (or something like nfsd does the equivalent), we 716 * want to report any writeback errors that occurred since the last fsync (or 717 * since the file was opened if there haven't been any). 718 * 719 * Grab the wb_err from the mapping. If it matches what we have in the file, 720 * then just quickly return 0. The file is all caught up. 721 * 722 * If it doesn't match, then take the mapping value, set the "seen" flag in 723 * it and try to swap it into place. If it works, or another task beat us 724 * to it with the new value, then update the f_wb_err and return the error 725 * portion. The error at this point must be reported via proper channels 726 * (a'la fsync, or NFS COMMIT operation, etc.). 727 * 728 * While we handle mapping->wb_err with atomic operations, the f_wb_err 729 * value is protected by the f_lock since we must ensure that it reflects 730 * the latest value swapped in for this file descriptor. 731 * 732 * Return: %0 on success, negative error code otherwise. 733 */ 734int file_check_and_advance_wb_err(struct file *file) 735{ 736 int err = 0; 737 errseq_t old = READ_ONCE(file->f_wb_err); 738 struct address_space *mapping = file->f_mapping; 739 740 /* Locklessly handle the common case where nothing has changed */ 741 if (errseq_check(&mapping->wb_err, old)) { 742 /* Something changed, must use slow path */ 743 spin_lock(&file->f_lock); 744 old = file->f_wb_err; 745 err = errseq_check_and_advance(&mapping->wb_err, 746 &file->f_wb_err); 747 trace_file_check_and_advance_wb_err(file, old); 748 spin_unlock(&file->f_lock); 749 } 750 751 /* 752 * We're mostly using this function as a drop in replacement for 753 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect 754 * that the legacy code would have had on these flags. 755 */ 756 clear_bit(AS_EIO, &mapping->flags); 757 clear_bit(AS_ENOSPC, &mapping->flags); 758 return err; 759} 760EXPORT_SYMBOL(file_check_and_advance_wb_err); 761 762/** 763 * file_write_and_wait_range - write out & wait on a file range 764 * @file: file pointing to address_space with pages 765 * @lstart: offset in bytes where the range starts 766 * @lend: offset in bytes where the range ends (inclusive) 767 * 768 * Write out and wait upon file offsets lstart->lend, inclusive. 769 * 770 * Note that @lend is inclusive (describes the last byte to be written) so 771 * that this function can be used to write to the very end-of-file (end = -1). 772 * 773 * After writing out and waiting on the data, we check and advance the 774 * f_wb_err cursor to the latest value, and return any errors detected there. 775 * 776 * Return: %0 on success, negative error code otherwise. 777 */ 778int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend) 779{ 780 int err = 0, err2; 781 struct address_space *mapping = file->f_mapping; 782 783 if (lend < lstart) 784 return 0; 785 786 if (mapping_needs_writeback(mapping)) { 787 err = filemap_fdatawrite_range(mapping, lstart, lend); 788 /* See comment of filemap_write_and_wait() */ 789 if (err != -EIO) 790 __filemap_fdatawait_range(mapping, lstart, lend); 791 } 792 err2 = file_check_and_advance_wb_err(file); 793 if (!err) 794 err = err2; 795 return err; 796} 797EXPORT_SYMBOL(file_write_and_wait_range); 798 799/** 800 * replace_page_cache_folio - replace a pagecache folio with a new one 801 * @old: folio to be replaced 802 * @new: folio to replace with 803 * 804 * This function replaces a folio in the pagecache with a new one. On 805 * success it acquires the pagecache reference for the new folio and 806 * drops it for the old folio. Both the old and new folios must be 807 * locked. This function does not add the new folio to the LRU, the 808 * caller must do that. 809 * 810 * The remove + add is atomic. This function cannot fail. 811 */ 812void replace_page_cache_folio(struct folio *old, struct folio *new) 813{ 814 struct address_space *mapping = old->mapping; 815 void (*free_folio)(struct folio *) = mapping->a_ops->free_folio; 816 pgoff_t offset = old->index; 817 XA_STATE(xas, &mapping->i_pages, offset); 818 819 VM_BUG_ON_FOLIO(!folio_test_locked(old), old); 820 VM_BUG_ON_FOLIO(!folio_test_locked(new), new); 821 VM_BUG_ON_FOLIO(new->mapping, new); 822 823 folio_get(new); 824 new->mapping = mapping; 825 new->index = offset; 826 827 mem_cgroup_replace_folio(old, new); 828 829 xas_lock_irq(&xas); 830 xas_store(&xas, new); 831 832 old->mapping = NULL; 833 /* hugetlb pages do not participate in page cache accounting. */ 834 if (!folio_test_hugetlb(old)) 835 lruvec_stat_sub_folio(old, NR_FILE_PAGES); 836 if (!folio_test_hugetlb(new)) 837 lruvec_stat_add_folio(new, NR_FILE_PAGES); 838 if (folio_test_swapbacked(old)) 839 lruvec_stat_sub_folio(old, NR_SHMEM); 840 if (folio_test_swapbacked(new)) 841 lruvec_stat_add_folio(new, NR_SHMEM); 842 xas_unlock_irq(&xas); 843 if (free_folio) 844 free_folio(old); 845 folio_put(old); 846} 847EXPORT_SYMBOL_GPL(replace_page_cache_folio); 848 849noinline int __filemap_add_folio(struct address_space *mapping, 850 struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) 851{ 852 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); 853 bool huge; 854 long nr; 855 unsigned int forder = folio_order(folio); 856 857 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 858 VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); 859 VM_BUG_ON_FOLIO(folio_order(folio) < mapping_min_folio_order(mapping), 860 folio); 861 mapping_set_update(&xas, mapping); 862 863 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); 864 huge = folio_test_hugetlb(folio); 865 nr = folio_nr_pages(folio); 866 867 gfp &= GFP_RECLAIM_MASK; 868 folio_ref_add(folio, nr); 869 folio->mapping = mapping; 870 folio->index = xas.xa_index; 871 872 for (;;) { 873 int order = -1; 874 void *entry, *old = NULL; 875 876 xas_lock_irq(&xas); 877 xas_for_each_conflict(&xas, entry) { 878 old = entry; 879 if (!xa_is_value(entry)) { 880 xas_set_err(&xas, -EEXIST); 881 goto unlock; 882 } 883 /* 884 * If a larger entry exists, 885 * it will be the first and only entry iterated. 886 */ 887 if (order == -1) 888 order = xas_get_order(&xas); 889 } 890 891 if (old) { 892 if (order > 0 && order > forder) { 893 unsigned int split_order = max(forder, 894 xas_try_split_min_order(order)); 895 896 /* How to handle large swap entries? */ 897 BUG_ON(shmem_mapping(mapping)); 898 899 while (order > forder) { 900 xas_set_order(&xas, index, split_order); 901 xas_try_split(&xas, old, order); 902 if (xas_error(&xas)) 903 goto unlock; 904 order = split_order; 905 split_order = 906 max(xas_try_split_min_order( 907 split_order), 908 forder); 909 } 910 xas_reset(&xas); 911 } 912 if (shadowp) 913 *shadowp = old; 914 } 915 916 xas_store(&xas, folio); 917 if (xas_error(&xas)) 918 goto unlock; 919 920 mapping->nrpages += nr; 921 922 /* hugetlb pages do not participate in page cache accounting */ 923 if (!huge) { 924 lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); 925 if (folio_test_pmd_mappable(folio)) 926 lruvec_stat_mod_folio(folio, 927 NR_FILE_THPS, nr); 928 } 929 930unlock: 931 xas_unlock_irq(&xas); 932 933 if (!xas_nomem(&xas, gfp)) 934 break; 935 } 936 937 if (xas_error(&xas)) 938 goto error; 939 940 trace_mm_filemap_add_to_page_cache(folio); 941 return 0; 942error: 943 folio->mapping = NULL; 944 /* Leave folio->index set: truncation relies upon it */ 945 folio_put_refs(folio, nr); 946 return xas_error(&xas); 947} 948ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO); 949 950int filemap_add_folio(struct address_space *mapping, struct folio *folio, 951 pgoff_t index, gfp_t gfp) 952{ 953 void *shadow = NULL; 954 int ret; 955 struct mem_cgroup *tmp; 956 bool kernel_file = test_bit(AS_KERNEL_FILE, &mapping->flags); 957 958 if (kernel_file) 959 tmp = set_active_memcg(root_mem_cgroup); 960 ret = mem_cgroup_charge(folio, NULL, gfp); 961 if (kernel_file) 962 set_active_memcg(tmp); 963 if (ret) 964 return ret; 965 966 __folio_set_locked(folio); 967 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); 968 if (unlikely(ret)) { 969 mem_cgroup_uncharge(folio); 970 __folio_clear_locked(folio); 971 } else { 972 /* 973 * The folio might have been evicted from cache only 974 * recently, in which case it should be activated like 975 * any other repeatedly accessed folio. 976 * The exception is folios getting rewritten; evicting other 977 * data from the working set, only to cache data that will 978 * get overwritten with something else, is a waste of memory. 979 */ 980 WARN_ON_ONCE(folio_test_active(folio)); 981 if (!(gfp & __GFP_WRITE) && shadow) 982 workingset_refault(folio, shadow); 983 folio_add_lru(folio); 984 if (kernel_file) 985 mod_node_page_state(folio_pgdat(folio), 986 NR_KERNEL_FILE_PAGES, 987 folio_nr_pages(folio)); 988 } 989 return ret; 990} 991EXPORT_SYMBOL_GPL(filemap_add_folio); 992 993#ifdef CONFIG_NUMA 994struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order, 995 struct mempolicy *policy) 996{ 997 int n; 998 struct folio *folio; 999 1000 if (policy) 1001 return folio_alloc_mpol_noprof(gfp, order, policy, 1002 NO_INTERLEAVE_INDEX, numa_node_id()); 1003 1004 if (cpuset_do_page_mem_spread()) { 1005 unsigned int cpuset_mems_cookie; 1006 do { 1007 cpuset_mems_cookie = read_mems_allowed_begin(); 1008 n = cpuset_mem_spread_node(); 1009 folio = __folio_alloc_node_noprof(gfp, order, n); 1010 } while (!folio && read_mems_allowed_retry(cpuset_mems_cookie)); 1011 1012 return folio; 1013 } 1014 return folio_alloc_noprof(gfp, order); 1015} 1016EXPORT_SYMBOL(filemap_alloc_folio_noprof); 1017#endif 1018 1019/* 1020 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings 1021 * 1022 * Lock exclusively invalidate_lock of any passed mapping that is not NULL. 1023 * 1024 * @mapping1: the first mapping to lock 1025 * @mapping2: the second mapping to lock 1026 */ 1027void filemap_invalidate_lock_two(struct address_space *mapping1, 1028 struct address_space *mapping2) 1029{ 1030 if (mapping1 > mapping2) 1031 swap(mapping1, mapping2); 1032 if (mapping1) 1033 down_write(&mapping1->invalidate_lock); 1034 if (mapping2 && mapping1 != mapping2) 1035 down_write_nested(&mapping2->invalidate_lock, 1); 1036} 1037EXPORT_SYMBOL(filemap_invalidate_lock_two); 1038 1039/* 1040 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings 1041 * 1042 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL. 1043 * 1044 * @mapping1: the first mapping to unlock 1045 * @mapping2: the second mapping to unlock 1046 */ 1047void filemap_invalidate_unlock_two(struct address_space *mapping1, 1048 struct address_space *mapping2) 1049{ 1050 if (mapping1) 1051 up_write(&mapping1->invalidate_lock); 1052 if (mapping2 && mapping1 != mapping2) 1053 up_write(&mapping2->invalidate_lock); 1054} 1055EXPORT_SYMBOL(filemap_invalidate_unlock_two); 1056 1057/* 1058 * In order to wait for pages to become available there must be 1059 * waitqueues associated with pages. By using a hash table of 1060 * waitqueues where the bucket discipline is to maintain all 1061 * waiters on the same queue and wake all when any of the pages 1062 * become available, and for the woken contexts to check to be 1063 * sure the appropriate page became available, this saves space 1064 * at a cost of "thundering herd" phenomena during rare hash 1065 * collisions. 1066 */ 1067#define PAGE_WAIT_TABLE_BITS 8 1068#define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS) 1069static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned; 1070 1071static wait_queue_head_t *folio_waitqueue(struct folio *folio) 1072{ 1073 return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)]; 1074} 1075 1076/* How many times do we accept lock stealing from under a waiter? */ 1077static int sysctl_page_lock_unfairness = 5; 1078static const struct ctl_table filemap_sysctl_table[] = { 1079 { 1080 .procname = "page_lock_unfairness", 1081 .data = &sysctl_page_lock_unfairness, 1082 .maxlen = sizeof(sysctl_page_lock_unfairness), 1083 .mode = 0644, 1084 .proc_handler = proc_dointvec_minmax, 1085 .extra1 = SYSCTL_ZERO, 1086 } 1087}; 1088 1089void __init pagecache_init(void) 1090{ 1091 int i; 1092 1093 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++) 1094 init_waitqueue_head(&folio_wait_table[i]); 1095 1096 page_writeback_init(); 1097 register_sysctl_init("vm", filemap_sysctl_table); 1098} 1099 1100/* 1101 * The page wait code treats the "wait->flags" somewhat unusually, because 1102 * we have multiple different kinds of waits, not just the usual "exclusive" 1103 * one. 1104 * 1105 * We have: 1106 * 1107 * (a) no special bits set: 1108 * 1109 * We're just waiting for the bit to be released, and when a waker 1110 * calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up, 1111 * and remove it from the wait queue. 1112 * 1113 * Simple and straightforward. 1114 * 1115 * (b) WQ_FLAG_EXCLUSIVE: 1116 * 1117 * The waiter is waiting to get the lock, and only one waiter should 1118 * be woken up to avoid any thundering herd behavior. We'll set the 1119 * WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue. 1120 * 1121 * This is the traditional exclusive wait. 1122 * 1123 * (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM: 1124 * 1125 * The waiter is waiting to get the bit, and additionally wants the 1126 * lock to be transferred to it for fair lock behavior. If the lock 1127 * cannot be taken, we stop walking the wait queue without waking 1128 * the waiter. 1129 * 1130 * This is the "fair lock handoff" case, and in addition to setting 1131 * WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see 1132 * that it now has the lock. 1133 */ 1134static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg) 1135{ 1136 unsigned int flags; 1137 struct wait_page_key *key = arg; 1138 struct wait_page_queue *wait_page 1139 = container_of(wait, struct wait_page_queue, wait); 1140 1141 if (!wake_page_match(wait_page, key)) 1142 return 0; 1143 1144 /* 1145 * If it's a lock handoff wait, we get the bit for it, and 1146 * stop walking (and do not wake it up) if we can't. 1147 */ 1148 flags = wait->flags; 1149 if (flags & WQ_FLAG_EXCLUSIVE) { 1150 if (test_bit(key->bit_nr, &key->folio->flags.f)) 1151 return -1; 1152 if (flags & WQ_FLAG_CUSTOM) { 1153 if (test_and_set_bit(key->bit_nr, &key->folio->flags.f)) 1154 return -1; 1155 flags |= WQ_FLAG_DONE; 1156 } 1157 } 1158 1159 /* 1160 * We are holding the wait-queue lock, but the waiter that 1161 * is waiting for this will be checking the flags without 1162 * any locking. 1163 * 1164 * So update the flags atomically, and wake up the waiter 1165 * afterwards to avoid any races. This store-release pairs 1166 * with the load-acquire in folio_wait_bit_common(). 1167 */ 1168 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); 1169 wake_up_state(wait->private, mode); 1170 1171 /* 1172 * Ok, we have successfully done what we're waiting for, 1173 * and we can unconditionally remove the wait entry. 1174 * 1175 * Note that this pairs with the "finish_wait()" in the 1176 * waiter, and has to be the absolute last thing we do. 1177 * After this list_del_init(&wait->entry) the wait entry 1178 * might be de-allocated and the process might even have 1179 * exited. 1180 */ 1181 list_del_init_careful(&wait->entry); 1182 return (flags & WQ_FLAG_EXCLUSIVE) != 0; 1183} 1184 1185static void folio_wake_bit(struct folio *folio, int bit_nr) 1186{ 1187 wait_queue_head_t *q = folio_waitqueue(folio); 1188 struct wait_page_key key; 1189 unsigned long flags; 1190 1191 key.folio = folio; 1192 key.bit_nr = bit_nr; 1193 key.page_match = 0; 1194 1195 spin_lock_irqsave(&q->lock, flags); 1196 __wake_up_locked_key(q, TASK_NORMAL, &key); 1197 1198 /* 1199 * It's possible to miss clearing waiters here, when we woke our page 1200 * waiters, but the hashed waitqueue has waiters for other pages on it. 1201 * That's okay, it's a rare case. The next waker will clear it. 1202 * 1203 * Note that, depending on the page pool (buddy, hugetlb, ZONE_DEVICE, 1204 * other), the flag may be cleared in the course of freeing the page; 1205 * but that is not required for correctness. 1206 */ 1207 if (!waitqueue_active(q) || !key.page_match) 1208 folio_clear_waiters(folio); 1209 1210 spin_unlock_irqrestore(&q->lock, flags); 1211} 1212 1213/* 1214 * A choice of three behaviors for folio_wait_bit_common(): 1215 */ 1216enum behavior { 1217 EXCLUSIVE, /* Hold ref to page and take the bit when woken, like 1218 * __folio_lock() waiting on then setting PG_locked. 1219 */ 1220 SHARED, /* Hold ref to page and check the bit when woken, like 1221 * folio_wait_writeback() waiting on PG_writeback. 1222 */ 1223 DROP, /* Drop ref to page before wait, no check when woken, 1224 * like folio_put_wait_locked() on PG_locked. 1225 */ 1226}; 1227 1228/* 1229 * Attempt to check (or get) the folio flag, and mark us done 1230 * if successful. 1231 */ 1232static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, 1233 struct wait_queue_entry *wait) 1234{ 1235 if (wait->flags & WQ_FLAG_EXCLUSIVE) { 1236 if (test_and_set_bit(bit_nr, &folio->flags.f)) 1237 return false; 1238 } else if (test_bit(bit_nr, &folio->flags.f)) 1239 return false; 1240 1241 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; 1242 return true; 1243} 1244 1245static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, 1246 int state, enum behavior behavior) 1247{ 1248 wait_queue_head_t *q = folio_waitqueue(folio); 1249 int unfairness = sysctl_page_lock_unfairness; 1250 struct wait_page_queue wait_page; 1251 wait_queue_entry_t *wait = &wait_page.wait; 1252 bool thrashing = false; 1253 unsigned long pflags; 1254 bool in_thrashing; 1255 1256 if (bit_nr == PG_locked && 1257 !folio_test_uptodate(folio) && folio_test_workingset(folio)) { 1258 delayacct_thrashing_start(&in_thrashing); 1259 psi_memstall_enter(&pflags); 1260 thrashing = true; 1261 } 1262 1263 init_wait(wait); 1264 wait->func = wake_page_function; 1265 wait_page.folio = folio; 1266 wait_page.bit_nr = bit_nr; 1267 1268repeat: 1269 wait->flags = 0; 1270 if (behavior == EXCLUSIVE) { 1271 wait->flags = WQ_FLAG_EXCLUSIVE; 1272 if (--unfairness < 0) 1273 wait->flags |= WQ_FLAG_CUSTOM; 1274 } 1275 1276 /* 1277 * Do one last check whether we can get the 1278 * page bit synchronously. 1279 * 1280 * Do the folio_set_waiters() marking before that 1281 * to let any waker we _just_ missed know they 1282 * need to wake us up (otherwise they'll never 1283 * even go to the slow case that looks at the 1284 * page queue), and add ourselves to the wait 1285 * queue if we need to sleep. 1286 * 1287 * This part needs to be done under the queue 1288 * lock to avoid races. 1289 */ 1290 spin_lock_irq(&q->lock); 1291 folio_set_waiters(folio); 1292 if (!folio_trylock_flag(folio, bit_nr, wait)) 1293 __add_wait_queue_entry_tail(q, wait); 1294 spin_unlock_irq(&q->lock); 1295 1296 /* 1297 * From now on, all the logic will be based on 1298 * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to 1299 * see whether the page bit testing has already 1300 * been done by the wake function. 1301 * 1302 * We can drop our reference to the folio. 1303 */ 1304 if (behavior == DROP) 1305 folio_put(folio); 1306 1307 /* 1308 * Note that until the "finish_wait()", or until 1309 * we see the WQ_FLAG_WOKEN flag, we need to 1310 * be very careful with the 'wait->flags', because 1311 * we may race with a waker that sets them. 1312 */ 1313 for (;;) { 1314 unsigned int flags; 1315 1316 set_current_state(state); 1317 1318 /* Loop until we've been woken or interrupted */ 1319 flags = smp_load_acquire(&wait->flags); 1320 if (!(flags & WQ_FLAG_WOKEN)) { 1321 if (signal_pending_state(state, current)) 1322 break; 1323 1324 io_schedule(); 1325 continue; 1326 } 1327 1328 /* If we were non-exclusive, we're done */ 1329 if (behavior != EXCLUSIVE) 1330 break; 1331 1332 /* If the waker got the lock for us, we're done */ 1333 if (flags & WQ_FLAG_DONE) 1334 break; 1335 1336 /* 1337 * Otherwise, if we're getting the lock, we need to 1338 * try to get it ourselves. 1339 * 1340 * And if that fails, we'll have to retry this all. 1341 */ 1342 if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0)))) 1343 goto repeat; 1344 1345 wait->flags |= WQ_FLAG_DONE; 1346 break; 1347 } 1348 1349 /* 1350 * If a signal happened, this 'finish_wait()' may remove the last 1351 * waiter from the wait-queues, but the folio waiters bit will remain 1352 * set. That's ok. The next wakeup will take care of it, and trying 1353 * to do it here would be difficult and prone to races. 1354 */ 1355 finish_wait(q, wait); 1356 1357 if (thrashing) { 1358 delayacct_thrashing_end(&in_thrashing); 1359 psi_memstall_leave(&pflags); 1360 } 1361 1362 /* 1363 * NOTE! The wait->flags weren't stable until we've done the 1364 * 'finish_wait()', and we could have exited the loop above due 1365 * to a signal, and had a wakeup event happen after the signal 1366 * test but before the 'finish_wait()'. 1367 * 1368 * So only after the finish_wait() can we reliably determine 1369 * if we got woken up or not, so we can now figure out the final 1370 * return value based on that state without races. 1371 * 1372 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive 1373 * waiter, but an exclusive one requires WQ_FLAG_DONE. 1374 */ 1375 if (behavior == EXCLUSIVE) 1376 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; 1377 1378 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; 1379} 1380 1381#ifdef CONFIG_MIGRATION 1382/** 1383 * softleaf_entry_wait_on_locked - Wait for a migration entry or 1384 * device_private entry to be removed. 1385 * @entry: migration or device_private swap entry. 1386 * @ptl: already locked ptl. This function will drop the lock. 1387 * 1388 * Wait for a migration entry referencing the given page, or device_private 1389 * entry referencing a dvice_private page to be unlocked. This is 1390 * equivalent to folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE) except 1391 * this can be called without taking a reference on the page. Instead this 1392 * should be called while holding the ptl for @entry referencing 1393 * the page. 1394 * 1395 * Returns after unlocking the ptl. 1396 * 1397 * This follows the same logic as folio_wait_bit_common() so see the comments 1398 * there. 1399 */ 1400void softleaf_entry_wait_on_locked(softleaf_t entry, spinlock_t *ptl) 1401 __releases(ptl) 1402{ 1403 struct wait_page_queue wait_page; 1404 wait_queue_entry_t *wait = &wait_page.wait; 1405 bool thrashing = false; 1406 unsigned long pflags; 1407 bool in_thrashing; 1408 wait_queue_head_t *q; 1409 struct folio *folio = softleaf_to_folio(entry); 1410 1411 q = folio_waitqueue(folio); 1412 if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { 1413 delayacct_thrashing_start(&in_thrashing); 1414 psi_memstall_enter(&pflags); 1415 thrashing = true; 1416 } 1417 1418 init_wait(wait); 1419 wait->func = wake_page_function; 1420 wait_page.folio = folio; 1421 wait_page.bit_nr = PG_locked; 1422 wait->flags = 0; 1423 1424 spin_lock_irq(&q->lock); 1425 folio_set_waiters(folio); 1426 if (!folio_trylock_flag(folio, PG_locked, wait)) 1427 __add_wait_queue_entry_tail(q, wait); 1428 spin_unlock_irq(&q->lock); 1429 1430 /* 1431 * If a migration entry exists for the page the migration path must hold 1432 * a valid reference to the page, and it must take the ptl to remove the 1433 * migration entry. So the page is valid until the ptl is dropped. 1434 * Similarly any path attempting to drop the last reference to a 1435 * device-private page needs to grab the ptl to remove the device-private 1436 * entry. 1437 */ 1438 spin_unlock(ptl); 1439 1440 for (;;) { 1441 unsigned int flags; 1442 1443 set_current_state(TASK_UNINTERRUPTIBLE); 1444 1445 /* Loop until we've been woken or interrupted */ 1446 flags = smp_load_acquire(&wait->flags); 1447 if (!(flags & WQ_FLAG_WOKEN)) { 1448 if (signal_pending_state(TASK_UNINTERRUPTIBLE, current)) 1449 break; 1450 1451 io_schedule(); 1452 continue; 1453 } 1454 break; 1455 } 1456 1457 finish_wait(q, wait); 1458 1459 if (thrashing) { 1460 delayacct_thrashing_end(&in_thrashing); 1461 psi_memstall_leave(&pflags); 1462 } 1463} 1464#endif 1465 1466void folio_wait_bit(struct folio *folio, int bit_nr) 1467{ 1468 folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); 1469} 1470EXPORT_SYMBOL(folio_wait_bit); 1471 1472int folio_wait_bit_killable(struct folio *folio, int bit_nr) 1473{ 1474 return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED); 1475} 1476EXPORT_SYMBOL(folio_wait_bit_killable); 1477 1478/** 1479 * folio_put_wait_locked - Drop a reference and wait for it to be unlocked 1480 * @folio: The folio to wait for. 1481 * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc). 1482 * 1483 * The caller should hold a reference on @folio. They expect the page to 1484 * become unlocked relatively soon, but do not wish to hold up migration 1485 * (for example) by holding the reference while waiting for the folio to 1486 * come unlocked. After this function returns, the caller should not 1487 * dereference @folio. 1488 * 1489 * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal. 1490 */ 1491static int folio_put_wait_locked(struct folio *folio, int state) 1492{ 1493 return folio_wait_bit_common(folio, PG_locked, state, DROP); 1494} 1495 1496/** 1497 * folio_unlock - Unlock a locked folio. 1498 * @folio: The folio. 1499 * 1500 * Unlocks the folio and wakes up any thread sleeping on the page lock. 1501 * 1502 * Context: May be called from interrupt or process context. May not be 1503 * called from NMI context. 1504 */ 1505void folio_unlock(struct folio *folio) 1506{ 1507 /* Bit 7 allows x86 to check the byte's sign bit */ 1508 BUILD_BUG_ON(PG_waiters != 7); 1509 BUILD_BUG_ON(PG_locked > 7); 1510 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1511 if (folio_xor_flags_has_waiters(folio, 1 << PG_locked)) 1512 folio_wake_bit(folio, PG_locked); 1513} 1514EXPORT_SYMBOL(folio_unlock); 1515 1516/** 1517 * folio_end_read - End read on a folio. 1518 * @folio: The folio. 1519 * @success: True if all reads completed successfully. 1520 * 1521 * When all reads against a folio have completed, filesystems should 1522 * call this function to let the pagecache know that no more reads 1523 * are outstanding. This will unlock the folio and wake up any thread 1524 * sleeping on the lock. The folio will also be marked uptodate if all 1525 * reads succeeded. 1526 * 1527 * Context: May be called from interrupt or process context. May not be 1528 * called from NMI context. 1529 */ 1530void folio_end_read(struct folio *folio, bool success) 1531{ 1532 unsigned long mask = 1 << PG_locked; 1533 1534 /* Must be in bottom byte for x86 to work */ 1535 BUILD_BUG_ON(PG_uptodate > 7); 1536 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1537 VM_BUG_ON_FOLIO(success && folio_test_uptodate(folio), folio); 1538 1539 if (likely(success)) 1540 mask |= 1 << PG_uptodate; 1541 if (folio_xor_flags_has_waiters(folio, mask)) 1542 folio_wake_bit(folio, PG_locked); 1543} 1544EXPORT_SYMBOL(folio_end_read); 1545 1546/** 1547 * folio_end_private_2 - Clear PG_private_2 and wake any waiters. 1548 * @folio: The folio. 1549 * 1550 * Clear the PG_private_2 bit on a folio and wake up any sleepers waiting for 1551 * it. The folio reference held for PG_private_2 being set is released. 1552 * 1553 * This is, for example, used when a netfs folio is being written to a local 1554 * disk cache, thereby allowing writes to the cache for the same folio to be 1555 * serialised. 1556 */ 1557void folio_end_private_2(struct folio *folio) 1558{ 1559 VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio); 1560 clear_bit_unlock(PG_private_2, folio_flags(folio, 0)); 1561 folio_wake_bit(folio, PG_private_2); 1562 folio_put(folio); 1563} 1564EXPORT_SYMBOL(folio_end_private_2); 1565 1566/** 1567 * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio. 1568 * @folio: The folio to wait on. 1569 * 1570 * Wait for PG_private_2 to be cleared on a folio. 1571 */ 1572void folio_wait_private_2(struct folio *folio) 1573{ 1574 while (folio_test_private_2(folio)) 1575 folio_wait_bit(folio, PG_private_2); 1576} 1577EXPORT_SYMBOL(folio_wait_private_2); 1578 1579/** 1580 * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio. 1581 * @folio: The folio to wait on. 1582 * 1583 * Wait for PG_private_2 to be cleared on a folio or until a fatal signal is 1584 * received by the calling task. 1585 * 1586 * Return: 1587 * - 0 if successful. 1588 * - -EINTR if a fatal signal was encountered. 1589 */ 1590int folio_wait_private_2_killable(struct folio *folio) 1591{ 1592 int ret = 0; 1593 1594 while (folio_test_private_2(folio)) { 1595 ret = folio_wait_bit_killable(folio, PG_private_2); 1596 if (ret < 0) 1597 break; 1598 } 1599 1600 return ret; 1601} 1602EXPORT_SYMBOL(folio_wait_private_2_killable); 1603 1604static void filemap_end_dropbehind(struct folio *folio) 1605{ 1606 struct address_space *mapping = folio->mapping; 1607 1608 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1609 1610 if (folio_test_writeback(folio) || folio_test_dirty(folio)) 1611 return; 1612 if (!folio_test_clear_dropbehind(folio)) 1613 return; 1614 if (mapping) 1615 folio_unmap_invalidate(mapping, folio, 0); 1616} 1617 1618/* 1619 * If folio was marked as dropbehind, then pages should be dropped when writeback 1620 * completes. Do that now. If we fail, it's likely because of a big folio - 1621 * just reset dropbehind for that case and latter completions should invalidate. 1622 */ 1623void folio_end_dropbehind(struct folio *folio) 1624{ 1625 if (!folio_test_dropbehind(folio)) 1626 return; 1627 1628 /* 1629 * Hitting !in_task() should not happen off RWF_DONTCACHE writeback, 1630 * but can happen if normal writeback just happens to find dirty folios 1631 * that were created as part of uncached writeback, and that writeback 1632 * would otherwise not need non-IRQ handling. Just skip the 1633 * invalidation in that case. 1634 */ 1635 if (in_task() && folio_trylock(folio)) { 1636 filemap_end_dropbehind(folio); 1637 folio_unlock(folio); 1638 } 1639} 1640EXPORT_SYMBOL_GPL(folio_end_dropbehind); 1641 1642/** 1643 * folio_end_writeback_no_dropbehind - End writeback against a folio. 1644 * @folio: The folio. 1645 * 1646 * The folio must actually be under writeback. 1647 * This call is intended for filesystems that need to defer dropbehind. 1648 * 1649 * Context: May be called from process or interrupt context. 1650 */ 1651void folio_end_writeback_no_dropbehind(struct folio *folio) 1652{ 1653 VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); 1654 1655 /* 1656 * folio_test_clear_reclaim() could be used here but it is an 1657 * atomic operation and overkill in this particular case. Failing 1658 * to shuffle a folio marked for immediate reclaim is too mild 1659 * a gain to justify taking an atomic operation penalty at the 1660 * end of every folio writeback. 1661 */ 1662 if (folio_test_reclaim(folio)) { 1663 folio_clear_reclaim(folio); 1664 folio_rotate_reclaimable(folio); 1665 } 1666 1667 if (__folio_end_writeback(folio)) 1668 folio_wake_bit(folio, PG_writeback); 1669 1670 acct_reclaim_writeback(folio); 1671} 1672EXPORT_SYMBOL_GPL(folio_end_writeback_no_dropbehind); 1673 1674/** 1675 * folio_end_writeback - End writeback against a folio. 1676 * @folio: The folio. 1677 * 1678 * The folio must actually be under writeback. 1679 * 1680 * Context: May be called from process or interrupt context. 1681 */ 1682void folio_end_writeback(struct folio *folio) 1683{ 1684 VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); 1685 1686 /* 1687 * Writeback does not hold a folio reference of its own, relying 1688 * on truncation to wait for the clearing of PG_writeback. 1689 * But here we must make sure that the folio is not freed and 1690 * reused before the folio_wake_bit(). 1691 */ 1692 folio_get(folio); 1693 folio_end_writeback_no_dropbehind(folio); 1694 folio_end_dropbehind(folio); 1695 folio_put(folio); 1696} 1697EXPORT_SYMBOL(folio_end_writeback); 1698 1699/** 1700 * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it. 1701 * @folio: The folio to lock 1702 */ 1703void __folio_lock(struct folio *folio) 1704{ 1705 folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE, 1706 EXCLUSIVE); 1707} 1708EXPORT_SYMBOL(__folio_lock); 1709 1710int __folio_lock_killable(struct folio *folio) 1711{ 1712 return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE, 1713 EXCLUSIVE); 1714} 1715EXPORT_SYMBOL_GPL(__folio_lock_killable); 1716 1717static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait) 1718{ 1719 struct wait_queue_head *q = folio_waitqueue(folio); 1720 int ret; 1721 1722 wait->folio = folio; 1723 wait->bit_nr = PG_locked; 1724 1725 spin_lock_irq(&q->lock); 1726 __add_wait_queue_entry_tail(q, &wait->wait); 1727 folio_set_waiters(folio); 1728 ret = !folio_trylock(folio); 1729 /* 1730 * If we were successful now, we know we're still on the 1731 * waitqueue as we're still under the lock. This means it's 1732 * safe to remove and return success, we know the callback 1733 * isn't going to trigger. 1734 */ 1735 if (!ret) 1736 __remove_wait_queue(q, &wait->wait); 1737 else 1738 ret = -EIOCBQUEUED; 1739 spin_unlock_irq(&q->lock); 1740 return ret; 1741} 1742 1743/* 1744 * Return values: 1745 * 0 - folio is locked. 1746 * non-zero - folio is not locked. 1747 * mmap_lock or per-VMA lock has been released (mmap_read_unlock() or 1748 * vma_end_read()), unless flags had both FAULT_FLAG_ALLOW_RETRY and 1749 * FAULT_FLAG_RETRY_NOWAIT set, in which case the lock is still held. 1750 * 1751 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 0 1752 * with the folio locked and the mmap_lock/per-VMA lock is left unperturbed. 1753 */ 1754vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf) 1755{ 1756 unsigned int flags = vmf->flags; 1757 1758 if (fault_flag_allow_retry_first(flags)) { 1759 /* 1760 * CAUTION! In this case, mmap_lock/per-VMA lock is not 1761 * released even though returning VM_FAULT_RETRY. 1762 */ 1763 if (flags & FAULT_FLAG_RETRY_NOWAIT) 1764 return VM_FAULT_RETRY; 1765 1766 release_fault_lock(vmf); 1767 if (flags & FAULT_FLAG_KILLABLE) 1768 folio_wait_locked_killable(folio); 1769 else 1770 folio_wait_locked(folio); 1771 return VM_FAULT_RETRY; 1772 } 1773 if (flags & FAULT_FLAG_KILLABLE) { 1774 bool ret; 1775 1776 ret = __folio_lock_killable(folio); 1777 if (ret) { 1778 release_fault_lock(vmf); 1779 return VM_FAULT_RETRY; 1780 } 1781 } else { 1782 __folio_lock(folio); 1783 } 1784 1785 return 0; 1786} 1787 1788/** 1789 * page_cache_next_miss() - Find the next gap in the page cache. 1790 * @mapping: Mapping. 1791 * @index: Index. 1792 * @max_scan: Maximum range to search. 1793 * 1794 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the 1795 * gap with the lowest index. 1796 * 1797 * This function may be called under the rcu_read_lock. However, this will 1798 * not atomically search a snapshot of the cache at a single point in time. 1799 * For example, if a gap is created at index 5, then subsequently a gap is 1800 * created at index 10, page_cache_next_miss covering both indices may 1801 * return 10 if called under the rcu_read_lock. 1802 * 1803 * Return: The index of the gap if found, otherwise an index outside the 1804 * range specified (in which case 'return - index >= max_scan' will be true). 1805 * In the rare case of index wrap-around, 0 will be returned. 1806 */ 1807pgoff_t page_cache_next_miss(struct address_space *mapping, 1808 pgoff_t index, unsigned long max_scan) 1809{ 1810 XA_STATE(xas, &mapping->i_pages, index); 1811 unsigned long nr = max_scan; 1812 1813 while (nr--) { 1814 void *entry = xas_next(&xas); 1815 if (!entry || xa_is_value(entry)) 1816 return xas.xa_index; 1817 if (xas.xa_index == 0) 1818 return 0; 1819 } 1820 1821 return index + max_scan; 1822} 1823EXPORT_SYMBOL(page_cache_next_miss); 1824 1825/** 1826 * page_cache_prev_miss() - Find the previous gap in the page cache. 1827 * @mapping: Mapping. 1828 * @index: Index. 1829 * @max_scan: Maximum range to search. 1830 * 1831 * Search the range [max(index - max_scan + 1, 0), index] for the 1832 * gap with the highest index. 1833 * 1834 * This function may be called under the rcu_read_lock. However, this will 1835 * not atomically search a snapshot of the cache at a single point in time. 1836 * For example, if a gap is created at index 10, then subsequently a gap is 1837 * created at index 5, page_cache_prev_miss() covering both indices may 1838 * return 5 if called under the rcu_read_lock. 1839 * 1840 * Return: The index of the gap if found, otherwise an index outside the 1841 * range specified (in which case 'index - return >= max_scan' will be true). 1842 * In the rare case of wrap-around, ULONG_MAX will be returned. 1843 */ 1844pgoff_t page_cache_prev_miss(struct address_space *mapping, 1845 pgoff_t index, unsigned long max_scan) 1846{ 1847 XA_STATE(xas, &mapping->i_pages, index); 1848 1849 while (max_scan--) { 1850 void *entry = xas_prev(&xas); 1851 if (!entry || xa_is_value(entry)) 1852 break; 1853 if (xas.xa_index == ULONG_MAX) 1854 break; 1855 } 1856 1857 return xas.xa_index; 1858} 1859EXPORT_SYMBOL(page_cache_prev_miss); 1860 1861/* 1862 * Lockless page cache protocol: 1863 * On the lookup side: 1864 * 1. Load the folio from i_pages 1865 * 2. Increment the refcount if it's not zero 1866 * 3. If the folio is not found by xas_reload(), put the refcount and retry 1867 * 1868 * On the removal side: 1869 * A. Freeze the page (by zeroing the refcount if nobody else has a reference) 1870 * B. Remove the page from i_pages 1871 * C. Return the page to the page allocator 1872 * 1873 * This means that any page may have its reference count temporarily 1874 * increased by a speculative page cache (or GUP-fast) lookup as it can 1875 * be allocated by another user before the RCU grace period expires. 1876 * Because the refcount temporarily acquired here may end up being the 1877 * last refcount on the page, any page allocation must be freeable by 1878 * folio_put(). 1879 */ 1880 1881/* 1882 * filemap_get_entry - Get a page cache entry. 1883 * @mapping: the address_space to search 1884 * @index: The page cache index. 1885 * 1886 * Looks up the page cache entry at @mapping & @index. If it is a folio, 1887 * it is returned with an increased refcount. If it is a shadow entry 1888 * of a previously evicted folio, or a swap entry from shmem/tmpfs, 1889 * it is returned without further action. 1890 * 1891 * Return: The folio, swap or shadow entry, %NULL if nothing is found. 1892 */ 1893void *filemap_get_entry(struct address_space *mapping, pgoff_t index) 1894{ 1895 XA_STATE(xas, &mapping->i_pages, index); 1896 struct folio *folio; 1897 1898 rcu_read_lock(); 1899repeat: 1900 xas_reset(&xas); 1901 folio = xas_load(&xas); 1902 if (xas_retry(&xas, folio)) 1903 goto repeat; 1904 /* 1905 * A shadow entry of a recently evicted page, or a swap entry from 1906 * shmem/tmpfs. Return it without attempting to raise page count. 1907 */ 1908 if (!folio || xa_is_value(folio)) 1909 goto out; 1910 1911 if (!folio_try_get(folio)) 1912 goto repeat; 1913 1914 if (unlikely(folio != xas_reload(&xas))) { 1915 folio_put(folio); 1916 goto repeat; 1917 } 1918out: 1919 rcu_read_unlock(); 1920 1921 return folio; 1922} 1923 1924/** 1925 * __filemap_get_folio_mpol - Find and get a reference to a folio. 1926 * @mapping: The address_space to search. 1927 * @index: The page index. 1928 * @fgp_flags: %FGP flags modify how the folio is returned. 1929 * @gfp: Memory allocation flags to use if %FGP_CREAT is specified. 1930 * @policy: NUMA memory allocation policy to follow. 1931 * 1932 * Looks up the page cache entry at @mapping & @index. 1933 * 1934 * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even 1935 * if the %GFP flags specified for %FGP_CREAT are atomic. 1936 * 1937 * If this function returns a folio, it is returned with an increased refcount. 1938 * 1939 * Return: The found folio or an ERR_PTR() otherwise. 1940 */ 1941struct folio *__filemap_get_folio_mpol(struct address_space *mapping, 1942 pgoff_t index, fgf_t fgp_flags, gfp_t gfp, struct mempolicy *policy) 1943{ 1944 struct folio *folio; 1945 1946repeat: 1947 folio = filemap_get_entry(mapping, index); 1948 if (xa_is_value(folio)) 1949 folio = NULL; 1950 if (!folio) 1951 goto no_page; 1952 1953 if (fgp_flags & FGP_LOCK) { 1954 if (fgp_flags & FGP_NOWAIT) { 1955 if (!folio_trylock(folio)) { 1956 folio_put(folio); 1957 return ERR_PTR(-EAGAIN); 1958 } 1959 } else { 1960 folio_lock(folio); 1961 } 1962 1963 /* Has the page been truncated? */ 1964 if (unlikely(folio->mapping != mapping)) { 1965 folio_unlock(folio); 1966 folio_put(folio); 1967 goto repeat; 1968 } 1969 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); 1970 } 1971 1972 if (fgp_flags & FGP_ACCESSED) 1973 folio_mark_accessed(folio); 1974 else if (fgp_flags & FGP_WRITE) { 1975 /* Clear idle flag for buffer write */ 1976 if (folio_test_idle(folio)) 1977 folio_clear_idle(folio); 1978 } 1979 1980 if (fgp_flags & FGP_STABLE) 1981 folio_wait_stable(folio); 1982no_page: 1983 if (!folio && (fgp_flags & FGP_CREAT)) { 1984 unsigned int min_order = mapping_min_folio_order(mapping); 1985 unsigned int order = max(min_order, FGF_GET_ORDER(fgp_flags)); 1986 int err; 1987 index = mapping_align_index(mapping, index); 1988 1989 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) 1990 gfp |= __GFP_WRITE; 1991 if (fgp_flags & FGP_NOFS) 1992 gfp &= ~__GFP_FS; 1993 if (fgp_flags & FGP_NOWAIT) { 1994 gfp &= ~GFP_KERNEL; 1995 gfp |= GFP_NOWAIT; 1996 } 1997 if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) 1998 fgp_flags |= FGP_LOCK; 1999 2000 if (order > mapping_max_folio_order(mapping)) 2001 order = mapping_max_folio_order(mapping); 2002 /* If we're not aligned, allocate a smaller folio */ 2003 if (index & ((1UL << order) - 1)) 2004 order = __ffs(index); 2005 2006 do { 2007 gfp_t alloc_gfp = gfp; 2008 2009 err = -ENOMEM; 2010 if (order > min_order) 2011 alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN; 2012 folio = filemap_alloc_folio(alloc_gfp, order, policy); 2013 if (!folio) 2014 continue; 2015 2016 /* Init accessed so avoid atomic mark_page_accessed later */ 2017 if (fgp_flags & FGP_ACCESSED) 2018 __folio_set_referenced(folio); 2019 if (fgp_flags & FGP_DONTCACHE) 2020 __folio_set_dropbehind(folio); 2021 2022 err = filemap_add_folio(mapping, folio, index, gfp); 2023 if (!err) 2024 break; 2025 folio_put(folio); 2026 folio = NULL; 2027 } while (order-- > min_order); 2028 2029 if (err == -EEXIST) 2030 goto repeat; 2031 if (err) { 2032 /* 2033 * When NOWAIT I/O fails to allocate folios this could 2034 * be due to a nonblocking memory allocation and not 2035 * because the system actually is out of memory. 2036 * Return -EAGAIN so that there caller retries in a 2037 * blocking fashion instead of propagating -ENOMEM 2038 * to the application. 2039 */ 2040 if ((fgp_flags & FGP_NOWAIT) && err == -ENOMEM) 2041 err = -EAGAIN; 2042 return ERR_PTR(err); 2043 } 2044 /* 2045 * filemap_add_folio locks the page, and for mmap 2046 * we expect an unlocked page. 2047 */ 2048 if (folio && (fgp_flags & FGP_FOR_MMAP)) 2049 folio_unlock(folio); 2050 } 2051 2052 if (!folio) 2053 return ERR_PTR(-ENOENT); 2054 /* not an uncached lookup, clear uncached if set */ 2055 if (folio_test_dropbehind(folio) && !(fgp_flags & FGP_DONTCACHE)) 2056 folio_clear_dropbehind(folio); 2057 return folio; 2058} 2059EXPORT_SYMBOL(__filemap_get_folio_mpol); 2060 2061static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, 2062 xa_mark_t mark) 2063{ 2064 struct folio *folio; 2065 2066retry: 2067 if (mark == XA_PRESENT) 2068 folio = xas_find(xas, max); 2069 else 2070 folio = xas_find_marked(xas, max, mark); 2071 2072 if (xas_retry(xas, folio)) 2073 goto retry; 2074 /* 2075 * A shadow entry of a recently evicted page, a swap 2076 * entry from shmem/tmpfs or a DAX entry. Return it 2077 * without attempting to raise page count. 2078 */ 2079 if (!folio || xa_is_value(folio)) 2080 return folio; 2081 2082 if (!folio_try_get(folio)) 2083 goto reset; 2084 2085 if (unlikely(folio != xas_reload(xas))) { 2086 folio_put(folio); 2087 goto reset; 2088 } 2089 2090 return folio; 2091reset: 2092 xas_reset(xas); 2093 goto retry; 2094} 2095 2096/** 2097 * find_get_entries - gang pagecache lookup 2098 * @mapping: The address_space to search 2099 * @start: The starting page cache index 2100 * @end: The final page index (inclusive). 2101 * @fbatch: Where the resulting entries are placed. 2102 * @indices: The cache indices corresponding to the entries in @entries 2103 * 2104 * find_get_entries() will search for and return a batch of entries in 2105 * the mapping. The entries are placed in @fbatch. find_get_entries() 2106 * takes a reference on any actual folios it returns. 2107 * 2108 * The entries have ascending indexes. The indices may not be consecutive 2109 * due to not-present entries or large folios. 2110 * 2111 * Any shadow entries of evicted folios, or swap entries from 2112 * shmem/tmpfs, are included in the returned array. 2113 * 2114 * Return: The number of entries which were found. 2115 */ 2116unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, 2117 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) 2118{ 2119 XA_STATE(xas, &mapping->i_pages, *start); 2120 struct folio *folio; 2121 2122 rcu_read_lock(); 2123 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { 2124 indices[fbatch->nr] = xas.xa_index; 2125 if (!folio_batch_add(fbatch, folio)) 2126 break; 2127 } 2128 2129 if (folio_batch_count(fbatch)) { 2130 unsigned long nr; 2131 int idx = folio_batch_count(fbatch) - 1; 2132 2133 folio = fbatch->folios[idx]; 2134 if (!xa_is_value(folio)) 2135 nr = folio_nr_pages(folio); 2136 else 2137 nr = 1 << xa_get_order(&mapping->i_pages, indices[idx]); 2138 *start = round_down(indices[idx] + nr, nr); 2139 } 2140 rcu_read_unlock(); 2141 2142 return folio_batch_count(fbatch); 2143} 2144 2145/** 2146 * find_lock_entries - Find a batch of pagecache entries. 2147 * @mapping: The address_space to search. 2148 * @start: The starting page cache index. 2149 * @end: The final page index (inclusive). 2150 * @fbatch: Where the resulting entries are placed. 2151 * @indices: The cache indices of the entries in @fbatch. 2152 * 2153 * find_lock_entries() will return a batch of entries from @mapping. 2154 * Swap, shadow and DAX entries are included. Folios are returned 2155 * locked and with an incremented refcount. Folios which are locked 2156 * by somebody else or under writeback are skipped. Folios which are 2157 * partially outside the range are not returned. 2158 * 2159 * The entries have ascending indexes. The indices may not be consecutive 2160 * due to not-present entries, large folios, folios which could not be 2161 * locked or folios under writeback. 2162 * 2163 * Return: The number of entries which were found. 2164 */ 2165unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, 2166 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) 2167{ 2168 XA_STATE(xas, &mapping->i_pages, *start); 2169 struct folio *folio; 2170 2171 rcu_read_lock(); 2172 while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { 2173 unsigned long base; 2174 unsigned long nr; 2175 2176 if (!xa_is_value(folio)) { 2177 nr = folio_nr_pages(folio); 2178 base = folio->index; 2179 /* Omit large folio which begins before the start */ 2180 if (base < *start) 2181 goto put; 2182 /* Omit large folio which extends beyond the end */ 2183 if (base + nr - 1 > end) 2184 goto put; 2185 if (!folio_trylock(folio)) 2186 goto put; 2187 if (folio->mapping != mapping || 2188 folio_test_writeback(folio)) 2189 goto unlock; 2190 VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), 2191 folio); 2192 } else { 2193 nr = 1 << xas_get_order(&xas); 2194 base = xas.xa_index & ~(nr - 1); 2195 /* Omit order>0 value which begins before the start */ 2196 if (base < *start) 2197 continue; 2198 /* Omit order>0 value which extends beyond the end */ 2199 if (base + nr - 1 > end) 2200 break; 2201 } 2202 2203 /* Update start now so that last update is correct on return */ 2204 *start = base + nr; 2205 indices[fbatch->nr] = xas.xa_index; 2206 if (!folio_batch_add(fbatch, folio)) 2207 break; 2208 continue; 2209unlock: 2210 folio_unlock(folio); 2211put: 2212 folio_put(folio); 2213 } 2214 rcu_read_unlock(); 2215 2216 return folio_batch_count(fbatch); 2217} 2218 2219/** 2220 * filemap_get_folios - Get a batch of folios 2221 * @mapping: The address_space to search 2222 * @start: The starting page index 2223 * @end: The final page index (inclusive) 2224 * @fbatch: The batch to fill. 2225 * 2226 * Search for and return a batch of folios in the mapping starting at 2227 * index @start and up to index @end (inclusive). The folios are returned 2228 * in @fbatch with an elevated reference count. 2229 * 2230 * Return: The number of folios which were found. 2231 * We also update @start to index the next folio for the traversal. 2232 */ 2233unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, 2234 pgoff_t end, struct folio_batch *fbatch) 2235{ 2236 return filemap_get_folios_tag(mapping, start, end, XA_PRESENT, fbatch); 2237} 2238EXPORT_SYMBOL(filemap_get_folios); 2239 2240/** 2241 * filemap_get_folios_contig - Get a batch of contiguous folios 2242 * @mapping: The address_space to search 2243 * @start: The starting page index 2244 * @end: The final page index (inclusive) 2245 * @fbatch: The batch to fill 2246 * 2247 * filemap_get_folios_contig() works exactly like filemap_get_folios(), 2248 * except the returned folios are guaranteed to be contiguous. This may 2249 * not return all contiguous folios if the batch gets filled up. 2250 * 2251 * Return: The number of folios found. 2252 * Also update @start to be positioned for traversal of the next folio. 2253 */ 2254 2255unsigned filemap_get_folios_contig(struct address_space *mapping, 2256 pgoff_t *start, pgoff_t end, struct folio_batch *fbatch) 2257{ 2258 XA_STATE(xas, &mapping->i_pages, *start); 2259 unsigned long nr; 2260 struct folio *folio; 2261 2262 rcu_read_lock(); 2263 2264 for (folio = xas_load(&xas); folio && xas.xa_index <= end; 2265 folio = xas_next(&xas)) { 2266 if (xas_retry(&xas, folio)) 2267 continue; 2268 /* 2269 * If the entry has been swapped out, we can stop looking. 2270 * No current caller is looking for DAX entries. 2271 */ 2272 if (xa_is_value(folio)) 2273 goto update_start; 2274 2275 /* If we landed in the middle of a THP, continue at its end. */ 2276 if (xa_is_sibling(folio)) 2277 goto update_start; 2278 2279 if (!folio_try_get(folio)) 2280 goto retry; 2281 2282 if (unlikely(folio != xas_reload(&xas))) 2283 goto put_folio; 2284 2285 if (!folio_batch_add(fbatch, folio)) { 2286 nr = folio_nr_pages(folio); 2287 *start = folio->index + nr; 2288 goto out; 2289 } 2290 xas_advance(&xas, folio_next_index(folio) - 1); 2291 continue; 2292put_folio: 2293 folio_put(folio); 2294 2295retry: 2296 xas_reset(&xas); 2297 } 2298 2299update_start: 2300 nr = folio_batch_count(fbatch); 2301 2302 if (nr) { 2303 folio = fbatch->folios[nr - 1]; 2304 *start = folio_next_index(folio); 2305 } 2306out: 2307 rcu_read_unlock(); 2308 return folio_batch_count(fbatch); 2309} 2310EXPORT_SYMBOL(filemap_get_folios_contig); 2311 2312/** 2313 * filemap_get_folios_tag - Get a batch of folios matching @tag 2314 * @mapping: The address_space to search 2315 * @start: The starting page index 2316 * @end: The final page index (inclusive) 2317 * @tag: The tag index 2318 * @fbatch: The batch to fill 2319 * 2320 * The first folio may start before @start; if it does, it will contain 2321 * @start. The final folio may extend beyond @end; if it does, it will 2322 * contain @end. The folios have ascending indices. There may be gaps 2323 * between the folios if there are indices which have no folio in the 2324 * page cache. If folios are added to or removed from the page cache 2325 * while this is running, they may or may not be found by this call. 2326 * Only returns folios that are tagged with @tag. 2327 * 2328 * Return: The number of folios found. 2329 * Also update @start to index the next folio for traversal. 2330 */ 2331unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, 2332 pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) 2333{ 2334 XA_STATE(xas, &mapping->i_pages, *start); 2335 struct folio *folio; 2336 2337 rcu_read_lock(); 2338 while ((folio = find_get_entry(&xas, end, tag)) != NULL) { 2339 /* 2340 * Shadow entries should never be tagged, but this iteration 2341 * is lockless so there is a window for page reclaim to evict 2342 * a page we saw tagged. Skip over it. 2343 */ 2344 if (xa_is_value(folio)) 2345 continue; 2346 if (!folio_batch_add(fbatch, folio)) { 2347 unsigned long nr = folio_nr_pages(folio); 2348 *start = folio->index + nr; 2349 goto out; 2350 } 2351 } 2352 /* 2353 * We come here when there is no page beyond @end. We take care to not 2354 * overflow the index @start as it confuses some of the callers. This 2355 * breaks the iteration when there is a page at index -1 but that is 2356 * already broke anyway. 2357 */ 2358 if (end == (pgoff_t)-1) 2359 *start = (pgoff_t)-1; 2360 else 2361 *start = end + 1; 2362out: 2363 rcu_read_unlock(); 2364 2365 return folio_batch_count(fbatch); 2366} 2367EXPORT_SYMBOL(filemap_get_folios_tag); 2368 2369/** 2370 * filemap_get_folios_dirty - Get a batch of dirty folios 2371 * @mapping: The address_space to search 2372 * @start: The starting folio index 2373 * @end: The final folio index (inclusive) 2374 * @fbatch: The batch to fill 2375 * 2376 * filemap_get_folios_dirty() works exactly like filemap_get_folios(), except 2377 * the returned folios are presumed to be dirty or undergoing writeback. Dirty 2378 * state is presumed because we don't block on folio lock nor want to miss 2379 * folios. Callers that need to can recheck state upon locking the folio. 2380 * 2381 * This may not return all dirty folios if the batch gets filled up. 2382 * 2383 * Return: The number of folios found. 2384 * Also update @start to be positioned for traversal of the next folio. 2385 */ 2386unsigned filemap_get_folios_dirty(struct address_space *mapping, pgoff_t *start, 2387 pgoff_t end, struct folio_batch *fbatch) 2388{ 2389 XA_STATE(xas, &mapping->i_pages, *start); 2390 struct folio *folio; 2391 2392 rcu_read_lock(); 2393 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { 2394 if (xa_is_value(folio)) 2395 continue; 2396 if (folio_trylock(folio)) { 2397 bool clean = !folio_test_dirty(folio) && 2398 !folio_test_writeback(folio); 2399 folio_unlock(folio); 2400 if (clean) { 2401 folio_put(folio); 2402 continue; 2403 } 2404 } 2405 if (!folio_batch_add(fbatch, folio)) { 2406 unsigned long nr = folio_nr_pages(folio); 2407 *start = folio->index + nr; 2408 goto out; 2409 } 2410 } 2411 /* 2412 * We come here when there is no folio beyond @end. We take care to not 2413 * overflow the index @start as it confuses some of the callers. This 2414 * breaks the iteration when there is a folio at index -1 but that is 2415 * already broke anyway. 2416 */ 2417 if (end == (pgoff_t)-1) 2418 *start = (pgoff_t)-1; 2419 else 2420 *start = end + 1; 2421out: 2422 rcu_read_unlock(); 2423 2424 return folio_batch_count(fbatch); 2425} 2426 2427/* 2428 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 2429 * a _large_ part of the i/o request. Imagine the worst scenario: 2430 * 2431 * ---R__________________________________________B__________ 2432 * ^ reading here ^ bad block(assume 4k) 2433 * 2434 * read(R) => miss => readahead(R...B) => media error => frustrating retries 2435 * => failing the whole request => read(R) => read(R+1) => 2436 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 2437 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 2438 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 2439 * 2440 * It is going insane. Fix it by quickly scaling down the readahead size. 2441 */ 2442static void shrink_readahead_size_eio(struct file_ra_state *ra) 2443{ 2444 ra->ra_pages /= 4; 2445} 2446 2447/* 2448 * filemap_get_read_batch - Get a batch of folios for read 2449 * 2450 * Get a batch of folios which represent a contiguous range of bytes in 2451 * the file. No exceptional entries will be returned. If @index is in 2452 * the middle of a folio, the entire folio will be returned. The last 2453 * folio in the batch may have the readahead flag set or the uptodate flag 2454 * clear so that the caller can take the appropriate action. 2455 */ 2456static void filemap_get_read_batch(struct address_space *mapping, 2457 pgoff_t index, pgoff_t max, struct folio_batch *fbatch) 2458{ 2459 XA_STATE(xas, &mapping->i_pages, index); 2460 struct folio *folio; 2461 2462 rcu_read_lock(); 2463 for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { 2464 if (xas_retry(&xas, folio)) 2465 continue; 2466 if (xas.xa_index > max || xa_is_value(folio)) 2467 break; 2468 if (xa_is_sibling(folio)) 2469 break; 2470 if (!folio_try_get(folio)) 2471 goto retry; 2472 2473 if (unlikely(folio != xas_reload(&xas))) 2474 goto put_folio; 2475 2476 if (!folio_batch_add(fbatch, folio)) 2477 break; 2478 if (!folio_test_uptodate(folio)) 2479 break; 2480 if (folio_test_readahead(folio)) 2481 break; 2482 xas_advance(&xas, folio_next_index(folio) - 1); 2483 continue; 2484put_folio: 2485 folio_put(folio); 2486retry: 2487 xas_reset(&xas); 2488 } 2489 rcu_read_unlock(); 2490} 2491 2492static int filemap_read_folio(struct file *file, filler_t filler, 2493 struct folio *folio) 2494{ 2495 bool workingset = folio_test_workingset(folio); 2496 unsigned long pflags; 2497 int error; 2498 2499 /* Start the actual read. The read will unlock the page. */ 2500 if (unlikely(workingset)) 2501 psi_memstall_enter(&pflags); 2502 error = filler(file, folio); 2503 if (unlikely(workingset)) 2504 psi_memstall_leave(&pflags); 2505 if (error) 2506 return error; 2507 2508 error = folio_wait_locked_killable(folio); 2509 if (error) 2510 return error; 2511 if (folio_test_uptodate(folio)) 2512 return 0; 2513 if (file) 2514 shrink_readahead_size_eio(&file->f_ra); 2515 return -EIO; 2516} 2517 2518static bool filemap_range_uptodate(struct address_space *mapping, 2519 loff_t pos, size_t count, struct folio *folio, 2520 bool need_uptodate) 2521{ 2522 if (folio_test_uptodate(folio)) 2523 return true; 2524 /* pipes can't handle partially uptodate pages */ 2525 if (need_uptodate) 2526 return false; 2527 if (!mapping->a_ops->is_partially_uptodate) 2528 return false; 2529 if (mapping->host->i_blkbits >= folio_shift(folio)) 2530 return false; 2531 2532 if (folio_pos(folio) > pos) { 2533 count -= folio_pos(folio) - pos; 2534 pos = 0; 2535 } else { 2536 pos -= folio_pos(folio); 2537 } 2538 2539 if (pos == 0 && count >= folio_size(folio)) 2540 return false; 2541 2542 return mapping->a_ops->is_partially_uptodate(folio, pos, count); 2543} 2544 2545static int filemap_update_page(struct kiocb *iocb, 2546 struct address_space *mapping, size_t count, 2547 struct folio *folio, bool need_uptodate) 2548{ 2549 int error; 2550 2551 if (iocb->ki_flags & IOCB_NOWAIT) { 2552 if (!filemap_invalidate_trylock_shared(mapping)) 2553 return -EAGAIN; 2554 } else { 2555 filemap_invalidate_lock_shared(mapping); 2556 } 2557 2558 if (!folio_trylock(folio)) { 2559 error = -EAGAIN; 2560 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) 2561 goto unlock_mapping; 2562 if (!(iocb->ki_flags & IOCB_WAITQ)) { 2563 filemap_invalidate_unlock_shared(mapping); 2564 /* 2565 * This is where we usually end up waiting for a 2566 * previously submitted readahead to finish. 2567 */ 2568 folio_put_wait_locked(folio, TASK_KILLABLE); 2569 return AOP_TRUNCATED_PAGE; 2570 } 2571 error = __folio_lock_async(folio, iocb->ki_waitq); 2572 if (error) 2573 goto unlock_mapping; 2574 } 2575 2576 error = AOP_TRUNCATED_PAGE; 2577 if (!folio->mapping) 2578 goto unlock; 2579 2580 error = 0; 2581 if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio, 2582 need_uptodate)) 2583 goto unlock; 2584 2585 error = -EAGAIN; 2586 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) 2587 goto unlock; 2588 2589 error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio, 2590 folio); 2591 goto unlock_mapping; 2592unlock: 2593 folio_unlock(folio); 2594unlock_mapping: 2595 filemap_invalidate_unlock_shared(mapping); 2596 if (error == AOP_TRUNCATED_PAGE) 2597 folio_put(folio); 2598 return error; 2599} 2600 2601static int filemap_create_folio(struct kiocb *iocb, struct folio_batch *fbatch) 2602{ 2603 struct address_space *mapping = iocb->ki_filp->f_mapping; 2604 struct folio *folio; 2605 int error; 2606 unsigned int min_order = mapping_min_folio_order(mapping); 2607 pgoff_t index; 2608 2609 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) 2610 return -EAGAIN; 2611 2612 folio = filemap_alloc_folio(mapping_gfp_mask(mapping), min_order, NULL); 2613 if (!folio) 2614 return -ENOMEM; 2615 if (iocb->ki_flags & IOCB_DONTCACHE) 2616 __folio_set_dropbehind(folio); 2617 2618 /* 2619 * Protect against truncate / hole punch. Grabbing invalidate_lock 2620 * here assures we cannot instantiate and bring uptodate new 2621 * pagecache folios after evicting page cache during truncate 2622 * and before actually freeing blocks. Note that we could 2623 * release invalidate_lock after inserting the folio into 2624 * the page cache as the locked folio would then be enough to 2625 * synchronize with hole punching. But there are code paths 2626 * such as filemap_update_page() filling in partially uptodate 2627 * pages or ->readahead() that need to hold invalidate_lock 2628 * while mapping blocks for IO so let's hold the lock here as 2629 * well to keep locking rules simple. 2630 */ 2631 filemap_invalidate_lock_shared(mapping); 2632 index = (iocb->ki_pos >> (PAGE_SHIFT + min_order)) << min_order; 2633 error = filemap_add_folio(mapping, folio, index, 2634 mapping_gfp_constraint(mapping, GFP_KERNEL)); 2635 if (error == -EEXIST) 2636 error = AOP_TRUNCATED_PAGE; 2637 if (error) 2638 goto error; 2639 2640 error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio, 2641 folio); 2642 if (error) 2643 goto error; 2644 2645 filemap_invalidate_unlock_shared(mapping); 2646 folio_batch_add(fbatch, folio); 2647 return 0; 2648error: 2649 filemap_invalidate_unlock_shared(mapping); 2650 folio_put(folio); 2651 return error; 2652} 2653 2654static int filemap_readahead(struct kiocb *iocb, struct file *file, 2655 struct address_space *mapping, struct folio *folio, 2656 pgoff_t last_index) 2657{ 2658 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index); 2659 2660 if (iocb->ki_flags & IOCB_NOIO) 2661 return -EAGAIN; 2662 if (iocb->ki_flags & IOCB_DONTCACHE) 2663 ractl.dropbehind = 1; 2664 page_cache_async_ra(&ractl, folio, last_index - folio->index); 2665 return 0; 2666} 2667 2668static int filemap_get_pages(struct kiocb *iocb, size_t count, 2669 struct folio_batch *fbatch, bool need_uptodate) 2670{ 2671 struct file *filp = iocb->ki_filp; 2672 struct address_space *mapping = filp->f_mapping; 2673 pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; 2674 pgoff_t last_index; 2675 struct folio *folio; 2676 unsigned int flags; 2677 int err = 0; 2678 2679 /* "last_index" is the index of the folio beyond the end of the read */ 2680 last_index = round_up(iocb->ki_pos + count, 2681 mapping_min_folio_nrbytes(mapping)) >> PAGE_SHIFT; 2682retry: 2683 if (fatal_signal_pending(current)) 2684 return -EINTR; 2685 2686 filemap_get_read_batch(mapping, index, last_index - 1, fbatch); 2687 if (!folio_batch_count(fbatch)) { 2688 DEFINE_READAHEAD(ractl, filp, &filp->f_ra, mapping, index); 2689 2690 if (iocb->ki_flags & IOCB_NOIO) 2691 return -EAGAIN; 2692 if (iocb->ki_flags & IOCB_NOWAIT) 2693 flags = memalloc_noio_save(); 2694 if (iocb->ki_flags & IOCB_DONTCACHE) 2695 ractl.dropbehind = 1; 2696 page_cache_sync_ra(&ractl, last_index - index); 2697 if (iocb->ki_flags & IOCB_NOWAIT) 2698 memalloc_noio_restore(flags); 2699 filemap_get_read_batch(mapping, index, last_index - 1, fbatch); 2700 } 2701 if (!folio_batch_count(fbatch)) { 2702 err = filemap_create_folio(iocb, fbatch); 2703 if (err == AOP_TRUNCATED_PAGE) 2704 goto retry; 2705 return err; 2706 } 2707 2708 folio = fbatch->folios[folio_batch_count(fbatch) - 1]; 2709 if (folio_test_readahead(folio)) { 2710 err = filemap_readahead(iocb, filp, mapping, folio, last_index); 2711 if (err) 2712 goto err; 2713 } 2714 if (!folio_test_uptodate(folio)) { 2715 if (folio_batch_count(fbatch) > 1) { 2716 err = -EAGAIN; 2717 goto err; 2718 } 2719 err = filemap_update_page(iocb, mapping, count, folio, 2720 need_uptodate); 2721 if (err) 2722 goto err; 2723 } 2724 2725 trace_mm_filemap_get_pages(mapping, index, last_index - 1); 2726 return 0; 2727err: 2728 if (err < 0) 2729 folio_put(folio); 2730 if (likely(--fbatch->nr)) 2731 return 0; 2732 if (err == AOP_TRUNCATED_PAGE) 2733 goto retry; 2734 return err; 2735} 2736 2737static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio) 2738{ 2739 unsigned int shift = folio_shift(folio); 2740 2741 return (pos1 >> shift == pos2 >> shift); 2742} 2743 2744static void filemap_end_dropbehind_read(struct folio *folio) 2745{ 2746 if (!folio_test_dropbehind(folio)) 2747 return; 2748 if (folio_test_writeback(folio) || folio_test_dirty(folio)) 2749 return; 2750 if (folio_trylock(folio)) { 2751 filemap_end_dropbehind(folio); 2752 folio_unlock(folio); 2753 } 2754} 2755 2756/** 2757 * filemap_read - Read data from the page cache. 2758 * @iocb: The iocb to read. 2759 * @iter: Destination for the data. 2760 * @already_read: Number of bytes already read by the caller. 2761 * 2762 * Copies data from the page cache. If the data is not currently present, 2763 * uses the readahead and read_folio address_space operations to fetch it. 2764 * 2765 * Return: Total number of bytes copied, including those already read by 2766 * the caller. If an error happens before any bytes are copied, returns 2767 * a negative error number. 2768 */ 2769ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, 2770 ssize_t already_read) 2771{ 2772 struct file *filp = iocb->ki_filp; 2773 struct file_ra_state *ra = &filp->f_ra; 2774 struct address_space *mapping = filp->f_mapping; 2775 struct inode *inode = mapping->host; 2776 struct folio_batch fbatch; 2777 int i, error = 0; 2778 bool writably_mapped; 2779 loff_t isize, end_offset; 2780 loff_t last_pos = ra->prev_pos; 2781 2782 if (unlikely(iocb->ki_pos < 0)) 2783 return -EINVAL; 2784 if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes)) 2785 return 0; 2786 if (unlikely(!iov_iter_count(iter))) 2787 return 0; 2788 2789 iov_iter_truncate(iter, inode->i_sb->s_maxbytes - iocb->ki_pos); 2790 folio_batch_init(&fbatch); 2791 2792 do { 2793 cond_resched(); 2794 2795 /* 2796 * If we've already successfully copied some data, then we 2797 * can no longer safely return -EIOCBQUEUED. Hence mark 2798 * an async read NOWAIT at that point. 2799 */ 2800 if ((iocb->ki_flags & IOCB_WAITQ) && already_read) 2801 iocb->ki_flags |= IOCB_NOWAIT; 2802 2803 if (unlikely(iocb->ki_pos >= i_size_read(inode))) 2804 break; 2805 2806 error = filemap_get_pages(iocb, iter->count, &fbatch, false); 2807 if (error < 0) 2808 break; 2809 2810 /* 2811 * i_size must be checked after we know the pages are Uptodate. 2812 * 2813 * Checking i_size after the check allows us to calculate 2814 * the correct value for "nr", which means the zero-filled 2815 * part of the page is not copied back to userspace (unless 2816 * another truncate extends the file - this is desired though). 2817 */ 2818 isize = i_size_read(inode); 2819 if (unlikely(iocb->ki_pos >= isize)) 2820 goto put_folios; 2821 end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); 2822 2823 /* 2824 * Once we start copying data, we don't want to be touching any 2825 * cachelines that might be contended: 2826 */ 2827 writably_mapped = mapping_writably_mapped(mapping); 2828 2829 /* 2830 * When a read accesses the same folio several times, only 2831 * mark it as accessed the first time. 2832 */ 2833 if (!pos_same_folio(iocb->ki_pos, last_pos - 1, 2834 fbatch.folios[0])) 2835 folio_mark_accessed(fbatch.folios[0]); 2836 2837 for (i = 0; i < folio_batch_count(&fbatch); i++) { 2838 struct folio *folio = fbatch.folios[i]; 2839 size_t fsize = folio_size(folio); 2840 size_t offset = iocb->ki_pos & (fsize - 1); 2841 size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, 2842 fsize - offset); 2843 size_t copied; 2844 2845 if (end_offset < folio_pos(folio)) 2846 break; 2847 if (i > 0) 2848 folio_mark_accessed(folio); 2849 /* 2850 * If users can be writing to this folio using arbitrary 2851 * virtual addresses, take care of potential aliasing 2852 * before reading the folio on the kernel side. 2853 */ 2854 if (writably_mapped) 2855 flush_dcache_folio(folio); 2856 2857 copied = copy_folio_to_iter(folio, offset, bytes, iter); 2858 2859 already_read += copied; 2860 iocb->ki_pos += copied; 2861 last_pos = iocb->ki_pos; 2862 2863 if (copied < bytes) { 2864 error = -EFAULT; 2865 break; 2866 } 2867 } 2868put_folios: 2869 for (i = 0; i < folio_batch_count(&fbatch); i++) { 2870 struct folio *folio = fbatch.folios[i]; 2871 2872 filemap_end_dropbehind_read(folio); 2873 folio_put(folio); 2874 } 2875 folio_batch_init(&fbatch); 2876 } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); 2877 2878 file_accessed(filp); 2879 ra->prev_pos = last_pos; 2880 return already_read ? already_read : error; 2881} 2882EXPORT_SYMBOL_GPL(filemap_read); 2883 2884int kiocb_write_and_wait(struct kiocb *iocb, size_t count) 2885{ 2886 struct address_space *mapping = iocb->ki_filp->f_mapping; 2887 loff_t pos = iocb->ki_pos; 2888 loff_t end = pos + count - 1; 2889 2890 if (iocb->ki_flags & IOCB_NOWAIT) { 2891 if (filemap_range_needs_writeback(mapping, pos, end)) 2892 return -EAGAIN; 2893 return 0; 2894 } 2895 2896 return filemap_write_and_wait_range(mapping, pos, end); 2897} 2898EXPORT_SYMBOL_GPL(kiocb_write_and_wait); 2899 2900int filemap_invalidate_pages(struct address_space *mapping, 2901 loff_t pos, loff_t end, bool nowait) 2902{ 2903 int ret; 2904 2905 if (nowait) { 2906 /* we could block if there are any pages in the range */ 2907 if (filemap_range_has_page(mapping, pos, end)) 2908 return -EAGAIN; 2909 } else { 2910 ret = filemap_write_and_wait_range(mapping, pos, end); 2911 if (ret) 2912 return ret; 2913 } 2914 2915 /* 2916 * After a write we want buffered reads to be sure to go to disk to get 2917 * the new data. We invalidate clean cached page from the region we're 2918 * about to write. We do this *before* the write so that we can return 2919 * without clobbering -EIOCBQUEUED from ->direct_IO(). 2920 */ 2921 return invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, 2922 end >> PAGE_SHIFT); 2923} 2924 2925int kiocb_invalidate_pages(struct kiocb *iocb, size_t count) 2926{ 2927 struct address_space *mapping = iocb->ki_filp->f_mapping; 2928 2929 return filemap_invalidate_pages(mapping, iocb->ki_pos, 2930 iocb->ki_pos + count - 1, 2931 iocb->ki_flags & IOCB_NOWAIT); 2932} 2933EXPORT_SYMBOL_GPL(kiocb_invalidate_pages); 2934 2935/** 2936 * generic_file_read_iter - generic filesystem read routine 2937 * @iocb: kernel I/O control block 2938 * @iter: destination for the data read 2939 * 2940 * This is the "read_iter()" routine for all filesystems 2941 * that can use the page cache directly. 2942 * 2943 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall 2944 * be returned when no data can be read without waiting for I/O requests 2945 * to complete; it doesn't prevent readahead. 2946 * 2947 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O 2948 * requests shall be made for the read or for readahead. When no data 2949 * can be read, -EAGAIN shall be returned. When readahead would be 2950 * triggered, a partial, possibly empty read shall be returned. 2951 * 2952 * Return: 2953 * * number of bytes copied, even for partial reads 2954 * * negative error code (or 0 if IOCB_NOIO) if nothing was read 2955 */ 2956ssize_t 2957generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) 2958{ 2959 size_t count = iov_iter_count(iter); 2960 ssize_t retval = 0; 2961 2962 if (!count) 2963 return 0; /* skip atime */ 2964 2965 if (iocb->ki_flags & IOCB_DIRECT) { 2966 struct file *file = iocb->ki_filp; 2967 struct address_space *mapping = file->f_mapping; 2968 struct inode *inode = mapping->host; 2969 2970 retval = kiocb_write_and_wait(iocb, count); 2971 if (retval < 0) 2972 return retval; 2973 file_accessed(file); 2974 2975 retval = mapping->a_ops->direct_IO(iocb, iter); 2976 if (retval >= 0) { 2977 iocb->ki_pos += retval; 2978 count -= retval; 2979 } 2980 if (retval != -EIOCBQUEUED) 2981 iov_iter_revert(iter, count - iov_iter_count(iter)); 2982 2983 /* 2984 * Btrfs can have a short DIO read if we encounter 2985 * compressed extents, so if there was an error, or if 2986 * we've already read everything we wanted to, or if 2987 * there was a short read because we hit EOF, go ahead 2988 * and return. Otherwise fallthrough to buffered io for 2989 * the rest of the read. Buffered reads will not work for 2990 * DAX files, so don't bother trying. 2991 */ 2992 if (retval < 0 || !count || IS_DAX(inode)) 2993 return retval; 2994 if (iocb->ki_pos >= i_size_read(inode)) 2995 return retval; 2996 } 2997 2998 return filemap_read(iocb, iter, retval); 2999} 3000EXPORT_SYMBOL(generic_file_read_iter); 3001 3002/* 3003 * Splice subpages from a folio into a pipe. 3004 */ 3005size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, 3006 struct folio *folio, loff_t fpos, size_t size) 3007{ 3008 struct page *page; 3009 size_t spliced = 0, offset = offset_in_folio(folio, fpos); 3010 3011 page = folio_page(folio, offset / PAGE_SIZE); 3012 size = min(size, folio_size(folio) - offset); 3013 offset %= PAGE_SIZE; 3014 3015 while (spliced < size && !pipe_is_full(pipe)) { 3016 struct pipe_buffer *buf = pipe_head_buf(pipe); 3017 size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced); 3018 3019 *buf = (struct pipe_buffer) { 3020 .ops = &page_cache_pipe_buf_ops, 3021 .page = page, 3022 .offset = offset, 3023 .len = part, 3024 }; 3025 folio_get(folio); 3026 pipe->head++; 3027 page++; 3028 spliced += part; 3029 offset = 0; 3030 } 3031 3032 return spliced; 3033} 3034 3035/** 3036 * filemap_splice_read - Splice data from a file's pagecache into a pipe 3037 * @in: The file to read from 3038 * @ppos: Pointer to the file position to read from 3039 * @pipe: The pipe to splice into 3040 * @len: The amount to splice 3041 * @flags: The SPLICE_F_* flags 3042 * 3043 * This function gets folios from a file's pagecache and splices them into the 3044 * pipe. Readahead will be called as necessary to fill more folios. This may 3045 * be used for blockdevs also. 3046 * 3047 * Return: On success, the number of bytes read will be returned and *@ppos 3048 * will be updated if appropriate; 0 will be returned if there is no more data 3049 * to be read; -EAGAIN will be returned if the pipe had no space, and some 3050 * other negative error code will be returned on error. A short read may occur 3051 * if the pipe has insufficient space, we reach the end of the data or we hit a 3052 * hole. 3053 */ 3054ssize_t filemap_splice_read(struct file *in, loff_t *ppos, 3055 struct pipe_inode_info *pipe, 3056 size_t len, unsigned int flags) 3057{ 3058 struct folio_batch fbatch; 3059 struct kiocb iocb; 3060 size_t total_spliced = 0, used, npages; 3061 loff_t isize, end_offset; 3062 bool writably_mapped; 3063 int i, error = 0; 3064 3065 if (unlikely(*ppos >= in->f_mapping->host->i_sb->s_maxbytes)) 3066 return 0; 3067 3068 init_sync_kiocb(&iocb, in); 3069 iocb.ki_pos = *ppos; 3070 3071 /* Work out how much data we can actually add into the pipe */ 3072 used = pipe_buf_usage(pipe); 3073 npages = max_t(ssize_t, pipe->max_usage - used, 0); 3074 len = min_t(size_t, len, npages * PAGE_SIZE); 3075 3076 folio_batch_init(&fbatch); 3077 3078 do { 3079 cond_resched(); 3080 3081 if (*ppos >= i_size_read(in->f_mapping->host)) 3082 break; 3083 3084 iocb.ki_pos = *ppos; 3085 error = filemap_get_pages(&iocb, len, &fbatch, true); 3086 if (error < 0) 3087 break; 3088 3089 /* 3090 * i_size must be checked after we know the pages are Uptodate. 3091 * 3092 * Checking i_size after the check allows us to calculate 3093 * the correct value for "nr", which means the zero-filled 3094 * part of the page is not copied back to userspace (unless 3095 * another truncate extends the file - this is desired though). 3096 */ 3097 isize = i_size_read(in->f_mapping->host); 3098 if (unlikely(*ppos >= isize)) 3099 break; 3100 end_offset = min_t(loff_t, isize, *ppos + len); 3101 3102 /* 3103 * Once we start copying data, we don't want to be touching any 3104 * cachelines that might be contended: 3105 */ 3106 writably_mapped = mapping_writably_mapped(in->f_mapping); 3107 3108 for (i = 0; i < folio_batch_count(&fbatch); i++) { 3109 struct folio *folio = fbatch.folios[i]; 3110 size_t n; 3111 3112 if (folio_pos(folio) >= end_offset) 3113 goto out; 3114 folio_mark_accessed(folio); 3115 3116 /* 3117 * If users can be writing to this folio using arbitrary 3118 * virtual addresses, take care of potential aliasing 3119 * before reading the folio on the kernel side. 3120 */ 3121 if (writably_mapped) 3122 flush_dcache_folio(folio); 3123 3124 n = min_t(loff_t, len, isize - *ppos); 3125 n = splice_folio_into_pipe(pipe, folio, *ppos, n); 3126 if (!n) 3127 goto out; 3128 len -= n; 3129 total_spliced += n; 3130 *ppos += n; 3131 in->f_ra.prev_pos = *ppos; 3132 if (pipe_is_full(pipe)) 3133 goto out; 3134 } 3135 3136 folio_batch_release(&fbatch); 3137 } while (len); 3138 3139out: 3140 folio_batch_release(&fbatch); 3141 file_accessed(in); 3142 3143 return total_spliced ? total_spliced : error; 3144} 3145EXPORT_SYMBOL(filemap_splice_read); 3146 3147static inline loff_t folio_seek_hole_data(struct xa_state *xas, 3148 struct address_space *mapping, struct folio *folio, 3149 loff_t start, loff_t end, bool seek_data) 3150{ 3151 const struct address_space_operations *ops = mapping->a_ops; 3152 size_t offset, bsz = i_blocksize(mapping->host); 3153 3154 if (xa_is_value(folio) || folio_test_uptodate(folio)) 3155 return seek_data ? start : end; 3156 if (!ops->is_partially_uptodate) 3157 return seek_data ? end : start; 3158 3159 xas_pause(xas); 3160 rcu_read_unlock(); 3161 folio_lock(folio); 3162 if (unlikely(folio->mapping != mapping)) 3163 goto unlock; 3164 3165 offset = offset_in_folio(folio, start) & ~(bsz - 1); 3166 3167 do { 3168 if (ops->is_partially_uptodate(folio, offset, bsz) == 3169 seek_data) 3170 break; 3171 start = (start + bsz) & ~((u64)bsz - 1); 3172 offset += bsz; 3173 } while (offset < folio_size(folio)); 3174unlock: 3175 folio_unlock(folio); 3176 rcu_read_lock(); 3177 return start; 3178} 3179 3180static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio) 3181{ 3182 if (xa_is_value(folio)) 3183 return PAGE_SIZE << xas_get_order(xas); 3184 return folio_size(folio); 3185} 3186 3187/** 3188 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache. 3189 * @mapping: Address space to search. 3190 * @start: First byte to consider. 3191 * @end: Limit of search (exclusive). 3192 * @whence: Either SEEK_HOLE or SEEK_DATA. 3193 * 3194 * If the page cache knows which blocks contain holes and which blocks 3195 * contain data, your filesystem can use this function to implement 3196 * SEEK_HOLE and SEEK_DATA. This is useful for filesystems which are 3197 * entirely memory-based such as tmpfs, and filesystems which support 3198 * unwritten extents. 3199 * 3200 * Return: The requested offset on success, or -ENXIO if @whence specifies 3201 * SEEK_DATA and there is no data after @start. There is an implicit hole 3202 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start 3203 * and @end contain data. 3204 */ 3205loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, 3206 loff_t end, int whence) 3207{ 3208 XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); 3209 pgoff_t max = (end - 1) >> PAGE_SHIFT; 3210 bool seek_data = (whence == SEEK_DATA); 3211 struct folio *folio; 3212 3213 if (end <= start) 3214 return -ENXIO; 3215 3216 rcu_read_lock(); 3217 while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { 3218 loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; 3219 size_t seek_size; 3220 3221 if (start < pos) { 3222 if (!seek_data) 3223 goto unlock; 3224 start = pos; 3225 } 3226 3227 seek_size = seek_folio_size(&xas, folio); 3228 pos = round_up((u64)pos + 1, seek_size); 3229 start = folio_seek_hole_data(&xas, mapping, folio, start, pos, 3230 seek_data); 3231 if (start < pos) 3232 goto unlock; 3233 if (start >= end) 3234 break; 3235 if (seek_size > PAGE_SIZE) 3236 xas_set(&xas, pos >> PAGE_SHIFT); 3237 if (!xa_is_value(folio)) 3238 folio_put(folio); 3239 } 3240 if (seek_data) 3241 start = -ENXIO; 3242unlock: 3243 rcu_read_unlock(); 3244 if (folio && !xa_is_value(folio)) 3245 folio_put(folio); 3246 if (start > end) 3247 return end; 3248 return start; 3249} 3250 3251#ifdef CONFIG_MMU 3252#define MMAP_LOTSAMISS (100) 3253/* 3254 * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock 3255 * @vmf - the vm_fault for this fault. 3256 * @folio - the folio to lock. 3257 * @fpin - the pointer to the file we may pin (or is already pinned). 3258 * 3259 * This works similar to lock_folio_or_retry in that it can drop the 3260 * mmap_lock. It differs in that it actually returns the folio locked 3261 * if it returns 1 and 0 if it couldn't lock the folio. If we did have 3262 * to drop the mmap_lock then fpin will point to the pinned file and 3263 * needs to be fput()'ed at a later point. 3264 */ 3265static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, 3266 struct file **fpin) 3267{ 3268 if (folio_trylock(folio)) 3269 return 1; 3270 3271 /* 3272 * NOTE! This will make us return with VM_FAULT_RETRY, but with 3273 * the fault lock still held. That's how FAULT_FLAG_RETRY_NOWAIT 3274 * is supposed to work. We have way too many special cases.. 3275 */ 3276 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 3277 return 0; 3278 3279 *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); 3280 if (vmf->flags & FAULT_FLAG_KILLABLE) { 3281 if (__folio_lock_killable(folio)) { 3282 /* 3283 * We didn't have the right flags to drop the 3284 * fault lock, but all fault_handlers only check 3285 * for fatal signals if we return VM_FAULT_RETRY, 3286 * so we need to drop the fault lock here and 3287 * return 0 if we don't have a fpin. 3288 */ 3289 if (*fpin == NULL) 3290 release_fault_lock(vmf); 3291 return 0; 3292 } 3293 } else 3294 __folio_lock(folio); 3295 3296 return 1; 3297} 3298 3299/* 3300 * Synchronous readahead happens when we don't even find a page in the page 3301 * cache at all. We don't want to perform IO under the mmap sem, so if we have 3302 * to drop the mmap sem we return the file that was pinned in order for us to do 3303 * that. If we didn't pin a file then we return NULL. The file that is 3304 * returned needs to be fput()'ed when we're done with it. 3305 */ 3306static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) 3307{ 3308 struct file *file = vmf->vma->vm_file; 3309 struct file_ra_state *ra = &file->f_ra; 3310 struct address_space *mapping = file->f_mapping; 3311 DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff); 3312 struct file *fpin = NULL; 3313 vm_flags_t vm_flags = vmf->vma->vm_flags; 3314 bool force_thp_readahead = false; 3315 unsigned short mmap_miss; 3316 3317 /* Use the readahead code, even if readahead is disabled */ 3318 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 3319 (vm_flags & VM_HUGEPAGE) && HPAGE_PMD_ORDER <= MAX_PAGECACHE_ORDER) 3320 force_thp_readahead = true; 3321 3322 if (!force_thp_readahead) { 3323 /* 3324 * If we don't want any read-ahead, don't bother. 3325 * VM_EXEC case below is already intended for random access. 3326 */ 3327 if ((vm_flags & (VM_RAND_READ | VM_EXEC)) == VM_RAND_READ) 3328 return fpin; 3329 3330 if (!ra->ra_pages) 3331 return fpin; 3332 3333 if (vm_flags & VM_SEQ_READ) { 3334 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3335 page_cache_sync_ra(&ractl, ra->ra_pages); 3336 return fpin; 3337 } 3338 } 3339 3340 if (!(vm_flags & VM_SEQ_READ)) { 3341 /* Avoid banging the cache line if not needed */ 3342 mmap_miss = READ_ONCE(ra->mmap_miss); 3343 if (mmap_miss < MMAP_LOTSAMISS * 10) 3344 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); 3345 3346 /* 3347 * Do we miss much more than hit in this file? If so, 3348 * stop bothering with read-ahead. It will only hurt. 3349 */ 3350 if (mmap_miss > MMAP_LOTSAMISS) 3351 return fpin; 3352 } 3353 3354 if (force_thp_readahead) { 3355 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3356 ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1); 3357 ra->size = HPAGE_PMD_NR; 3358 /* 3359 * Fetch two PMD folios, so we get the chance to actually 3360 * readahead, unless we've been told not to. 3361 */ 3362 if (!(vm_flags & VM_RAND_READ)) 3363 ra->size *= 2; 3364 ra->async_size = HPAGE_PMD_NR; 3365 ra->order = HPAGE_PMD_ORDER; 3366 page_cache_ra_order(&ractl, ra); 3367 return fpin; 3368 } 3369 3370 if (vm_flags & VM_EXEC) { 3371 /* 3372 * Allow arch to request a preferred minimum folio order for 3373 * executable memory. This can often be beneficial to 3374 * performance if (e.g.) arm64 can contpte-map the folio. 3375 * Executable memory rarely benefits from readahead, due to its 3376 * random access nature, so set async_size to 0. 3377 * 3378 * Limit to the boundaries of the VMA to avoid reading in any 3379 * pad that might exist between sections, which would be a waste 3380 * of memory. 3381 */ 3382 struct vm_area_struct *vma = vmf->vma; 3383 unsigned long start = vma->vm_pgoff; 3384 unsigned long end = start + vma_pages(vma); 3385 unsigned long ra_end; 3386 3387 ra->order = exec_folio_order(); 3388 ra->start = round_down(vmf->pgoff, 1UL << ra->order); 3389 ra->start = max(ra->start, start); 3390 ra_end = round_up(ra->start + ra->ra_pages, 1UL << ra->order); 3391 ra_end = min(ra_end, end); 3392 ra->size = ra_end - ra->start; 3393 ra->async_size = 0; 3394 } else { 3395 /* 3396 * mmap read-around 3397 */ 3398 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); 3399 ra->size = ra->ra_pages; 3400 ra->async_size = ra->ra_pages / 4; 3401 ra->order = 0; 3402 } 3403 3404 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3405 ractl._index = ra->start; 3406 page_cache_ra_order(&ractl, ra); 3407 return fpin; 3408} 3409 3410/* 3411 * Asynchronous readahead happens when we find the page and PG_readahead, 3412 * so we want to possibly extend the readahead further. We return the file that 3413 * was pinned if we have to drop the mmap_lock in order to do IO. 3414 */ 3415static struct file *do_async_mmap_readahead(struct vm_fault *vmf, 3416 struct folio *folio) 3417{ 3418 struct file *file = vmf->vma->vm_file; 3419 struct file_ra_state *ra = &file->f_ra; 3420 DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff); 3421 struct file *fpin = NULL; 3422 unsigned short mmap_miss; 3423 3424 /* If we don't want any read-ahead, don't bother */ 3425 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) 3426 return fpin; 3427 3428 /* 3429 * If the folio is locked, we're likely racing against another fault. 3430 * Don't touch the mmap_miss counter to avoid decreasing it multiple 3431 * times for a single folio and break the balance with mmap_miss 3432 * increase in do_sync_mmap_readahead(). 3433 */ 3434 if (likely(!folio_test_locked(folio))) { 3435 mmap_miss = READ_ONCE(ra->mmap_miss); 3436 if (mmap_miss) 3437 WRITE_ONCE(ra->mmap_miss, --mmap_miss); 3438 } 3439 3440 if (folio_test_readahead(folio)) { 3441 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3442 page_cache_async_ra(&ractl, folio, ra->ra_pages); 3443 } 3444 return fpin; 3445} 3446 3447static vm_fault_t filemap_fault_recheck_pte_none(struct vm_fault *vmf) 3448{ 3449 struct vm_area_struct *vma = vmf->vma; 3450 vm_fault_t ret = 0; 3451 pte_t *ptep; 3452 3453 /* 3454 * We might have COW'ed a pagecache folio and might now have an mlocked 3455 * anon folio mapped. The original pagecache folio is not mlocked and 3456 * might have been evicted. During a read+clear/modify/write update of 3457 * the PTE, such as done in do_numa_page()/change_pte_range(), we 3458 * temporarily clear the PTE under PT lock and might detect it here as 3459 * "none" when not holding the PT lock. 3460 * 3461 * Not rechecking the PTE under PT lock could result in an unexpected 3462 * major fault in an mlock'ed region. Recheck only for this special 3463 * scenario while holding the PT lock, to not degrade non-mlocked 3464 * scenarios. Recheck the PTE without PT lock firstly, thereby reducing 3465 * the number of times we hold PT lock. 3466 */ 3467 if (!(vma->vm_flags & VM_LOCKED)) 3468 return 0; 3469 3470 if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)) 3471 return 0; 3472 3473 ptep = pte_offset_map_ro_nolock(vma->vm_mm, vmf->pmd, vmf->address, 3474 &vmf->ptl); 3475 if (unlikely(!ptep)) 3476 return VM_FAULT_NOPAGE; 3477 3478 if (unlikely(!pte_none(ptep_get_lockless(ptep)))) { 3479 ret = VM_FAULT_NOPAGE; 3480 } else { 3481 spin_lock(vmf->ptl); 3482 if (unlikely(!pte_none(ptep_get(ptep)))) 3483 ret = VM_FAULT_NOPAGE; 3484 spin_unlock(vmf->ptl); 3485 } 3486 pte_unmap(ptep); 3487 return ret; 3488} 3489 3490/** 3491 * filemap_fault - read in file data for page fault handling 3492 * @vmf: struct vm_fault containing details of the fault 3493 * 3494 * filemap_fault() is invoked via the vma operations vector for a 3495 * mapped memory region to read in file data during a page fault. 3496 * 3497 * The goto's are kind of ugly, but this streamlines the normal case of having 3498 * it in the page cache, and handles the special cases reasonably without 3499 * having a lot of duplicated code. 3500 * 3501 * vma->vm_mm->mmap_lock must be held on entry. 3502 * 3503 * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock 3504 * may be dropped before doing I/O or by lock_folio_maybe_drop_mmap(). 3505 * 3506 * If our return value does not have VM_FAULT_RETRY set, the mmap_lock 3507 * has not been released. 3508 * 3509 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. 3510 * 3511 * Return: bitwise-OR of %VM_FAULT_ codes. 3512 */ 3513vm_fault_t filemap_fault(struct vm_fault *vmf) 3514{ 3515 int error; 3516 struct file *file = vmf->vma->vm_file; 3517 struct file *fpin = NULL; 3518 struct address_space *mapping = file->f_mapping; 3519 struct inode *inode = mapping->host; 3520 pgoff_t max_idx, index = vmf->pgoff; 3521 struct folio *folio; 3522 vm_fault_t ret = 0; 3523 bool mapping_locked = false; 3524 3525 max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 3526 if (unlikely(index >= max_idx)) 3527 return VM_FAULT_SIGBUS; 3528 3529 trace_mm_filemap_fault(mapping, index); 3530 3531 /* 3532 * Do we have something in the page cache already? 3533 */ 3534 folio = filemap_get_folio(mapping, index); 3535 if (likely(!IS_ERR(folio))) { 3536 /* 3537 * We found the page, so try async readahead before waiting for 3538 * the lock. 3539 */ 3540 if (!(vmf->flags & FAULT_FLAG_TRIED)) 3541 fpin = do_async_mmap_readahead(vmf, folio); 3542 if (unlikely(!folio_test_uptodate(folio))) { 3543 filemap_invalidate_lock_shared(mapping); 3544 mapping_locked = true; 3545 } 3546 } else { 3547 ret = filemap_fault_recheck_pte_none(vmf); 3548 if (unlikely(ret)) 3549 return ret; 3550 3551 /* No page in the page cache at all */ 3552 count_vm_event(PGMAJFAULT); 3553 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); 3554 ret = VM_FAULT_MAJOR; 3555 fpin = do_sync_mmap_readahead(vmf); 3556retry_find: 3557 /* 3558 * See comment in filemap_create_folio() why we need 3559 * invalidate_lock 3560 */ 3561 if (!mapping_locked) { 3562 filemap_invalidate_lock_shared(mapping); 3563 mapping_locked = true; 3564 } 3565 folio = __filemap_get_folio(mapping, index, 3566 FGP_CREAT|FGP_FOR_MMAP, 3567 vmf->gfp_mask); 3568 if (IS_ERR(folio)) { 3569 if (fpin) 3570 goto out_retry; 3571 filemap_invalidate_unlock_shared(mapping); 3572 return VM_FAULT_OOM; 3573 } 3574 } 3575 3576 if (!lock_folio_maybe_drop_mmap(vmf, folio, &fpin)) 3577 goto out_retry; 3578 3579 /* Did it get truncated? */ 3580 if (unlikely(folio->mapping != mapping)) { 3581 folio_unlock(folio); 3582 folio_put(folio); 3583 goto retry_find; 3584 } 3585 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); 3586 3587 /* 3588 * We have a locked folio in the page cache, now we need to check 3589 * that it's up-to-date. If not, it is going to be due to an error, 3590 * or because readahead was otherwise unable to retrieve it. 3591 */ 3592 if (unlikely(!folio_test_uptodate(folio))) { 3593 /* 3594 * If the invalidate lock is not held, the folio was in cache 3595 * and uptodate and now it is not. Strange but possible since we 3596 * didn't hold the page lock all the time. Let's drop 3597 * everything, get the invalidate lock and try again. 3598 */ 3599 if (!mapping_locked) { 3600 folio_unlock(folio); 3601 folio_put(folio); 3602 goto retry_find; 3603 } 3604 3605 /* 3606 * OK, the folio is really not uptodate. This can be because the 3607 * VMA has the VM_RAND_READ flag set, or because an error 3608 * arose. Let's read it in directly. 3609 */ 3610 goto page_not_uptodate; 3611 } 3612 3613 /* 3614 * We've made it this far and we had to drop our mmap_lock, now is the 3615 * time to return to the upper layer and have it re-find the vma and 3616 * redo the fault. 3617 */ 3618 if (fpin) { 3619 folio_unlock(folio); 3620 goto out_retry; 3621 } 3622 if (mapping_locked) 3623 filemap_invalidate_unlock_shared(mapping); 3624 3625 /* 3626 * Found the page and have a reference on it. 3627 * We must recheck i_size under page lock. 3628 */ 3629 max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 3630 if (unlikely(index >= max_idx)) { 3631 folio_unlock(folio); 3632 folio_put(folio); 3633 return VM_FAULT_SIGBUS; 3634 } 3635 3636 vmf->page = folio_file_page(folio, index); 3637 return ret | VM_FAULT_LOCKED; 3638 3639page_not_uptodate: 3640 /* 3641 * Umm, take care of errors if the page isn't up-to-date. 3642 * Try to re-read it _once_. We do this synchronously, 3643 * because there really aren't any performance issues here 3644 * and we need to check for errors. 3645 */ 3646 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3647 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); 3648 if (fpin) 3649 goto out_retry; 3650 folio_put(folio); 3651 3652 if (!error || error == AOP_TRUNCATED_PAGE) 3653 goto retry_find; 3654 filemap_invalidate_unlock_shared(mapping); 3655 3656 return VM_FAULT_SIGBUS; 3657 3658out_retry: 3659 /* 3660 * We dropped the mmap_lock, we need to return to the fault handler to 3661 * re-find the vma and come back and find our hopefully still populated 3662 * page. 3663 */ 3664 if (!IS_ERR(folio)) 3665 folio_put(folio); 3666 if (mapping_locked) 3667 filemap_invalidate_unlock_shared(mapping); 3668 if (fpin) 3669 fput(fpin); 3670 return ret | VM_FAULT_RETRY; 3671} 3672EXPORT_SYMBOL(filemap_fault); 3673 3674static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, 3675 pgoff_t start) 3676{ 3677 struct mm_struct *mm = vmf->vma->vm_mm; 3678 3679 /* Huge page is mapped? No need to proceed. */ 3680 if (pmd_trans_huge(*vmf->pmd)) { 3681 folio_unlock(folio); 3682 folio_put(folio); 3683 return true; 3684 } 3685 3686 if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) { 3687 struct page *page = folio_file_page(folio, start); 3688 vm_fault_t ret = do_set_pmd(vmf, folio, page); 3689 if (!ret) { 3690 /* The page is mapped successfully, reference consumed. */ 3691 folio_unlock(folio); 3692 return true; 3693 } 3694 } 3695 3696 if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) 3697 pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); 3698 3699 return false; 3700} 3701 3702static struct folio *next_uptodate_folio(struct xa_state *xas, 3703 struct address_space *mapping, pgoff_t end_pgoff) 3704{ 3705 struct folio *folio = xas_next_entry(xas, end_pgoff); 3706 unsigned long max_idx; 3707 3708 do { 3709 if (!folio) 3710 return NULL; 3711 if (xas_retry(xas, folio)) 3712 continue; 3713 if (xa_is_value(folio)) 3714 continue; 3715 if (!folio_try_get(folio)) 3716 continue; 3717 if (folio_test_locked(folio)) 3718 goto skip; 3719 /* Has the page moved or been split? */ 3720 if (unlikely(folio != xas_reload(xas))) 3721 goto skip; 3722 if (!folio_test_uptodate(folio) || folio_test_readahead(folio)) 3723 goto skip; 3724 if (!folio_trylock(folio)) 3725 goto skip; 3726 if (folio->mapping != mapping) 3727 goto unlock; 3728 if (!folio_test_uptodate(folio)) 3729 goto unlock; 3730 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3731 if (xas->xa_index >= max_idx) 3732 goto unlock; 3733 return folio; 3734unlock: 3735 folio_unlock(folio); 3736skip: 3737 folio_put(folio); 3738 } while ((folio = xas_next_entry(xas, end_pgoff)) != NULL); 3739 3740 return NULL; 3741} 3742 3743/* 3744 * Map page range [start_page, start_page + nr_pages) of folio. 3745 * start_page is gotten from start by folio_page(folio, start) 3746 */ 3747static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, 3748 struct folio *folio, unsigned long start, 3749 unsigned long addr, unsigned int nr_pages, 3750 unsigned long *rss, unsigned short *mmap_miss, 3751 pgoff_t file_end) 3752{ 3753 struct address_space *mapping = folio->mapping; 3754 unsigned int ref_from_caller = 1; 3755 vm_fault_t ret = 0; 3756 struct page *page = folio_page(folio, start); 3757 unsigned int count = 0; 3758 pte_t *old_ptep = vmf->pte; 3759 unsigned long addr0; 3760 3761 /* 3762 * Map the large folio fully where possible: 3763 * 3764 * - The folio is fully within size of the file or belong 3765 * to shmem/tmpfs; 3766 * - The folio doesn't cross VMA boundary; 3767 * - The folio doesn't cross page table boundary; 3768 */ 3769 addr0 = addr - start * PAGE_SIZE; 3770 if ((file_end >= folio_next_index(folio) || shmem_mapping(mapping)) && 3771 folio_within_vma(folio, vmf->vma) && 3772 (addr0 & PMD_MASK) == ((addr0 + folio_size(folio) - 1) & PMD_MASK)) { 3773 vmf->pte -= start; 3774 page -= start; 3775 addr = addr0; 3776 nr_pages = folio_nr_pages(folio); 3777 } 3778 3779 do { 3780 if (PageHWPoison(page + count)) 3781 goto skip; 3782 3783 /* 3784 * If there are too many folios that are recently evicted 3785 * in a file, they will probably continue to be evicted. 3786 * In such situation, read-ahead is only a waste of IO. 3787 * Don't decrease mmap_miss in this scenario to make sure 3788 * we can stop read-ahead. 3789 */ 3790 if (!folio_test_workingset(folio)) 3791 (*mmap_miss)++; 3792 3793 /* 3794 * NOTE: If there're PTE markers, we'll leave them to be 3795 * handled in the specific fault path, and it'll prohibit the 3796 * fault-around logic. 3797 */ 3798 if (!pte_none(ptep_get(&vmf->pte[count]))) 3799 goto skip; 3800 3801 count++; 3802 continue; 3803skip: 3804 if (count) { 3805 set_pte_range(vmf, folio, page, count, addr); 3806 *rss += count; 3807 folio_ref_add(folio, count - ref_from_caller); 3808 ref_from_caller = 0; 3809 if (in_range(vmf->address, addr, count * PAGE_SIZE)) 3810 ret = VM_FAULT_NOPAGE; 3811 } 3812 3813 count++; 3814 page += count; 3815 vmf->pte += count; 3816 addr += count * PAGE_SIZE; 3817 count = 0; 3818 } while (--nr_pages > 0); 3819 3820 if (count) { 3821 set_pte_range(vmf, folio, page, count, addr); 3822 *rss += count; 3823 folio_ref_add(folio, count - ref_from_caller); 3824 ref_from_caller = 0; 3825 if (in_range(vmf->address, addr, count * PAGE_SIZE)) 3826 ret = VM_FAULT_NOPAGE; 3827 } 3828 3829 vmf->pte = old_ptep; 3830 if (ref_from_caller) 3831 /* Locked folios cannot get truncated. */ 3832 folio_ref_dec(folio); 3833 3834 return ret; 3835} 3836 3837static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, 3838 struct folio *folio, unsigned long addr, 3839 unsigned long *rss, unsigned short *mmap_miss) 3840{ 3841 vm_fault_t ret = 0; 3842 struct page *page = &folio->page; 3843 3844 if (PageHWPoison(page)) 3845 goto out; 3846 3847 /* See comment of filemap_map_folio_range() */ 3848 if (!folio_test_workingset(folio)) 3849 (*mmap_miss)++; 3850 3851 /* 3852 * NOTE: If there're PTE markers, we'll leave them to be 3853 * handled in the specific fault path, and it'll prohibit 3854 * the fault-around logic. 3855 */ 3856 if (!pte_none(ptep_get(vmf->pte))) 3857 goto out; 3858 3859 if (vmf->address == addr) 3860 ret = VM_FAULT_NOPAGE; 3861 3862 set_pte_range(vmf, folio, page, 1, addr); 3863 (*rss)++; 3864 return ret; 3865 3866out: 3867 /* Locked folios cannot get truncated. */ 3868 folio_ref_dec(folio); 3869 return ret; 3870} 3871 3872vm_fault_t filemap_map_pages(struct vm_fault *vmf, 3873 pgoff_t start_pgoff, pgoff_t end_pgoff) 3874{ 3875 struct vm_area_struct *vma = vmf->vma; 3876 struct file *file = vma->vm_file; 3877 struct address_space *mapping = file->f_mapping; 3878 pgoff_t file_end, last_pgoff = start_pgoff; 3879 unsigned long addr; 3880 XA_STATE(xas, &mapping->i_pages, start_pgoff); 3881 struct folio *folio; 3882 vm_fault_t ret = 0; 3883 unsigned long rss = 0; 3884 unsigned int nr_pages = 0, folio_type; 3885 unsigned short mmap_miss = 0, mmap_miss_saved; 3886 3887 /* 3888 * Recalculate end_pgoff based on file_end before calling 3889 * next_uptodate_folio() to avoid races with concurrent 3890 * truncation. 3891 */ 3892 file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1; 3893 end_pgoff = min(end_pgoff, file_end); 3894 3895 rcu_read_lock(); 3896 folio = next_uptodate_folio(&xas, mapping, end_pgoff); 3897 if (!folio) 3898 goto out; 3899 3900 /* 3901 * Do not allow to map with PMD across i_size to preserve 3902 * SIGBUS semantics. 3903 * 3904 * Make an exception for shmem/tmpfs that for long time 3905 * intentionally mapped with PMDs across i_size. 3906 */ 3907 if ((file_end >= folio_next_index(folio) || shmem_mapping(mapping)) && 3908 filemap_map_pmd(vmf, folio, start_pgoff)) { 3909 ret = VM_FAULT_NOPAGE; 3910 goto out; 3911 } 3912 3913 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); 3914 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); 3915 if (!vmf->pte) { 3916 folio_unlock(folio); 3917 folio_put(folio); 3918 goto out; 3919 } 3920 3921 folio_type = mm_counter_file(folio); 3922 do { 3923 unsigned long end; 3924 3925 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; 3926 vmf->pte += xas.xa_index - last_pgoff; 3927 last_pgoff = xas.xa_index; 3928 end = folio_next_index(folio) - 1; 3929 nr_pages = min(end, end_pgoff) - xas.xa_index + 1; 3930 3931 if (!folio_test_large(folio)) 3932 ret |= filemap_map_order0_folio(vmf, 3933 folio, addr, &rss, &mmap_miss); 3934 else 3935 ret |= filemap_map_folio_range(vmf, folio, 3936 xas.xa_index - folio->index, addr, 3937 nr_pages, &rss, &mmap_miss, file_end); 3938 3939 folio_unlock(folio); 3940 } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL); 3941 add_mm_counter(vma->vm_mm, folio_type, rss); 3942 pte_unmap_unlock(vmf->pte, vmf->ptl); 3943 trace_mm_filemap_map_pages(mapping, start_pgoff, end_pgoff); 3944out: 3945 rcu_read_unlock(); 3946 3947 mmap_miss_saved = READ_ONCE(file->f_ra.mmap_miss); 3948 if (mmap_miss >= mmap_miss_saved) 3949 WRITE_ONCE(file->f_ra.mmap_miss, 0); 3950 else 3951 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss_saved - mmap_miss); 3952 3953 return ret; 3954} 3955EXPORT_SYMBOL(filemap_map_pages); 3956 3957vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 3958{ 3959 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 3960 struct folio *folio = page_folio(vmf->page); 3961 vm_fault_t ret = VM_FAULT_LOCKED; 3962 3963 sb_start_pagefault(mapping->host->i_sb); 3964 file_update_time(vmf->vma->vm_file); 3965 folio_lock(folio); 3966 if (folio->mapping != mapping) { 3967 folio_unlock(folio); 3968 ret = VM_FAULT_NOPAGE; 3969 goto out; 3970 } 3971 /* 3972 * We mark the folio dirty already here so that when freeze is in 3973 * progress, we are guaranteed that writeback during freezing will 3974 * see the dirty folio and writeprotect it again. 3975 */ 3976 folio_mark_dirty(folio); 3977 folio_wait_stable(folio); 3978out: 3979 sb_end_pagefault(mapping->host->i_sb); 3980 return ret; 3981} 3982 3983const struct vm_operations_struct generic_file_vm_ops = { 3984 .fault = filemap_fault, 3985 .map_pages = filemap_map_pages, 3986 .page_mkwrite = filemap_page_mkwrite, 3987}; 3988 3989/* This is used for a general mmap of a disk file */ 3990 3991int generic_file_mmap(struct file *file, struct vm_area_struct *vma) 3992{ 3993 struct address_space *mapping = file->f_mapping; 3994 3995 if (!mapping->a_ops->read_folio) 3996 return -ENOEXEC; 3997 file_accessed(file); 3998 vma->vm_ops = &generic_file_vm_ops; 3999 return 0; 4000} 4001 4002int generic_file_mmap_prepare(struct vm_area_desc *desc) 4003{ 4004 struct file *file = desc->file; 4005 struct address_space *mapping = file->f_mapping; 4006 4007 if (!mapping->a_ops->read_folio) 4008 return -ENOEXEC; 4009 file_accessed(file); 4010 desc->vm_ops = &generic_file_vm_ops; 4011 return 0; 4012} 4013 4014/* 4015 * This is for filesystems which do not implement ->writepage. 4016 */ 4017int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 4018{ 4019 if (vma_is_shared_maywrite(vma)) 4020 return -EINVAL; 4021 return generic_file_mmap(file, vma); 4022} 4023 4024int generic_file_readonly_mmap_prepare(struct vm_area_desc *desc) 4025{ 4026 if (is_shared_maywrite(desc->vm_flags)) 4027 return -EINVAL; 4028 return generic_file_mmap_prepare(desc); 4029} 4030#else 4031vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 4032{ 4033 return VM_FAULT_SIGBUS; 4034} 4035int generic_file_mmap(struct file *file, struct vm_area_struct *vma) 4036{ 4037 return -ENOSYS; 4038} 4039int generic_file_mmap_prepare(struct vm_area_desc *desc) 4040{ 4041 return -ENOSYS; 4042} 4043int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 4044{ 4045 return -ENOSYS; 4046} 4047int generic_file_readonly_mmap_prepare(struct vm_area_desc *desc) 4048{ 4049 return -ENOSYS; 4050} 4051#endif /* CONFIG_MMU */ 4052 4053EXPORT_SYMBOL(filemap_page_mkwrite); 4054EXPORT_SYMBOL(generic_file_mmap); 4055EXPORT_SYMBOL(generic_file_mmap_prepare); 4056EXPORT_SYMBOL(generic_file_readonly_mmap); 4057EXPORT_SYMBOL(generic_file_readonly_mmap_prepare); 4058 4059static struct folio *do_read_cache_folio(struct address_space *mapping, 4060 pgoff_t index, filler_t filler, struct file *file, gfp_t gfp) 4061{ 4062 struct folio *folio; 4063 int err; 4064 4065 if (!filler) 4066 filler = mapping->a_ops->read_folio; 4067repeat: 4068 folio = filemap_get_folio(mapping, index); 4069 if (IS_ERR(folio)) { 4070 folio = filemap_alloc_folio(gfp, mapping_min_folio_order(mapping), NULL); 4071 if (!folio) 4072 return ERR_PTR(-ENOMEM); 4073 index = mapping_align_index(mapping, index); 4074 err = filemap_add_folio(mapping, folio, index, gfp); 4075 if (unlikely(err)) { 4076 folio_put(folio); 4077 if (err == -EEXIST) 4078 goto repeat; 4079 /* Presumably ENOMEM for xarray node */ 4080 return ERR_PTR(err); 4081 } 4082 4083 goto filler; 4084 } 4085 if (folio_test_uptodate(folio)) 4086 goto out; 4087 4088 if (!folio_trylock(folio)) { 4089 folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); 4090 goto repeat; 4091 } 4092 4093 /* Folio was truncated from mapping */ 4094 if (!folio->mapping) { 4095 folio_unlock(folio); 4096 folio_put(folio); 4097 goto repeat; 4098 } 4099 4100 /* Someone else locked and filled the page in a very small window */ 4101 if (folio_test_uptodate(folio)) { 4102 folio_unlock(folio); 4103 goto out; 4104 } 4105 4106filler: 4107 err = filemap_read_folio(file, filler, folio); 4108 if (err) { 4109 folio_put(folio); 4110 if (err == AOP_TRUNCATED_PAGE) 4111 goto repeat; 4112 return ERR_PTR(err); 4113 } 4114 4115out: 4116 folio_mark_accessed(folio); 4117 return folio; 4118} 4119 4120/** 4121 * read_cache_folio - Read into page cache, fill it if needed. 4122 * @mapping: The address_space to read from. 4123 * @index: The index to read. 4124 * @filler: Function to perform the read, or NULL to use aops->read_folio(). 4125 * @file: Passed to filler function, may be NULL if not required. 4126 * 4127 * Read one page into the page cache. If it succeeds, the folio returned 4128 * will contain @index, but it may not be the first page of the folio. 4129 * 4130 * If the filler function returns an error, it will be returned to the 4131 * caller. 4132 * 4133 * Context: May sleep. Expects mapping->invalidate_lock to be held. 4134 * Return: An uptodate folio on success, ERR_PTR() on failure. 4135 */ 4136struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, 4137 filler_t filler, struct file *file) 4138{ 4139 return do_read_cache_folio(mapping, index, filler, file, 4140 mapping_gfp_mask(mapping)); 4141} 4142EXPORT_SYMBOL(read_cache_folio); 4143 4144/** 4145 * mapping_read_folio_gfp - Read into page cache, using specified allocation flags. 4146 * @mapping: The address_space for the folio. 4147 * @index: The index that the allocated folio will contain. 4148 * @gfp: The page allocator flags to use if allocating. 4149 * 4150 * This is the same as "read_cache_folio(mapping, index, NULL, NULL)", but with 4151 * any new memory allocations done using the specified allocation flags. 4152 * 4153 * The most likely error from this function is EIO, but ENOMEM is 4154 * possible and so is EINTR. If ->read_folio returns another error, 4155 * that will be returned to the caller. 4156 * 4157 * The function expects mapping->invalidate_lock to be already held. 4158 * 4159 * Return: Uptodate folio on success, ERR_PTR() on failure. 4160 */ 4161struct folio *mapping_read_folio_gfp(struct address_space *mapping, 4162 pgoff_t index, gfp_t gfp) 4163{ 4164 return do_read_cache_folio(mapping, index, NULL, NULL, gfp); 4165} 4166EXPORT_SYMBOL(mapping_read_folio_gfp); 4167 4168static struct page *do_read_cache_page(struct address_space *mapping, 4169 pgoff_t index, filler_t *filler, struct file *file, gfp_t gfp) 4170{ 4171 struct folio *folio; 4172 4173 folio = do_read_cache_folio(mapping, index, filler, file, gfp); 4174 if (IS_ERR(folio)) 4175 return &folio->page; 4176 return folio_file_page(folio, index); 4177} 4178 4179struct page *read_cache_page(struct address_space *mapping, 4180 pgoff_t index, filler_t *filler, struct file *file) 4181{ 4182 return do_read_cache_page(mapping, index, filler, file, 4183 mapping_gfp_mask(mapping)); 4184} 4185EXPORT_SYMBOL(read_cache_page); 4186 4187/** 4188 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 4189 * @mapping: the page's address_space 4190 * @index: the page index 4191 * @gfp: the page allocator flags to use if allocating 4192 * 4193 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 4194 * any new page allocations done using the specified allocation flags. 4195 * 4196 * If the page does not get brought uptodate, return -EIO. 4197 * 4198 * The function expects mapping->invalidate_lock to be already held. 4199 * 4200 * Return: up to date page on success, ERR_PTR() on failure. 4201 */ 4202struct page *read_cache_page_gfp(struct address_space *mapping, 4203 pgoff_t index, 4204 gfp_t gfp) 4205{ 4206 return do_read_cache_page(mapping, index, NULL, NULL, gfp); 4207} 4208EXPORT_SYMBOL(read_cache_page_gfp); 4209 4210/* 4211 * Warn about a page cache invalidation failure during a direct I/O write. 4212 */ 4213static void dio_warn_stale_pagecache(struct file *filp) 4214{ 4215 static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST); 4216 char pathname[128]; 4217 char *path; 4218 4219 errseq_set(&filp->f_mapping->wb_err, -EIO); 4220 if (__ratelimit(&_rs)) { 4221 path = file_path(filp, pathname, sizeof(pathname)); 4222 if (IS_ERR(path)) 4223 path = "(unknown)"; 4224 pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n"); 4225 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, 4226 current->comm); 4227 } 4228} 4229 4230void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count) 4231{ 4232 struct address_space *mapping = iocb->ki_filp->f_mapping; 4233 4234 if (mapping->nrpages && 4235 invalidate_inode_pages2_range(mapping, 4236 iocb->ki_pos >> PAGE_SHIFT, 4237 (iocb->ki_pos + count - 1) >> PAGE_SHIFT)) 4238 dio_warn_stale_pagecache(iocb->ki_filp); 4239} 4240 4241ssize_t 4242generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) 4243{ 4244 struct address_space *mapping = iocb->ki_filp->f_mapping; 4245 size_t write_len = iov_iter_count(from); 4246 ssize_t written; 4247 4248 /* 4249 * If a page can not be invalidated, return 0 to fall back 4250 * to buffered write. 4251 */ 4252 written = kiocb_invalidate_pages(iocb, write_len); 4253 if (written) { 4254 if (written == -EBUSY) 4255 return 0; 4256 return written; 4257 } 4258 4259 written = mapping->a_ops->direct_IO(iocb, from); 4260 4261 /* 4262 * Finally, try again to invalidate clean pages which might have been 4263 * cached by non-direct readahead, or faulted in by get_user_pages() 4264 * if the source of the write was an mmap'ed region of the file 4265 * we're writing. Either one is a pretty crazy thing to do, 4266 * so we don't support it 100%. If this invalidation 4267 * fails, tough, the write still worked... 4268 * 4269 * Most of the time we do not need this since dio_complete() will do 4270 * the invalidation for us. However there are some file systems that 4271 * do not end up with dio_complete() being called, so let's not break 4272 * them by removing it completely. 4273 * 4274 * Noticeable example is a blkdev_direct_IO(). 4275 * 4276 * Skip invalidation for async writes or if mapping has no pages. 4277 */ 4278 if (written > 0) { 4279 struct inode *inode = mapping->host; 4280 loff_t pos = iocb->ki_pos; 4281 4282 kiocb_invalidate_post_direct_write(iocb, written); 4283 pos += written; 4284 write_len -= written; 4285 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 4286 i_size_write(inode, pos); 4287 mark_inode_dirty(inode); 4288 } 4289 iocb->ki_pos = pos; 4290 } 4291 if (written != -EIOCBQUEUED) 4292 iov_iter_revert(from, write_len - iov_iter_count(from)); 4293 return written; 4294} 4295EXPORT_SYMBOL(generic_file_direct_write); 4296 4297ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) 4298{ 4299 struct file *file = iocb->ki_filp; 4300 loff_t pos = iocb->ki_pos; 4301 struct address_space *mapping = file->f_mapping; 4302 const struct address_space_operations *a_ops = mapping->a_ops; 4303 size_t chunk = mapping_max_folio_size(mapping); 4304 long status = 0; 4305 ssize_t written = 0; 4306 4307 do { 4308 struct folio *folio; 4309 size_t offset; /* Offset into folio */ 4310 size_t bytes; /* Bytes to write to folio */ 4311 size_t copied; /* Bytes copied from user */ 4312 void *fsdata = NULL; 4313 4314 bytes = iov_iter_count(i); 4315retry: 4316 offset = pos & (chunk - 1); 4317 bytes = min(chunk - offset, bytes); 4318 balance_dirty_pages_ratelimited(mapping); 4319 4320 if (fatal_signal_pending(current)) { 4321 status = -EINTR; 4322 break; 4323 } 4324 4325 status = a_ops->write_begin(iocb, mapping, pos, bytes, 4326 &folio, &fsdata); 4327 if (unlikely(status < 0)) 4328 break; 4329 4330 offset = offset_in_folio(folio, pos); 4331 if (bytes > folio_size(folio) - offset) 4332 bytes = folio_size(folio) - offset; 4333 4334 if (mapping_writably_mapped(mapping)) 4335 flush_dcache_folio(folio); 4336 4337 /* 4338 * Faults here on mmap()s can recurse into arbitrary 4339 * filesystem code. Lots of locks are held that can 4340 * deadlock. Use an atomic copy to avoid deadlocking 4341 * in page fault handling. 4342 */ 4343 copied = copy_folio_from_iter_atomic(folio, offset, bytes, i); 4344 flush_dcache_folio(folio); 4345 4346 status = a_ops->write_end(iocb, mapping, pos, bytes, copied, 4347 folio, fsdata); 4348 if (unlikely(status != copied)) { 4349 iov_iter_revert(i, copied - max(status, 0L)); 4350 if (unlikely(status < 0)) 4351 break; 4352 } 4353 cond_resched(); 4354 4355 if (unlikely(status == 0)) { 4356 /* 4357 * A short copy made ->write_end() reject the 4358 * thing entirely. Might be memory poisoning 4359 * halfway through, might be a race with munmap, 4360 * might be severe memory pressure. 4361 */ 4362 if (chunk > PAGE_SIZE) 4363 chunk /= 2; 4364 if (copied) { 4365 bytes = copied; 4366 goto retry; 4367 } 4368 4369 /* 4370 * 'folio' is now unlocked and faults on it can be 4371 * handled. Ensure forward progress by trying to 4372 * fault it in now. 4373 */ 4374 if (fault_in_iov_iter_readable(i, bytes) == bytes) { 4375 status = -EFAULT; 4376 break; 4377 } 4378 } else { 4379 pos += status; 4380 written += status; 4381 } 4382 } while (iov_iter_count(i)); 4383 4384 if (!written) 4385 return status; 4386 iocb->ki_pos += written; 4387 return written; 4388} 4389EXPORT_SYMBOL(generic_perform_write); 4390 4391/** 4392 * __generic_file_write_iter - write data to a file 4393 * @iocb: IO state structure (file, offset, etc.) 4394 * @from: iov_iter with data to write 4395 * 4396 * This function does all the work needed for actually writing data to a 4397 * file. It does all basic checks, removes SUID from the file, updates 4398 * modification times and calls proper subroutines depending on whether we 4399 * do direct IO or a standard buffered write. 4400 * 4401 * It expects i_rwsem to be grabbed unless we work on a block device or similar 4402 * object which does not need locking at all. 4403 * 4404 * This function does *not* take care of syncing data in case of O_SYNC write. 4405 * A caller has to handle it. This is mainly due to the fact that we want to 4406 * avoid syncing under i_rwsem. 4407 * 4408 * Return: 4409 * * number of bytes written, even for truncated writes 4410 * * negative error code if no data has been written at all 4411 */ 4412ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 4413{ 4414 struct file *file = iocb->ki_filp; 4415 struct address_space *mapping = file->f_mapping; 4416 struct inode *inode = mapping->host; 4417 ssize_t ret; 4418 4419 ret = file_remove_privs(file); 4420 if (ret) 4421 return ret; 4422 4423 ret = file_update_time(file); 4424 if (ret) 4425 return ret; 4426 4427 if (iocb->ki_flags & IOCB_DIRECT) { 4428 ret = generic_file_direct_write(iocb, from); 4429 /* 4430 * If the write stopped short of completing, fall back to 4431 * buffered writes. Some filesystems do this for writes to 4432 * holes, for example. For DAX files, a buffered write will 4433 * not succeed (even if it did, DAX does not handle dirty 4434 * page-cache pages correctly). 4435 */ 4436 if (ret < 0 || !iov_iter_count(from) || IS_DAX(inode)) 4437 return ret; 4438 return direct_write_fallback(iocb, from, ret, 4439 generic_perform_write(iocb, from)); 4440 } 4441 4442 return generic_perform_write(iocb, from); 4443} 4444EXPORT_SYMBOL(__generic_file_write_iter); 4445 4446/** 4447 * generic_file_write_iter - write data to a file 4448 * @iocb: IO state structure 4449 * @from: iov_iter with data to write 4450 * 4451 * This is a wrapper around __generic_file_write_iter() to be used by most 4452 * filesystems. It takes care of syncing the file in case of O_SYNC file 4453 * and acquires i_rwsem as needed. 4454 * Return: 4455 * * negative error code if no data has been written at all of 4456 * vfs_fsync_range() failed for a synchronous write 4457 * * number of bytes written, even for truncated writes 4458 */ 4459ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 4460{ 4461 struct file *file = iocb->ki_filp; 4462 struct inode *inode = file->f_mapping->host; 4463 ssize_t ret; 4464 4465 inode_lock(inode); 4466 ret = generic_write_checks(iocb, from); 4467 if (ret > 0) 4468 ret = __generic_file_write_iter(iocb, from); 4469 inode_unlock(inode); 4470 4471 if (ret > 0) 4472 ret = generic_write_sync(iocb, ret); 4473 return ret; 4474} 4475EXPORT_SYMBOL(generic_file_write_iter); 4476 4477/** 4478 * filemap_release_folio() - Release fs-specific metadata on a folio. 4479 * @folio: The folio which the kernel is trying to free. 4480 * @gfp: Memory allocation flags (and I/O mode). 4481 * 4482 * The address_space is trying to release any data attached to a folio 4483 * (presumably at folio->private). 4484 * 4485 * This will also be called if the private_2 flag is set on a page, 4486 * indicating that the folio has other metadata associated with it. 4487 * 4488 * The @gfp argument specifies whether I/O may be performed to release 4489 * this page (__GFP_IO), and whether the call may block 4490 * (__GFP_RECLAIM & __GFP_FS). 4491 * 4492 * Return: %true if the release was successful, otherwise %false. 4493 */ 4494bool filemap_release_folio(struct folio *folio, gfp_t gfp) 4495{ 4496 struct address_space * const mapping = folio->mapping; 4497 4498 BUG_ON(!folio_test_locked(folio)); 4499 if (!folio_needs_release(folio)) 4500 return true; 4501 if (folio_test_writeback(folio)) 4502 return false; 4503 4504 if (mapping && mapping->a_ops->release_folio) 4505 return mapping->a_ops->release_folio(folio, gfp); 4506 return try_to_free_buffers(folio); 4507} 4508EXPORT_SYMBOL(filemap_release_folio); 4509 4510/** 4511 * filemap_invalidate_inode - Invalidate/forcibly write back a range of an inode's pagecache 4512 * @inode: The inode to flush 4513 * @flush: Set to write back rather than simply invalidate. 4514 * @start: First byte to in range. 4515 * @end: Last byte in range (inclusive), or LLONG_MAX for everything from start 4516 * onwards. 4517 * 4518 * Invalidate all the folios on an inode that contribute to the specified 4519 * range, possibly writing them back first. Whilst the operation is 4520 * undertaken, the invalidate lock is held to prevent new folios from being 4521 * installed. 4522 */ 4523int filemap_invalidate_inode(struct inode *inode, bool flush, 4524 loff_t start, loff_t end) 4525{ 4526 struct address_space *mapping = inode->i_mapping; 4527 pgoff_t first = start >> PAGE_SHIFT; 4528 pgoff_t last = end >> PAGE_SHIFT; 4529 pgoff_t nr = end == LLONG_MAX ? ULONG_MAX : last - first + 1; 4530 4531 if (!mapping || !mapping->nrpages || end < start) 4532 goto out; 4533 4534 /* Prevent new folios from being added to the inode. */ 4535 filemap_invalidate_lock(mapping); 4536 4537 if (!mapping->nrpages) 4538 goto unlock; 4539 4540 unmap_mapping_pages(mapping, first, nr, false); 4541 4542 /* Write back the data if we're asked to. */ 4543 if (flush) 4544 filemap_fdatawrite_range(mapping, start, end); 4545 4546 /* Wait for writeback to complete on all folios and discard. */ 4547 invalidate_inode_pages2_range(mapping, start / PAGE_SIZE, end / PAGE_SIZE); 4548 4549unlock: 4550 filemap_invalidate_unlock(mapping); 4551out: 4552 return filemap_check_errors(mapping); 4553} 4554EXPORT_SYMBOL_GPL(filemap_invalidate_inode); 4555 4556#ifdef CONFIG_CACHESTAT_SYSCALL 4557/** 4558 * filemap_cachestat() - compute the page cache statistics of a mapping 4559 * @mapping: The mapping to compute the statistics for. 4560 * @first_index: The starting page cache index. 4561 * @last_index: The final page index (inclusive). 4562 * @cs: the cachestat struct to write the result to. 4563 * 4564 * This will query the page cache statistics of a mapping in the 4565 * page range of [first_index, last_index] (inclusive). The statistics 4566 * queried include: number of dirty pages, number of pages marked for 4567 * writeback, and the number of (recently) evicted pages. 4568 */ 4569static void filemap_cachestat(struct address_space *mapping, 4570 pgoff_t first_index, pgoff_t last_index, struct cachestat *cs) 4571{ 4572 XA_STATE(xas, &mapping->i_pages, first_index); 4573 struct folio *folio; 4574 4575 /* Flush stats (and potentially sleep) outside the RCU read section. */ 4576 mem_cgroup_flush_stats_ratelimited(NULL); 4577 4578 rcu_read_lock(); 4579 xas_for_each(&xas, folio, last_index) { 4580 int order; 4581 unsigned long nr_pages; 4582 pgoff_t folio_first_index, folio_last_index; 4583 4584 /* 4585 * Don't deref the folio. It is not pinned, and might 4586 * get freed (and reused) underneath us. 4587 * 4588 * We *could* pin it, but that would be expensive for 4589 * what should be a fast and lightweight syscall. 4590 * 4591 * Instead, derive all information of interest from 4592 * the rcu-protected xarray. 4593 */ 4594 4595 if (xas_retry(&xas, folio)) 4596 continue; 4597 4598 order = xas_get_order(&xas); 4599 nr_pages = 1 << order; 4600 folio_first_index = round_down(xas.xa_index, 1 << order); 4601 folio_last_index = folio_first_index + nr_pages - 1; 4602 4603 /* Folios might straddle the range boundaries, only count covered pages */ 4604 if (folio_first_index < first_index) 4605 nr_pages -= first_index - folio_first_index; 4606 4607 if (folio_last_index > last_index) 4608 nr_pages -= folio_last_index - last_index; 4609 4610 if (xa_is_value(folio)) { 4611 /* page is evicted */ 4612 void *shadow = (void *)folio; 4613 bool workingset; /* not used */ 4614 4615 cs->nr_evicted += nr_pages; 4616 4617#ifdef CONFIG_SWAP /* implies CONFIG_MMU */ 4618 if (shmem_mapping(mapping)) { 4619 /* shmem file - in swap cache */ 4620 swp_entry_t swp = radix_to_swp_entry(folio); 4621 4622 /* swapin error results in poisoned entry */ 4623 if (!softleaf_is_swap(swp)) 4624 goto resched; 4625 4626 /* 4627 * Getting a swap entry from the shmem 4628 * inode means we beat 4629 * shmem_unuse(). rcu_read_lock() 4630 * ensures swapoff waits for us before 4631 * freeing the swapper space. However, 4632 * we can race with swapping and 4633 * invalidation, so there might not be 4634 * a shadow in the swapcache (yet). 4635 */ 4636 shadow = swap_cache_get_shadow(swp); 4637 if (!shadow) 4638 goto resched; 4639 } 4640#endif 4641 if (workingset_test_recent(shadow, true, &workingset, false)) 4642 cs->nr_recently_evicted += nr_pages; 4643 4644 goto resched; 4645 } 4646 4647 /* page is in cache */ 4648 cs->nr_cache += nr_pages; 4649 4650 if (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY)) 4651 cs->nr_dirty += nr_pages; 4652 4653 if (xas_get_mark(&xas, PAGECACHE_TAG_WRITEBACK)) 4654 cs->nr_writeback += nr_pages; 4655 4656resched: 4657 if (need_resched()) { 4658 xas_pause(&xas); 4659 cond_resched_rcu(); 4660 } 4661 } 4662 rcu_read_unlock(); 4663} 4664 4665/* 4666 * See mincore: reveal pagecache information only for files 4667 * that the calling process has write access to, or could (if 4668 * tried) open for writing. 4669 */ 4670static inline bool can_do_cachestat(struct file *f) 4671{ 4672 if (f->f_mode & FMODE_WRITE) 4673 return true; 4674 if (inode_owner_or_capable(file_mnt_idmap(f), file_inode(f))) 4675 return true; 4676 return file_permission(f, MAY_WRITE) == 0; 4677} 4678 4679/* 4680 * The cachestat(2) system call. 4681 * 4682 * cachestat() returns the page cache statistics of a file in the 4683 * bytes range specified by `off` and `len`: number of cached pages, 4684 * number of dirty pages, number of pages marked for writeback, 4685 * number of evicted pages, and number of recently evicted pages. 4686 * 4687 * An evicted page is a page that is previously in the page cache 4688 * but has been evicted since. A page is recently evicted if its last 4689 * eviction was recent enough that its reentry to the cache would 4690 * indicate that it is actively being used by the system, and that 4691 * there is memory pressure on the system. 4692 * 4693 * `off` and `len` must be non-negative integers. If `len` > 0, 4694 * the queried range is [`off`, `off` + `len`]. If `len` == 0, 4695 * we will query in the range from `off` to the end of the file. 4696 * 4697 * The `flags` argument is unused for now, but is included for future 4698 * extensibility. User should pass 0 (i.e no flag specified). 4699 * 4700 * Currently, hugetlbfs is not supported. 4701 * 4702 * Because the status of a page can change after cachestat() checks it 4703 * but before it returns to the application, the returned values may 4704 * contain stale information. 4705 * 4706 * return values: 4707 * zero - success 4708 * -EFAULT - cstat or cstat_range points to an illegal address 4709 * -EINVAL - invalid flags 4710 * -EBADF - invalid file descriptor 4711 * -EOPNOTSUPP - file descriptor is of a hugetlbfs file 4712 */ 4713SYSCALL_DEFINE4(cachestat, unsigned int, fd, 4714 struct cachestat_range __user *, cstat_range, 4715 struct cachestat __user *, cstat, unsigned int, flags) 4716{ 4717 CLASS(fd, f)(fd); 4718 struct address_space *mapping; 4719 struct cachestat_range csr; 4720 struct cachestat cs; 4721 pgoff_t first_index, last_index; 4722 4723 if (fd_empty(f)) 4724 return -EBADF; 4725 4726 if (copy_from_user(&csr, cstat_range, 4727 sizeof(struct cachestat_range))) 4728 return -EFAULT; 4729 4730 /* hugetlbfs is not supported */ 4731 if (is_file_hugepages(fd_file(f))) 4732 return -EOPNOTSUPP; 4733 4734 if (!can_do_cachestat(fd_file(f))) 4735 return -EPERM; 4736 4737 if (flags != 0) 4738 return -EINVAL; 4739 4740 first_index = csr.off >> PAGE_SHIFT; 4741 last_index = 4742 csr.len == 0 ? ULONG_MAX : (csr.off + csr.len - 1) >> PAGE_SHIFT; 4743 memset(&cs, 0, sizeof(struct cachestat)); 4744 mapping = fd_file(f)->f_mapping; 4745 filemap_cachestat(mapping, first_index, last_index, &cs); 4746 4747 if (copy_to_user(cstat, &cs, sizeof(struct cachestat))) 4748 return -EFAULT; 4749 4750 return 0; 4751} 4752#endif /* CONFIG_CACHESTAT_SYSCALL */
~/
Read-only file preview.