pagemap.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _LINUX_PAGEMAP_H
   3#define _LINUX_PAGEMAP_H
   4
   5/*
   6 * Copyright 1995 Linus Torvalds
   7 */
   8#include <linux/mm.h>
   9#include <linux/fs.h>
  10#include <linux/list.h>
  11#include <linux/highmem.h>
  12#include <linux/compiler.h>
  13#include <linux/uaccess.h>
  14#include <linux/gfp.h>
  15#include <linux/bitops.h>
  16#include <linux/hardirq.h> /* for in_interrupt() */
  17#include <linux/hugetlb_inline.h>
  18
  19struct folio_batch;
  20
  21unsigned long invalidate_mapping_pages(struct address_space *mapping,
  22					pgoff_t start, pgoff_t end);
  23
  24static inline void invalidate_remote_inode(struct inode *inode)
  25{
  26	if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
  27	    S_ISLNK(inode->i_mode))
  28		invalidate_mapping_pages(inode->i_mapping, 0, -1);
  29}
  30int invalidate_inode_pages2(struct address_space *mapping);
  31int invalidate_inode_pages2_range(struct address_space *mapping,
  32		pgoff_t start, pgoff_t end);
  33int kiocb_invalidate_pages(struct kiocb *iocb, size_t count);
  34void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count);
  35int filemap_invalidate_pages(struct address_space *mapping,
  36			     loff_t pos, loff_t end, bool nowait);
  37
  38int write_inode_now(struct inode *, int sync);
  39int filemap_fdatawrite(struct address_space *);
  40int filemap_flush(struct address_space *);
  41int filemap_flush_nr(struct address_space *mapping, long *nr_to_write);
  42int filemap_fdatawait_keep_errors(struct address_space *mapping);
  43int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend);
  44int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
  45		loff_t start_byte, loff_t end_byte);
  46int filemap_invalidate_inode(struct inode *inode, bool flush,
  47			     loff_t start, loff_t end);
  48
  49static inline int filemap_fdatawait(struct address_space *mapping)
  50{
  51	return filemap_fdatawait_range(mapping, 0, LLONG_MAX);
  52}
  53
  54bool filemap_range_has_page(struct address_space *, loff_t lstart, loff_t lend);
  55int filemap_write_and_wait_range(struct address_space *mapping,
  56		loff_t lstart, loff_t lend);
  57int filemap_fdatawrite_range(struct address_space *mapping,
  58		loff_t start, loff_t end);
  59int filemap_check_errors(struct address_space *mapping);
  60void __filemap_set_wb_err(struct address_space *mapping, int err);
  61int kiocb_write_and_wait(struct kiocb *iocb, size_t count);
  62
  63static inline int filemap_write_and_wait(struct address_space *mapping)
  64{
  65	return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);
  66}
  67
  68/**
  69 * filemap_set_wb_err - set a writeback error on an address_space
  70 * @mapping: mapping in which to set writeback error
  71 * @err: error to be set in mapping
  72 *
  73 * When writeback fails in some way, we must record that error so that
  74 * userspace can be informed when fsync and the like are called.  We endeavor
  75 * to report errors on any file that was open at the time of the error.  Some
  76 * internal callers also need to know when writeback errors have occurred.
  77 *
  78 * When a writeback error occurs, most filesystems will want to call
  79 * filemap_set_wb_err to record the error in the mapping so that it will be
  80 * automatically reported whenever fsync is called on the file.
  81 */
  82static inline void filemap_set_wb_err(struct address_space *mapping, int err)
  83{
  84	/* Fastpath for common case of no error */
  85	if (unlikely(err))
  86		__filemap_set_wb_err(mapping, err);
  87}
  88
  89/**
  90 * filemap_check_wb_err - has an error occurred since the mark was sampled?
  91 * @mapping: mapping to check for writeback errors
  92 * @since: previously-sampled errseq_t
  93 *
  94 * Grab the errseq_t value from the mapping, and see if it has changed "since"
  95 * the given value was sampled.
  96 *
  97 * If it has then report the latest error set, otherwise return 0.
  98 */
  99static inline int filemap_check_wb_err(struct address_space *mapping,
 100					errseq_t since)
 101{
 102	return errseq_check(&mapping->wb_err, since);
 103}
 104
 105/**
 106 * filemap_sample_wb_err - sample the current errseq_t to test for later errors
 107 * @mapping: mapping to be sampled
 108 *
 109 * Writeback errors are always reported relative to a particular sample point
 110 * in the past. This function provides those sample points.
 111 */
 112static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)
 113{
 114	return errseq_sample(&mapping->wb_err);
 115}
 116
 117/**
 118 * file_sample_sb_err - sample the current errseq_t to test for later errors
 119 * @file: file pointer to be sampled
 120 *
 121 * Grab the most current superblock-level errseq_t value for the given
 122 * struct file.
 123 */
 124static inline errseq_t file_sample_sb_err(struct file *file)
 125{
 126	return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);
 127}
 128
 129/*
 130 * Flush file data before changing attributes.  Caller must hold any locks
 131 * required to prevent further writes to this file until we're done setting
 132 * flags.
 133 */
 134static inline int inode_drain_writes(struct inode *inode)
 135{
 136	inode_dio_wait(inode);
 137	return filemap_write_and_wait(inode->i_mapping);
 138}
 139
 140static inline bool mapping_empty(const struct address_space *mapping)
 141{
 142	return xa_empty(&mapping->i_pages);
 143}
 144
 145/*
 146 * mapping_shrinkable - test if page cache state allows inode reclaim
 147 * @mapping: the page cache mapping
 148 *
 149 * This checks the mapping's cache state for the pupose of inode
 150 * reclaim and LRU management.
 151 *
 152 * The caller is expected to hold the i_lock, but is not required to
 153 * hold the i_pages lock, which usually protects cache state. That's
 154 * because the i_lock and the list_lru lock that protect the inode and
 155 * its LRU state don't nest inside the irq-safe i_pages lock.
 156 *
 157 * Cache deletions are performed under the i_lock, which ensures that
 158 * when an inode goes empty, it will reliably get queued on the LRU.
 159 *
 160 * Cache additions do not acquire the i_lock and may race with this
 161 * check, in which case we'll report the inode as shrinkable when it
 162 * has cache pages. This is okay: the shrinker also checks the
 163 * refcount and the referenced bit, which will be elevated or set in
 164 * the process of adding new cache pages to an inode.
 165 */
 166static inline bool mapping_shrinkable(const struct address_space *mapping)
 167{
 168	void *head;
 169
 170	/*
 171	 * On highmem systems, there could be lowmem pressure from the
 172	 * inodes before there is highmem pressure from the page
 173	 * cache. Make inodes shrinkable regardless of cache state.
 174	 */
 175	if (IS_ENABLED(CONFIG_HIGHMEM))
 176		return true;
 177
 178	/* Cache completely empty? Shrink away. */
 179	head = rcu_access_pointer(mapping->i_pages.xa_head);
 180	if (!head)
 181		return true;
 182
 183	/*
 184	 * The xarray stores single offset-0 entries directly in the
 185	 * head pointer, which allows non-resident page cache entries
 186	 * to escape the shadow shrinker's list of xarray nodes. The
 187	 * inode shrinker needs to pick them up under memory pressure.
 188	 */
 189	if (!xa_is_node(head) && xa_is_value(head))
 190		return true;
 191
 192	return false;
 193}
 194
 195/*
 196 * Bits in mapping->flags.
 197 */
 198enum mapping_flags {
 199	AS_EIO		= 0,	/* IO error on async write */
 200	AS_ENOSPC	= 1,	/* ENOSPC on async write */
 201	AS_MM_ALL_LOCKS	= 2,	/* under mm_take_all_locks() */
 202	AS_UNEVICTABLE	= 3,	/* e.g., ramdisk, SHM_LOCK */
 203	AS_EXITING	= 4, 	/* final truncate in progress */
 204	/* writeback related tags are not used */
 205	AS_NO_WRITEBACK_TAGS = 5,
 206	AS_RELEASE_ALWAYS = 6,	/* Call ->release_folio(), even if no private data */
 207	AS_STABLE_WRITES = 7,	/* must wait for writeback before modifying
 208				   folio contents */
 209	AS_INACCESSIBLE = 8,	/* Do not attempt direct R/W access to the mapping */
 210	AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM = 9,
 211	AS_KERNEL_FILE = 10,	/* mapping for a fake kernel file that shouldn't
 212				   account usage to user cgroups */
 213	/* Bits 16-25 are used for FOLIO_ORDER */
 214	AS_FOLIO_ORDER_BITS = 5,
 215	AS_FOLIO_ORDER_MIN = 16,
 216	AS_FOLIO_ORDER_MAX = AS_FOLIO_ORDER_MIN + AS_FOLIO_ORDER_BITS,
 217};
 218
 219#define AS_FOLIO_ORDER_BITS_MASK ((1u << AS_FOLIO_ORDER_BITS) - 1)
 220#define AS_FOLIO_ORDER_MIN_MASK (AS_FOLIO_ORDER_BITS_MASK << AS_FOLIO_ORDER_MIN)
 221#define AS_FOLIO_ORDER_MAX_MASK (AS_FOLIO_ORDER_BITS_MASK << AS_FOLIO_ORDER_MAX)
 222#define AS_FOLIO_ORDER_MASK (AS_FOLIO_ORDER_MIN_MASK | AS_FOLIO_ORDER_MAX_MASK)
 223
 224/**
 225 * mapping_set_error - record a writeback error in the address_space
 226 * @mapping: the mapping in which an error should be set
 227 * @error: the error to set in the mapping
 228 *
 229 * When writeback fails in some way, we must record that error so that
 230 * userspace can be informed when fsync and the like are called.  We endeavor
 231 * to report errors on any file that was open at the time of the error.  Some
 232 * internal callers also need to know when writeback errors have occurred.
 233 *
 234 * When a writeback error occurs, most filesystems will want to call
 235 * mapping_set_error to record the error in the mapping so that it can be
 236 * reported when the application calls fsync(2).
 237 */
 238static inline void mapping_set_error(struct address_space *mapping, int error)
 239{
 240	if (likely(!error))
 241		return;
 242
 243	/* Record in wb_err for checkers using errseq_t based tracking */
 244	__filemap_set_wb_err(mapping, error);
 245
 246	/* Record it in superblock */
 247	if (mapping->host)
 248		errseq_set(&mapping->host->i_sb->s_wb_err, error);
 249
 250	/* Record it in flags for now, for legacy callers */
 251	if (error == -ENOSPC)
 252		set_bit(AS_ENOSPC, &mapping->flags);
 253	else
 254		set_bit(AS_EIO, &mapping->flags);
 255}
 256
 257static inline void mapping_set_unevictable(struct address_space *mapping)
 258{
 259	set_bit(AS_UNEVICTABLE, &mapping->flags);
 260}
 261
 262static inline void mapping_clear_unevictable(struct address_space *mapping)
 263{
 264	clear_bit(AS_UNEVICTABLE, &mapping->flags);
 265}
 266
 267static inline bool mapping_unevictable(const struct address_space *mapping)
 268{
 269	return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);
 270}
 271
 272static inline void mapping_set_exiting(struct address_space *mapping)
 273{
 274	set_bit(AS_EXITING, &mapping->flags);
 275}
 276
 277static inline int mapping_exiting(const struct address_space *mapping)
 278{
 279	return test_bit(AS_EXITING, &mapping->flags);
 280}
 281
 282static inline void mapping_set_no_writeback_tags(struct address_space *mapping)
 283{
 284	set_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);
 285}
 286
 287static inline int mapping_use_writeback_tags(const struct address_space *mapping)
 288{
 289	return !test_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);
 290}
 291
 292static inline bool mapping_release_always(const struct address_space *mapping)
 293{
 294	return test_bit(AS_RELEASE_ALWAYS, &mapping->flags);
 295}
 296
 297static inline void mapping_set_release_always(struct address_space *mapping)
 298{
 299	set_bit(AS_RELEASE_ALWAYS, &mapping->flags);
 300}
 301
 302static inline void mapping_clear_release_always(struct address_space *mapping)
 303{
 304	clear_bit(AS_RELEASE_ALWAYS, &mapping->flags);
 305}
 306
 307static inline bool mapping_stable_writes(const struct address_space *mapping)
 308{
 309	return test_bit(AS_STABLE_WRITES, &mapping->flags);
 310}
 311
 312static inline void mapping_set_stable_writes(struct address_space *mapping)
 313{
 314	set_bit(AS_STABLE_WRITES, &mapping->flags);
 315}
 316
 317static inline void mapping_clear_stable_writes(struct address_space *mapping)
 318{
 319	clear_bit(AS_STABLE_WRITES, &mapping->flags);
 320}
 321
 322static inline void mapping_set_inaccessible(struct address_space *mapping)
 323{
 324	/*
 325	 * It's expected inaccessible mappings are also unevictable. Compaction
 326	 * migrate scanner (isolate_migratepages_block()) relies on this to
 327	 * reduce page locking.
 328	 */
 329	set_bit(AS_UNEVICTABLE, &mapping->flags);
 330	set_bit(AS_INACCESSIBLE, &mapping->flags);
 331}
 332
 333static inline bool mapping_inaccessible(const struct address_space *mapping)
 334{
 335	return test_bit(AS_INACCESSIBLE, &mapping->flags);
 336}
 337
 338static inline void mapping_set_writeback_may_deadlock_on_reclaim(struct address_space *mapping)
 339{
 340	set_bit(AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM, &mapping->flags);
 341}
 342
 343static inline bool mapping_writeback_may_deadlock_on_reclaim(const struct address_space *mapping)
 344{
 345	return test_bit(AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM, &mapping->flags);
 346}
 347
 348static inline gfp_t mapping_gfp_mask(const struct address_space *mapping)
 349{
 350	return mapping->gfp_mask;
 351}
 352
 353/* Restricts the given gfp_mask to what the mapping allows. */
 354static inline gfp_t mapping_gfp_constraint(const struct address_space *mapping,
 355		gfp_t gfp_mask)
 356{
 357	return mapping_gfp_mask(mapping) & gfp_mask;
 358}
 359
 360/*
 361 * This is non-atomic.  Only to be used before the mapping is activated.
 362 * Probably needs a barrier...
 363 */
 364static inline void mapping_set_gfp_mask(struct address_space *m, gfp_t mask)
 365{
 366	m->gfp_mask = mask;
 367}
 368
 369/*
 370 * There are some parts of the kernel which assume that PMD entries
 371 * are exactly HPAGE_PMD_ORDER.  Those should be fixed, but until then,
 372 * limit the maximum allocation order to PMD size.  I'm not aware of any
 373 * assumptions about maximum order if THP are disabled, but 8 seems like
 374 * a good order (that's 1MB if you're using 4kB pages)
 375 */
 376#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 377#define PREFERRED_MAX_PAGECACHE_ORDER	HPAGE_PMD_ORDER
 378#else
 379#define PREFERRED_MAX_PAGECACHE_ORDER	8
 380#endif
 381
 382/*
 383 * xas_split_alloc() does not support arbitrary orders. This implies no
 384 * 512MB THP on ARM64 with 64KB base page size.
 385 */
 386#define MAX_XAS_ORDER		(XA_CHUNK_SHIFT * 2 - 1)
 387#define MAX_PAGECACHE_ORDER	min(MAX_XAS_ORDER, PREFERRED_MAX_PAGECACHE_ORDER)
 388
 389/*
 390 * mapping_max_folio_size_supported() - Check the max folio size supported
 391 *
 392 * The filesystem should call this function at mount time if there is a
 393 * requirement on the folio mapping size in the page cache.
 394 */
 395static inline size_t mapping_max_folio_size_supported(void)
 396{
 397	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
 398		return 1U << (PAGE_SHIFT + MAX_PAGECACHE_ORDER);
 399	return PAGE_SIZE;
 400}
 401
 402/*
 403 * mapping_set_folio_order_range() - Set the orders supported by a file.
 404 * @mapping: The address space of the file.
 405 * @min: Minimum folio order (between 0-MAX_PAGECACHE_ORDER inclusive).
 406 * @max: Maximum folio order (between @min-MAX_PAGECACHE_ORDER inclusive).
 407 *
 408 * The filesystem should call this function in its inode constructor to
 409 * indicate which base size (min) and maximum size (max) of folio the VFS
 410 * can use to cache the contents of the file.  This should only be used
 411 * if the filesystem needs special handling of folio sizes (ie there is
 412 * something the core cannot know).
 413 * Do not tune it based on, eg, i_size.
 414 *
 415 * Context: This should not be called while the inode is active as it
 416 * is non-atomic.
 417 */
 418static inline void mapping_set_folio_order_range(struct address_space *mapping,
 419						 unsigned int min,
 420						 unsigned int max)
 421{
 422	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
 423		return;
 424
 425	if (min > MAX_PAGECACHE_ORDER)
 426		min = MAX_PAGECACHE_ORDER;
 427
 428	if (max > MAX_PAGECACHE_ORDER)
 429		max = MAX_PAGECACHE_ORDER;
 430
 431	if (max < min)
 432		max = min;
 433
 434	mapping->flags = (mapping->flags & ~AS_FOLIO_ORDER_MASK) |
 435		(min << AS_FOLIO_ORDER_MIN) | (max << AS_FOLIO_ORDER_MAX);
 436}
 437
 438static inline void mapping_set_folio_min_order(struct address_space *mapping,
 439					       unsigned int min)
 440{
 441	mapping_set_folio_order_range(mapping, min, MAX_PAGECACHE_ORDER);
 442}
 443
 444/**
 445 * mapping_set_large_folios() - Indicate the file supports large folios.
 446 * @mapping: The address space of the file.
 447 *
 448 * The filesystem should call this function in its inode constructor to
 449 * indicate that the VFS can use large folios to cache the contents of
 450 * the file.
 451 *
 452 * Context: This should not be called while the inode is active as it
 453 * is non-atomic.
 454 */
 455static inline void mapping_set_large_folios(struct address_space *mapping)
 456{
 457	mapping_set_folio_order_range(mapping, 0, MAX_PAGECACHE_ORDER);
 458}
 459
 460static inline unsigned int
 461mapping_max_folio_order(const struct address_space *mapping)
 462{
 463	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
 464		return 0;
 465	return (mapping->flags & AS_FOLIO_ORDER_MAX_MASK) >> AS_FOLIO_ORDER_MAX;
 466}
 467
 468static inline unsigned int
 469mapping_min_folio_order(const struct address_space *mapping)
 470{
 471	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
 472		return 0;
 473	return (mapping->flags & AS_FOLIO_ORDER_MIN_MASK) >> AS_FOLIO_ORDER_MIN;
 474}
 475
 476static inline unsigned long
 477mapping_min_folio_nrpages(const struct address_space *mapping)
 478{
 479	return 1UL << mapping_min_folio_order(mapping);
 480}
 481
 482static inline unsigned long
 483mapping_min_folio_nrbytes(const struct address_space *mapping)
 484{
 485	return mapping_min_folio_nrpages(mapping) << PAGE_SHIFT;
 486}
 487
 488/**
 489 * mapping_align_index() - Align index for this mapping.
 490 * @mapping: The address_space.
 491 * @index: The page index.
 492 *
 493 * The index of a folio must be naturally aligned.  If you are adding a
 494 * new folio to the page cache and need to know what index to give it,
 495 * call this function.
 496 */
 497static inline pgoff_t mapping_align_index(const struct address_space *mapping,
 498					  pgoff_t index)
 499{
 500	return round_down(index, mapping_min_folio_nrpages(mapping));
 501}
 502
 503/*
 504 * Large folio support currently depends on THP.  These dependencies are
 505 * being worked on but are not yet fixed.
 506 */
 507static inline bool mapping_large_folio_support(const struct address_space *mapping)
 508{
 509	/* AS_FOLIO_ORDER is only reasonable for pagecache folios */
 510	VM_WARN_ONCE((unsigned long)mapping & FOLIO_MAPPING_ANON,
 511			"Anonymous mapping always supports large folio");
 512
 513	return mapping_max_folio_order(mapping) > 0;
 514}
 515
 516/* Return the maximum folio size for this pagecache mapping, in bytes. */
 517static inline size_t mapping_max_folio_size(const struct address_space *mapping)
 518{
 519	return PAGE_SIZE << mapping_max_folio_order(mapping);
 520}
 521
 522static inline int filemap_nr_thps(const struct address_space *mapping)
 523{
 524#ifdef CONFIG_READ_ONLY_THP_FOR_FS
 525	return atomic_read(&mapping->nr_thps);
 526#else
 527	return 0;
 528#endif
 529}
 530
 531static inline void filemap_nr_thps_inc(struct address_space *mapping)
 532{
 533#ifdef CONFIG_READ_ONLY_THP_FOR_FS
 534	if (!mapping_large_folio_support(mapping))
 535		atomic_inc(&mapping->nr_thps);
 536#else
 537	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
 538#endif
 539}
 540
 541static inline void filemap_nr_thps_dec(struct address_space *mapping)
 542{
 543#ifdef CONFIG_READ_ONLY_THP_FOR_FS
 544	if (!mapping_large_folio_support(mapping))
 545		atomic_dec(&mapping->nr_thps);
 546#else
 547	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
 548#endif
 549}
 550
 551struct address_space *folio_mapping(const struct folio *folio);
 552
 553/**
 554 * folio_flush_mapping - Find the file mapping this folio belongs to.
 555 * @folio: The folio.
 556 *
 557 * For folios which are in the page cache, return the mapping that this
 558 * page belongs to.  Anonymous folios return NULL, even if they're in
 559 * the swap cache.  Other kinds of folio also return NULL.
 560 *
 561 * This is ONLY used by architecture cache flushing code.  If you aren't
 562 * writing cache flushing code, you want either folio_mapping() or
 563 * folio_file_mapping().
 564 */
 565static inline struct address_space *folio_flush_mapping(struct folio *folio)
 566{
 567	if (unlikely(folio_test_swapcache(folio)))
 568		return NULL;
 569
 570	return folio_mapping(folio);
 571}
 572
 573/**
 574 * folio_inode - Get the host inode for this folio.
 575 * @folio: The folio.
 576 *
 577 * For folios which are in the page cache, return the inode that this folio
 578 * belongs to.
 579 *
 580 * Do not call this for folios which aren't in the page cache.
 581 */
 582static inline struct inode *folio_inode(struct folio *folio)
 583{
 584	return folio->mapping->host;
 585}
 586
 587/**
 588 * folio_attach_private - Attach private data to a folio.
 589 * @folio: Folio to attach data to.
 590 * @data: Data to attach to folio.
 591 *
 592 * Attaching private data to a folio increments the page's reference count.
 593 * The data must be detached before the folio will be freed.
 594 */
 595static inline void folio_attach_private(struct folio *folio, void *data)
 596{
 597	folio_get(folio);
 598	folio->private = data;
 599	folio_set_private(folio);
 600}
 601
 602/**
 603 * folio_change_private - Change private data on a folio.
 604 * @folio: Folio to change the data on.
 605 * @data: Data to set on the folio.
 606 *
 607 * Change the private data attached to a folio and return the old
 608 * data.  The page must previously have had data attached and the data
 609 * must be detached before the folio will be freed.
 610 *
 611 * Return: Data that was previously attached to the folio.
 612 */
 613static inline void *folio_change_private(struct folio *folio, void *data)
 614{
 615	void *old = folio_get_private(folio);
 616
 617	folio->private = data;
 618	return old;
 619}
 620
 621/**
 622 * folio_detach_private - Detach private data from a folio.
 623 * @folio: Folio to detach data from.
 624 *
 625 * Removes the data that was previously attached to the folio and decrements
 626 * the refcount on the page.
 627 *
 628 * Return: Data that was attached to the folio.
 629 */
 630static inline void *folio_detach_private(struct folio *folio)
 631{
 632	void *data = folio_get_private(folio);
 633
 634	if (!folio_test_private(folio))
 635		return NULL;
 636	folio_clear_private(folio);
 637	folio->private = NULL;
 638	folio_put(folio);
 639
 640	return data;
 641}
 642
 643static inline void attach_page_private(struct page *page, void *data)
 644{
 645	folio_attach_private(page_folio(page), data);
 646}
 647
 648static inline void *detach_page_private(struct page *page)
 649{
 650	return folio_detach_private(page_folio(page));
 651}
 652
 653#ifdef CONFIG_NUMA
 654struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order,
 655		struct mempolicy *policy);
 656#else
 657static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order,
 658		struct mempolicy *policy)
 659{
 660	return folio_alloc_noprof(gfp, order);
 661}
 662#endif
 663
 664#define filemap_alloc_folio(...)				\
 665	alloc_hooks(filemap_alloc_folio_noprof(__VA_ARGS__))
 666
 667static inline struct page *__page_cache_alloc(gfp_t gfp)
 668{
 669	return &filemap_alloc_folio(gfp, 0, NULL)->page;
 670}
 671
 672static inline gfp_t readahead_gfp_mask(struct address_space *x)
 673{
 674	return mapping_gfp_mask(x) | __GFP_NORETRY | __GFP_NOWARN;
 675}
 676
 677typedef int filler_t(struct file *, struct folio *);
 678
 679pgoff_t page_cache_next_miss(struct address_space *mapping,
 680			     pgoff_t index, unsigned long max_scan);
 681pgoff_t page_cache_prev_miss(struct address_space *mapping,
 682			     pgoff_t index, unsigned long max_scan);
 683
 684/**
 685 * typedef fgf_t - Flags for getting folios from the page cache.
 686 *
 687 * Most users of the page cache will not need to use these flags;
 688 * there are convenience functions such as filemap_get_folio() and
 689 * filemap_lock_folio().  For users which need more control over exactly
 690 * what is done with the folios, these flags to __filemap_get_folio()
 691 * are available.
 692 *
 693 * * %FGP_ACCESSED - The folio will be marked accessed.
 694 * * %FGP_LOCK - The folio is returned locked.
 695 * * %FGP_CREAT - If no folio is present then a new folio is allocated,
 696 *   added to the page cache and the VM's LRU list.  The folio is
 697 *   returned locked.
 698 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
 699 *   folio is already in cache.  If the folio was allocated, unlock it
 700 *   before returning so the caller can do the same dance.
 701 * * %FGP_WRITE - The folio will be written to by the caller.
 702 * * %FGP_NOFS - __GFP_FS will get cleared in gfp.
 703 * * %FGP_NOWAIT - Don't block on the folio lock.
 704 * * %FGP_STABLE - Wait for the folio to be stable (finished writeback)
 705 * * %FGP_DONTCACHE - Uncached buffered IO
 706 * * %FGP_WRITEBEGIN - The flags to use in a filesystem write_begin()
 707 *   implementation.
 708 */
 709typedef unsigned int __bitwise fgf_t;
 710
 711#define FGP_ACCESSED		((__force fgf_t)0x00000001)
 712#define FGP_LOCK		((__force fgf_t)0x00000002)
 713#define FGP_CREAT		((__force fgf_t)0x00000004)
 714#define FGP_WRITE		((__force fgf_t)0x00000008)
 715#define FGP_NOFS		((__force fgf_t)0x00000010)
 716#define FGP_NOWAIT		((__force fgf_t)0x00000020)
 717#define FGP_FOR_MMAP		((__force fgf_t)0x00000040)
 718#define FGP_STABLE		((__force fgf_t)0x00000080)
 719#define FGP_DONTCACHE		((__force fgf_t)0x00000100)
 720#define FGF_GET_ORDER(fgf)	(((__force unsigned)fgf) >> 26)	/* top 6 bits */
 721
 722#define FGP_WRITEBEGIN		(FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE)
 723
 724static inline unsigned int filemap_get_order(size_t size)
 725{
 726	unsigned int shift = ilog2(size);
 727
 728	if (shift <= PAGE_SHIFT)
 729		return 0;
 730
 731	return shift - PAGE_SHIFT;
 732}
 733
 734/**
 735 * fgf_set_order - Encode a length in the fgf_t flags.
 736 * @size: The suggested size of the folio to create.
 737 *
 738 * The caller of __filemap_get_folio() can use this to suggest a preferred
 739 * size for the folio that is created.  If there is already a folio at
 740 * the index, it will be returned, no matter what its size.  If a folio
 741 * is freshly created, it may be of a different size than requested
 742 * due to alignment constraints, memory pressure, or the presence of
 743 * other folios at nearby indices.
 744 */
 745static inline fgf_t fgf_set_order(size_t size)
 746{
 747	unsigned int order = filemap_get_order(size);
 748
 749	if (!order)
 750		return 0;
 751	return (__force fgf_t)(order << 26);
 752}
 753
 754void *filemap_get_entry(struct address_space *mapping, pgoff_t index);
 755struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
 756		pgoff_t index, fgf_t fgf_flags, gfp_t gfp, struct mempolicy *policy);
 757struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
 758		fgf_t fgp_flags, gfp_t gfp);
 759
 760static inline struct folio *__filemap_get_folio(struct address_space *mapping,
 761		pgoff_t index, fgf_t fgf_flags, gfp_t gfp)
 762{
 763	return __filemap_get_folio_mpol(mapping, index, fgf_flags, gfp, NULL);
 764}
 765
 766/**
 767 * write_begin_get_folio - Get folio for write_begin with flags.
 768 * @iocb: The kiocb passed from write_begin (may be NULL).
 769 * @mapping: The address space to search.
 770 * @index: The page cache index.
 771 * @len: Length of data being written.
 772 *
 773 * This is a helper for filesystem write_begin() implementations.
 774 * It wraps __filemap_get_folio(), setting appropriate flags in
 775 * the write begin context.
 776 *
 777 * Return: A folio or an ERR_PTR.
 778 */
 779static inline struct folio *write_begin_get_folio(const struct kiocb *iocb,
 780		  struct address_space *mapping, pgoff_t index, size_t len)
 781{
 782        fgf_t fgp_flags = FGP_WRITEBEGIN;
 783
 784        fgp_flags |= fgf_set_order(len);
 785
 786        if (iocb && iocb->ki_flags & IOCB_DONTCACHE)
 787                fgp_flags |= FGP_DONTCACHE;
 788
 789        return __filemap_get_folio(mapping, index, fgp_flags,
 790                                   mapping_gfp_mask(mapping));
 791}
 792
 793/**
 794 * filemap_get_folio - Find and get a folio.
 795 * @mapping: The address_space to search.
 796 * @index: The page index.
 797 *
 798 * Looks up the page cache entry at @mapping & @index.  If a folio is
 799 * present, it is returned with an increased refcount.
 800 *
 801 * Return: A folio or ERR_PTR(-ENOENT) if there is no folio in the cache for
 802 * this index.  Will not return a shadow, swap or DAX entry.
 803 */
 804static inline struct folio *filemap_get_folio(struct address_space *mapping,
 805					pgoff_t index)
 806{
 807	return __filemap_get_folio(mapping, index, 0, 0);
 808}
 809
 810/**
 811 * filemap_lock_folio - Find and lock a folio.
 812 * @mapping: The address_space to search.
 813 * @index: The page index.
 814 *
 815 * Looks up the page cache entry at @mapping & @index.  If a folio is
 816 * present, it is returned locked with an increased refcount.
 817 *
 818 * Context: May sleep.
 819 * Return: A folio or ERR_PTR(-ENOENT) if there is no folio in the cache for
 820 * this index.  Will not return a shadow, swap or DAX entry.
 821 */
 822static inline struct folio *filemap_lock_folio(struct address_space *mapping,
 823					pgoff_t index)
 824{
 825	return __filemap_get_folio(mapping, index, FGP_LOCK, 0);
 826}
 827
 828/**
 829 * filemap_grab_folio - grab a folio from the page cache
 830 * @mapping: The address space to search
 831 * @index: The page index
 832 *
 833 * Looks up the page cache entry at @mapping & @index. If no folio is found,
 834 * a new folio is created. The folio is locked, marked as accessed, and
 835 * returned.
 836 *
 837 * Return: A found or created folio. ERR_PTR(-ENOMEM) if no folio is found
 838 * and failed to create a folio.
 839 */
 840static inline struct folio *filemap_grab_folio(struct address_space *mapping,
 841					pgoff_t index)
 842{
 843	return __filemap_get_folio(mapping, index,
 844			FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
 845			mapping_gfp_mask(mapping));
 846}
 847
 848/**
 849 * find_get_page - find and get a page reference
 850 * @mapping: the address_space to search
 851 * @offset: the page index
 852 *
 853 * Looks up the page cache slot at @mapping & @offset.  If there is a
 854 * page cache page, it is returned with an increased refcount.
 855 *
 856 * Otherwise, %NULL is returned.
 857 */
 858static inline struct page *find_get_page(struct address_space *mapping,
 859					pgoff_t offset)
 860{
 861	return pagecache_get_page(mapping, offset, 0, 0);
 862}
 863
 864static inline struct page *find_get_page_flags(struct address_space *mapping,
 865					pgoff_t offset, fgf_t fgp_flags)
 866{
 867	return pagecache_get_page(mapping, offset, fgp_flags, 0);
 868}
 869
 870/**
 871 * find_lock_page - locate, pin and lock a pagecache page
 872 * @mapping: the address_space to search
 873 * @index: the page index
 874 *
 875 * Looks up the page cache entry at @mapping & @index.  If there is a
 876 * page cache page, it is returned locked and with an increased
 877 * refcount.
 878 *
 879 * Context: May sleep.
 880 * Return: A struct page or %NULL if there is no page in the cache for this
 881 * index.
 882 */
 883static inline struct page *find_lock_page(struct address_space *mapping,
 884					pgoff_t index)
 885{
 886	return pagecache_get_page(mapping, index, FGP_LOCK, 0);
 887}
 888
 889/**
 890 * find_or_create_page - locate or add a pagecache page
 891 * @mapping: the page's address_space
 892 * @index: the page's index into the mapping
 893 * @gfp_mask: page allocation mode
 894 *
 895 * Looks up the page cache slot at @mapping & @offset.  If there is a
 896 * page cache page, it is returned locked and with an increased
 897 * refcount.
 898 *
 899 * If the page is not present, a new page is allocated using @gfp_mask
 900 * and added to the page cache and the VM's LRU list.  The page is
 901 * returned locked and with an increased refcount.
 902 *
 903 * On memory exhaustion, %NULL is returned.
 904 *
 905 * find_or_create_page() may sleep, even if @gfp_flags specifies an
 906 * atomic allocation!
 907 */
 908static inline struct page *find_or_create_page(struct address_space *mapping,
 909					pgoff_t index, gfp_t gfp_mask)
 910{
 911	return pagecache_get_page(mapping, index,
 912					FGP_LOCK|FGP_ACCESSED|FGP_CREAT,
 913					gfp_mask);
 914}
 915
 916/**
 917 * grab_cache_page_nowait - returns locked page at given index in given cache
 918 * @mapping: target address_space
 919 * @index: the page index
 920 *
 921 * Returns locked page at given index in given cache, creating it if
 922 * needed, but do not wait if the page is locked or to reclaim memory.
 923 * This is intended for speculative data generators, where the data can
 924 * be regenerated if the page couldn't be grabbed.  This routine should
 925 * be safe to call while holding the lock for another page.
 926 *
 927 * Clear __GFP_FS when allocating the page to avoid recursion into the fs
 928 * and deadlock against the caller's locked page.
 929 */
 930static inline struct page *grab_cache_page_nowait(struct address_space *mapping,
 931				pgoff_t index)
 932{
 933	return pagecache_get_page(mapping, index,
 934			FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
 935			mapping_gfp_mask(mapping));
 936}
 937
 938/**
 939 * folio_next_index - Get the index of the next folio.
 940 * @folio: The current folio.
 941 *
 942 * Return: The index of the folio which follows this folio in the file.
 943 */
 944static inline pgoff_t folio_next_index(const struct folio *folio)
 945{
 946	return folio->index + folio_nr_pages(folio);
 947}
 948
 949/**
 950 * folio_next_pos - Get the file position of the next folio.
 951 * @folio: The current folio.
 952 *
 953 * Return: The position of the folio which follows this folio in the file.
 954 */
 955static inline loff_t folio_next_pos(const struct folio *folio)
 956{
 957	return (loff_t)folio_next_index(folio) << PAGE_SHIFT;
 958}
 959
 960/**
 961 * folio_file_page - The page for a particular index.
 962 * @folio: The folio which contains this index.
 963 * @index: The index we want to look up.
 964 *
 965 * Sometimes after looking up a folio in the page cache, we need to
 966 * obtain the specific page for an index (eg a page fault).
 967 *
 968 * Return: The page containing the file data for this index.
 969 */
 970static inline struct page *folio_file_page(struct folio *folio, pgoff_t index)
 971{
 972	return folio_page(folio, index & (folio_nr_pages(folio) - 1));
 973}
 974
 975/**
 976 * folio_contains - Does this folio contain this index?
 977 * @folio: The folio.
 978 * @index: The page index within the file.
 979 *
 980 * Context: The caller should have the folio locked and ensure
 981 * e.g., shmem did not move this folio to the swap cache.
 982 * Return: true or false.
 983 */
 984static inline bool folio_contains(const struct folio *folio, pgoff_t index)
 985{
 986	VM_WARN_ON_ONCE_FOLIO(folio_test_swapcache(folio), folio);
 987	return index - folio->index < folio_nr_pages(folio);
 988}
 989
 990unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
 991		pgoff_t end, struct folio_batch *fbatch);
 992unsigned filemap_get_folios_contig(struct address_space *mapping,
 993		pgoff_t *start, pgoff_t end, struct folio_batch *fbatch);
 994unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,
 995		pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch);
 996unsigned filemap_get_folios_dirty(struct address_space *mapping,
 997		pgoff_t *start, pgoff_t end, struct folio_batch *fbatch);
 998
 999struct folio *read_cache_folio(struct address_space *, pgoff_t index,
1000		filler_t *filler, struct file *file);
1001struct folio *mapping_read_folio_gfp(struct address_space *, pgoff_t index,
1002		gfp_t flags);
1003struct page *read_cache_page(struct address_space *, pgoff_t index,
1004		filler_t *filler, struct file *file);
1005extern struct page * read_cache_page_gfp(struct address_space *mapping,
1006				pgoff_t index, gfp_t gfp_mask);
1007
1008static inline struct page *read_mapping_page(struct address_space *mapping,
1009				pgoff_t index, struct file *file)
1010{
1011	return read_cache_page(mapping, index, NULL, file);
1012}
1013
1014static inline struct folio *read_mapping_folio(struct address_space *mapping,
1015				pgoff_t index, struct file *file)
1016{
1017	return read_cache_folio(mapping, index, NULL, file);
1018}
1019
1020/**
1021 * page_pgoff - Calculate the logical page offset of this page.
1022 * @folio: The folio containing this page.
1023 * @page: The page which we need the offset of.
1024 *
1025 * For file pages, this is the offset from the beginning of the file
1026 * in units of PAGE_SIZE.  For anonymous pages, this is the offset from
1027 * the beginning of the anon_vma in units of PAGE_SIZE.  This will
1028 * return nonsense for KSM pages.
1029 *
1030 * Context: Caller must have a reference on the folio or otherwise
1031 * prevent it from being split or freed.
1032 *
1033 * Return: The offset in units of PAGE_SIZE.
1034 */
1035static inline pgoff_t page_pgoff(const struct folio *folio,
1036		const struct page *page)
1037{
1038	return folio->index + folio_page_idx(folio, page);
1039}
1040
1041/**
1042 * folio_pos - Returns the byte position of this folio in its file.
1043 * @folio: The folio.
1044 */
1045static inline loff_t folio_pos(const struct folio *folio)
1046{
1047	return ((loff_t)folio->index) * PAGE_SIZE;
1048}
1049
1050/*
1051 * Return byte-offset into filesystem object for page.
1052 */
1053static inline loff_t page_offset(struct page *page)
1054{
1055	struct folio *folio = page_folio(page);
1056
1057	return folio_pos(folio) + folio_page_idx(folio, page) * PAGE_SIZE;
1058}
1059
1060/*
1061 * Get the offset in PAGE_SIZE (even for hugetlb folios).
1062 */
1063static inline pgoff_t folio_pgoff(const struct folio *folio)
1064{
1065	return folio->index;
1066}
1067
1068static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,
1069					const unsigned long address)
1070{
1071	pgoff_t pgoff;
1072	pgoff = (address - vma->vm_start) >> PAGE_SHIFT;
1073	pgoff += vma->vm_pgoff;
1074	return pgoff;
1075}
1076
1077struct wait_page_key {
1078	struct folio *folio;
1079	int bit_nr;
1080	int page_match;
1081};
1082
1083struct wait_page_queue {
1084	struct folio *folio;
1085	int bit_nr;
1086	wait_queue_entry_t wait;
1087};
1088
1089static inline bool wake_page_match(struct wait_page_queue *wait_page,
1090				  struct wait_page_key *key)
1091{
1092	if (wait_page->folio != key->folio)
1093	       return false;
1094	key->page_match = 1;
1095
1096	if (wait_page->bit_nr != key->bit_nr)
1097		return false;
1098
1099	return true;
1100}
1101
1102void __folio_lock(struct folio *folio);
1103int __folio_lock_killable(struct folio *folio);
1104vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf);
1105void unlock_page(struct page *page);
1106void folio_unlock(struct folio *folio);
1107
1108/**
1109 * folio_trylock() - Attempt to lock a folio.
1110 * @folio: The folio to attempt to lock.
1111 *
1112 * Sometimes it is undesirable to wait for a folio to be unlocked (eg
1113 * when the locks are being taken in the wrong order, or if making
1114 * progress through a batch of folios is more important than processing
1115 * them in order).  Usually folio_lock() is the correct function to call.
1116 *
1117 * Context: Any context.
1118 * Return: Whether the lock was successfully acquired.
1119 */
1120static inline bool folio_trylock(struct folio *folio)
1121{
1122	return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0)));
1123}
1124
1125/*
1126 * Return true if the page was successfully locked
1127 */
1128static inline bool trylock_page(struct page *page)
1129{
1130	return folio_trylock(page_folio(page));
1131}
1132
1133/**
1134 * folio_lock() - Lock this folio.
1135 * @folio: The folio to lock.
1136 *
1137 * The folio lock protects against many things, probably more than it
1138 * should.  It is primarily held while a folio is being brought uptodate,
1139 * either from its backing file or from swap.  It is also held while a
1140 * folio is being truncated from its address_space, so holding the lock
1141 * is sufficient to keep folio->mapping stable.
1142 *
1143 * The folio lock is also held while write() is modifying the page to
1144 * provide POSIX atomicity guarantees (as long as the write does not
1145 * cross a page boundary).  Other modifications to the data in the folio
1146 * do not hold the folio lock and can race with writes, eg DMA and stores
1147 * to mapped pages.
1148 *
1149 * Context: May sleep.  If you need to acquire the locks of two or
1150 * more folios, they must be in order of ascending index, if they are
1151 * in the same address_space.  If they are in different address_spaces,
1152 * acquire the lock of the folio which belongs to the address_space which
1153 * has the lowest address in memory first.
1154 */
1155static inline void folio_lock(struct folio *folio)
1156{
1157	might_sleep();
1158	if (!folio_trylock(folio))
1159		__folio_lock(folio);
1160}
1161
1162/**
1163 * lock_page() - Lock the folio containing this page.
1164 * @page: The page to lock.
1165 *
1166 * See folio_lock() for a description of what the lock protects.
1167 * This is a legacy function and new code should probably use folio_lock()
1168 * instead.
1169 *
1170 * Context: May sleep.  Pages in the same folio share a lock, so do not
1171 * attempt to lock two pages which share a folio.
1172 */
1173static inline void lock_page(struct page *page)
1174{
1175	struct folio *folio;
1176	might_sleep();
1177
1178	folio = page_folio(page);
1179	if (!folio_trylock(folio))
1180		__folio_lock(folio);
1181}
1182
1183/**
1184 * folio_lock_killable() - Lock this folio, interruptible by a fatal signal.
1185 * @folio: The folio to lock.
1186 *
1187 * Attempts to lock the folio, like folio_lock(), except that the sleep
1188 * to acquire the lock is interruptible by a fatal signal.
1189 *
1190 * Context: May sleep; see folio_lock().
1191 * Return: 0 if the lock was acquired; -EINTR if a fatal signal was received.
1192 */
1193static inline int folio_lock_killable(struct folio *folio)
1194{
1195	might_sleep();
1196	if (!folio_trylock(folio))
1197		return __folio_lock_killable(folio);
1198	return 0;
1199}
1200
1201/*
1202 * folio_lock_or_retry - Lock the folio, unless this would block and the
1203 * caller indicated that it can handle a retry.
1204 *
1205 * Return value and mmap_lock implications depend on flags; see
1206 * __folio_lock_or_retry().
1207 */
1208static inline vm_fault_t folio_lock_or_retry(struct folio *folio,
1209					     struct vm_fault *vmf)
1210{
1211	might_sleep();
1212	if (!folio_trylock(folio))
1213		return __folio_lock_or_retry(folio, vmf);
1214	return 0;
1215}
1216
1217/*
1218 * This is exported only for folio_wait_locked/folio_wait_writeback, etc.,
1219 * and should not be used directly.
1220 */
1221void folio_wait_bit(struct folio *folio, int bit_nr);
1222int folio_wait_bit_killable(struct folio *folio, int bit_nr);
1223
1224/* 
1225 * Wait for a folio to be unlocked.
1226 *
1227 * This must be called with the caller "holding" the folio,
1228 * ie with increased folio reference count so that the folio won't
1229 * go away during the wait.
1230 */
1231static inline void folio_wait_locked(struct folio *folio)
1232{
1233	if (folio_test_locked(folio))
1234		folio_wait_bit(folio, PG_locked);
1235}
1236
1237static inline int folio_wait_locked_killable(struct folio *folio)
1238{
1239	if (!folio_test_locked(folio))
1240		return 0;
1241	return folio_wait_bit_killable(folio, PG_locked);
1242}
1243
1244void folio_end_read(struct folio *folio, bool success);
1245void wait_on_page_writeback(struct page *page);
1246void folio_wait_writeback(struct folio *folio);
1247int folio_wait_writeback_killable(struct folio *folio);
1248void end_page_writeback(struct page *page);
1249void folio_end_writeback(struct folio *folio);
1250void folio_end_writeback_no_dropbehind(struct folio *folio);
1251void folio_end_dropbehind(struct folio *folio);
1252void folio_wait_stable(struct folio *folio);
1253void __folio_mark_dirty(struct folio *folio, struct address_space *, int warn);
1254void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb);
1255void __folio_cancel_dirty(struct folio *folio);
1256static inline void folio_cancel_dirty(struct folio *folio)
1257{
1258	/* Avoid atomic ops, locking, etc. when not actually needed. */
1259	if (folio_test_dirty(folio))
1260		__folio_cancel_dirty(folio);
1261}
1262bool folio_clear_dirty_for_io(struct folio *folio);
1263bool clear_page_dirty_for_io(struct page *page);
1264void folio_invalidate(struct folio *folio, size_t offset, size_t length);
1265bool noop_dirty_folio(struct address_space *mapping, struct folio *folio);
1266
1267#ifdef CONFIG_MIGRATION
1268int filemap_migrate_folio(struct address_space *mapping, struct folio *dst,
1269		struct folio *src, enum migrate_mode mode);
1270#else
1271#define filemap_migrate_folio NULL
1272#endif
1273void folio_end_private_2(struct folio *folio);
1274void folio_wait_private_2(struct folio *folio);
1275int folio_wait_private_2_killable(struct folio *folio);
1276
1277/*
1278 * Fault in userspace address range.
1279 */
1280size_t fault_in_writeable(char __user *uaddr, size_t size);
1281size_t fault_in_subpage_writeable(char __user *uaddr, size_t size);
1282size_t fault_in_safe_writeable(const char __user *uaddr, size_t size);
1283size_t fault_in_readable(const char __user *uaddr, size_t size);
1284
1285int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
1286		pgoff_t index, gfp_t gfp);
1287int filemap_add_folio(struct address_space *mapping, struct folio *folio,
1288		pgoff_t index, gfp_t gfp);
1289void filemap_remove_folio(struct folio *folio);
1290void __filemap_remove_folio(struct folio *folio, void *shadow);
1291void replace_page_cache_folio(struct folio *old, struct folio *new);
1292void delete_from_page_cache_batch(struct address_space *mapping,
1293				  struct folio_batch *fbatch);
1294bool filemap_release_folio(struct folio *folio, gfp_t gfp);
1295loff_t mapping_seek_hole_data(struct address_space *, loff_t start, loff_t end,
1296		int whence);
1297
1298/* Must be non-static for BPF error injection */
1299int __filemap_add_folio(struct address_space *mapping, struct folio *folio,
1300		pgoff_t index, gfp_t gfp, void **shadowp);
1301
1302bool filemap_range_has_writeback(struct address_space *mapping,
1303				 loff_t start_byte, loff_t end_byte);
1304
1305/**
1306 * filemap_range_needs_writeback - check if range potentially needs writeback
1307 * @mapping:           address space within which to check
1308 * @start_byte:        offset in bytes where the range starts
1309 * @end_byte:          offset in bytes where the range ends (inclusive)
1310 *
1311 * Find at least one page in the range supplied, usually used to check if
1312 * direct writing in this range will trigger a writeback. Used by O_DIRECT
1313 * read/write with IOCB_NOWAIT, to see if the caller needs to do
1314 * filemap_write_and_wait_range() before proceeding.
1315 *
1316 * Return: %true if the caller should do filemap_write_and_wait_range() before
1317 * doing O_DIRECT to a page in this range, %false otherwise.
1318 */
1319static inline bool filemap_range_needs_writeback(struct address_space *mapping,
1320						 loff_t start_byte,
1321						 loff_t end_byte)
1322{
1323	if (!mapping->nrpages)
1324		return false;
1325	if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
1326	    !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
1327		return false;
1328	return filemap_range_has_writeback(mapping, start_byte, end_byte);
1329}
1330
1331/**
1332 * struct readahead_control - Describes a readahead request.
1333 *
1334 * A readahead request is for consecutive pages.  Filesystems which
1335 * implement the ->readahead method should call readahead_folio() or
1336 * __readahead_batch() in a loop and attempt to start reads into each
1337 * folio in the request.
1338 *
1339 * Most of the fields in this struct are private and should be accessed
1340 * by the functions below.
1341 *
1342 * @file: The file, used primarily by network filesystems for authentication.
1343 *	  May be NULL if invoked internally by the filesystem.
1344 * @mapping: Readahead this filesystem object.
1345 * @ra: File readahead state.  May be NULL.
1346 */
1347struct readahead_control {
1348	struct file *file;
1349	struct address_space *mapping;
1350	struct file_ra_state *ra;
1351/* private: use the readahead_* accessors instead */
1352	pgoff_t _index;
1353	unsigned int _nr_pages;
1354	unsigned int _batch_count;
1355	bool dropbehind;
1356	bool _workingset;
1357	unsigned long _pflags;
1358};
1359
1360#define DEFINE_READAHEAD(ractl, f, r, m, i)				\
1361	struct readahead_control ractl = {				\
1362		.file = f,						\
1363		.mapping = m,						\
1364		.ra = r,						\
1365		._index = i,						\
1366	}
1367
1368#define VM_READAHEAD_PAGES	(SZ_128K / PAGE_SIZE)
1369
1370void page_cache_ra_unbounded(struct readahead_control *,
1371		unsigned long nr_to_read, unsigned long lookahead_count);
1372void page_cache_sync_ra(struct readahead_control *, unsigned long req_count);
1373void page_cache_async_ra(struct readahead_control *, struct folio *,
1374		unsigned long req_count);
1375void readahead_expand(struct readahead_control *ractl,
1376		      loff_t new_start, size_t new_len);
1377
1378/**
1379 * page_cache_sync_readahead - generic file readahead
1380 * @mapping: address_space which holds the pagecache and I/O vectors
1381 * @ra: file_ra_state which holds the readahead state
1382 * @file: Used by the filesystem for authentication.
1383 * @index: Index of first page to be read.
1384 * @req_count: Total number of pages being read by the caller.
1385 *
1386 * page_cache_sync_readahead() should be called when a cache miss happened:
1387 * it will submit the read.  The readahead logic may decide to piggyback more
1388 * pages onto the read request if access patterns suggest it will improve
1389 * performance.
1390 */
1391static inline
1392void page_cache_sync_readahead(struct address_space *mapping,
1393		struct file_ra_state *ra, struct file *file, pgoff_t index,
1394		unsigned long req_count)
1395{
1396	DEFINE_READAHEAD(ractl, file, ra, mapping, index);
1397	page_cache_sync_ra(&ractl, req_count);
1398}
1399
1400/**
1401 * page_cache_async_readahead - file readahead for marked pages
1402 * @mapping: address_space which holds the pagecache and I/O vectors
1403 * @ra: file_ra_state which holds the readahead state
1404 * @file: Used by the filesystem for authentication.
1405 * @folio: The folio which triggered the readahead call.
1406 * @req_count: Total number of pages being read by the caller.
1407 *
1408 * page_cache_async_readahead() should be called when a page is used which
1409 * is marked as PageReadahead; this is a marker to suggest that the application
1410 * has used up enough of the readahead window that we should start pulling in
1411 * more pages.
1412 */
1413static inline
1414void page_cache_async_readahead(struct address_space *mapping,
1415		struct file_ra_state *ra, struct file *file,
1416		struct folio *folio, unsigned long req_count)
1417{
1418	DEFINE_READAHEAD(ractl, file, ra, mapping, folio->index);
1419	page_cache_async_ra(&ractl, folio, req_count);
1420}
1421
1422static inline struct folio *__readahead_folio(struct readahead_control *ractl)
1423{
1424	struct folio *folio;
1425
1426	BUG_ON(ractl->_batch_count > ractl->_nr_pages);
1427	ractl->_nr_pages -= ractl->_batch_count;
1428	ractl->_index += ractl->_batch_count;
1429
1430	if (!ractl->_nr_pages) {
1431		ractl->_batch_count = 0;
1432		return NULL;
1433	}
1434
1435	folio = xa_load(&ractl->mapping->i_pages, ractl->_index);
1436	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1437	ractl->_batch_count = folio_nr_pages(folio);
1438
1439	return folio;
1440}
1441
1442/**
1443 * readahead_folio - Get the next folio to read.
1444 * @ractl: The current readahead request.
1445 *
1446 * Context: The folio is locked.  The caller should unlock the folio once
1447 * all I/O to that folio has completed.
1448 * Return: A pointer to the next folio, or %NULL if we are done.
1449 */
1450static inline struct folio *readahead_folio(struct readahead_control *ractl)
1451{
1452	struct folio *folio = __readahead_folio(ractl);
1453
1454	if (folio)
1455		folio_put(folio);
1456	return folio;
1457}
1458
1459static inline unsigned int __readahead_batch(struct readahead_control *rac,
1460		struct page **array, unsigned int array_sz)
1461{
1462	unsigned int i = 0;
1463	XA_STATE(xas, &rac->mapping->i_pages, 0);
1464	struct folio *folio;
1465
1466	BUG_ON(rac->_batch_count > rac->_nr_pages);
1467	rac->_nr_pages -= rac->_batch_count;
1468	rac->_index += rac->_batch_count;
1469	rac->_batch_count = 0;
1470
1471	xas_set(&xas, rac->_index);
1472	rcu_read_lock();
1473	xas_for_each(&xas, folio, rac->_index + rac->_nr_pages - 1) {
1474		if (xas_retry(&xas, folio))
1475			continue;
1476		VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1477		array[i++] = folio_page(folio, 0);
1478		rac->_batch_count += folio_nr_pages(folio);
1479		if (i == array_sz)
1480			break;
1481	}
1482	rcu_read_unlock();
1483
1484	return i;
1485}
1486
1487/**
1488 * readahead_pos - The byte offset into the file of this readahead request.
1489 * @rac: The readahead request.
1490 */
1491static inline loff_t readahead_pos(const struct readahead_control *rac)
1492{
1493	return (loff_t)rac->_index * PAGE_SIZE;
1494}
1495
1496/**
1497 * readahead_length - The number of bytes in this readahead request.
1498 * @rac: The readahead request.
1499 */
1500static inline size_t readahead_length(const struct readahead_control *rac)
1501{
1502	return rac->_nr_pages * PAGE_SIZE;
1503}
1504
1505/**
1506 * readahead_index - The index of the first page in this readahead request.
1507 * @rac: The readahead request.
1508 */
1509static inline pgoff_t readahead_index(const struct readahead_control *rac)
1510{
1511	return rac->_index;
1512}
1513
1514/**
1515 * readahead_count - The number of pages in this readahead request.
1516 * @rac: The readahead request.
1517 */
1518static inline unsigned int readahead_count(const struct readahead_control *rac)
1519{
1520	return rac->_nr_pages;
1521}
1522
1523/**
1524 * readahead_batch_length - The number of bytes in the current batch.
1525 * @rac: The readahead request.
1526 */
1527static inline size_t readahead_batch_length(const struct readahead_control *rac)
1528{
1529	return rac->_batch_count * PAGE_SIZE;
1530}
1531
1532static inline unsigned long dir_pages(const struct inode *inode)
1533{
1534	return (unsigned long)(inode->i_size + PAGE_SIZE - 1) >>
1535			       PAGE_SHIFT;
1536}
1537
1538/**
1539 * folio_mkwrite_check_truncate - check if folio was truncated
1540 * @folio: the folio to check
1541 * @inode: the inode to check the folio against
1542 *
1543 * Return: the number of bytes in the folio up to EOF,
1544 * or -EFAULT if the folio was truncated.
1545 */
1546static inline ssize_t folio_mkwrite_check_truncate(const struct folio *folio,
1547						   const struct inode *inode)
1548{
1549	loff_t size = i_size_read(inode);
1550	pgoff_t index = size >> PAGE_SHIFT;
1551	size_t offset = offset_in_folio(folio, size);
1552
1553	if (!folio->mapping)
1554		return -EFAULT;
1555
1556	/* folio is wholly inside EOF */
1557	if (folio_next_index(folio) - 1 < index)
1558		return folio_size(folio);
1559	/* folio is wholly past EOF */
1560	if (folio->index > index || !offset)
1561		return -EFAULT;
1562	/* folio is partially inside EOF */
1563	return offset;
1564}
1565
1566/**
1567 * i_blocks_per_folio - How many blocks fit in this folio.
1568 * @inode: The inode which contains the blocks.
1569 * @folio: The folio.
1570 *
1571 * If the block size is larger than the size of this folio, return zero.
1572 *
1573 * Context: The caller should hold a refcount on the folio to prevent it
1574 * from being split.
1575 * Return: The number of filesystem blocks covered by this folio.
1576 */
1577static inline
1578unsigned int i_blocks_per_folio(const struct inode *inode,
1579				const struct folio *folio)
1580{
1581	return folio_size(folio) >> inode->i_blkbits;
1582}
1583#endif /* _LINUX_PAGEMAP_H */