FILE
vmscan.c
File preview with syntax highlighting.
linux-6.19.14/mm/vmscan.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 4 * 5 * Swap reorganised 29.12.95, Stephen Tweedie. 6 * kswapd added: 7.1.96 sct 7 * Removed kswapd_ctl limits, and swap out as many pages as needed 8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 9 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com). 10 * Multiqueue VM started 5.8.00, Rik van Riel. 11 */ 12 13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15#include <linux/mm.h> 16#include <linux/sched/mm.h> 17#include <linux/module.h> 18#include <linux/gfp.h> 19#include <linux/kernel_stat.h> 20#include <linux/swap.h> 21#include <linux/pagemap.h> 22#include <linux/init.h> 23#include <linux/highmem.h> 24#include <linux/vmpressure.h> 25#include <linux/vmstat.h> 26#include <linux/file.h> 27#include <linux/writeback.h> 28#include <linux/blkdev.h> 29#include <linux/buffer_head.h> /* for buffer_heads_over_limit */ 30#include <linux/mm_inline.h> 31#include <linux/backing-dev.h> 32#include <linux/rmap.h> 33#include <linux/topology.h> 34#include <linux/cpu.h> 35#include <linux/cpuset.h> 36#include <linux/compaction.h> 37#include <linux/notifier.h> 38#include <linux/delay.h> 39#include <linux/kthread.h> 40#include <linux/freezer.h> 41#include <linux/memcontrol.h> 42#include <linux/migrate.h> 43#include <linux/delayacct.h> 44#include <linux/sysctl.h> 45#include <linux/memory-tiers.h> 46#include <linux/oom.h> 47#include <linux/pagevec.h> 48#include <linux/prefetch.h> 49#include <linux/printk.h> 50#include <linux/dax.h> 51#include <linux/psi.h> 52#include <linux/pagewalk.h> 53#include <linux/shmem_fs.h> 54#include <linux/ctype.h> 55#include <linux/debugfs.h> 56#include <linux/khugepaged.h> 57#include <linux/rculist_nulls.h> 58#include <linux/random.h> 59#include <linux/mmu_notifier.h> 60#include <linux/parser.h> 61 62#include <asm/tlbflush.h> 63#include <asm/div64.h> 64 65#include <linux/swapops.h> 66#include <linux/balloon_compaction.h> 67#include <linux/sched/sysctl.h> 68 69#include "internal.h" 70#include "swap.h" 71 72#define CREATE_TRACE_POINTS 73#include <trace/events/vmscan.h> 74 75struct scan_control { 76 /* How many pages shrink_list() should reclaim */ 77 unsigned long nr_to_reclaim; 78 79 /* 80 * Nodemask of nodes allowed by the caller. If NULL, all nodes 81 * are scanned. 82 */ 83 nodemask_t *nodemask; 84 85 /* 86 * The memory cgroup that hit its limit and as a result is the 87 * primary target of this reclaim invocation. 88 */ 89 struct mem_cgroup *target_mem_cgroup; 90 91 /* 92 * Scan pressure balancing between anon and file LRUs 93 */ 94 unsigned long anon_cost; 95 unsigned long file_cost; 96 97 /* Swappiness value for proactive reclaim. Always use sc_swappiness()! */ 98 int *proactive_swappiness; 99 100 /* Can active folios be deactivated as part of reclaim? */ 101#define DEACTIVATE_ANON 1 102#define DEACTIVATE_FILE 2 103 unsigned int may_deactivate:2; 104 unsigned int force_deactivate:1; 105 unsigned int skipped_deactivate:1; 106 107 /* Writepage batching in laptop mode; RECLAIM_WRITE */ 108 unsigned int may_writepage:1; 109 110 /* Can mapped folios be reclaimed? */ 111 unsigned int may_unmap:1; 112 113 /* Can folios be swapped as part of reclaim? */ 114 unsigned int may_swap:1; 115 116 /* Not allow cache_trim_mode to be turned on as part of reclaim? */ 117 unsigned int no_cache_trim_mode:1; 118 119 /* Has cache_trim_mode failed at least once? */ 120 unsigned int cache_trim_mode_failed:1; 121 122 /* Proactive reclaim invoked by userspace */ 123 unsigned int proactive:1; 124 125 /* 126 * Cgroup memory below memory.low is protected as long as we 127 * don't threaten to OOM. If any cgroup is reclaimed at 128 * reduced force or passed over entirely due to its memory.low 129 * setting (memcg_low_skipped), and nothing is reclaimed as a 130 * result, then go back for one more cycle that reclaims the protected 131 * memory (memcg_low_reclaim) to avert OOM. 132 */ 133 unsigned int memcg_low_reclaim:1; 134 unsigned int memcg_low_skipped:1; 135 136 /* Shared cgroup tree walk failed, rescan the whole tree */ 137 unsigned int memcg_full_walk:1; 138 139 unsigned int hibernation_mode:1; 140 141 /* One of the zones is ready for compaction */ 142 unsigned int compaction_ready:1; 143 144 /* There is easily reclaimable cold cache in the current node */ 145 unsigned int cache_trim_mode:1; 146 147 /* The file folios on the current node are dangerously low */ 148 unsigned int file_is_tiny:1; 149 150 /* Always discard instead of demoting to lower tier memory */ 151 unsigned int no_demotion:1; 152 153 /* Allocation order */ 154 s8 order; 155 156 /* Scan (total_size >> priority) pages at once */ 157 s8 priority; 158 159 /* The highest zone to isolate folios for reclaim from */ 160 s8 reclaim_idx; 161 162 /* This context's GFP mask */ 163 gfp_t gfp_mask; 164 165 /* Incremented by the number of inactive pages that were scanned */ 166 unsigned long nr_scanned; 167 168 /* Number of pages freed so far during a call to shrink_zones() */ 169 unsigned long nr_reclaimed; 170 171 struct { 172 unsigned int dirty; 173 unsigned int unqueued_dirty; 174 unsigned int congested; 175 unsigned int writeback; 176 unsigned int immediate; 177 unsigned int file_taken; 178 unsigned int taken; 179 } nr; 180 181 /* for recording the reclaimed slab by now */ 182 struct reclaim_state reclaim_state; 183}; 184 185#ifdef ARCH_HAS_PREFETCHW 186#define prefetchw_prev_lru_folio(_folio, _base, _field) \ 187 do { \ 188 if ((_folio)->lru.prev != _base) { \ 189 struct folio *prev; \ 190 \ 191 prev = lru_to_folio(&(_folio->lru)); \ 192 prefetchw(&prev->_field); \ 193 } \ 194 } while (0) 195#else 196#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) 197#endif 198 199/* 200 * From 0 .. MAX_SWAPPINESS. Higher means more swappy. 201 */ 202int vm_swappiness = 60; 203 204#ifdef CONFIG_MEMCG 205 206/* Returns true for reclaim through cgroup limits or cgroup interfaces. */ 207static bool cgroup_reclaim(struct scan_control *sc) 208{ 209 return sc->target_mem_cgroup; 210} 211 212/* 213 * Returns true for reclaim on the root cgroup. This is true for direct 214 * allocator reclaim and reclaim through cgroup interfaces on the root cgroup. 215 */ 216static bool root_reclaim(struct scan_control *sc) 217{ 218 return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup); 219} 220 221/** 222 * writeback_throttling_sane - is the usual dirty throttling mechanism available? 223 * @sc: scan_control in question 224 * 225 * The normal page dirty throttling mechanism in balance_dirty_pages() is 226 * completely broken with the legacy memcg and direct stalling in 227 * shrink_folio_list() is used for throttling instead, which lacks all the 228 * niceties such as fairness, adaptive pausing, bandwidth proportional 229 * allocation and configurability. 230 * 231 * This function tests whether the vmscan currently in progress can assume 232 * that the normal dirty throttling mechanism is operational. 233 */ 234static bool writeback_throttling_sane(struct scan_control *sc) 235{ 236 if (!cgroup_reclaim(sc)) 237 return true; 238#ifdef CONFIG_CGROUP_WRITEBACK 239 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 240 return true; 241#endif 242 return false; 243} 244 245static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) 246{ 247 if (sc->proactive && sc->proactive_swappiness) 248 return *sc->proactive_swappiness; 249 return mem_cgroup_swappiness(memcg); 250} 251#else 252static bool cgroup_reclaim(struct scan_control *sc) 253{ 254 return false; 255} 256 257static bool root_reclaim(struct scan_control *sc) 258{ 259 return true; 260} 261 262static bool writeback_throttling_sane(struct scan_control *sc) 263{ 264 return true; 265} 266 267static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) 268{ 269 return READ_ONCE(vm_swappiness); 270} 271#endif 272 273/* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to 274 * and including the specified highidx 275 * @zone: The current zone in the iterator 276 * @pgdat: The pgdat which node_zones are being iterated 277 * @idx: The index variable 278 * @highidx: The index of the highest zone to return 279 * 280 * This macro iterates through all managed zones up to and including the specified highidx. 281 * The zone iterator enters an invalid state after macro call and must be reinitialized 282 * before it can be used again. 283 */ 284#define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx) \ 285 for ((idx) = 0, (zone) = (pgdat)->node_zones; \ 286 (idx) <= (highidx); \ 287 (idx)++, (zone)++) \ 288 if (!managed_zone(zone)) \ 289 continue; \ 290 else 291 292static void set_task_reclaim_state(struct task_struct *task, 293 struct reclaim_state *rs) 294{ 295 /* Check for an overwrite */ 296 WARN_ON_ONCE(rs && task->reclaim_state); 297 298 /* Check for the nulling of an already-nulled member */ 299 WARN_ON_ONCE(!rs && !task->reclaim_state); 300 301 task->reclaim_state = rs; 302} 303 304/* 305 * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to 306 * scan_control->nr_reclaimed. 307 */ 308static void flush_reclaim_state(struct scan_control *sc) 309{ 310 /* 311 * Currently, reclaim_state->reclaimed includes three types of pages 312 * freed outside of vmscan: 313 * (1) Slab pages. 314 * (2) Clean file pages from pruned inodes (on highmem systems). 315 * (3) XFS freed buffer pages. 316 * 317 * For all of these cases, we cannot universally link the pages to a 318 * single memcg. For example, a memcg-aware shrinker can free one object 319 * charged to the target memcg, causing an entire page to be freed. 320 * If we count the entire page as reclaimed from the memcg, we end up 321 * overestimating the reclaimed amount (potentially under-reclaiming). 322 * 323 * Only count such pages for global reclaim to prevent under-reclaiming 324 * from the target memcg; preventing unnecessary retries during memcg 325 * charging and false positives from proactive reclaim. 326 * 327 * For uncommon cases where the freed pages were actually mostly 328 * charged to the target memcg, we end up underestimating the reclaimed 329 * amount. This should be fine. The freed pages will be uncharged 330 * anyway, even if they are not counted here properly, and we will be 331 * able to make forward progress in charging (which is usually in a 332 * retry loop). 333 * 334 * We can go one step further, and report the uncharged objcg pages in 335 * memcg reclaim, to make reporting more accurate and reduce 336 * underestimation, but it's probably not worth the complexity for now. 337 */ 338 if (current->reclaim_state && root_reclaim(sc)) { 339 sc->nr_reclaimed += current->reclaim_state->reclaimed; 340 current->reclaim_state->reclaimed = 0; 341 } 342} 343 344static bool can_demote(int nid, struct scan_control *sc, 345 struct mem_cgroup *memcg) 346{ 347 struct pglist_data *pgdat = NODE_DATA(nid); 348 nodemask_t allowed_mask; 349 350 if (!pgdat || !numa_demotion_enabled) 351 return false; 352 if (sc && sc->no_demotion) 353 return false; 354 355 node_get_allowed_targets(pgdat, &allowed_mask); 356 if (nodes_empty(allowed_mask)) 357 return false; 358 359 /* Filter out nodes that are not in cgroup's mems_allowed. */ 360 mem_cgroup_node_filter_allowed(memcg, &allowed_mask); 361 return !nodes_empty(allowed_mask); 362} 363 364static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, 365 int nid, 366 struct scan_control *sc) 367{ 368 if (memcg == NULL) { 369 /* 370 * For non-memcg reclaim, is there 371 * space in any swap device? 372 */ 373 if (get_nr_swap_pages() > 0) 374 return true; 375 } else { 376 /* Is the memcg below its swap limit? */ 377 if (mem_cgroup_get_nr_swap_pages(memcg) > 0) 378 return true; 379 } 380 381 /* 382 * The page can not be swapped. 383 * 384 * Can it be reclaimed from this node via demotion? 385 */ 386 return can_demote(nid, sc, memcg); 387} 388 389/* 390 * This misses isolated folios which are not accounted for to save counters. 391 * As the data only determines if reclaim or compaction continues, it is 392 * not expected that isolated folios will be a dominating factor. 393 */ 394unsigned long zone_reclaimable_pages(struct zone *zone) 395{ 396 unsigned long nr; 397 398 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + 399 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); 400 if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) 401 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + 402 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); 403 404 return nr; 405} 406 407/** 408 * lruvec_lru_size - Returns the number of pages on the given LRU list. 409 * @lruvec: lru vector 410 * @lru: lru to use 411 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) 412 */ 413static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 414 int zone_idx) 415{ 416 unsigned long size = 0; 417 int zid; 418 struct zone *zone; 419 420 for_each_managed_zone_pgdat(zone, lruvec_pgdat(lruvec), zid, zone_idx) { 421 if (!mem_cgroup_disabled()) 422 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); 423 else 424 size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); 425 } 426 return size; 427} 428 429static unsigned long drop_slab_node(int nid) 430{ 431 unsigned long freed = 0; 432 struct mem_cgroup *memcg = NULL; 433 434 memcg = mem_cgroup_iter(NULL, NULL, NULL); 435 do { 436 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); 437 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 438 439 return freed; 440} 441 442void drop_slab(void) 443{ 444 int nid; 445 int shift = 0; 446 unsigned long freed; 447 448 do { 449 freed = 0; 450 for_each_online_node(nid) { 451 if (fatal_signal_pending(current)) 452 return; 453 454 freed += drop_slab_node(nid); 455 } 456 } while ((freed >> shift++) > 1); 457} 458 459#define CHECK_RECLAIMER_OFFSET(type) \ 460 do { \ 461 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \ 462 PGDEMOTE_##type - PGDEMOTE_KSWAPD); \ 463 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \ 464 PGSCAN_##type - PGSCAN_KSWAPD); \ 465 } while (0) 466 467static int reclaimer_offset(struct scan_control *sc) 468{ 469 CHECK_RECLAIMER_OFFSET(DIRECT); 470 CHECK_RECLAIMER_OFFSET(KHUGEPAGED); 471 CHECK_RECLAIMER_OFFSET(PROACTIVE); 472 473 if (current_is_kswapd()) 474 return 0; 475 if (current_is_khugepaged()) 476 return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD; 477 if (sc->proactive) 478 return PGSTEAL_PROACTIVE - PGSTEAL_KSWAPD; 479 return PGSTEAL_DIRECT - PGSTEAL_KSWAPD; 480} 481 482/* 483 * We detected a synchronous write error writing a folio out. Probably 484 * -ENOSPC. We need to propagate that into the address_space for a subsequent 485 * fsync(), msync() or close(). 486 * 487 * The tricky part is that after writepage we cannot touch the mapping: nothing 488 * prevents it from being freed up. But we have a ref on the folio and once 489 * that folio is locked, the mapping is pinned. 490 * 491 * We're allowed to run sleeping folio_lock() here because we know the caller has 492 * __GFP_FS. 493 */ 494static void handle_write_error(struct address_space *mapping, 495 struct folio *folio, int error) 496{ 497 folio_lock(folio); 498 if (folio_mapping(folio) == mapping) 499 mapping_set_error(mapping, error); 500 folio_unlock(folio); 501} 502 503static bool skip_throttle_noprogress(pg_data_t *pgdat) 504{ 505 int reclaimable = 0, write_pending = 0; 506 int i; 507 struct zone *zone; 508 /* 509 * If kswapd is disabled, reschedule if necessary but do not 510 * throttle as the system is likely near OOM. 511 */ 512 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES) 513 return true; 514 515 /* 516 * If there are a lot of dirty/writeback folios then do not 517 * throttle as throttling will occur when the folios cycle 518 * towards the end of the LRU if still under writeback. 519 */ 520 for_each_managed_zone_pgdat(zone, pgdat, i, MAX_NR_ZONES - 1) { 521 reclaimable += zone_reclaimable_pages(zone); 522 write_pending += zone_page_state_snapshot(zone, 523 NR_ZONE_WRITE_PENDING); 524 } 525 if (2 * write_pending <= reclaimable) 526 return true; 527 528 return false; 529} 530 531void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) 532{ 533 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; 534 long timeout, ret; 535 DEFINE_WAIT(wait); 536 537 /* 538 * Do not throttle user workers, kthreads other than kswapd or 539 * workqueues. They may be required for reclaim to make 540 * forward progress (e.g. journalling workqueues or kthreads). 541 */ 542 if (!current_is_kswapd() && 543 current->flags & (PF_USER_WORKER|PF_KTHREAD)) { 544 cond_resched(); 545 return; 546 } 547 548 /* 549 * These figures are pulled out of thin air. 550 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many 551 * parallel reclaimers which is a short-lived event so the timeout is 552 * short. Failing to make progress or waiting on writeback are 553 * potentially long-lived events so use a longer timeout. This is shaky 554 * logic as a failure to make progress could be due to anything from 555 * writeback to a slow device to excessive referenced folios at the tail 556 * of the inactive LRU. 557 */ 558 switch(reason) { 559 case VMSCAN_THROTTLE_WRITEBACK: 560 timeout = HZ/10; 561 562 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { 563 WRITE_ONCE(pgdat->nr_reclaim_start, 564 node_page_state(pgdat, NR_THROTTLED_WRITTEN)); 565 } 566 567 break; 568 case VMSCAN_THROTTLE_CONGESTED: 569 fallthrough; 570 case VMSCAN_THROTTLE_NOPROGRESS: 571 if (skip_throttle_noprogress(pgdat)) { 572 cond_resched(); 573 return; 574 } 575 576 timeout = 1; 577 578 break; 579 case VMSCAN_THROTTLE_ISOLATED: 580 timeout = HZ/50; 581 break; 582 default: 583 WARN_ON_ONCE(1); 584 timeout = HZ; 585 break; 586 } 587 588 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 589 ret = schedule_timeout(timeout); 590 finish_wait(wqh, &wait); 591 592 if (reason == VMSCAN_THROTTLE_WRITEBACK) 593 atomic_dec(&pgdat->nr_writeback_throttled); 594 595 trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), 596 jiffies_to_usecs(timeout - ret), 597 reason); 598} 599 600/* 601 * Account for folios written if tasks are throttled waiting on dirty 602 * folios to clean. If enough folios have been cleaned since throttling 603 * started then wakeup the throttled tasks. 604 */ 605void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 606 int nr_throttled) 607{ 608 unsigned long nr_written; 609 610 node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); 611 612 /* 613 * This is an inaccurate read as the per-cpu deltas may not 614 * be synchronised. However, given that the system is 615 * writeback throttled, it is not worth taking the penalty 616 * of getting an accurate count. At worst, the throttle 617 * timeout guarantees forward progress. 618 */ 619 nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - 620 READ_ONCE(pgdat->nr_reclaim_start); 621 622 if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) 623 wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); 624} 625 626/* possible outcome of pageout() */ 627typedef enum { 628 /* failed to write folio out, folio is locked */ 629 PAGE_KEEP, 630 /* move folio to the active list, folio is locked */ 631 PAGE_ACTIVATE, 632 /* folio has been sent to the disk successfully, folio is unlocked */ 633 PAGE_SUCCESS, 634 /* folio is clean and locked */ 635 PAGE_CLEAN, 636} pageout_t; 637 638static pageout_t writeout(struct folio *folio, struct address_space *mapping, 639 struct swap_iocb **plug, struct list_head *folio_list) 640{ 641 int res; 642 643 folio_set_reclaim(folio); 644 645 /* 646 * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled 647 * or we failed to allocate contiguous swap entries, in which case 648 * the split out folios get added back to folio_list. 649 */ 650 if (shmem_mapping(mapping)) 651 res = shmem_writeout(folio, plug, folio_list); 652 else 653 res = swap_writeout(folio, plug); 654 655 if (res < 0) 656 handle_write_error(mapping, folio, res); 657 if (res == AOP_WRITEPAGE_ACTIVATE) { 658 folio_clear_reclaim(folio); 659 return PAGE_ACTIVATE; 660 } 661 662 /* synchronous write? */ 663 if (!folio_test_writeback(folio)) 664 folio_clear_reclaim(folio); 665 666 trace_mm_vmscan_write_folio(folio); 667 node_stat_add_folio(folio, NR_VMSCAN_WRITE); 668 return PAGE_SUCCESS; 669} 670 671/* 672 * pageout is called by shrink_folio_list() for each dirty folio. 673 */ 674static pageout_t pageout(struct folio *folio, struct address_space *mapping, 675 struct swap_iocb **plug, struct list_head *folio_list) 676{ 677 /* 678 * We no longer attempt to writeback filesystem folios here, other 679 * than tmpfs/shmem. That's taken care of in page-writeback. 680 * If we find a dirty filesystem folio at the end of the LRU list, 681 * typically that means the filesystem is saturating the storage 682 * with contiguous writes and telling it to write a folio here 683 * would only make the situation worse by injecting an element 684 * of random access. 685 * 686 * If the folio is swapcache, write it back even if that would 687 * block, for some throttling. This happens by accident, because 688 * swap_backing_dev_info is bust: it doesn't reflect the 689 * congestion state of the swapdevs. Easy to fix, if needed. 690 * 691 * A freeable shmem or swapcache folio is referenced only by the 692 * caller that isolated the folio and the page cache. 693 */ 694 if (folio_ref_count(folio) != 1 + folio_nr_pages(folio) || !mapping) 695 return PAGE_KEEP; 696 if (!shmem_mapping(mapping) && !folio_test_anon(folio)) 697 return PAGE_ACTIVATE; 698 if (!folio_clear_dirty_for_io(folio)) 699 return PAGE_CLEAN; 700 return writeout(folio, mapping, plug, folio_list); 701} 702 703/* 704 * Same as remove_mapping, but if the folio is removed from the mapping, it 705 * gets returned with a refcount of 0. 706 */ 707static int __remove_mapping(struct address_space *mapping, struct folio *folio, 708 bool reclaimed, struct mem_cgroup *target_memcg) 709{ 710 int refcount; 711 void *shadow = NULL; 712 struct swap_cluster_info *ci; 713 714 BUG_ON(!folio_test_locked(folio)); 715 BUG_ON(mapping != folio_mapping(folio)); 716 717 if (folio_test_swapcache(folio)) { 718 ci = swap_cluster_get_and_lock_irq(folio); 719 } else { 720 spin_lock(&mapping->host->i_lock); 721 xa_lock_irq(&mapping->i_pages); 722 } 723 724 /* 725 * The non racy check for a busy folio. 726 * 727 * Must be careful with the order of the tests. When someone has 728 * a ref to the folio, it may be possible that they dirty it then 729 * drop the reference. So if the dirty flag is tested before the 730 * refcount here, then the following race may occur: 731 * 732 * get_user_pages(&page); 733 * [user mapping goes away] 734 * write_to(page); 735 * !folio_test_dirty(folio) [good] 736 * folio_set_dirty(folio); 737 * folio_put(folio); 738 * !refcount(folio) [good, discard it] 739 * 740 * [oops, our write_to data is lost] 741 * 742 * Reversing the order of the tests ensures such a situation cannot 743 * escape unnoticed. The smp_rmb is needed to ensure the folio->flags 744 * load is not satisfied before that of folio->_refcount. 745 * 746 * Note that if the dirty flag is always set via folio_mark_dirty, 747 * and thus under the i_pages lock, then this ordering is not required. 748 */ 749 refcount = 1 + folio_nr_pages(folio); 750 if (!folio_ref_freeze(folio, refcount)) 751 goto cannot_free; 752 /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ 753 if (unlikely(folio_test_dirty(folio))) { 754 folio_ref_unfreeze(folio, refcount); 755 goto cannot_free; 756 } 757 758 if (folio_test_swapcache(folio)) { 759 swp_entry_t swap = folio->swap; 760 761 if (reclaimed && !mapping_exiting(mapping)) 762 shadow = workingset_eviction(folio, target_memcg); 763 __swap_cache_del_folio(ci, folio, swap, shadow); 764 memcg1_swapout(folio, swap); 765 swap_cluster_unlock_irq(ci); 766 put_swap_folio(folio, swap); 767 } else { 768 void (*free_folio)(struct folio *); 769 770 free_folio = mapping->a_ops->free_folio; 771 /* 772 * Remember a shadow entry for reclaimed file cache in 773 * order to detect refaults, thus thrashing, later on. 774 * 775 * But don't store shadows in an address space that is 776 * already exiting. This is not just an optimization, 777 * inode reclaim needs to empty out the radix tree or 778 * the nodes are lost. Don't plant shadows behind its 779 * back. 780 * 781 * We also don't store shadows for DAX mappings because the 782 * only page cache folios found in these are zero pages 783 * covering holes, and because we don't want to mix DAX 784 * exceptional entries and shadow exceptional entries in the 785 * same address_space. 786 */ 787 if (reclaimed && folio_is_file_lru(folio) && 788 !mapping_exiting(mapping) && !dax_mapping(mapping)) 789 shadow = workingset_eviction(folio, target_memcg); 790 __filemap_remove_folio(folio, shadow); 791 xa_unlock_irq(&mapping->i_pages); 792 if (mapping_shrinkable(mapping)) 793 inode_lru_list_add(mapping->host); 794 spin_unlock(&mapping->host->i_lock); 795 796 if (free_folio) 797 free_folio(folio); 798 } 799 800 return 1; 801 802cannot_free: 803 if (folio_test_swapcache(folio)) { 804 swap_cluster_unlock_irq(ci); 805 } else { 806 xa_unlock_irq(&mapping->i_pages); 807 spin_unlock(&mapping->host->i_lock); 808 } 809 return 0; 810} 811 812/** 813 * remove_mapping() - Attempt to remove a folio from its mapping. 814 * @mapping: The address space. 815 * @folio: The folio to remove. 816 * 817 * If the folio is dirty, under writeback or if someone else has a ref 818 * on it, removal will fail. 819 * Return: The number of pages removed from the mapping. 0 if the folio 820 * could not be removed. 821 * Context: The caller should have a single refcount on the folio and 822 * hold its lock. 823 */ 824long remove_mapping(struct address_space *mapping, struct folio *folio) 825{ 826 if (__remove_mapping(mapping, folio, false, NULL)) { 827 /* 828 * Unfreezing the refcount with 1 effectively 829 * drops the pagecache ref for us without requiring another 830 * atomic operation. 831 */ 832 folio_ref_unfreeze(folio, 1); 833 return folio_nr_pages(folio); 834 } 835 return 0; 836} 837 838/** 839 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. 840 * @folio: Folio to be returned to an LRU list. 841 * 842 * Add previously isolated @folio to appropriate LRU list. 843 * The folio may still be unevictable for other reasons. 844 * 845 * Context: lru_lock must not be held, interrupts must be enabled. 846 */ 847void folio_putback_lru(struct folio *folio) 848{ 849 folio_add_lru(folio); 850 folio_put(folio); /* drop ref from isolate */ 851} 852 853enum folio_references { 854 FOLIOREF_RECLAIM, 855 FOLIOREF_RECLAIM_CLEAN, 856 FOLIOREF_KEEP, 857 FOLIOREF_ACTIVATE, 858}; 859 860#ifdef CONFIG_LRU_GEN 861/* 862 * Only used on a mapped folio in the eviction (rmap walk) path, where promotion 863 * needs to be done by taking the folio off the LRU list and then adding it back 864 * with PG_active set. In contrast, the aging (page table walk) path uses 865 * folio_update_gen(). 866 */ 867static bool lru_gen_set_refs(struct folio *folio) 868{ 869 /* see the comment on LRU_REFS_FLAGS */ 870 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { 871 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); 872 return false; 873 } 874 875 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); 876 return true; 877} 878#else 879static bool lru_gen_set_refs(struct folio *folio) 880{ 881 return false; 882} 883#endif /* CONFIG_LRU_GEN */ 884 885static enum folio_references folio_check_references(struct folio *folio, 886 struct scan_control *sc) 887{ 888 int referenced_ptes, referenced_folio; 889 vm_flags_t vm_flags; 890 891 referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, 892 &vm_flags); 893 894 /* 895 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. 896 * Let the folio, now marked Mlocked, be moved to the unevictable list. 897 */ 898 if (vm_flags & VM_LOCKED) 899 return FOLIOREF_ACTIVATE; 900 901 /* 902 * There are two cases to consider. 903 * 1) Rmap lock contention: rotate. 904 * 2) Skip the non-shared swapbacked folio mapped solely by 905 * the exiting or OOM-reaped process. 906 */ 907 if (referenced_ptes == -1) 908 return FOLIOREF_KEEP; 909 910 if (lru_gen_enabled()) { 911 if (!referenced_ptes) 912 return FOLIOREF_RECLAIM; 913 914 return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; 915 } 916 917 referenced_folio = folio_test_clear_referenced(folio); 918 919 if (referenced_ptes) { 920 /* 921 * All mapped folios start out with page table 922 * references from the instantiating fault, so we need 923 * to look twice if a mapped file/anon folio is used more 924 * than once. 925 * 926 * Mark it and spare it for another trip around the 927 * inactive list. Another page table reference will 928 * lead to its activation. 929 * 930 * Note: the mark is set for activated folios as well 931 * so that recently deactivated but used folios are 932 * quickly recovered. 933 */ 934 folio_set_referenced(folio); 935 936 if (referenced_folio || referenced_ptes > 1) 937 return FOLIOREF_ACTIVATE; 938 939 /* 940 * Activate file-backed executable folios after first usage. 941 */ 942 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) 943 return FOLIOREF_ACTIVATE; 944 945 return FOLIOREF_KEEP; 946 } 947 948 /* Reclaim if clean, defer dirty folios to writeback */ 949 if (referenced_folio && folio_is_file_lru(folio)) 950 return FOLIOREF_RECLAIM_CLEAN; 951 952 return FOLIOREF_RECLAIM; 953} 954 955/* Check if a folio is dirty or under writeback */ 956static void folio_check_dirty_writeback(struct folio *folio, 957 bool *dirty, bool *writeback) 958{ 959 struct address_space *mapping; 960 961 /* 962 * Anonymous folios are not handled by flushers and must be written 963 * from reclaim context. Do not stall reclaim based on them. 964 * MADV_FREE anonymous folios are put into inactive file list too. 965 * They could be mistakenly treated as file lru. So further anon 966 * test is needed. 967 */ 968 if (!folio_is_file_lru(folio) || 969 (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { 970 *dirty = false; 971 *writeback = false; 972 return; 973 } 974 975 /* By default assume that the folio flags are accurate */ 976 *dirty = folio_test_dirty(folio); 977 *writeback = folio_test_writeback(folio); 978 979 /* Verify dirty/writeback state if the filesystem supports it */ 980 if (!folio_test_private(folio)) 981 return; 982 983 mapping = folio_mapping(folio); 984 if (mapping && mapping->a_ops->is_dirty_writeback) 985 mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); 986} 987 988static struct folio *alloc_demote_folio(struct folio *src, 989 unsigned long private) 990{ 991 struct folio *dst; 992 nodemask_t *allowed_mask; 993 struct migration_target_control *mtc; 994 995 mtc = (struct migration_target_control *)private; 996 997 allowed_mask = mtc->nmask; 998 /* 999 * make sure we allocate from the target node first also trying to 1000 * demote or reclaim pages from the target node via kswapd if we are 1001 * low on free memory on target node. If we don't do this and if 1002 * we have free memory on the slower(lower) memtier, we would start 1003 * allocating pages from slower(lower) memory tiers without even forcing 1004 * a demotion of cold pages from the target memtier. This can result 1005 * in the kernel placing hot pages in slower(lower) memory tiers. 1006 */ 1007 mtc->nmask = NULL; 1008 mtc->gfp_mask |= __GFP_THISNODE; 1009 dst = alloc_migration_target(src, (unsigned long)mtc); 1010 if (dst) 1011 return dst; 1012 1013 mtc->gfp_mask &= ~__GFP_THISNODE; 1014 mtc->nmask = allowed_mask; 1015 1016 return alloc_migration_target(src, (unsigned long)mtc); 1017} 1018 1019/* 1020 * Take folios on @demote_folios and attempt to demote them to another node. 1021 * Folios which are not demoted are left on @demote_folios. 1022 */ 1023static unsigned int demote_folio_list(struct list_head *demote_folios, 1024 struct pglist_data *pgdat, 1025 struct mem_cgroup *memcg) 1026{ 1027 int target_nid; 1028 unsigned int nr_succeeded; 1029 nodemask_t allowed_mask; 1030 1031 struct migration_target_control mtc = { 1032 /* 1033 * Allocate from 'node', or fail quickly and quietly. 1034 * When this happens, 'page' will likely just be discarded 1035 * instead of migrated. 1036 */ 1037 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | 1038 __GFP_NOMEMALLOC | GFP_NOWAIT, 1039 .nmask = &allowed_mask, 1040 .reason = MR_DEMOTION, 1041 }; 1042 1043 if (list_empty(demote_folios)) 1044 return 0; 1045 1046 node_get_allowed_targets(pgdat, &allowed_mask); 1047 mem_cgroup_node_filter_allowed(memcg, &allowed_mask); 1048 if (nodes_empty(allowed_mask)) 1049 return 0; 1050 1051 target_nid = next_demotion_node(pgdat->node_id); 1052 if (target_nid == NUMA_NO_NODE) 1053 /* No lower-tier nodes or nodes were hot-unplugged. */ 1054 return 0; 1055 if (!node_isset(target_nid, allowed_mask)) 1056 target_nid = node_random(&allowed_mask); 1057 mtc.nid = target_nid; 1058 1059 /* Demotion ignores all cpuset and mempolicy settings */ 1060 migrate_pages(demote_folios, alloc_demote_folio, NULL, 1061 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, 1062 &nr_succeeded); 1063 1064 return nr_succeeded; 1065} 1066 1067static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) 1068{ 1069 if (gfp_mask & __GFP_FS) 1070 return true; 1071 if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) 1072 return false; 1073 /* 1074 * We can "enter_fs" for swap-cache with only __GFP_IO 1075 * providing this isn't SWP_FS_OPS. 1076 * ->flags can be updated non-atomicially (scan_swap_map_slots), 1077 * but that will never affect SWP_FS_OPS, so the data_race 1078 * is safe. 1079 */ 1080 return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); 1081} 1082 1083/* 1084 * shrink_folio_list() returns the number of reclaimed pages 1085 */ 1086static unsigned int shrink_folio_list(struct list_head *folio_list, 1087 struct pglist_data *pgdat, struct scan_control *sc, 1088 struct reclaim_stat *stat, bool ignore_references, 1089 struct mem_cgroup *memcg) 1090{ 1091 struct folio_batch free_folios; 1092 LIST_HEAD(ret_folios); 1093 LIST_HEAD(demote_folios); 1094 unsigned int nr_reclaimed = 0, nr_demoted = 0; 1095 unsigned int pgactivate = 0; 1096 bool do_demote_pass; 1097 struct swap_iocb *plug = NULL; 1098 1099 folio_batch_init(&free_folios); 1100 memset(stat, 0, sizeof(*stat)); 1101 cond_resched(); 1102 do_demote_pass = can_demote(pgdat->node_id, sc, memcg); 1103 1104retry: 1105 while (!list_empty(folio_list)) { 1106 struct address_space *mapping; 1107 struct folio *folio; 1108 enum folio_references references = FOLIOREF_RECLAIM; 1109 bool dirty, writeback; 1110 unsigned int nr_pages; 1111 1112 cond_resched(); 1113 1114 folio = lru_to_folio(folio_list); 1115 list_del(&folio->lru); 1116 1117 if (!folio_trylock(folio)) 1118 goto keep; 1119 1120 if (folio_contain_hwpoisoned_page(folio)) { 1121 /* 1122 * unmap_poisoned_folio() can't handle large 1123 * folio, just skip it. memory_failure() will 1124 * handle it if the UCE is triggered again. 1125 */ 1126 if (folio_test_large(folio)) 1127 goto keep_locked; 1128 1129 unmap_poisoned_folio(folio, folio_pfn(folio), false); 1130 folio_unlock(folio); 1131 folio_put(folio); 1132 continue; 1133 } 1134 1135 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1136 1137 nr_pages = folio_nr_pages(folio); 1138 1139 /* Account the number of base pages */ 1140 sc->nr_scanned += nr_pages; 1141 1142 if (unlikely(!folio_evictable(folio))) 1143 goto activate_locked; 1144 1145 if (!sc->may_unmap && folio_mapped(folio)) 1146 goto keep_locked; 1147 1148 /* 1149 * The number of dirty pages determines if a node is marked 1150 * reclaim_congested. kswapd will stall and start writing 1151 * folios if the tail of the LRU is all dirty unqueued folios. 1152 */ 1153 folio_check_dirty_writeback(folio, &dirty, &writeback); 1154 if (dirty || writeback) 1155 stat->nr_dirty += nr_pages; 1156 1157 if (dirty && !writeback) 1158 stat->nr_unqueued_dirty += nr_pages; 1159 1160 /* 1161 * Treat this folio as congested if folios are cycling 1162 * through the LRU so quickly that the folios marked 1163 * for immediate reclaim are making it to the end of 1164 * the LRU a second time. 1165 */ 1166 if (writeback && folio_test_reclaim(folio)) 1167 stat->nr_congested += nr_pages; 1168 1169 /* 1170 * If a folio at the tail of the LRU is under writeback, there 1171 * are three cases to consider. 1172 * 1173 * 1) If reclaim is encountering an excessive number 1174 * of folios under writeback and this folio has both 1175 * the writeback and reclaim flags set, then it 1176 * indicates that folios are being queued for I/O but 1177 * are being recycled through the LRU before the I/O 1178 * can complete. Waiting on the folio itself risks an 1179 * indefinite stall if it is impossible to writeback 1180 * the folio due to I/O error or disconnected storage 1181 * so instead note that the LRU is being scanned too 1182 * quickly and the caller can stall after the folio 1183 * list has been processed. 1184 * 1185 * 2) Global or new memcg reclaim encounters a folio that is 1186 * not marked for immediate reclaim, or the caller does not 1187 * have __GFP_FS (or __GFP_IO if it's simply going to swap, 1188 * not to fs), or the folio belongs to a mapping where 1189 * waiting on writeback during reclaim may lead to a deadlock. 1190 * In this case mark the folio for immediate reclaim and 1191 * continue scanning. 1192 * 1193 * Require may_enter_fs() because we would wait on fs, which 1194 * may not have submitted I/O yet. And the loop driver might 1195 * enter reclaim, and deadlock if it waits on a folio for 1196 * which it is needed to do the write (loop masks off 1197 * __GFP_IO|__GFP_FS for this reason); but more thought 1198 * would probably show more reasons. 1199 * 1200 * 3) Legacy memcg encounters a folio that already has the 1201 * reclaim flag set. memcg does not have any dirty folio 1202 * throttling so we could easily OOM just because too many 1203 * folios are in writeback and there is nothing else to 1204 * reclaim. Wait for the writeback to complete. 1205 * 1206 * In cases 1) and 2) we activate the folios to get them out of 1207 * the way while we continue scanning for clean folios on the 1208 * inactive list and refilling from the active list. The 1209 * observation here is that waiting for disk writes is more 1210 * expensive than potentially causing reloads down the line. 1211 * Since they're marked for immediate reclaim, they won't put 1212 * memory pressure on the cache working set any longer than it 1213 * takes to write them to disk. 1214 */ 1215 if (folio_test_writeback(folio)) { 1216 mapping = folio_mapping(folio); 1217 1218 /* Case 1 above */ 1219 if (current_is_kswapd() && 1220 folio_test_reclaim(folio) && 1221 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { 1222 stat->nr_immediate += nr_pages; 1223 goto activate_locked; 1224 1225 /* Case 2 above */ 1226 } else if (writeback_throttling_sane(sc) || 1227 !folio_test_reclaim(folio) || 1228 !may_enter_fs(folio, sc->gfp_mask) || 1229 (mapping && 1230 mapping_writeback_may_deadlock_on_reclaim(mapping))) { 1231 /* 1232 * This is slightly racy - 1233 * folio_end_writeback() might have 1234 * just cleared the reclaim flag, then 1235 * setting the reclaim flag here ends up 1236 * interpreted as the readahead flag - but 1237 * that does not matter enough to care. 1238 * What we do want is for this folio to 1239 * have the reclaim flag set next time 1240 * memcg reclaim reaches the tests above, 1241 * so it will then wait for writeback to 1242 * avoid OOM; and it's also appropriate 1243 * in global reclaim. 1244 */ 1245 folio_set_reclaim(folio); 1246 stat->nr_writeback += nr_pages; 1247 goto activate_locked; 1248 1249 /* Case 3 above */ 1250 } else { 1251 folio_unlock(folio); 1252 folio_wait_writeback(folio); 1253 /* then go back and try same folio again */ 1254 list_add_tail(&folio->lru, folio_list); 1255 continue; 1256 } 1257 } 1258 1259 if (!ignore_references) 1260 references = folio_check_references(folio, sc); 1261 1262 switch (references) { 1263 case FOLIOREF_ACTIVATE: 1264 goto activate_locked; 1265 case FOLIOREF_KEEP: 1266 stat->nr_ref_keep += nr_pages; 1267 goto keep_locked; 1268 case FOLIOREF_RECLAIM: 1269 case FOLIOREF_RECLAIM_CLEAN: 1270 ; /* try to reclaim the folio below */ 1271 } 1272 1273 /* 1274 * Before reclaiming the folio, try to relocate 1275 * its contents to another node. 1276 */ 1277 if (do_demote_pass && 1278 (thp_migration_supported() || !folio_test_large(folio))) { 1279 list_add(&folio->lru, &demote_folios); 1280 folio_unlock(folio); 1281 continue; 1282 } 1283 1284 /* 1285 * Anonymous process memory has backing store? 1286 * Try to allocate it some swap space here. 1287 * Lazyfree folio could be freed directly 1288 */ 1289 if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { 1290 if (!folio_test_swapcache(folio)) { 1291 if (!(sc->gfp_mask & __GFP_IO)) 1292 goto keep_locked; 1293 if (folio_maybe_dma_pinned(folio)) 1294 goto keep_locked; 1295 if (folio_test_large(folio)) { 1296 /* cannot split folio, skip it */ 1297 if (folio_expected_ref_count(folio) != 1298 folio_ref_count(folio) - 1) 1299 goto activate_locked; 1300 /* 1301 * Split partially mapped folios right away. 1302 * We can free the unmapped pages without IO. 1303 */ 1304 if (data_race(!list_empty(&folio->_deferred_list) && 1305 folio_test_partially_mapped(folio)) && 1306 split_folio_to_list(folio, folio_list)) 1307 goto activate_locked; 1308 } 1309 if (folio_alloc_swap(folio)) { 1310 int __maybe_unused order = folio_order(folio); 1311 1312 if (!folio_test_large(folio)) 1313 goto activate_locked_split; 1314 /* Fallback to swap normal pages */ 1315 if (split_folio_to_list(folio, folio_list)) 1316 goto activate_locked; 1317#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1318 if (nr_pages >= HPAGE_PMD_NR) { 1319 count_memcg_folio_events(folio, 1320 THP_SWPOUT_FALLBACK, 1); 1321 count_vm_event(THP_SWPOUT_FALLBACK); 1322 } 1323#endif 1324 count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK); 1325 if (folio_alloc_swap(folio)) 1326 goto activate_locked_split; 1327 } 1328 /* 1329 * Normally the folio will be dirtied in unmap because its 1330 * pte should be dirty. A special case is MADV_FREE page. The 1331 * page's pte could have dirty bit cleared but the folio's 1332 * SwapBacked flag is still set because clearing the dirty bit 1333 * and SwapBacked flag has no lock protected. For such folio, 1334 * unmap will not set dirty bit for it, so folio reclaim will 1335 * not write the folio out. This can cause data corruption when 1336 * the folio is swapped in later. Always setting the dirty flag 1337 * for the folio solves the problem. 1338 */ 1339 folio_mark_dirty(folio); 1340 } 1341 } 1342 1343 /* 1344 * If the folio was split above, the tail pages will make 1345 * their own pass through this function and be accounted 1346 * then. 1347 */ 1348 if ((nr_pages > 1) && !folio_test_large(folio)) { 1349 sc->nr_scanned -= (nr_pages - 1); 1350 nr_pages = 1; 1351 } 1352 1353 /* 1354 * The folio is mapped into the page tables of one or more 1355 * processes. Try to unmap it here. 1356 */ 1357 if (folio_mapped(folio)) { 1358 enum ttu_flags flags = TTU_BATCH_FLUSH; 1359 bool was_swapbacked = folio_test_swapbacked(folio); 1360 1361 if (folio_test_pmd_mappable(folio)) 1362 flags |= TTU_SPLIT_HUGE_PMD; 1363 /* 1364 * Without TTU_SYNC, try_to_unmap will only begin to 1365 * hold PTL from the first present PTE within a large 1366 * folio. Some initial PTEs might be skipped due to 1367 * races with parallel PTE writes in which PTEs can be 1368 * cleared temporarily before being written new present 1369 * values. This will lead to a large folio is still 1370 * mapped while some subpages have been partially 1371 * unmapped after try_to_unmap; TTU_SYNC helps 1372 * try_to_unmap acquire PTL from the first PTE, 1373 * eliminating the influence of temporary PTE values. 1374 */ 1375 if (folio_test_large(folio)) 1376 flags |= TTU_SYNC; 1377 1378 try_to_unmap(folio, flags); 1379 if (folio_mapped(folio)) { 1380 stat->nr_unmap_fail += nr_pages; 1381 if (!was_swapbacked && 1382 folio_test_swapbacked(folio)) 1383 stat->nr_lazyfree_fail += nr_pages; 1384 goto activate_locked; 1385 } 1386 } 1387 1388 /* 1389 * Folio is unmapped now so it cannot be newly pinned anymore. 1390 * No point in trying to reclaim folio if it is pinned. 1391 * Furthermore we don't want to reclaim underlying fs metadata 1392 * if the folio is pinned and thus potentially modified by the 1393 * pinning process as that may upset the filesystem. 1394 */ 1395 if (folio_maybe_dma_pinned(folio)) 1396 goto activate_locked; 1397 1398 mapping = folio_mapping(folio); 1399 if (folio_test_dirty(folio)) { 1400 if (folio_is_file_lru(folio)) { 1401 /* 1402 * Immediately reclaim when written back. 1403 * Similar in principle to folio_deactivate() 1404 * except we already have the folio isolated 1405 * and know it's dirty 1406 */ 1407 node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, 1408 nr_pages); 1409 if (!folio_test_reclaim(folio)) 1410 folio_set_reclaim(folio); 1411 1412 goto activate_locked; 1413 } 1414 1415 if (references == FOLIOREF_RECLAIM_CLEAN) 1416 goto keep_locked; 1417 if (!may_enter_fs(folio, sc->gfp_mask)) 1418 goto keep_locked; 1419 if (!sc->may_writepage) 1420 goto keep_locked; 1421 1422 /* 1423 * Folio is dirty. Flush the TLB if a writable entry 1424 * potentially exists to avoid CPU writes after I/O 1425 * starts and then write it out here. 1426 */ 1427 try_to_unmap_flush_dirty(); 1428 switch (pageout(folio, mapping, &plug, folio_list)) { 1429 case PAGE_KEEP: 1430 goto keep_locked; 1431 case PAGE_ACTIVATE: 1432 /* 1433 * If shmem folio is split when writeback to swap, 1434 * the tail pages will make their own pass through 1435 * this function and be accounted then. 1436 */ 1437 if (nr_pages > 1 && !folio_test_large(folio)) { 1438 sc->nr_scanned -= (nr_pages - 1); 1439 nr_pages = 1; 1440 } 1441 goto activate_locked; 1442 case PAGE_SUCCESS: 1443 if (nr_pages > 1 && !folio_test_large(folio)) { 1444 sc->nr_scanned -= (nr_pages - 1); 1445 nr_pages = 1; 1446 } 1447 stat->nr_pageout += nr_pages; 1448 1449 if (folio_test_writeback(folio)) 1450 goto keep; 1451 if (folio_test_dirty(folio)) 1452 goto keep; 1453 1454 /* 1455 * A synchronous write - probably a ramdisk. Go 1456 * ahead and try to reclaim the folio. 1457 */ 1458 if (!folio_trylock(folio)) 1459 goto keep; 1460 if (folio_test_dirty(folio) || 1461 folio_test_writeback(folio)) 1462 goto keep_locked; 1463 mapping = folio_mapping(folio); 1464 fallthrough; 1465 case PAGE_CLEAN: 1466 ; /* try to free the folio below */ 1467 } 1468 } 1469 1470 /* 1471 * If the folio has buffers, try to free the buffer 1472 * mappings associated with this folio. If we succeed 1473 * we try to free the folio as well. 1474 * 1475 * We do this even if the folio is dirty. 1476 * filemap_release_folio() does not perform I/O, but it 1477 * is possible for a folio to have the dirty flag set, 1478 * but it is actually clean (all its buffers are clean). 1479 * This happens if the buffers were written out directly, 1480 * with submit_bh(). ext3 will do this, as well as 1481 * the blockdev mapping. filemap_release_folio() will 1482 * discover that cleanness and will drop the buffers 1483 * and mark the folio clean - it can be freed. 1484 * 1485 * Rarely, folios can have buffers and no ->mapping. 1486 * These are the folios which were not successfully 1487 * invalidated in truncate_cleanup_folio(). We try to 1488 * drop those buffers here and if that worked, and the 1489 * folio is no longer mapped into process address space 1490 * (refcount == 1) it can be freed. Otherwise, leave 1491 * the folio on the LRU so it is swappable. 1492 */ 1493 if (folio_needs_release(folio)) { 1494 if (!filemap_release_folio(folio, sc->gfp_mask)) 1495 goto activate_locked; 1496 if (!mapping && folio_ref_count(folio) == 1) { 1497 folio_unlock(folio); 1498 if (folio_put_testzero(folio)) 1499 goto free_it; 1500 else { 1501 /* 1502 * rare race with speculative reference. 1503 * the speculative reference will free 1504 * this folio shortly, so we may 1505 * increment nr_reclaimed here (and 1506 * leave it off the LRU). 1507 */ 1508 nr_reclaimed += nr_pages; 1509 continue; 1510 } 1511 } 1512 } 1513 1514 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { 1515 /* follow __remove_mapping for reference */ 1516 if (!folio_ref_freeze(folio, 1)) 1517 goto keep_locked; 1518 /* 1519 * The folio has only one reference left, which is 1520 * from the isolation. After the caller puts the 1521 * folio back on the lru and drops the reference, the 1522 * folio will be freed anyway. It doesn't matter 1523 * which lru it goes on. So we don't bother checking 1524 * the dirty flag here. 1525 */ 1526 count_vm_events(PGLAZYFREED, nr_pages); 1527 count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); 1528 } else if (!mapping || !__remove_mapping(mapping, folio, true, 1529 sc->target_mem_cgroup)) 1530 goto keep_locked; 1531 1532 folio_unlock(folio); 1533free_it: 1534 /* 1535 * Folio may get swapped out as a whole, need to account 1536 * all pages in it. 1537 */ 1538 nr_reclaimed += nr_pages; 1539 1540 folio_unqueue_deferred_split(folio); 1541 if (folio_batch_add(&free_folios, folio) == 0) { 1542 mem_cgroup_uncharge_folios(&free_folios); 1543 try_to_unmap_flush(); 1544 free_unref_folios(&free_folios); 1545 } 1546 continue; 1547 1548activate_locked_split: 1549 /* 1550 * The tail pages that are failed to add into swap cache 1551 * reach here. Fixup nr_scanned and nr_pages. 1552 */ 1553 if (nr_pages > 1) { 1554 sc->nr_scanned -= (nr_pages - 1); 1555 nr_pages = 1; 1556 } 1557activate_locked: 1558 /* Not a candidate for swapping, so reclaim swap space. */ 1559 if (folio_test_swapcache(folio) && 1560 (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) 1561 folio_free_swap(folio); 1562 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1563 if (!folio_test_mlocked(folio)) { 1564 int type = folio_is_file_lru(folio); 1565 folio_set_active(folio); 1566 stat->nr_activate[type] += nr_pages; 1567 count_memcg_folio_events(folio, PGACTIVATE, nr_pages); 1568 } 1569keep_locked: 1570 folio_unlock(folio); 1571keep: 1572 list_add(&folio->lru, &ret_folios); 1573 VM_BUG_ON_FOLIO(folio_test_lru(folio) || 1574 folio_test_unevictable(folio), folio); 1575 } 1576 /* 'folio_list' is always empty here */ 1577 1578 /* Migrate folios selected for demotion */ 1579 nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg); 1580 nr_reclaimed += nr_demoted; 1581 stat->nr_demoted += nr_demoted; 1582 /* Folios that could not be demoted are still in @demote_folios */ 1583 if (!list_empty(&demote_folios)) { 1584 /* Folios which weren't demoted go back on @folio_list */ 1585 list_splice_init(&demote_folios, folio_list); 1586 1587 /* 1588 * goto retry to reclaim the undemoted folios in folio_list if 1589 * desired. 1590 * 1591 * Reclaiming directly from top tier nodes is not often desired 1592 * due to it breaking the LRU ordering: in general memory 1593 * should be reclaimed from lower tier nodes and demoted from 1594 * top tier nodes. 1595 * 1596 * However, disabling reclaim from top tier nodes entirely 1597 * would cause ooms in edge scenarios where lower tier memory 1598 * is unreclaimable for whatever reason, eg memory being 1599 * mlocked or too hot to reclaim. We can disable reclaim 1600 * from top tier nodes in proactive reclaim though as that is 1601 * not real memory pressure. 1602 */ 1603 if (!sc->proactive) { 1604 do_demote_pass = false; 1605 goto retry; 1606 } 1607 } 1608 1609 pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; 1610 1611 mem_cgroup_uncharge_folios(&free_folios); 1612 try_to_unmap_flush(); 1613 free_unref_folios(&free_folios); 1614 1615 list_splice(&ret_folios, folio_list); 1616 count_vm_events(PGACTIVATE, pgactivate); 1617 1618 if (plug) 1619 swap_write_unplug(plug); 1620 return nr_reclaimed; 1621} 1622 1623unsigned int reclaim_clean_pages_from_list(struct zone *zone, 1624 struct list_head *folio_list) 1625{ 1626 struct scan_control sc = { 1627 .gfp_mask = GFP_KERNEL, 1628 .may_unmap = 1, 1629 }; 1630 struct reclaim_stat stat; 1631 unsigned int nr_reclaimed; 1632 struct folio *folio, *next; 1633 LIST_HEAD(clean_folios); 1634 unsigned int noreclaim_flag; 1635 1636 list_for_each_entry_safe(folio, next, folio_list, lru) { 1637 /* TODO: these pages should not even appear in this list. */ 1638 if (page_has_movable_ops(&folio->page)) 1639 continue; 1640 if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && 1641 !folio_test_dirty(folio) && !folio_test_unevictable(folio)) { 1642 folio_clear_active(folio); 1643 list_move(&folio->lru, &clean_folios); 1644 } 1645 } 1646 1647 /* 1648 * We should be safe here since we are only dealing with file pages and 1649 * we are not kswapd and therefore cannot write dirty file pages. But 1650 * call memalloc_noreclaim_save() anyway, just in case these conditions 1651 * change in the future. 1652 */ 1653 noreclaim_flag = memalloc_noreclaim_save(); 1654 nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, 1655 &stat, true, NULL); 1656 memalloc_noreclaim_restore(noreclaim_flag); 1657 1658 list_splice(&clean_folios, folio_list); 1659 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 1660 -(long)nr_reclaimed); 1661 /* 1662 * Since lazyfree pages are isolated from file LRU from the beginning, 1663 * they will rotate back to anonymous LRU in the end if it failed to 1664 * discard so isolated count will be mismatched. 1665 * Compensate the isolated count for both LRU lists. 1666 */ 1667 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, 1668 stat.nr_lazyfree_fail); 1669 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 1670 -(long)stat.nr_lazyfree_fail); 1671 return nr_reclaimed; 1672} 1673 1674/* 1675 * Update LRU sizes after isolating pages. The LRU size updates must 1676 * be complete before mem_cgroup_update_lru_size due to a sanity check. 1677 */ 1678static __always_inline void update_lru_sizes(struct lruvec *lruvec, 1679 enum lru_list lru, unsigned long *nr_zone_taken) 1680{ 1681 int zid; 1682 1683 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1684 if (!nr_zone_taken[zid]) 1685 continue; 1686 1687 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); 1688 } 1689 1690} 1691 1692/* 1693 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. 1694 * 1695 * lruvec->lru_lock is heavily contended. Some of the functions that 1696 * shrink the lists perform better by taking out a batch of pages 1697 * and working on them outside the LRU lock. 1698 * 1699 * For pagecache intensive workloads, this function is the hottest 1700 * spot in the kernel (apart from copy_*_user functions). 1701 * 1702 * Lru_lock must be held before calling this function. 1703 * 1704 * @nr_to_scan: The number of eligible pages to look through on the list. 1705 * @lruvec: The LRU vector to pull pages from. 1706 * @dst: The temp list to put pages on to. 1707 * @nr_scanned: The number of pages that were scanned. 1708 * @sc: The scan_control struct for this reclaim session 1709 * @lru: LRU list id for isolating 1710 * 1711 * returns how many pages were moved onto *@dst. 1712 */ 1713static unsigned long isolate_lru_folios(unsigned long nr_to_scan, 1714 struct lruvec *lruvec, struct list_head *dst, 1715 unsigned long *nr_scanned, struct scan_control *sc, 1716 enum lru_list lru) 1717{ 1718 struct list_head *src = &lruvec->lists[lru]; 1719 unsigned long nr_taken = 0; 1720 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; 1721 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; 1722 unsigned long skipped = 0, total_scan = 0, scan = 0; 1723 unsigned long nr_pages; 1724 unsigned long max_nr_skipped = 0; 1725 LIST_HEAD(folios_skipped); 1726 1727 while (scan < nr_to_scan && !list_empty(src)) { 1728 struct list_head *move_to = src; 1729 struct folio *folio; 1730 1731 folio = lru_to_folio(src); 1732 prefetchw_prev_lru_folio(folio, src, flags); 1733 1734 nr_pages = folio_nr_pages(folio); 1735 total_scan += nr_pages; 1736 1737 /* Using max_nr_skipped to prevent hard LOCKUP*/ 1738 if (max_nr_skipped < SWAP_CLUSTER_MAX_SKIPPED && 1739 (folio_zonenum(folio) > sc->reclaim_idx)) { 1740 nr_skipped[folio_zonenum(folio)] += nr_pages; 1741 move_to = &folios_skipped; 1742 max_nr_skipped++; 1743 goto move; 1744 } 1745 1746 /* 1747 * Do not count skipped folios because that makes the function 1748 * return with no isolated folios if the LRU mostly contains 1749 * ineligible folios. This causes the VM to not reclaim any 1750 * folios, triggering a premature OOM. 1751 * Account all pages in a folio. 1752 */ 1753 scan += nr_pages; 1754 1755 if (!folio_test_lru(folio)) 1756 goto move; 1757 if (!sc->may_unmap && folio_mapped(folio)) 1758 goto move; 1759 1760 /* 1761 * Be careful not to clear the lru flag until after we're 1762 * sure the folio is not being freed elsewhere -- the 1763 * folio release code relies on it. 1764 */ 1765 if (unlikely(!folio_try_get(folio))) 1766 goto move; 1767 1768 if (!folio_test_clear_lru(folio)) { 1769 /* Another thread is already isolating this folio */ 1770 folio_put(folio); 1771 goto move; 1772 } 1773 1774 nr_taken += nr_pages; 1775 nr_zone_taken[folio_zonenum(folio)] += nr_pages; 1776 move_to = dst; 1777move: 1778 list_move(&folio->lru, move_to); 1779 } 1780 1781 /* 1782 * Splice any skipped folios to the start of the LRU list. Note that 1783 * this disrupts the LRU order when reclaiming for lower zones but 1784 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX 1785 * scanning would soon rescan the same folios to skip and waste lots 1786 * of cpu cycles. 1787 */ 1788 if (!list_empty(&folios_skipped)) { 1789 int zid; 1790 1791 list_splice(&folios_skipped, src); 1792 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1793 if (!nr_skipped[zid]) 1794 continue; 1795 1796 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); 1797 skipped += nr_skipped[zid]; 1798 } 1799 } 1800 *nr_scanned = total_scan; 1801 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, 1802 total_scan, skipped, nr_taken, lru); 1803 update_lru_sizes(lruvec, lru, nr_zone_taken); 1804 return nr_taken; 1805} 1806 1807/** 1808 * folio_isolate_lru() - Try to isolate a folio from its LRU list. 1809 * @folio: Folio to isolate from its LRU list. 1810 * 1811 * Isolate a @folio from an LRU list and adjust the vmstat statistic 1812 * corresponding to whatever LRU list the folio was on. 1813 * 1814 * The folio will have its LRU flag cleared. If it was found on the 1815 * active list, it will have the Active flag set. If it was found on the 1816 * unevictable list, it will have the Unevictable flag set. These flags 1817 * may need to be cleared by the caller before letting the page go. 1818 * 1819 * Context: 1820 * 1821 * (1) Must be called with an elevated refcount on the folio. This is a 1822 * fundamental difference from isolate_lru_folios() (which is called 1823 * without a stable reference). 1824 * (2) The lru_lock must not be held. 1825 * (3) Interrupts must be enabled. 1826 * 1827 * Return: true if the folio was removed from an LRU list. 1828 * false if the folio was not on an LRU list. 1829 */ 1830bool folio_isolate_lru(struct folio *folio) 1831{ 1832 bool ret = false; 1833 1834 VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); 1835 1836 if (folio_test_clear_lru(folio)) { 1837 struct lruvec *lruvec; 1838 1839 folio_get(folio); 1840 lruvec = folio_lruvec_lock_irq(folio); 1841 lruvec_del_folio(lruvec, folio); 1842 unlock_page_lruvec_irq(lruvec); 1843 ret = true; 1844 } 1845 1846 return ret; 1847} 1848 1849/* 1850 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and 1851 * then get rescheduled. When there are massive number of tasks doing page 1852 * allocation, such sleeping direct reclaimers may keep piling up on each CPU, 1853 * the LRU list will go small and be scanned faster than necessary, leading to 1854 * unnecessary swapping, thrashing and OOM. 1855 */ 1856static bool too_many_isolated(struct pglist_data *pgdat, int file, 1857 struct scan_control *sc) 1858{ 1859 unsigned long inactive, isolated; 1860 bool too_many; 1861 1862 if (current_is_kswapd()) 1863 return false; 1864 1865 if (!writeback_throttling_sane(sc)) 1866 return false; 1867 1868 if (file) { 1869 inactive = node_page_state(pgdat, NR_INACTIVE_FILE); 1870 isolated = node_page_state(pgdat, NR_ISOLATED_FILE); 1871 } else { 1872 inactive = node_page_state(pgdat, NR_INACTIVE_ANON); 1873 isolated = node_page_state(pgdat, NR_ISOLATED_ANON); 1874 } 1875 1876 /* 1877 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they 1878 * won't get blocked by normal direct-reclaimers, forming a circular 1879 * deadlock. 1880 */ 1881 if (gfp_has_io_fs(sc->gfp_mask)) 1882 inactive >>= 3; 1883 1884 too_many = isolated > inactive; 1885 1886 /* Wake up tasks throttled due to too_many_isolated. */ 1887 if (!too_many) 1888 wake_throttle_isolated(pgdat); 1889 1890 return too_many; 1891} 1892 1893/* 1894 * move_folios_to_lru() moves folios from private @list to appropriate LRU list. 1895 * 1896 * Returns the number of pages moved to the given lruvec. 1897 */ 1898static unsigned int move_folios_to_lru(struct lruvec *lruvec, 1899 struct list_head *list) 1900{ 1901 int nr_pages, nr_moved = 0; 1902 struct folio_batch free_folios; 1903 1904 folio_batch_init(&free_folios); 1905 while (!list_empty(list)) { 1906 struct folio *folio = lru_to_folio(list); 1907 1908 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 1909 list_del(&folio->lru); 1910 if (unlikely(!folio_evictable(folio))) { 1911 spin_unlock_irq(&lruvec->lru_lock); 1912 folio_putback_lru(folio); 1913 spin_lock_irq(&lruvec->lru_lock); 1914 continue; 1915 } 1916 1917 /* 1918 * The folio_set_lru needs to be kept here for list integrity. 1919 * Otherwise: 1920 * #0 move_folios_to_lru #1 release_pages 1921 * if (!folio_put_testzero()) 1922 * if (folio_put_testzero()) 1923 * !lru //skip lru_lock 1924 * folio_set_lru() 1925 * list_add(&folio->lru,) 1926 * list_add(&folio->lru,) 1927 */ 1928 folio_set_lru(folio); 1929 1930 if (unlikely(folio_put_testzero(folio))) { 1931 __folio_clear_lru_flags(folio); 1932 1933 folio_unqueue_deferred_split(folio); 1934 if (folio_batch_add(&free_folios, folio) == 0) { 1935 spin_unlock_irq(&lruvec->lru_lock); 1936 mem_cgroup_uncharge_folios(&free_folios); 1937 free_unref_folios(&free_folios); 1938 spin_lock_irq(&lruvec->lru_lock); 1939 } 1940 1941 continue; 1942 } 1943 1944 /* 1945 * All pages were isolated from the same lruvec (and isolation 1946 * inhibits memcg migration). 1947 */ 1948 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); 1949 lruvec_add_folio(lruvec, folio); 1950 nr_pages = folio_nr_pages(folio); 1951 nr_moved += nr_pages; 1952 if (folio_test_active(folio)) 1953 workingset_age_nonresident(lruvec, nr_pages); 1954 } 1955 1956 if (free_folios.nr) { 1957 spin_unlock_irq(&lruvec->lru_lock); 1958 mem_cgroup_uncharge_folios(&free_folios); 1959 free_unref_folios(&free_folios); 1960 spin_lock_irq(&lruvec->lru_lock); 1961 } 1962 1963 return nr_moved; 1964} 1965 1966/* 1967 * If a kernel thread (such as nfsd for loop-back mounts) services a backing 1968 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case 1969 * we should not throttle. Otherwise it is safe to do so. 1970 */ 1971static int current_may_throttle(void) 1972{ 1973 return !(current->flags & PF_LOCAL_THROTTLE); 1974} 1975 1976/* 1977 * shrink_inactive_list() is a helper for shrink_node(). It returns the number 1978 * of reclaimed pages 1979 */ 1980static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 1981 struct lruvec *lruvec, struct scan_control *sc, 1982 enum lru_list lru) 1983{ 1984 LIST_HEAD(folio_list); 1985 unsigned long nr_scanned; 1986 unsigned int nr_reclaimed = 0; 1987 unsigned long nr_taken; 1988 struct reclaim_stat stat; 1989 bool file = is_file_lru(lru); 1990 enum vm_event_item item; 1991 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 1992 bool stalled = false; 1993 1994 while (unlikely(too_many_isolated(pgdat, file, sc))) { 1995 if (stalled) 1996 return 0; 1997 1998 /* wait a bit for the reclaimer. */ 1999 stalled = true; 2000 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); 2001 2002 /* We are about to die and free our memory. Return now. */ 2003 if (fatal_signal_pending(current)) 2004 return SWAP_CLUSTER_MAX; 2005 } 2006 2007 lru_add_drain(); 2008 2009 spin_lock_irq(&lruvec->lru_lock); 2010 2011 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, 2012 &nr_scanned, sc, lru); 2013 2014 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2015 item = PGSCAN_KSWAPD + reclaimer_offset(sc); 2016 if (!cgroup_reclaim(sc)) 2017 __count_vm_events(item, nr_scanned); 2018 count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); 2019 __count_vm_events(PGSCAN_ANON + file, nr_scanned); 2020 2021 spin_unlock_irq(&lruvec->lru_lock); 2022 2023 if (nr_taken == 0) 2024 return 0; 2025 2026 nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false, 2027 lruvec_memcg(lruvec)); 2028 2029 spin_lock_irq(&lruvec->lru_lock); 2030 move_folios_to_lru(lruvec, &folio_list); 2031 2032 mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), 2033 stat.nr_demoted); 2034 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2035 item = PGSTEAL_KSWAPD + reclaimer_offset(sc); 2036 if (!cgroup_reclaim(sc)) 2037 __count_vm_events(item, nr_reclaimed); 2038 count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); 2039 __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); 2040 2041 lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout, 2042 nr_scanned - nr_reclaimed); 2043 2044 /* 2045 * If dirty folios are scanned that are not queued for IO, it 2046 * implies that flushers are not doing their job. This can 2047 * happen when memory pressure pushes dirty folios to the end of 2048 * the LRU before the dirty limits are breached and the dirty 2049 * data has expired. It can also happen when the proportion of 2050 * dirty folios grows not through writes but through memory 2051 * pressure reclaiming all the clean cache. And in some cases, 2052 * the flushers simply cannot keep up with the allocation 2053 * rate. Nudge the flusher threads in case they are asleep. 2054 */ 2055 if (stat.nr_unqueued_dirty == nr_taken) { 2056 wakeup_flusher_threads(WB_REASON_VMSCAN); 2057 /* 2058 * For cgroupv1 dirty throttling is achieved by waking up 2059 * the kernel flusher here and later waiting on folios 2060 * which are in writeback to finish (see shrink_folio_list()). 2061 * 2062 * Flusher may not be able to issue writeback quickly 2063 * enough for cgroupv1 writeback throttling to work 2064 * on a large system. 2065 */ 2066 if (!writeback_throttling_sane(sc)) 2067 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 2068 } 2069 2070 sc->nr.dirty += stat.nr_dirty; 2071 sc->nr.congested += stat.nr_congested; 2072 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 2073 sc->nr.writeback += stat.nr_writeback; 2074 sc->nr.immediate += stat.nr_immediate; 2075 sc->nr.taken += nr_taken; 2076 if (file) 2077 sc->nr.file_taken += nr_taken; 2078 2079 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 2080 nr_scanned, nr_reclaimed, &stat, sc->priority, file); 2081 return nr_reclaimed; 2082} 2083 2084/* 2085 * shrink_active_list() moves folios from the active LRU to the inactive LRU. 2086 * 2087 * We move them the other way if the folio is referenced by one or more 2088 * processes. 2089 * 2090 * If the folios are mostly unmapped, the processing is fast and it is 2091 * appropriate to hold lru_lock across the whole operation. But if 2092 * the folios are mapped, the processing is slow (folio_referenced()), so 2093 * we should drop lru_lock around each folio. It's impossible to balance 2094 * this, so instead we remove the folios from the LRU while processing them. 2095 * It is safe to rely on the active flag against the non-LRU folios in here 2096 * because nobody will play with that bit on a non-LRU folio. 2097 * 2098 * The downside is that we have to touch folio->_refcount against each folio. 2099 * But we had to alter folio->flags anyway. 2100 */ 2101static void shrink_active_list(unsigned long nr_to_scan, 2102 struct lruvec *lruvec, 2103 struct scan_control *sc, 2104 enum lru_list lru) 2105{ 2106 unsigned long nr_taken; 2107 unsigned long nr_scanned; 2108 vm_flags_t vm_flags; 2109 LIST_HEAD(l_hold); /* The folios which were snipped off */ 2110 LIST_HEAD(l_active); 2111 LIST_HEAD(l_inactive); 2112 unsigned nr_deactivate, nr_activate; 2113 unsigned nr_rotated = 0; 2114 bool file = is_file_lru(lru); 2115 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2116 2117 lru_add_drain(); 2118 2119 spin_lock_irq(&lruvec->lru_lock); 2120 2121 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, 2122 &nr_scanned, sc, lru); 2123 2124 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2125 2126 if (!cgroup_reclaim(sc)) 2127 __count_vm_events(PGREFILL, nr_scanned); 2128 count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); 2129 2130 spin_unlock_irq(&lruvec->lru_lock); 2131 2132 while (!list_empty(&l_hold)) { 2133 struct folio *folio; 2134 2135 cond_resched(); 2136 folio = lru_to_folio(&l_hold); 2137 list_del(&folio->lru); 2138 2139 if (unlikely(!folio_evictable(folio))) { 2140 folio_putback_lru(folio); 2141 continue; 2142 } 2143 2144 if (unlikely(buffer_heads_over_limit)) { 2145 if (folio_needs_release(folio) && 2146 folio_trylock(folio)) { 2147 filemap_release_folio(folio, 0); 2148 folio_unlock(folio); 2149 } 2150 } 2151 2152 /* Referenced or rmap lock contention: rotate */ 2153 if (folio_referenced(folio, 0, sc->target_mem_cgroup, 2154 &vm_flags) != 0) { 2155 /* 2156 * Identify referenced, file-backed active folios and 2157 * give them one more trip around the active list. So 2158 * that executable code get better chances to stay in 2159 * memory under moderate memory pressure. Anon folios 2160 * are not likely to be evicted by use-once streaming 2161 * IO, plus JVM can create lots of anon VM_EXEC folios, 2162 * so we ignore them here. 2163 */ 2164 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { 2165 nr_rotated += folio_nr_pages(folio); 2166 list_add(&folio->lru, &l_active); 2167 continue; 2168 } 2169 } 2170 2171 folio_clear_active(folio); /* we are de-activating */ 2172 folio_set_workingset(folio); 2173 list_add(&folio->lru, &l_inactive); 2174 } 2175 2176 /* 2177 * Move folios back to the lru list. 2178 */ 2179 spin_lock_irq(&lruvec->lru_lock); 2180 2181 nr_activate = move_folios_to_lru(lruvec, &l_active); 2182 nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); 2183 2184 __count_vm_events(PGDEACTIVATE, nr_deactivate); 2185 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2186 2187 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2188 2189 lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated); 2190 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2191 nr_deactivate, nr_rotated, sc->priority, file); 2192} 2193 2194static unsigned int reclaim_folio_list(struct list_head *folio_list, 2195 struct pglist_data *pgdat) 2196{ 2197 struct reclaim_stat stat; 2198 unsigned int nr_reclaimed; 2199 struct folio *folio; 2200 struct scan_control sc = { 2201 .gfp_mask = GFP_KERNEL, 2202 .may_writepage = 1, 2203 .may_unmap = 1, 2204 .may_swap = 1, 2205 .no_demotion = 1, 2206 }; 2207 2208 nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &stat, true, NULL); 2209 while (!list_empty(folio_list)) { 2210 folio = lru_to_folio(folio_list); 2211 list_del(&folio->lru); 2212 folio_putback_lru(folio); 2213 } 2214 trace_mm_vmscan_reclaim_pages(pgdat->node_id, sc.nr_scanned, nr_reclaimed, &stat); 2215 2216 return nr_reclaimed; 2217} 2218 2219unsigned long reclaim_pages(struct list_head *folio_list) 2220{ 2221 int nid; 2222 unsigned int nr_reclaimed = 0; 2223 LIST_HEAD(node_folio_list); 2224 unsigned int noreclaim_flag; 2225 2226 if (list_empty(folio_list)) 2227 return nr_reclaimed; 2228 2229 noreclaim_flag = memalloc_noreclaim_save(); 2230 2231 nid = folio_nid(lru_to_folio(folio_list)); 2232 do { 2233 struct folio *folio = lru_to_folio(folio_list); 2234 2235 if (nid == folio_nid(folio)) { 2236 folio_clear_active(folio); 2237 list_move(&folio->lru, &node_folio_list); 2238 continue; 2239 } 2240 2241 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2242 nid = folio_nid(lru_to_folio(folio_list)); 2243 } while (!list_empty(folio_list)); 2244 2245 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2246 2247 memalloc_noreclaim_restore(noreclaim_flag); 2248 2249 return nr_reclaimed; 2250} 2251 2252static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, 2253 struct lruvec *lruvec, struct scan_control *sc) 2254{ 2255 if (is_active_lru(lru)) { 2256 if (sc->may_deactivate & (1 << is_file_lru(lru))) 2257 shrink_active_list(nr_to_scan, lruvec, sc, lru); 2258 else 2259 sc->skipped_deactivate = 1; 2260 return 0; 2261 } 2262 2263 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); 2264} 2265 2266/* 2267 * The inactive anon list should be small enough that the VM never has 2268 * to do too much work. 2269 * 2270 * The inactive file list should be small enough to leave most memory 2271 * to the established workingset on the scan-resistant active list, 2272 * but large enough to avoid thrashing the aggregate readahead window. 2273 * 2274 * Both inactive lists should also be large enough that each inactive 2275 * folio has a chance to be referenced again before it is reclaimed. 2276 * 2277 * If that fails and refaulting is observed, the inactive list grows. 2278 * 2279 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios 2280 * on this LRU, maintained by the pageout code. An inactive_ratio 2281 * of 3 means 3:1 or 25% of the folios are kept on the inactive list. 2282 * 2283 * total target max 2284 * memory ratio inactive 2285 * ------------------------------------- 2286 * 10MB 1 5MB 2287 * 100MB 1 50MB 2288 * 1GB 3 250MB 2289 * 10GB 10 0.9GB 2290 * 100GB 31 3GB 2291 * 1TB 101 10GB 2292 * 10TB 320 32GB 2293 */ 2294static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) 2295{ 2296 enum lru_list active_lru = inactive_lru + LRU_ACTIVE; 2297 unsigned long inactive, active; 2298 unsigned long inactive_ratio; 2299 unsigned long gb; 2300 2301 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); 2302 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); 2303 2304 gb = (inactive + active) >> (30 - PAGE_SHIFT); 2305 if (gb) 2306 inactive_ratio = int_sqrt(10 * gb); 2307 else 2308 inactive_ratio = 1; 2309 2310 return inactive * inactive_ratio < active; 2311} 2312 2313enum scan_balance { 2314 SCAN_EQUAL, 2315 SCAN_FRACT, 2316 SCAN_ANON, 2317 SCAN_FILE, 2318}; 2319 2320static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) 2321{ 2322 unsigned long file; 2323 struct lruvec *target_lruvec; 2324 2325 if (lru_gen_enabled()) 2326 return; 2327 2328 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 2329 2330 /* 2331 * Flush the memory cgroup stats in rate-limited way as we don't need 2332 * most accurate stats here. We may switch to regular stats flushing 2333 * in the future once it is cheap enough. 2334 */ 2335 mem_cgroup_flush_stats_ratelimited(sc->target_mem_cgroup); 2336 2337 /* 2338 * Determine the scan balance between anon and file LRUs. 2339 */ 2340 spin_lock_irq(&target_lruvec->lru_lock); 2341 sc->anon_cost = target_lruvec->anon_cost; 2342 sc->file_cost = target_lruvec->file_cost; 2343 spin_unlock_irq(&target_lruvec->lru_lock); 2344 2345 /* 2346 * Target desirable inactive:active list ratios for the anon 2347 * and file LRU lists. 2348 */ 2349 if (!sc->force_deactivate) { 2350 unsigned long refaults; 2351 2352 /* 2353 * When refaults are being observed, it means a new 2354 * workingset is being established. Deactivate to get 2355 * rid of any stale active pages quickly. 2356 */ 2357 refaults = lruvec_page_state(target_lruvec, 2358 WORKINGSET_ACTIVATE_ANON); 2359 if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || 2360 inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) 2361 sc->may_deactivate |= DEACTIVATE_ANON; 2362 else 2363 sc->may_deactivate &= ~DEACTIVATE_ANON; 2364 2365 refaults = lruvec_page_state(target_lruvec, 2366 WORKINGSET_ACTIVATE_FILE); 2367 if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || 2368 inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) 2369 sc->may_deactivate |= DEACTIVATE_FILE; 2370 else 2371 sc->may_deactivate &= ~DEACTIVATE_FILE; 2372 } else 2373 sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; 2374 2375 /* 2376 * If we have plenty of inactive file pages that aren't 2377 * thrashing, try to reclaim those first before touching 2378 * anonymous pages. 2379 */ 2380 file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); 2381 if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE) && 2382 !sc->no_cache_trim_mode) 2383 sc->cache_trim_mode = 1; 2384 else 2385 sc->cache_trim_mode = 0; 2386 2387 /* 2388 * Prevent the reclaimer from falling into the cache trap: as 2389 * cache pages start out inactive, every cache fault will tip 2390 * the scan balance towards the file LRU. And as the file LRU 2391 * shrinks, so does the window for rotation from references. 2392 * This means we have a runaway feedback loop where a tiny 2393 * thrashing file LRU becomes infinitely more attractive than 2394 * anon pages. Try to detect this based on file LRU size. 2395 */ 2396 if (!cgroup_reclaim(sc)) { 2397 unsigned long total_high_wmark = 0; 2398 unsigned long free, anon; 2399 int z; 2400 struct zone *zone; 2401 2402 free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); 2403 file = node_page_state(pgdat, NR_ACTIVE_FILE) + 2404 node_page_state(pgdat, NR_INACTIVE_FILE); 2405 2406 for_each_managed_zone_pgdat(zone, pgdat, z, MAX_NR_ZONES - 1) { 2407 total_high_wmark += high_wmark_pages(zone); 2408 } 2409 2410 /* 2411 * Consider anon: if that's low too, this isn't a 2412 * runaway file reclaim problem, but rather just 2413 * extreme pressure. Reclaim as per usual then. 2414 */ 2415 anon = node_page_state(pgdat, NR_INACTIVE_ANON); 2416 2417 sc->file_is_tiny = 2418 file + free <= total_high_wmark && 2419 !(sc->may_deactivate & DEACTIVATE_ANON) && 2420 anon >> sc->priority; 2421 } 2422} 2423 2424static inline void calculate_pressure_balance(struct scan_control *sc, 2425 int swappiness, u64 *fraction, u64 *denominator) 2426{ 2427 unsigned long anon_cost, file_cost, total_cost; 2428 unsigned long ap, fp; 2429 2430 /* 2431 * Calculate the pressure balance between anon and file pages. 2432 * 2433 * The amount of pressure we put on each LRU is inversely 2434 * proportional to the cost of reclaiming each list, as 2435 * determined by the share of pages that are refaulting, times 2436 * the relative IO cost of bringing back a swapped out 2437 * anonymous page vs reloading a filesystem page (swappiness). 2438 * 2439 * Although we limit that influence to ensure no list gets 2440 * left behind completely: at least a third of the pressure is 2441 * applied, before swappiness. 2442 * 2443 * With swappiness at 100, anon and file have equal IO cost. 2444 */ 2445 total_cost = sc->anon_cost + sc->file_cost; 2446 anon_cost = total_cost + sc->anon_cost; 2447 file_cost = total_cost + sc->file_cost; 2448 total_cost = anon_cost + file_cost; 2449 2450 ap = swappiness * (total_cost + 1); 2451 ap /= anon_cost + 1; 2452 2453 fp = (MAX_SWAPPINESS - swappiness) * (total_cost + 1); 2454 fp /= file_cost + 1; 2455 2456 fraction[WORKINGSET_ANON] = ap; 2457 fraction[WORKINGSET_FILE] = fp; 2458 *denominator = ap + fp; 2459} 2460 2461static unsigned long apply_proportional_protection(struct mem_cgroup *memcg, 2462 struct scan_control *sc, unsigned long scan) 2463{ 2464 unsigned long min, low; 2465 2466 mem_cgroup_protection(sc->target_mem_cgroup, memcg, &min, &low); 2467 2468 if (min || low) { 2469 /* 2470 * Scale a cgroup's reclaim pressure by proportioning 2471 * its current usage to its memory.low or memory.min 2472 * setting. 2473 * 2474 * This is important, as otherwise scanning aggression 2475 * becomes extremely binary -- from nothing as we 2476 * approach the memory protection threshold, to totally 2477 * nominal as we exceed it. This results in requiring 2478 * setting extremely liberal protection thresholds. It 2479 * also means we simply get no protection at all if we 2480 * set it too low, which is not ideal. 2481 * 2482 * If there is any protection in place, we reduce scan 2483 * pressure by how much of the total memory used is 2484 * within protection thresholds. 2485 * 2486 * There is one special case: in the first reclaim pass, 2487 * we skip over all groups that are within their low 2488 * protection. If that fails to reclaim enough pages to 2489 * satisfy the reclaim goal, we come back and override 2490 * the best-effort low protection. However, we still 2491 * ideally want to honor how well-behaved groups are in 2492 * that case instead of simply punishing them all 2493 * equally. As such, we reclaim them based on how much 2494 * memory they are using, reducing the scan pressure 2495 * again by how much of the total memory used is under 2496 * hard protection. 2497 */ 2498 unsigned long cgroup_size = mem_cgroup_size(memcg); 2499 unsigned long protection; 2500 2501 /* memory.low scaling, make sure we retry before OOM */ 2502 if (!sc->memcg_low_reclaim && low > min) { 2503 protection = low; 2504 sc->memcg_low_skipped = 1; 2505 } else { 2506 protection = min; 2507 } 2508 2509 /* Avoid TOCTOU with earlier protection check */ 2510 cgroup_size = max(cgroup_size, protection); 2511 2512 scan -= scan * protection / (cgroup_size + 1); 2513 2514 /* 2515 * Minimally target SWAP_CLUSTER_MAX pages to keep 2516 * reclaim moving forwards, avoiding decrementing 2517 * sc->priority further than desirable. 2518 */ 2519 scan = max(scan, SWAP_CLUSTER_MAX); 2520 } 2521 return scan; 2522} 2523 2524/* 2525 * Determine how aggressively the anon and file LRU lists should be 2526 * scanned. 2527 * 2528 * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan 2529 * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan 2530 */ 2531static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, 2532 unsigned long *nr) 2533{ 2534 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2535 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2536 int swappiness = sc_swappiness(sc, memcg); 2537 u64 fraction[ANON_AND_FILE]; 2538 u64 denominator = 0; /* gcc */ 2539 enum scan_balance scan_balance; 2540 enum lru_list lru; 2541 2542 /* If we have no swap space, do not bother scanning anon folios. */ 2543 if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { 2544 scan_balance = SCAN_FILE; 2545 goto out; 2546 } 2547 2548 /* 2549 * Global reclaim will swap to prevent OOM even with no 2550 * swappiness, but memcg users want to use this knob to 2551 * disable swapping for individual groups completely when 2552 * using the memory controller's swap limit feature would be 2553 * too expensive. 2554 */ 2555 if (cgroup_reclaim(sc) && !swappiness) { 2556 scan_balance = SCAN_FILE; 2557 goto out; 2558 } 2559 2560 /* Proactive reclaim initiated by userspace for anonymous memory only */ 2561 if (swappiness == SWAPPINESS_ANON_ONLY) { 2562 WARN_ON_ONCE(!sc->proactive); 2563 scan_balance = SCAN_ANON; 2564 goto out; 2565 } 2566 2567 /* 2568 * Do not apply any pressure balancing cleverness when the 2569 * system is close to OOM, scan both anon and file equally 2570 * (unless the swappiness setting disagrees with swapping). 2571 */ 2572 if (!sc->priority && swappiness) { 2573 scan_balance = SCAN_EQUAL; 2574 goto out; 2575 } 2576 2577 /* 2578 * If the system is almost out of file pages, force-scan anon. 2579 */ 2580 if (sc->file_is_tiny) { 2581 scan_balance = SCAN_ANON; 2582 goto out; 2583 } 2584 2585 /* 2586 * If there is enough inactive page cache, we do not reclaim 2587 * anything from the anonymous working right now to make sure 2588 * a streaming file access pattern doesn't cause swapping. 2589 */ 2590 if (sc->cache_trim_mode) { 2591 scan_balance = SCAN_FILE; 2592 goto out; 2593 } 2594 2595 scan_balance = SCAN_FRACT; 2596 calculate_pressure_balance(sc, swappiness, fraction, &denominator); 2597 2598out: 2599 for_each_evictable_lru(lru) { 2600 bool file = is_file_lru(lru); 2601 unsigned long lruvec_size; 2602 unsigned long scan; 2603 2604 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); 2605 scan = apply_proportional_protection(memcg, sc, lruvec_size); 2606 scan >>= sc->priority; 2607 2608 /* 2609 * If the cgroup's already been deleted, make sure to 2610 * scrape out the remaining cache. 2611 */ 2612 if (!scan && !mem_cgroup_online(memcg)) 2613 scan = min(lruvec_size, SWAP_CLUSTER_MAX); 2614 2615 switch (scan_balance) { 2616 case SCAN_EQUAL: 2617 /* Scan lists relative to size */ 2618 break; 2619 case SCAN_FRACT: 2620 /* 2621 * Scan types proportional to swappiness and 2622 * their relative recent reclaim efficiency. 2623 * Make sure we don't miss the last page on 2624 * the offlined memory cgroups because of a 2625 * round-off error. 2626 */ 2627 scan = mem_cgroup_online(memcg) ? 2628 div64_u64(scan * fraction[file], denominator) : 2629 DIV64_U64_ROUND_UP(scan * fraction[file], 2630 denominator); 2631 break; 2632 case SCAN_FILE: 2633 case SCAN_ANON: 2634 /* Scan one type exclusively */ 2635 if ((scan_balance == SCAN_FILE) != file) 2636 scan = 0; 2637 break; 2638 default: 2639 /* Look ma, no brain */ 2640 BUG(); 2641 } 2642 2643 nr[lru] = scan; 2644 } 2645} 2646 2647/* 2648 * Anonymous LRU management is a waste if there is 2649 * ultimately no way to reclaim the memory. 2650 */ 2651static bool can_age_anon_pages(struct lruvec *lruvec, 2652 struct scan_control *sc) 2653{ 2654 /* Aging the anon LRU is valuable if swap is present: */ 2655 if (total_swap_pages > 0) 2656 return true; 2657 2658 /* Also valuable if anon pages can be demoted: */ 2659 return can_demote(lruvec_pgdat(lruvec)->node_id, sc, 2660 lruvec_memcg(lruvec)); 2661} 2662 2663#ifdef CONFIG_LRU_GEN 2664 2665#ifdef CONFIG_LRU_GEN_ENABLED 2666DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); 2667#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) 2668#else 2669DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); 2670#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) 2671#endif 2672 2673static bool should_walk_mmu(void) 2674{ 2675 return arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK); 2676} 2677 2678static bool should_clear_pmd_young(void) 2679{ 2680 return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG); 2681} 2682 2683/****************************************************************************** 2684 * shorthand helpers 2685 ******************************************************************************/ 2686 2687#define DEFINE_MAX_SEQ(lruvec) \ 2688 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) 2689 2690#define DEFINE_MIN_SEQ(lruvec) \ 2691 unsigned long min_seq[ANON_AND_FILE] = { \ 2692 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ 2693 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ 2694 } 2695 2696/* Get the min/max evictable type based on swappiness */ 2697#define min_type(swappiness) (!(swappiness)) 2698#define max_type(swappiness) ((swappiness) < SWAPPINESS_ANON_ONLY) 2699 2700#define evictable_min_seq(min_seq, swappiness) \ 2701 min((min_seq)[min_type(swappiness)], (min_seq)[max_type(swappiness)]) 2702 2703#define for_each_gen_type_zone(gen, type, zone) \ 2704 for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ 2705 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ 2706 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) 2707 2708#define for_each_evictable_type(type, swappiness) \ 2709 for ((type) = min_type(swappiness); (type) <= max_type(swappiness); (type)++) 2710 2711#define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS) 2712#define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS) 2713 2714static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) 2715{ 2716 struct pglist_data *pgdat = NODE_DATA(nid); 2717 2718#ifdef CONFIG_MEMCG 2719 if (memcg) { 2720 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; 2721 2722 /* see the comment in mem_cgroup_lruvec() */ 2723 if (!lruvec->pgdat) 2724 lruvec->pgdat = pgdat; 2725 2726 return lruvec; 2727 } 2728#endif 2729 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 2730 2731 return &pgdat->__lruvec; 2732} 2733 2734static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) 2735{ 2736 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2737 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2738 2739 if (!sc->may_swap) 2740 return 0; 2741 2742 if (!can_demote(pgdat->node_id, sc, memcg) && 2743 mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) 2744 return 0; 2745 2746 return sc_swappiness(sc, memcg); 2747} 2748 2749static int get_nr_gens(struct lruvec *lruvec, int type) 2750{ 2751 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; 2752} 2753 2754static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) 2755{ 2756 int type; 2757 2758 for (type = 0; type < ANON_AND_FILE; type++) { 2759 int n = get_nr_gens(lruvec, type); 2760 2761 if (n < MIN_NR_GENS || n > MAX_NR_GENS) 2762 return false; 2763 } 2764 2765 return true; 2766} 2767 2768/****************************************************************************** 2769 * Bloom filters 2770 ******************************************************************************/ 2771 2772/* 2773 * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when 2774 * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of 2775 * bits in a bitmap, k is the number of hash functions and n is the number of 2776 * inserted items. 2777 * 2778 * Page table walkers use one of the two filters to reduce their search space. 2779 * To get rid of non-leaf entries that no longer have enough leaf entries, the 2780 * aging uses the double-buffering technique to flip to the other filter each 2781 * time it produces a new generation. For non-leaf entries that have enough 2782 * leaf entries, the aging carries them over to the next generation in 2783 * walk_pmd_range(); the eviction also report them when walking the rmap 2784 * in lru_gen_look_around(). 2785 * 2786 * For future optimizations: 2787 * 1. It's not necessary to keep both filters all the time. The spare one can be 2788 * freed after the RCU grace period and reallocated if needed again. 2789 * 2. And when reallocating, it's worth scaling its size according to the number 2790 * of inserted entries in the other filter, to reduce the memory overhead on 2791 * small systems and false positives on large systems. 2792 * 3. Jenkins' hash function is an alternative to Knuth's. 2793 */ 2794#define BLOOM_FILTER_SHIFT 15 2795 2796static inline int filter_gen_from_seq(unsigned long seq) 2797{ 2798 return seq % NR_BLOOM_FILTERS; 2799} 2800 2801static void get_item_key(void *item, int *key) 2802{ 2803 u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); 2804 2805 BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); 2806 2807 key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); 2808 key[1] = hash >> BLOOM_FILTER_SHIFT; 2809} 2810 2811static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, 2812 void *item) 2813{ 2814 int key[2]; 2815 unsigned long *filter; 2816 int gen = filter_gen_from_seq(seq); 2817 2818 filter = READ_ONCE(mm_state->filters[gen]); 2819 if (!filter) 2820 return true; 2821 2822 get_item_key(item, key); 2823 2824 return test_bit(key[0], filter) && test_bit(key[1], filter); 2825} 2826 2827static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, 2828 void *item) 2829{ 2830 int key[2]; 2831 unsigned long *filter; 2832 int gen = filter_gen_from_seq(seq); 2833 2834 filter = READ_ONCE(mm_state->filters[gen]); 2835 if (!filter) 2836 return; 2837 2838 get_item_key(item, key); 2839 2840 if (!test_bit(key[0], filter)) 2841 set_bit(key[0], filter); 2842 if (!test_bit(key[1], filter)) 2843 set_bit(key[1], filter); 2844} 2845 2846static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) 2847{ 2848 unsigned long *filter; 2849 int gen = filter_gen_from_seq(seq); 2850 2851 filter = mm_state->filters[gen]; 2852 if (filter) { 2853 bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); 2854 return; 2855 } 2856 2857 filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), 2858 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 2859 WRITE_ONCE(mm_state->filters[gen], filter); 2860} 2861 2862/****************************************************************************** 2863 * mm_struct list 2864 ******************************************************************************/ 2865 2866#ifdef CONFIG_LRU_GEN_WALKS_MMU 2867 2868static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 2869{ 2870 static struct lru_gen_mm_list mm_list = { 2871 .fifo = LIST_HEAD_INIT(mm_list.fifo), 2872 .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), 2873 }; 2874 2875#ifdef CONFIG_MEMCG 2876 if (memcg) 2877 return &memcg->mm_list; 2878#endif 2879 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 2880 2881 return &mm_list; 2882} 2883 2884static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec) 2885{ 2886 return &lruvec->mm_state; 2887} 2888 2889static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk) 2890{ 2891 int key; 2892 struct mm_struct *mm; 2893 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 2894 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); 2895 2896 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); 2897 key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); 2898 2899 if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) 2900 return NULL; 2901 2902 clear_bit(key, &mm->lru_gen.bitmap); 2903 2904 return mmget_not_zero(mm) ? mm : NULL; 2905} 2906 2907void lru_gen_add_mm(struct mm_struct *mm) 2908{ 2909 int nid; 2910 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); 2911 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 2912 2913 VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); 2914#ifdef CONFIG_MEMCG 2915 VM_WARN_ON_ONCE(mm->lru_gen.memcg); 2916 mm->lru_gen.memcg = memcg; 2917#endif 2918 spin_lock(&mm_list->lock); 2919 2920 for_each_node_state(nid, N_MEMORY) { 2921 struct lruvec *lruvec = get_lruvec(memcg, nid); 2922 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 2923 2924 /* the first addition since the last iteration */ 2925 if (mm_state->tail == &mm_list->fifo) 2926 mm_state->tail = &mm->lru_gen.list; 2927 } 2928 2929 list_add_tail(&mm->lru_gen.list, &mm_list->fifo); 2930 2931 spin_unlock(&mm_list->lock); 2932} 2933 2934void lru_gen_del_mm(struct mm_struct *mm) 2935{ 2936 int nid; 2937 struct lru_gen_mm_list *mm_list; 2938 struct mem_cgroup *memcg = NULL; 2939 2940 if (list_empty(&mm->lru_gen.list)) 2941 return; 2942 2943#ifdef CONFIG_MEMCG 2944 memcg = mm->lru_gen.memcg; 2945#endif 2946 mm_list = get_mm_list(memcg); 2947 2948 spin_lock(&mm_list->lock); 2949 2950 for_each_node(nid) { 2951 struct lruvec *lruvec = get_lruvec(memcg, nid); 2952 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 2953 2954 /* where the current iteration continues after */ 2955 if (mm_state->head == &mm->lru_gen.list) 2956 mm_state->head = mm_state->head->prev; 2957 2958 /* where the last iteration ended before */ 2959 if (mm_state->tail == &mm->lru_gen.list) 2960 mm_state->tail = mm_state->tail->next; 2961 } 2962 2963 list_del_init(&mm->lru_gen.list); 2964 2965 spin_unlock(&mm_list->lock); 2966 2967#ifdef CONFIG_MEMCG 2968 mem_cgroup_put(mm->lru_gen.memcg); 2969 mm->lru_gen.memcg = NULL; 2970#endif 2971} 2972 2973#ifdef CONFIG_MEMCG 2974void lru_gen_migrate_mm(struct mm_struct *mm) 2975{ 2976 struct mem_cgroup *memcg; 2977 struct task_struct *task = rcu_dereference_protected(mm->owner, true); 2978 2979 VM_WARN_ON_ONCE(task->mm != mm); 2980 lockdep_assert_held(&task->alloc_lock); 2981 2982 /* for mm_update_next_owner() */ 2983 if (mem_cgroup_disabled()) 2984 return; 2985 2986 /* migration can happen before addition */ 2987 if (!mm->lru_gen.memcg) 2988 return; 2989 2990 rcu_read_lock(); 2991 memcg = mem_cgroup_from_task(task); 2992 rcu_read_unlock(); 2993 if (memcg == mm->lru_gen.memcg) 2994 return; 2995 2996 VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); 2997 2998 lru_gen_del_mm(mm); 2999 lru_gen_add_mm(mm); 3000} 3001#endif 3002 3003#else /* !CONFIG_LRU_GEN_WALKS_MMU */ 3004 3005static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 3006{ 3007 return NULL; 3008} 3009 3010static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec) 3011{ 3012 return NULL; 3013} 3014 3015static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk) 3016{ 3017 return NULL; 3018} 3019 3020#endif 3021 3022static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) 3023{ 3024 int i; 3025 int hist; 3026 struct lruvec *lruvec = walk->lruvec; 3027 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3028 3029 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); 3030 3031 hist = lru_hist_from_seq(walk->seq); 3032 3033 for (i = 0; i < NR_MM_STATS; i++) { 3034 WRITE_ONCE(mm_state->stats[hist][i], 3035 mm_state->stats[hist][i] + walk->mm_stats[i]); 3036 walk->mm_stats[i] = 0; 3037 } 3038 3039 if (NR_HIST_GENS > 1 && last) { 3040 hist = lru_hist_from_seq(walk->seq + 1); 3041 3042 for (i = 0; i < NR_MM_STATS; i++) 3043 WRITE_ONCE(mm_state->stats[hist][i], 0); 3044 } 3045} 3046 3047static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **iter) 3048{ 3049 bool first = false; 3050 bool last = false; 3051 struct mm_struct *mm = NULL; 3052 struct lruvec *lruvec = walk->lruvec; 3053 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3054 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3055 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3056 3057 /* 3058 * mm_state->seq is incremented after each iteration of mm_list. There 3059 * are three interesting cases for this page table walker: 3060 * 1. It tries to start a new iteration with a stale max_seq: there is 3061 * nothing left to do. 3062 * 2. It started the next iteration: it needs to reset the Bloom filter 3063 * so that a fresh set of PTE tables can be recorded. 3064 * 3. It ended the current iteration: it needs to reset the mm stats 3065 * counters and tell its caller to increment max_seq. 3066 */ 3067 spin_lock(&mm_list->lock); 3068 3069 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq); 3070 3071 if (walk->seq <= mm_state->seq) 3072 goto done; 3073 3074 if (!mm_state->head) 3075 mm_state->head = &mm_list->fifo; 3076 3077 if (mm_state->head == &mm_list->fifo) 3078 first = true; 3079 3080 do { 3081 mm_state->head = mm_state->head->next; 3082 if (mm_state->head == &mm_list->fifo) { 3083 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3084 last = true; 3085 break; 3086 } 3087 3088 /* force scan for those added after the last iteration */ 3089 if (!mm_state->tail || mm_state->tail == mm_state->head) { 3090 mm_state->tail = mm_state->head->next; 3091 walk->force_scan = true; 3092 } 3093 } while (!(mm = get_next_mm(walk))); 3094done: 3095 if (*iter || last) 3096 reset_mm_stats(walk, last); 3097 3098 spin_unlock(&mm_list->lock); 3099 3100 if (mm && first) 3101 reset_bloom_filter(mm_state, walk->seq + 1); 3102 3103 if (*iter) 3104 mmput_async(*iter); 3105 3106 *iter = mm; 3107 3108 return last; 3109} 3110 3111static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq) 3112{ 3113 bool success = false; 3114 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3115 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3116 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3117 3118 spin_lock(&mm_list->lock); 3119 3120 VM_WARN_ON_ONCE(mm_state->seq + 1 < seq); 3121 3122 if (seq > mm_state->seq) { 3123 mm_state->head = NULL; 3124 mm_state->tail = NULL; 3125 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3126 success = true; 3127 } 3128 3129 spin_unlock(&mm_list->lock); 3130 3131 return success; 3132} 3133 3134/****************************************************************************** 3135 * PID controller 3136 ******************************************************************************/ 3137 3138/* 3139 * A feedback loop based on Proportional-Integral-Derivative (PID) controller. 3140 * 3141 * The P term is refaulted/(evicted+protected) from a tier in the generation 3142 * currently being evicted; the I term is the exponential moving average of the 3143 * P term over the generations previously evicted, using the smoothing factor 3144 * 1/2; the D term isn't supported. 3145 * 3146 * The setpoint (SP) is always the first tier of one type; the process variable 3147 * (PV) is either any tier of the other type or any other tier of the same 3148 * type. 3149 * 3150 * The error is the difference between the SP and the PV; the correction is to 3151 * turn off protection when SP>PV or turn on protection when SP<PV. 3152 * 3153 * For future optimizations: 3154 * 1. The D term may discount the other two terms over time so that long-lived 3155 * generations can resist stale information. 3156 */ 3157struct ctrl_pos { 3158 unsigned long refaulted; 3159 unsigned long total; 3160 int gain; 3161}; 3162 3163static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, 3164 struct ctrl_pos *pos) 3165{ 3166 int i; 3167 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3168 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3169 3170 pos->gain = gain; 3171 pos->refaulted = pos->total = 0; 3172 3173 for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) { 3174 pos->refaulted += lrugen->avg_refaulted[type][i] + 3175 atomic_long_read(&lrugen->refaulted[hist][type][i]); 3176 pos->total += lrugen->avg_total[type][i] + 3177 lrugen->protected[hist][type][i] + 3178 atomic_long_read(&lrugen->evicted[hist][type][i]); 3179 } 3180} 3181 3182static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) 3183{ 3184 int hist, tier; 3185 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3186 bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; 3187 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; 3188 3189 lockdep_assert_held(&lruvec->lru_lock); 3190 3191 if (!carryover && !clear) 3192 return; 3193 3194 hist = lru_hist_from_seq(seq); 3195 3196 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 3197 if (carryover) { 3198 unsigned long sum; 3199 3200 sum = lrugen->avg_refaulted[type][tier] + 3201 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3202 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); 3203 3204 sum = lrugen->avg_total[type][tier] + 3205 lrugen->protected[hist][type][tier] + 3206 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3207 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); 3208 } 3209 3210 if (clear) { 3211 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); 3212 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); 3213 WRITE_ONCE(lrugen->protected[hist][type][tier], 0); 3214 } 3215 } 3216} 3217 3218static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) 3219{ 3220 /* 3221 * Return true if the PV has a limited number of refaults or a lower 3222 * refaulted/total than the SP. 3223 */ 3224 return pv->refaulted < MIN_LRU_BATCH || 3225 pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= 3226 (sp->refaulted + 1) * pv->total * pv->gain; 3227} 3228 3229/****************************************************************************** 3230 * the aging 3231 ******************************************************************************/ 3232 3233/* promote pages accessed through page tables */ 3234static int folio_update_gen(struct folio *folio, int gen) 3235{ 3236 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); 3237 3238 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); 3239 3240 /* see the comment on LRU_REFS_FLAGS */ 3241 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { 3242 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); 3243 return -1; 3244 } 3245 3246 do { 3247 /* lru_gen_del_folio() has isolated this page? */ 3248 if (!(old_flags & LRU_GEN_MASK)) 3249 return -1; 3250 3251 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); 3252 new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset); 3253 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); 3254 3255 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3256} 3257 3258/* protect pages accessed multiple times through file descriptors */ 3259static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) 3260{ 3261 int type = folio_is_file_lru(folio); 3262 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3263 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3264 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); 3265 3266 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); 3267 3268 do { 3269 new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3270 /* folio_update_gen() has promoted this page? */ 3271 if (new_gen >= 0 && new_gen != old_gen) 3272 return new_gen; 3273 3274 new_gen = (old_gen + 1) % MAX_NR_GENS; 3275 3276 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); 3277 new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; 3278 /* for folio_end_writeback() */ 3279 if (reclaiming) 3280 new_flags |= BIT(PG_reclaim); 3281 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); 3282 3283 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 3284 3285 return new_gen; 3286} 3287 3288static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, 3289 int old_gen, int new_gen) 3290{ 3291 int type = folio_is_file_lru(folio); 3292 int zone = folio_zonenum(folio); 3293 int delta = folio_nr_pages(folio); 3294 3295 VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); 3296 VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); 3297 3298 walk->batched++; 3299 3300 walk->nr_pages[old_gen][type][zone] -= delta; 3301 walk->nr_pages[new_gen][type][zone] += delta; 3302} 3303 3304static void reset_batch_size(struct lru_gen_mm_walk *walk) 3305{ 3306 int gen, type, zone; 3307 struct lruvec *lruvec = walk->lruvec; 3308 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3309 3310 walk->batched = 0; 3311 3312 for_each_gen_type_zone(gen, type, zone) { 3313 enum lru_list lru = type * LRU_INACTIVE_FILE; 3314 int delta = walk->nr_pages[gen][type][zone]; 3315 3316 if (!delta) 3317 continue; 3318 3319 walk->nr_pages[gen][type][zone] = 0; 3320 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], 3321 lrugen->nr_pages[gen][type][zone] + delta); 3322 3323 if (lru_gen_is_active(lruvec, gen)) 3324 lru += LRU_ACTIVE; 3325 __update_lru_size(lruvec, lru, zone, delta); 3326 } 3327} 3328 3329static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) 3330{ 3331 struct address_space *mapping; 3332 struct vm_area_struct *vma = args->vma; 3333 struct lru_gen_mm_walk *walk = args->private; 3334 3335 if (!vma_is_accessible(vma)) 3336 return true; 3337 3338 if (is_vm_hugetlb_page(vma)) 3339 return true; 3340 3341 if (!vma_has_recency(vma)) 3342 return true; 3343 3344 if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) 3345 return true; 3346 3347 if (vma == get_gate_vma(vma->vm_mm)) 3348 return true; 3349 3350 if (vma_is_anonymous(vma)) 3351 return !walk->swappiness; 3352 3353 if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) 3354 return true; 3355 3356 mapping = vma->vm_file->f_mapping; 3357 if (mapping_unevictable(mapping)) 3358 return true; 3359 3360 if (shmem_mapping(mapping)) 3361 return !walk->swappiness; 3362 3363 if (walk->swappiness > MAX_SWAPPINESS) 3364 return true; 3365 3366 /* to exclude special mappings like dax, etc. */ 3367 return !mapping->a_ops->read_folio; 3368} 3369 3370/* 3371 * Some userspace memory allocators map many single-page VMAs. Instead of 3372 * returning back to the PGD table for each of such VMAs, finish an entire PMD 3373 * table to reduce zigzags and improve cache performance. 3374 */ 3375static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, 3376 unsigned long *vm_start, unsigned long *vm_end) 3377{ 3378 unsigned long start = round_up(*vm_end, size); 3379 unsigned long end = (start | ~mask) + 1; 3380 VMA_ITERATOR(vmi, args->mm, start); 3381 3382 VM_WARN_ON_ONCE(mask & size); 3383 VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); 3384 3385 for_each_vma(vmi, args->vma) { 3386 if (end && end <= args->vma->vm_start) 3387 return false; 3388 3389 if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) 3390 continue; 3391 3392 *vm_start = max(start, args->vma->vm_start); 3393 *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; 3394 3395 return true; 3396 } 3397 3398 return false; 3399} 3400 3401static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr, 3402 struct pglist_data *pgdat) 3403{ 3404 unsigned long pfn = pte_pfn(pte); 3405 3406 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3407 3408 if (!pte_present(pte) || is_zero_pfn(pfn)) 3409 return -1; 3410 3411 if (WARN_ON_ONCE(pte_special(pte))) 3412 return -1; 3413 3414 if (!pte_young(pte) && !mm_has_notifiers(vma->vm_mm)) 3415 return -1; 3416 3417 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3418 return -1; 3419 3420 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3421 return -1; 3422 3423 return pfn; 3424} 3425 3426static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr, 3427 struct pglist_data *pgdat) 3428{ 3429 unsigned long pfn = pmd_pfn(pmd); 3430 3431 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3432 3433 if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) 3434 return -1; 3435 3436 if (!pmd_young(pmd) && !mm_has_notifiers(vma->vm_mm)) 3437 return -1; 3438 3439 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3440 return -1; 3441 3442 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3443 return -1; 3444 3445 return pfn; 3446} 3447 3448static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, 3449 struct pglist_data *pgdat) 3450{ 3451 struct folio *folio = pfn_folio(pfn); 3452 3453 if (folio_lru_gen(folio) < 0) 3454 return NULL; 3455 3456 if (folio_nid(folio) != pgdat->node_id) 3457 return NULL; 3458 3459 if (folio_memcg(folio) != memcg) 3460 return NULL; 3461 3462 return folio; 3463} 3464 3465static bool suitable_to_scan(int total, int young) 3466{ 3467 int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); 3468 3469 /* suitable if the average number of young PTEs per cacheline is >=1 */ 3470 return young * n >= total; 3471} 3472 3473static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, 3474 int new_gen, bool dirty) 3475{ 3476 int old_gen; 3477 3478 if (!folio) 3479 return; 3480 3481 if (dirty && !folio_test_dirty(folio) && 3482 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 3483 !folio_test_swapcache(folio))) 3484 folio_mark_dirty(folio); 3485 3486 if (walk) { 3487 old_gen = folio_update_gen(folio, new_gen); 3488 if (old_gen >= 0 && old_gen != new_gen) 3489 update_batch_size(walk, folio, old_gen, new_gen); 3490 } else if (lru_gen_set_refs(folio)) { 3491 old_gen = folio_lru_gen(folio); 3492 if (old_gen >= 0 && old_gen != new_gen) 3493 folio_activate(folio); 3494 } 3495} 3496 3497static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 3498 struct mm_walk *args) 3499{ 3500 int i; 3501 bool dirty; 3502 pte_t *pte; 3503 spinlock_t *ptl; 3504 unsigned long addr; 3505 int total = 0; 3506 int young = 0; 3507 struct folio *last = NULL; 3508 struct lru_gen_mm_walk *walk = args->private; 3509 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3510 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3511 DEFINE_MAX_SEQ(walk->lruvec); 3512 int gen = lru_gen_from_seq(max_seq); 3513 pmd_t pmdval; 3514 3515 pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); 3516 if (!pte) 3517 return false; 3518 3519 if (!spin_trylock(ptl)) { 3520 pte_unmap(pte); 3521 return true; 3522 } 3523 3524 if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmd)))) { 3525 pte_unmap_unlock(pte, ptl); 3526 return false; 3527 } 3528 3529 arch_enter_lazy_mmu_mode(); 3530restart: 3531 for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { 3532 unsigned long pfn; 3533 struct folio *folio; 3534 pte_t ptent = ptep_get(pte + i); 3535 3536 total++; 3537 walk->mm_stats[MM_LEAF_TOTAL]++; 3538 3539 pfn = get_pte_pfn(ptent, args->vma, addr, pgdat); 3540 if (pfn == -1) 3541 continue; 3542 3543 folio = get_pfn_folio(pfn, memcg, pgdat); 3544 if (!folio) 3545 continue; 3546 3547 if (!ptep_clear_young_notify(args->vma, addr, pte + i)) 3548 continue; 3549 3550 if (last != folio) { 3551 walk_update_folio(walk, last, gen, dirty); 3552 3553 last = folio; 3554 dirty = false; 3555 } 3556 3557 if (pte_dirty(ptent)) 3558 dirty = true; 3559 3560 young++; 3561 walk->mm_stats[MM_LEAF_YOUNG]++; 3562 } 3563 3564 walk_update_folio(walk, last, gen, dirty); 3565 last = NULL; 3566 3567 if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) 3568 goto restart; 3569 3570 arch_leave_lazy_mmu_mode(); 3571 pte_unmap_unlock(pte, ptl); 3572 3573 return suitable_to_scan(total, young); 3574} 3575 3576static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma, 3577 struct mm_walk *args, unsigned long *bitmap, unsigned long *first) 3578{ 3579 int i; 3580 bool dirty; 3581 pmd_t *pmd; 3582 spinlock_t *ptl; 3583 struct folio *last = NULL; 3584 struct lru_gen_mm_walk *walk = args->private; 3585 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3586 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3587 DEFINE_MAX_SEQ(walk->lruvec); 3588 int gen = lru_gen_from_seq(max_seq); 3589 3590 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3591 3592 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ 3593 if (*first == -1) { 3594 *first = addr; 3595 bitmap_zero(bitmap, MIN_LRU_BATCH); 3596 return; 3597 } 3598 3599 i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); 3600 if (i && i <= MIN_LRU_BATCH) { 3601 __set_bit(i - 1, bitmap); 3602 return; 3603 } 3604 3605 pmd = pmd_offset(pud, *first); 3606 3607 ptl = pmd_lockptr(args->mm, pmd); 3608 if (!spin_trylock(ptl)) 3609 goto done; 3610 3611 arch_enter_lazy_mmu_mode(); 3612 3613 do { 3614 unsigned long pfn; 3615 struct folio *folio; 3616 3617 /* don't round down the first address */ 3618 addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; 3619 3620 if (!pmd_present(pmd[i])) 3621 goto next; 3622 3623 if (!pmd_trans_huge(pmd[i])) { 3624 if (!walk->force_scan && should_clear_pmd_young() && 3625 !mm_has_notifiers(args->mm)) 3626 pmdp_test_and_clear_young(vma, addr, pmd + i); 3627 goto next; 3628 } 3629 3630 pfn = get_pmd_pfn(pmd[i], vma, addr, pgdat); 3631 if (pfn == -1) 3632 goto next; 3633 3634 folio = get_pfn_folio(pfn, memcg, pgdat); 3635 if (!folio) 3636 goto next; 3637 3638 if (!pmdp_clear_young_notify(vma, addr, pmd + i)) 3639 goto next; 3640 3641 if (last != folio) { 3642 walk_update_folio(walk, last, gen, dirty); 3643 3644 last = folio; 3645 dirty = false; 3646 } 3647 3648 if (pmd_dirty(pmd[i])) 3649 dirty = true; 3650 3651 walk->mm_stats[MM_LEAF_YOUNG]++; 3652next: 3653 i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; 3654 } while (i <= MIN_LRU_BATCH); 3655 3656 walk_update_folio(walk, last, gen, dirty); 3657 3658 arch_leave_lazy_mmu_mode(); 3659 spin_unlock(ptl); 3660done: 3661 *first = -1; 3662} 3663 3664static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 3665 struct mm_walk *args) 3666{ 3667 int i; 3668 pmd_t *pmd; 3669 unsigned long next; 3670 unsigned long addr; 3671 struct vm_area_struct *vma; 3672 DECLARE_BITMAP(bitmap, MIN_LRU_BATCH); 3673 unsigned long first = -1; 3674 struct lru_gen_mm_walk *walk = args->private; 3675 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); 3676 3677 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3678 3679 /* 3680 * Finish an entire PMD in two passes: the first only reaches to PTE 3681 * tables to avoid taking the PMD lock; the second, if necessary, takes 3682 * the PMD lock to clear the accessed bit in PMD entries. 3683 */ 3684 pmd = pmd_offset(pud, start & PUD_MASK); 3685restart: 3686 /* walk_pte_range() may call get_next_vma() */ 3687 vma = args->vma; 3688 for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { 3689 pmd_t val = pmdp_get_lockless(pmd + i); 3690 3691 next = pmd_addr_end(addr, end); 3692 3693 if (!pmd_present(val) || is_huge_zero_pmd(val)) { 3694 walk->mm_stats[MM_LEAF_TOTAL]++; 3695 continue; 3696 } 3697 3698 if (pmd_trans_huge(val)) { 3699 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3700 unsigned long pfn = get_pmd_pfn(val, vma, addr, pgdat); 3701 3702 walk->mm_stats[MM_LEAF_TOTAL]++; 3703 3704 if (pfn != -1) 3705 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); 3706 continue; 3707 } 3708 3709 if (!walk->force_scan && should_clear_pmd_young() && 3710 !mm_has_notifiers(args->mm)) { 3711 if (!pmd_young(val)) 3712 continue; 3713 3714 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); 3715 } 3716 3717 if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) 3718 continue; 3719 3720 walk->mm_stats[MM_NONLEAF_FOUND]++; 3721 3722 if (!walk_pte_range(&val, addr, next, args)) 3723 continue; 3724 3725 walk->mm_stats[MM_NONLEAF_ADDED]++; 3726 3727 /* carry over to the next generation */ 3728 update_bloom_filter(mm_state, walk->seq + 1, pmd + i); 3729 } 3730 3731 walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); 3732 3733 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) 3734 goto restart; 3735} 3736 3737static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, 3738 struct mm_walk *args) 3739{ 3740 int i; 3741 pud_t *pud; 3742 unsigned long addr; 3743 unsigned long next; 3744 struct lru_gen_mm_walk *walk = args->private; 3745 3746 VM_WARN_ON_ONCE(p4d_leaf(*p4d)); 3747 3748 pud = pud_offset(p4d, start & P4D_MASK); 3749restart: 3750 for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { 3751 pud_t val = pudp_get(pud + i); 3752 3753 next = pud_addr_end(addr, end); 3754 3755 if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) 3756 continue; 3757 3758 walk_pmd_range(&val, addr, next, args); 3759 3760 if (need_resched() || walk->batched >= MAX_LRU_BATCH) { 3761 end = (addr | ~PUD_MASK) + 1; 3762 goto done; 3763 } 3764 } 3765 3766 if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) 3767 goto restart; 3768 3769 end = round_up(end, P4D_SIZE); 3770done: 3771 if (!end || !args->vma) 3772 return 1; 3773 3774 walk->next_addr = max(end, args->vma->vm_start); 3775 3776 return -EAGAIN; 3777} 3778 3779static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) 3780{ 3781 static const struct mm_walk_ops mm_walk_ops = { 3782 .test_walk = should_skip_vma, 3783 .p4d_entry = walk_pud_range, 3784 .walk_lock = PGWALK_RDLOCK, 3785 }; 3786 int err; 3787 struct lruvec *lruvec = walk->lruvec; 3788 3789 walk->next_addr = FIRST_USER_ADDRESS; 3790 3791 do { 3792 DEFINE_MAX_SEQ(lruvec); 3793 3794 err = -EBUSY; 3795 3796 /* another thread might have called inc_max_seq() */ 3797 if (walk->seq != max_seq) 3798 break; 3799 3800 /* the caller might be holding the lock for write */ 3801 if (mmap_read_trylock(mm)) { 3802 err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); 3803 3804 mmap_read_unlock(mm); 3805 } 3806 3807 if (walk->batched) { 3808 spin_lock_irq(&lruvec->lru_lock); 3809 reset_batch_size(walk); 3810 spin_unlock_irq(&lruvec->lru_lock); 3811 } 3812 3813 cond_resched(); 3814 } while (err == -EAGAIN); 3815} 3816 3817static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc) 3818{ 3819 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 3820 3821 if (pgdat && current_is_kswapd()) { 3822 VM_WARN_ON_ONCE(walk); 3823 3824 walk = &pgdat->mm_walk; 3825 } else if (!walk && force_alloc) { 3826 VM_WARN_ON_ONCE(current_is_kswapd()); 3827 3828 walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 3829 } 3830 3831 current->reclaim_state->mm_walk = walk; 3832 3833 return walk; 3834} 3835 3836static void clear_mm_walk(void) 3837{ 3838 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 3839 3840 VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); 3841 VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); 3842 3843 current->reclaim_state->mm_walk = NULL; 3844 3845 if (!current_is_kswapd()) 3846 kfree(walk); 3847} 3848 3849static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) 3850{ 3851 int zone; 3852 int remaining = MAX_LRU_BATCH; 3853 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3854 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3855 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3856 3857 /* For file type, skip the check if swappiness is anon only */ 3858 if (type && (swappiness == SWAPPINESS_ANON_ONLY)) 3859 goto done; 3860 3861 /* For anon type, skip the check if swappiness is zero (file only) */ 3862 if (!type && !swappiness) 3863 goto done; 3864 3865 /* prevent cold/hot inversion if the type is evictable */ 3866 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 3867 struct list_head *head = &lrugen->folios[old_gen][type][zone]; 3868 3869 while (!list_empty(head)) { 3870 struct folio *folio = lru_to_folio(head); 3871 int refs = folio_lru_refs(folio); 3872 bool workingset = folio_test_workingset(folio); 3873 3874 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 3875 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 3876 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 3877 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 3878 3879 new_gen = folio_inc_gen(lruvec, folio, false); 3880 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); 3881 3882 /* don't count the workingset being lazily promoted */ 3883 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { 3884 int tier = lru_tier_from_refs(refs, workingset); 3885 int delta = folio_nr_pages(folio); 3886 3887 WRITE_ONCE(lrugen->protected[hist][type][tier], 3888 lrugen->protected[hist][type][tier] + delta); 3889 } 3890 3891 if (!--remaining) 3892 return false; 3893 } 3894 } 3895done: 3896 reset_ctrl_pos(lruvec, type, true); 3897 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); 3898 3899 return true; 3900} 3901 3902static bool try_to_inc_min_seq(struct lruvec *lruvec, int swappiness) 3903{ 3904 int gen, type, zone; 3905 bool success = false; 3906 bool seq_inc_flag = false; 3907 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3908 DEFINE_MIN_SEQ(lruvec); 3909 3910 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 3911 3912 /* find the oldest populated generation */ 3913 for_each_evictable_type(type, swappiness) { 3914 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { 3915 gen = lru_gen_from_seq(min_seq[type]); 3916 3917 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 3918 if (!list_empty(&lrugen->folios[gen][type][zone])) 3919 goto next; 3920 } 3921 3922 min_seq[type]++; 3923 seq_inc_flag = true; 3924 } 3925next: 3926 ; 3927 } 3928 3929 /* 3930 * If min_seq[type] of both anonymous and file is not increased, 3931 * we can directly return false to avoid unnecessary checking 3932 * overhead later. 3933 */ 3934 if (!seq_inc_flag) 3935 return success; 3936 3937 /* see the comment on lru_gen_folio */ 3938 if (swappiness && swappiness <= MAX_SWAPPINESS) { 3939 unsigned long seq = lrugen->max_seq - MIN_NR_GENS; 3940 3941 if (min_seq[LRU_GEN_ANON] > seq && min_seq[LRU_GEN_FILE] < seq) 3942 min_seq[LRU_GEN_ANON] = seq; 3943 else if (min_seq[LRU_GEN_FILE] > seq && min_seq[LRU_GEN_ANON] < seq) 3944 min_seq[LRU_GEN_FILE] = seq; 3945 } 3946 3947 for_each_evictable_type(type, swappiness) { 3948 if (min_seq[type] <= lrugen->min_seq[type]) 3949 continue; 3950 3951 reset_ctrl_pos(lruvec, type, true); 3952 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); 3953 success = true; 3954 } 3955 3956 return success; 3957} 3958 3959static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness) 3960{ 3961 bool success; 3962 int prev, next; 3963 int type, zone; 3964 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3965restart: 3966 if (seq < READ_ONCE(lrugen->max_seq)) 3967 return false; 3968 3969 spin_lock_irq(&lruvec->lru_lock); 3970 3971 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 3972 3973 success = seq == lrugen->max_seq; 3974 if (!success) 3975 goto unlock; 3976 3977 for (type = 0; type < ANON_AND_FILE; type++) { 3978 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) 3979 continue; 3980 3981 if (inc_min_seq(lruvec, type, swappiness)) 3982 continue; 3983 3984 spin_unlock_irq(&lruvec->lru_lock); 3985 cond_resched(); 3986 goto restart; 3987 } 3988 3989 /* 3990 * Update the active/inactive LRU sizes for compatibility. Both sides of 3991 * the current max_seq need to be covered, since max_seq+1 can overlap 3992 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do 3993 * overlap, cold/hot inversion happens. 3994 */ 3995 prev = lru_gen_from_seq(lrugen->max_seq - 1); 3996 next = lru_gen_from_seq(lrugen->max_seq + 1); 3997 3998 for (type = 0; type < ANON_AND_FILE; type++) { 3999 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4000 enum lru_list lru = type * LRU_INACTIVE_FILE; 4001 long delta = lrugen->nr_pages[prev][type][zone] - 4002 lrugen->nr_pages[next][type][zone]; 4003 4004 if (!delta) 4005 continue; 4006 4007 __update_lru_size(lruvec, lru, zone, delta); 4008 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); 4009 } 4010 } 4011 4012 for (type = 0; type < ANON_AND_FILE; type++) 4013 reset_ctrl_pos(lruvec, type, false); 4014 4015 WRITE_ONCE(lrugen->timestamps[next], jiffies); 4016 /* make sure preceding modifications appear */ 4017 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); 4018unlock: 4019 spin_unlock_irq(&lruvec->lru_lock); 4020 4021 return success; 4022} 4023 4024static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq, 4025 int swappiness, bool force_scan) 4026{ 4027 bool success; 4028 struct lru_gen_mm_walk *walk; 4029 struct mm_struct *mm = NULL; 4030 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4031 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 4032 4033 VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq)); 4034 4035 if (!mm_state) 4036 return inc_max_seq(lruvec, seq, swappiness); 4037 4038 /* see the comment in iterate_mm_list() */ 4039 if (seq <= READ_ONCE(mm_state->seq)) 4040 return false; 4041 4042 /* 4043 * If the hardware doesn't automatically set the accessed bit, fallback 4044 * to lru_gen_look_around(), which only clears the accessed bit in a 4045 * handful of PTEs. Spreading the work out over a period of time usually 4046 * is less efficient, but it avoids bursty page faults. 4047 */ 4048 if (!should_walk_mmu()) { 4049 success = iterate_mm_list_nowalk(lruvec, seq); 4050 goto done; 4051 } 4052 4053 walk = set_mm_walk(NULL, true); 4054 if (!walk) { 4055 success = iterate_mm_list_nowalk(lruvec, seq); 4056 goto done; 4057 } 4058 4059 walk->lruvec = lruvec; 4060 walk->seq = seq; 4061 walk->swappiness = swappiness; 4062 walk->force_scan = force_scan; 4063 4064 do { 4065 success = iterate_mm_list(walk, &mm); 4066 if (mm) 4067 walk_mm(mm, walk); 4068 } while (mm); 4069done: 4070 if (success) { 4071 success = inc_max_seq(lruvec, seq, swappiness); 4072 WARN_ON_ONCE(!success); 4073 } 4074 4075 return success; 4076} 4077 4078/****************************************************************************** 4079 * working set protection 4080 ******************************************************************************/ 4081 4082static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc) 4083{ 4084 int priority; 4085 unsigned long reclaimable; 4086 4087 if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH) 4088 return; 4089 /* 4090 * Determine the initial priority based on 4091 * (total >> priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, 4092 * where reclaimed_to_scanned_ratio = inactive / total. 4093 */ 4094 reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE); 4095 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 4096 reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON); 4097 4098 /* round down reclaimable and round up sc->nr_to_reclaim */ 4099 priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1); 4100 4101 /* 4102 * The estimation is based on LRU pages only, so cap it to prevent 4103 * overshoots of shrinker objects by large margins. 4104 */ 4105 sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY); 4106} 4107 4108static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) 4109{ 4110 int gen, type, zone; 4111 unsigned long total = 0; 4112 int swappiness = get_swappiness(lruvec, sc); 4113 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4114 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4115 DEFINE_MAX_SEQ(lruvec); 4116 DEFINE_MIN_SEQ(lruvec); 4117 4118 for_each_evictable_type(type, swappiness) { 4119 unsigned long seq; 4120 4121 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4122 gen = lru_gen_from_seq(seq); 4123 4124 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4125 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4126 } 4127 } 4128 4129 /* whether the size is big enough to be helpful */ 4130 return mem_cgroup_online(memcg) ? (total >> sc->priority) : total; 4131} 4132 4133static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc, 4134 unsigned long min_ttl) 4135{ 4136 int gen; 4137 unsigned long birth; 4138 int swappiness = get_swappiness(lruvec, sc); 4139 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4140 DEFINE_MIN_SEQ(lruvec); 4141 4142 if (mem_cgroup_below_min(NULL, memcg)) 4143 return false; 4144 4145 if (!lruvec_is_sizable(lruvec, sc)) 4146 return false; 4147 4148 gen = lru_gen_from_seq(evictable_min_seq(min_seq, swappiness)); 4149 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 4150 4151 return time_is_before_jiffies(birth + min_ttl); 4152} 4153 4154/* to protect the working set of the last N jiffies */ 4155static unsigned long lru_gen_min_ttl __read_mostly; 4156 4157static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 4158{ 4159 struct mem_cgroup *memcg; 4160 unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); 4161 bool reclaimable = !min_ttl; 4162 4163 VM_WARN_ON_ONCE(!current_is_kswapd()); 4164 4165 set_initial_priority(pgdat, sc); 4166 4167 memcg = mem_cgroup_iter(NULL, NULL, NULL); 4168 do { 4169 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4170 4171 mem_cgroup_calculate_protection(NULL, memcg); 4172 4173 if (!reclaimable) 4174 reclaimable = lruvec_is_reclaimable(lruvec, sc, min_ttl); 4175 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 4176 4177 /* 4178 * The main goal is to OOM kill if every generation from all memcgs is 4179 * younger than min_ttl. However, another possibility is all memcgs are 4180 * either too small or below min. 4181 */ 4182 if (!reclaimable && mutex_trylock(&oom_lock)) { 4183 struct oom_control oc = { 4184 .gfp_mask = sc->gfp_mask, 4185 }; 4186 4187 out_of_memory(&oc); 4188 4189 mutex_unlock(&oom_lock); 4190 } 4191} 4192 4193/****************************************************************************** 4194 * rmap/PT walk feedback 4195 ******************************************************************************/ 4196 4197/* 4198 * This function exploits spatial locality when shrink_folio_list() walks the 4199 * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If 4200 * the scan was done cacheline efficiently, it adds the PMD entry pointing to 4201 * the PTE table to the Bloom filter. This forms a feedback loop between the 4202 * eviction and the aging. 4203 */ 4204bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw) 4205{ 4206 int i; 4207 bool dirty; 4208 unsigned long start; 4209 unsigned long end; 4210 struct lru_gen_mm_walk *walk; 4211 struct folio *last = NULL; 4212 int young = 1; 4213 pte_t *pte = pvmw->pte; 4214 unsigned long addr = pvmw->address; 4215 struct vm_area_struct *vma = pvmw->vma; 4216 struct folio *folio = pfn_folio(pvmw->pfn); 4217 struct mem_cgroup *memcg = folio_memcg(folio); 4218 struct pglist_data *pgdat = folio_pgdat(folio); 4219 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4220 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 4221 DEFINE_MAX_SEQ(lruvec); 4222 int gen = lru_gen_from_seq(max_seq); 4223 4224 lockdep_assert_held(pvmw->ptl); 4225 VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); 4226 4227 if (!ptep_clear_young_notify(vma, addr, pte)) 4228 return false; 4229 4230 if (spin_is_contended(pvmw->ptl)) 4231 return true; 4232 4233 /* exclude special VMAs containing anon pages from COW */ 4234 if (vma->vm_flags & VM_SPECIAL) 4235 return true; 4236 4237 /* avoid taking the LRU lock under the PTL when possible */ 4238 walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; 4239 4240 start = max(addr & PMD_MASK, vma->vm_start); 4241 end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; 4242 4243 if (end - start == PAGE_SIZE) 4244 return true; 4245 4246 if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { 4247 if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) 4248 end = start + MIN_LRU_BATCH * PAGE_SIZE; 4249 else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) 4250 start = end - MIN_LRU_BATCH * PAGE_SIZE; 4251 else { 4252 start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; 4253 end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; 4254 } 4255 } 4256 4257 arch_enter_lazy_mmu_mode(); 4258 4259 pte -= (addr - start) / PAGE_SIZE; 4260 4261 for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { 4262 unsigned long pfn; 4263 pte_t ptent = ptep_get(pte + i); 4264 4265 pfn = get_pte_pfn(ptent, vma, addr, pgdat); 4266 if (pfn == -1) 4267 continue; 4268 4269 folio = get_pfn_folio(pfn, memcg, pgdat); 4270 if (!folio) 4271 continue; 4272 4273 if (!ptep_clear_young_notify(vma, addr, pte + i)) 4274 continue; 4275 4276 if (last != folio) { 4277 walk_update_folio(walk, last, gen, dirty); 4278 4279 last = folio; 4280 dirty = false; 4281 } 4282 4283 if (pte_dirty(ptent)) 4284 dirty = true; 4285 4286 young++; 4287 } 4288 4289 walk_update_folio(walk, last, gen, dirty); 4290 4291 arch_leave_lazy_mmu_mode(); 4292 4293 /* feedback from rmap walkers to page table walkers */ 4294 if (mm_state && suitable_to_scan(i, young)) 4295 update_bloom_filter(mm_state, max_seq, pvmw->pmd); 4296 4297 return true; 4298} 4299 4300/****************************************************************************** 4301 * memcg LRU 4302 ******************************************************************************/ 4303 4304/* see the comment on MEMCG_NR_GENS */ 4305enum { 4306 MEMCG_LRU_NOP, 4307 MEMCG_LRU_HEAD, 4308 MEMCG_LRU_TAIL, 4309 MEMCG_LRU_OLD, 4310 MEMCG_LRU_YOUNG, 4311}; 4312 4313static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) 4314{ 4315 int seg; 4316 int old, new; 4317 unsigned long flags; 4318 int bin = get_random_u32_below(MEMCG_NR_BINS); 4319 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4320 4321 spin_lock_irqsave(&pgdat->memcg_lru.lock, flags); 4322 4323 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list)); 4324 4325 seg = 0; 4326 new = old = lruvec->lrugen.gen; 4327 4328 /* see the comment on MEMCG_NR_GENS */ 4329 if (op == MEMCG_LRU_HEAD) 4330 seg = MEMCG_LRU_HEAD; 4331 else if (op == MEMCG_LRU_TAIL) 4332 seg = MEMCG_LRU_TAIL; 4333 else if (op == MEMCG_LRU_OLD) 4334 new = get_memcg_gen(pgdat->memcg_lru.seq); 4335 else if (op == MEMCG_LRU_YOUNG) 4336 new = get_memcg_gen(pgdat->memcg_lru.seq + 1); 4337 else 4338 VM_WARN_ON_ONCE(true); 4339 4340 WRITE_ONCE(lruvec->lrugen.seg, seg); 4341 WRITE_ONCE(lruvec->lrugen.gen, new); 4342 4343 hlist_nulls_del_rcu(&lruvec->lrugen.list); 4344 4345 if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) 4346 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); 4347 else 4348 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); 4349 4350 pgdat->memcg_lru.nr_memcgs[old]--; 4351 pgdat->memcg_lru.nr_memcgs[new]++; 4352 4353 if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq)) 4354 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); 4355 4356 spin_unlock_irqrestore(&pgdat->memcg_lru.lock, flags); 4357} 4358 4359#ifdef CONFIG_MEMCG 4360 4361void lru_gen_online_memcg(struct mem_cgroup *memcg) 4362{ 4363 int gen; 4364 int nid; 4365 int bin = get_random_u32_below(MEMCG_NR_BINS); 4366 4367 for_each_node(nid) { 4368 struct pglist_data *pgdat = NODE_DATA(nid); 4369 struct lruvec *lruvec = get_lruvec(memcg, nid); 4370 4371 spin_lock_irq(&pgdat->memcg_lru.lock); 4372 4373 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list)); 4374 4375 gen = get_memcg_gen(pgdat->memcg_lru.seq); 4376 4377 lruvec->lrugen.gen = gen; 4378 4379 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); 4380 pgdat->memcg_lru.nr_memcgs[gen]++; 4381 4382 spin_unlock_irq(&pgdat->memcg_lru.lock); 4383 } 4384} 4385 4386void lru_gen_offline_memcg(struct mem_cgroup *memcg) 4387{ 4388 int nid; 4389 4390 for_each_node(nid) { 4391 struct lruvec *lruvec = get_lruvec(memcg, nid); 4392 4393 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD); 4394 } 4395} 4396 4397void lru_gen_release_memcg(struct mem_cgroup *memcg) 4398{ 4399 int gen; 4400 int nid; 4401 4402 for_each_node(nid) { 4403 struct pglist_data *pgdat = NODE_DATA(nid); 4404 struct lruvec *lruvec = get_lruvec(memcg, nid); 4405 4406 spin_lock_irq(&pgdat->memcg_lru.lock); 4407 4408 if (hlist_nulls_unhashed(&lruvec->lrugen.list)) 4409 goto unlock; 4410 4411 gen = lruvec->lrugen.gen; 4412 4413 hlist_nulls_del_init_rcu(&lruvec->lrugen.list); 4414 pgdat->memcg_lru.nr_memcgs[gen]--; 4415 4416 if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq)) 4417 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); 4418unlock: 4419 spin_unlock_irq(&pgdat->memcg_lru.lock); 4420 } 4421} 4422 4423void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) 4424{ 4425 struct lruvec *lruvec = get_lruvec(memcg, nid); 4426 4427 /* see the comment on MEMCG_NR_GENS */ 4428 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD) 4429 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); 4430} 4431 4432#endif /* CONFIG_MEMCG */ 4433 4434/****************************************************************************** 4435 * the eviction 4436 ******************************************************************************/ 4437 4438static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, 4439 int tier_idx) 4440{ 4441 bool success; 4442 bool dirty, writeback; 4443 int gen = folio_lru_gen(folio); 4444 int type = folio_is_file_lru(folio); 4445 int zone = folio_zonenum(folio); 4446 int delta = folio_nr_pages(folio); 4447 int refs = folio_lru_refs(folio); 4448 bool workingset = folio_test_workingset(folio); 4449 int tier = lru_tier_from_refs(refs, workingset); 4450 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4451 4452 VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); 4453 4454 /* unevictable */ 4455 if (!folio_evictable(folio)) { 4456 success = lru_gen_del_folio(lruvec, folio, true); 4457 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4458 folio_set_unevictable(folio); 4459 lruvec_add_folio(lruvec, folio); 4460 __count_vm_events(UNEVICTABLE_PGCULLED, delta); 4461 return true; 4462 } 4463 4464 /* promoted */ 4465 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { 4466 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4467 return true; 4468 } 4469 4470 /* protected */ 4471 if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) { 4472 gen = folio_inc_gen(lruvec, folio, false); 4473 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4474 4475 /* don't count the workingset being lazily promoted */ 4476 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { 4477 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 4478 4479 WRITE_ONCE(lrugen->protected[hist][type][tier], 4480 lrugen->protected[hist][type][tier] + delta); 4481 } 4482 return true; 4483 } 4484 4485 /* ineligible */ 4486 if (zone > sc->reclaim_idx) { 4487 gen = folio_inc_gen(lruvec, folio, false); 4488 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); 4489 return true; 4490 } 4491 4492 dirty = folio_test_dirty(folio); 4493 writeback = folio_test_writeback(folio); 4494 if (type == LRU_GEN_FILE && dirty) { 4495 sc->nr.file_taken += delta; 4496 if (!writeback) 4497 sc->nr.unqueued_dirty += delta; 4498 } 4499 4500 /* waiting for writeback */ 4501 if (writeback || (type == LRU_GEN_FILE && dirty)) { 4502 gen = folio_inc_gen(lruvec, folio, true); 4503 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4504 return true; 4505 } 4506 4507 return false; 4508} 4509 4510static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) 4511{ 4512 bool success; 4513 4514 /* swap constrained */ 4515 if (!(sc->gfp_mask & __GFP_IO) && 4516 (folio_test_dirty(folio) || 4517 (folio_test_anon(folio) && !folio_test_swapcache(folio)))) 4518 return false; 4519 4520 /* raced with release_pages() */ 4521 if (!folio_try_get(folio)) 4522 return false; 4523 4524 /* raced with another isolation */ 4525 if (!folio_test_clear_lru(folio)) { 4526 folio_put(folio); 4527 return false; 4528 } 4529 4530 /* see the comment on LRU_REFS_FLAGS */ 4531 if (!folio_test_referenced(folio)) 4532 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); 4533 4534 /* for shrink_folio_list() */ 4535 folio_clear_reclaim(folio); 4536 4537 success = lru_gen_del_folio(lruvec, folio, true); 4538 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4539 4540 return true; 4541} 4542 4543static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, 4544 struct scan_control *sc, int type, int tier, 4545 struct list_head *list) 4546{ 4547 int i; 4548 int gen; 4549 enum vm_event_item item; 4550 int sorted = 0; 4551 int scanned = 0; 4552 int isolated = 0; 4553 int skipped = 0; 4554 int scan_batch = min(nr_to_scan, MAX_LRU_BATCH); 4555 int remaining = scan_batch; 4556 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4557 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4558 4559 VM_WARN_ON_ONCE(!list_empty(list)); 4560 4561 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) 4562 return 0; 4563 4564 gen = lru_gen_from_seq(lrugen->min_seq[type]); 4565 4566 for (i = MAX_NR_ZONES; i > 0; i--) { 4567 LIST_HEAD(moved); 4568 int skipped_zone = 0; 4569 int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES; 4570 struct list_head *head = &lrugen->folios[gen][type][zone]; 4571 4572 while (!list_empty(head)) { 4573 struct folio *folio = lru_to_folio(head); 4574 int delta = folio_nr_pages(folio); 4575 4576 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4577 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4578 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4579 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4580 4581 scanned += delta; 4582 4583 if (sort_folio(lruvec, folio, sc, tier)) 4584 sorted += delta; 4585 else if (isolate_folio(lruvec, folio, sc)) { 4586 list_add(&folio->lru, list); 4587 isolated += delta; 4588 } else { 4589 list_move(&folio->lru, &moved); 4590 skipped_zone += delta; 4591 } 4592 4593 if (!--remaining || max(isolated, skipped_zone) >= MIN_LRU_BATCH) 4594 break; 4595 } 4596 4597 if (skipped_zone) { 4598 list_splice(&moved, head); 4599 __count_zid_vm_events(PGSCAN_SKIP, zone, skipped_zone); 4600 skipped += skipped_zone; 4601 } 4602 4603 if (!remaining || isolated >= MIN_LRU_BATCH) 4604 break; 4605 } 4606 4607 item = PGSCAN_KSWAPD + reclaimer_offset(sc); 4608 if (!cgroup_reclaim(sc)) { 4609 __count_vm_events(item, isolated); 4610 __count_vm_events(PGREFILL, sorted); 4611 } 4612 count_memcg_events(memcg, item, isolated); 4613 count_memcg_events(memcg, PGREFILL, sorted); 4614 __count_vm_events(PGSCAN_ANON + type, isolated); 4615 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, scan_batch, 4616 scanned, skipped, isolated, 4617 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); 4618 if (type == LRU_GEN_FILE) 4619 sc->nr.file_taken += isolated; 4620 /* 4621 * There might not be eligible folios due to reclaim_idx. Check the 4622 * remaining to prevent livelock if it's not making progress. 4623 */ 4624 return isolated || !remaining ? scanned : 0; 4625} 4626 4627static int get_tier_idx(struct lruvec *lruvec, int type) 4628{ 4629 int tier; 4630 struct ctrl_pos sp, pv; 4631 4632 /* 4633 * To leave a margin for fluctuations, use a larger gain factor (2:3). 4634 * This value is chosen because any other tier would have at least twice 4635 * as many refaults as the first tier. 4636 */ 4637 read_ctrl_pos(lruvec, type, 0, 2, &sp); 4638 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4639 read_ctrl_pos(lruvec, type, tier, 3, &pv); 4640 if (!positive_ctrl_err(&sp, &pv)) 4641 break; 4642 } 4643 4644 return tier - 1; 4645} 4646 4647static int get_type_to_scan(struct lruvec *lruvec, int swappiness) 4648{ 4649 struct ctrl_pos sp, pv; 4650 4651 if (swappiness <= MIN_SWAPPINESS + 1) 4652 return LRU_GEN_FILE; 4653 4654 if (swappiness >= MAX_SWAPPINESS) 4655 return LRU_GEN_ANON; 4656 /* 4657 * Compare the sum of all tiers of anon with that of file to determine 4658 * which type to scan. 4659 */ 4660 read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp); 4661 read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv); 4662 4663 return positive_ctrl_err(&sp, &pv); 4664} 4665 4666static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec, 4667 struct scan_control *sc, int swappiness, 4668 int *type_scanned, struct list_head *list) 4669{ 4670 int i; 4671 int type = get_type_to_scan(lruvec, swappiness); 4672 4673 for_each_evictable_type(i, swappiness) { 4674 int scanned; 4675 int tier = get_tier_idx(lruvec, type); 4676 4677 *type_scanned = type; 4678 4679 scanned = scan_folios(nr_to_scan, lruvec, sc, type, tier, list); 4680 if (scanned) 4681 return scanned; 4682 4683 type = !type; 4684 } 4685 4686 return 0; 4687} 4688 4689static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, 4690 struct scan_control *sc, int swappiness) 4691{ 4692 int type; 4693 int scanned; 4694 int reclaimed; 4695 LIST_HEAD(list); 4696 LIST_HEAD(clean); 4697 struct folio *folio; 4698 struct folio *next; 4699 enum vm_event_item item; 4700 struct reclaim_stat stat; 4701 struct lru_gen_mm_walk *walk; 4702 bool skip_retry = false; 4703 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4704 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4705 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4706 4707 spin_lock_irq(&lruvec->lru_lock); 4708 4709 scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &type, &list); 4710 4711 scanned += try_to_inc_min_seq(lruvec, swappiness); 4712 4713 if (evictable_min_seq(lrugen->min_seq, swappiness) + MIN_NR_GENS > lrugen->max_seq) 4714 scanned = 0; 4715 4716 spin_unlock_irq(&lruvec->lru_lock); 4717 4718 if (list_empty(&list)) 4719 return scanned; 4720retry: 4721 reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg); 4722 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 4723 sc->nr_reclaimed += reclaimed; 4724 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 4725 scanned, reclaimed, &stat, sc->priority, 4726 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); 4727 4728 list_for_each_entry_safe_reverse(folio, next, &list, lru) { 4729 DEFINE_MIN_SEQ(lruvec); 4730 4731 if (!folio_evictable(folio)) { 4732 list_del(&folio->lru); 4733 folio_putback_lru(folio); 4734 continue; 4735 } 4736 4737 /* retry folios that may have missed folio_rotate_reclaimable() */ 4738 if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) && 4739 !folio_test_dirty(folio) && !folio_test_writeback(folio)) { 4740 list_move(&folio->lru, &clean); 4741 continue; 4742 } 4743 4744 /* don't add rejected folios to the oldest generation */ 4745 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) 4746 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active)); 4747 } 4748 4749 spin_lock_irq(&lruvec->lru_lock); 4750 4751 move_folios_to_lru(lruvec, &list); 4752 4753 walk = current->reclaim_state->mm_walk; 4754 if (walk && walk->batched) { 4755 walk->lruvec = lruvec; 4756 reset_batch_size(walk); 4757 } 4758 4759 mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), 4760 stat.nr_demoted); 4761 4762 item = PGSTEAL_KSWAPD + reclaimer_offset(sc); 4763 if (!cgroup_reclaim(sc)) 4764 __count_vm_events(item, reclaimed); 4765 count_memcg_events(memcg, item, reclaimed); 4766 __count_vm_events(PGSTEAL_ANON + type, reclaimed); 4767 4768 spin_unlock_irq(&lruvec->lru_lock); 4769 4770 list_splice_init(&clean, &list); 4771 4772 if (!list_empty(&list)) { 4773 skip_retry = true; 4774 goto retry; 4775 } 4776 4777 return scanned; 4778} 4779 4780static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, 4781 int swappiness, unsigned long *nr_to_scan) 4782{ 4783 int gen, type, zone; 4784 unsigned long size = 0; 4785 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4786 DEFINE_MIN_SEQ(lruvec); 4787 4788 *nr_to_scan = 0; 4789 /* have to run aging, since eviction is not possible anymore */ 4790 if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq) 4791 return true; 4792 4793 for_each_evictable_type(type, swappiness) { 4794 unsigned long seq; 4795 4796 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4797 gen = lru_gen_from_seq(seq); 4798 4799 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4800 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4801 } 4802 } 4803 4804 *nr_to_scan = size; 4805 /* better to run aging even though eviction is still possible */ 4806 return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq; 4807} 4808 4809/* 4810 * For future optimizations: 4811 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg 4812 * reclaim. 4813 */ 4814static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness) 4815{ 4816 bool success; 4817 unsigned long nr_to_scan; 4818 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4819 DEFINE_MAX_SEQ(lruvec); 4820 4821 if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) 4822 return -1; 4823 4824 success = should_run_aging(lruvec, max_seq, swappiness, &nr_to_scan); 4825 4826 /* try to scrape all its memory if this memcg was deleted */ 4827 if (nr_to_scan && !mem_cgroup_online(memcg)) 4828 return nr_to_scan; 4829 4830 nr_to_scan = apply_proportional_protection(memcg, sc, nr_to_scan); 4831 4832 /* try to get away with not aging at the default priority */ 4833 if (!success || sc->priority == DEF_PRIORITY) 4834 return nr_to_scan >> sc->priority; 4835 4836 /* stop scanning this lruvec as it's low on cold folios */ 4837 return try_to_inc_max_seq(lruvec, max_seq, swappiness, false) ? -1 : 0; 4838} 4839 4840static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc) 4841{ 4842 int i; 4843 enum zone_watermarks mark; 4844 4845 /* don't abort memcg reclaim to ensure fairness */ 4846 if (!root_reclaim(sc)) 4847 return false; 4848 4849 if (sc->nr_reclaimed >= max(sc->nr_to_reclaim, compact_gap(sc->order))) 4850 return true; 4851 4852 /* check the order to exclude compaction-induced reclaim */ 4853 if (!current_is_kswapd() || sc->order) 4854 return false; 4855 4856 mark = sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ? 4857 WMARK_PROMO : WMARK_HIGH; 4858 4859 for (i = 0; i <= sc->reclaim_idx; i++) { 4860 struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; 4861 unsigned long size = wmark_pages(zone, mark) + MIN_LRU_BATCH; 4862 4863 if (managed_zone(zone) && !zone_watermark_ok(zone, 0, size, sc->reclaim_idx, 0)) 4864 return false; 4865 } 4866 4867 /* kswapd should abort if all eligible zones are safe */ 4868 return true; 4869} 4870 4871static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 4872{ 4873 long nr_to_scan; 4874 unsigned long scanned = 0; 4875 int swappiness = get_swappiness(lruvec, sc); 4876 4877 while (true) { 4878 int delta; 4879 4880 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); 4881 if (nr_to_scan <= 0) 4882 break; 4883 4884 delta = evict_folios(nr_to_scan, lruvec, sc, swappiness); 4885 if (!delta) 4886 break; 4887 4888 scanned += delta; 4889 if (scanned >= nr_to_scan) 4890 break; 4891 4892 if (should_abort_scan(lruvec, sc)) 4893 break; 4894 4895 cond_resched(); 4896 } 4897 4898 /* 4899 * If too many file cache in the coldest generation can't be evicted 4900 * due to being dirty, wake up the flusher. 4901 */ 4902 if (sc->nr.unqueued_dirty && sc->nr.unqueued_dirty == sc->nr.file_taken) 4903 wakeup_flusher_threads(WB_REASON_VMSCAN); 4904 4905 /* whether this lruvec should be rotated */ 4906 return nr_to_scan < 0; 4907} 4908 4909static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) 4910{ 4911 bool success; 4912 unsigned long scanned = sc->nr_scanned; 4913 unsigned long reclaimed = sc->nr_reclaimed; 4914 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4915 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4916 4917 /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ 4918 if (mem_cgroup_below_min(NULL, memcg)) 4919 return MEMCG_LRU_YOUNG; 4920 4921 if (mem_cgroup_below_low(NULL, memcg)) { 4922 /* see the comment on MEMCG_NR_GENS */ 4923 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL) 4924 return MEMCG_LRU_TAIL; 4925 4926 memcg_memory_event(memcg, MEMCG_LOW); 4927 } 4928 4929 success = try_to_shrink_lruvec(lruvec, sc); 4930 4931 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority); 4932 4933 if (!sc->proactive) 4934 vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned, 4935 sc->nr_reclaimed - reclaimed); 4936 4937 flush_reclaim_state(sc); 4938 4939 if (success && mem_cgroup_online(memcg)) 4940 return MEMCG_LRU_YOUNG; 4941 4942 if (!success && lruvec_is_sizable(lruvec, sc)) 4943 return 0; 4944 4945 /* one retry if offlined or too small */ 4946 return READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL ? 4947 MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG; 4948} 4949 4950static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) 4951{ 4952 int op; 4953 int gen; 4954 int bin; 4955 int first_bin; 4956 struct lruvec *lruvec; 4957 struct lru_gen_folio *lrugen; 4958 struct mem_cgroup *memcg; 4959 struct hlist_nulls_node *pos; 4960 4961 gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq)); 4962 bin = first_bin = get_random_u32_below(MEMCG_NR_BINS); 4963restart: 4964 op = 0; 4965 memcg = NULL; 4966 4967 rcu_read_lock(); 4968 4969 hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) { 4970 if (op) { 4971 lru_gen_rotate_memcg(lruvec, op); 4972 op = 0; 4973 } 4974 4975 mem_cgroup_put(memcg); 4976 memcg = NULL; 4977 4978 if (gen != READ_ONCE(lrugen->gen)) 4979 continue; 4980 4981 lruvec = container_of(lrugen, struct lruvec, lrugen); 4982 memcg = lruvec_memcg(lruvec); 4983 4984 if (!mem_cgroup_tryget(memcg)) { 4985 lru_gen_release_memcg(memcg); 4986 memcg = NULL; 4987 continue; 4988 } 4989 4990 rcu_read_unlock(); 4991 4992 op = shrink_one(lruvec, sc); 4993 4994 rcu_read_lock(); 4995 4996 if (should_abort_scan(lruvec, sc)) 4997 break; 4998 } 4999 5000 rcu_read_unlock(); 5001 5002 if (op) 5003 lru_gen_rotate_memcg(lruvec, op); 5004 5005 mem_cgroup_put(memcg); 5006 5007 if (!is_a_nulls(pos)) 5008 return; 5009 5010 /* restart if raced with lru_gen_rotate_memcg() */ 5011 if (gen != get_nulls_value(pos)) 5012 goto restart; 5013 5014 /* try the rest of the bins of the current generation */ 5015 bin = get_memcg_bin(bin + 1); 5016 if (bin != first_bin) 5017 goto restart; 5018} 5019 5020static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5021{ 5022 struct blk_plug plug; 5023 5024 VM_WARN_ON_ONCE(root_reclaim(sc)); 5025 VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap); 5026 5027 lru_add_drain(); 5028 5029 blk_start_plug(&plug); 5030 5031 set_mm_walk(NULL, sc->proactive); 5032 5033 if (try_to_shrink_lruvec(lruvec, sc)) 5034 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG); 5035 5036 clear_mm_walk(); 5037 5038 blk_finish_plug(&plug); 5039} 5040 5041static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) 5042{ 5043 struct blk_plug plug; 5044 unsigned long reclaimed = sc->nr_reclaimed; 5045 5046 VM_WARN_ON_ONCE(!root_reclaim(sc)); 5047 5048 /* 5049 * Unmapped clean folios are already prioritized. Scanning for more of 5050 * them is likely futile and can cause high reclaim latency when there 5051 * is a large number of memcgs. 5052 */ 5053 if (!sc->may_writepage || !sc->may_unmap) 5054 goto done; 5055 5056 lru_add_drain(); 5057 5058 blk_start_plug(&plug); 5059 5060 set_mm_walk(pgdat, sc->proactive); 5061 5062 set_initial_priority(pgdat, sc); 5063 5064 if (current_is_kswapd()) 5065 sc->nr_reclaimed = 0; 5066 5067 if (mem_cgroup_disabled()) 5068 shrink_one(&pgdat->__lruvec, sc); 5069 else 5070 shrink_many(pgdat, sc); 5071 5072 if (current_is_kswapd()) 5073 sc->nr_reclaimed += reclaimed; 5074 5075 clear_mm_walk(); 5076 5077 blk_finish_plug(&plug); 5078done: 5079 if (sc->nr_reclaimed > reclaimed) 5080 atomic_set(&pgdat->kswapd_failures, 0); 5081} 5082 5083/****************************************************************************** 5084 * state change 5085 ******************************************************************************/ 5086 5087static bool __maybe_unused state_is_valid(struct lruvec *lruvec) 5088{ 5089 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5090 5091 if (lrugen->enabled) { 5092 enum lru_list lru; 5093 5094 for_each_evictable_lru(lru) { 5095 if (!list_empty(&lruvec->lists[lru])) 5096 return false; 5097 } 5098 } else { 5099 int gen, type, zone; 5100 5101 for_each_gen_type_zone(gen, type, zone) { 5102 if (!list_empty(&lrugen->folios[gen][type][zone])) 5103 return false; 5104 } 5105 } 5106 5107 return true; 5108} 5109 5110static bool fill_evictable(struct lruvec *lruvec) 5111{ 5112 enum lru_list lru; 5113 int remaining = MAX_LRU_BATCH; 5114 5115 for_each_evictable_lru(lru) { 5116 int type = is_file_lru(lru); 5117 bool active = is_active_lru(lru); 5118 struct list_head *head = &lruvec->lists[lru]; 5119 5120 while (!list_empty(head)) { 5121 bool success; 5122 struct folio *folio = lru_to_folio(head); 5123 5124 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5125 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); 5126 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5127 VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); 5128 5129 lruvec_del_folio(lruvec, folio); 5130 success = lru_gen_add_folio(lruvec, folio, false); 5131 VM_WARN_ON_ONCE(!success); 5132 5133 if (!--remaining) 5134 return false; 5135 } 5136 } 5137 5138 return true; 5139} 5140 5141static bool drain_evictable(struct lruvec *lruvec) 5142{ 5143 int gen, type, zone; 5144 int remaining = MAX_LRU_BATCH; 5145 5146 for_each_gen_type_zone(gen, type, zone) { 5147 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; 5148 5149 while (!list_empty(head)) { 5150 bool success; 5151 struct folio *folio = lru_to_folio(head); 5152 5153 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5154 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 5155 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5156 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 5157 5158 success = lru_gen_del_folio(lruvec, folio, false); 5159 VM_WARN_ON_ONCE(!success); 5160 lruvec_add_folio(lruvec, folio); 5161 5162 if (!--remaining) 5163 return false; 5164 } 5165 } 5166 5167 return true; 5168} 5169 5170static void lru_gen_change_state(bool enabled) 5171{ 5172 static DEFINE_MUTEX(state_mutex); 5173 5174 struct mem_cgroup *memcg; 5175 5176 cgroup_lock(); 5177 cpus_read_lock(); 5178 get_online_mems(); 5179 mutex_lock(&state_mutex); 5180 5181 if (enabled == lru_gen_enabled()) 5182 goto unlock; 5183 5184 if (enabled) 5185 static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5186 else 5187 static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5188 5189 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5190 do { 5191 int nid; 5192 5193 for_each_node(nid) { 5194 struct lruvec *lruvec = get_lruvec(memcg, nid); 5195 5196 spin_lock_irq(&lruvec->lru_lock); 5197 5198 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 5199 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); 5200 5201 lruvec->lrugen.enabled = enabled; 5202 5203 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { 5204 spin_unlock_irq(&lruvec->lru_lock); 5205 cond_resched(); 5206 spin_lock_irq(&lruvec->lru_lock); 5207 } 5208 5209 spin_unlock_irq(&lruvec->lru_lock); 5210 } 5211 5212 cond_resched(); 5213 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5214unlock: 5215 mutex_unlock(&state_mutex); 5216 put_online_mems(); 5217 cpus_read_unlock(); 5218 cgroup_unlock(); 5219} 5220 5221/****************************************************************************** 5222 * sysfs interface 5223 ******************************************************************************/ 5224 5225static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5226{ 5227 return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); 5228} 5229 5230/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5231static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr, 5232 const char *buf, size_t len) 5233{ 5234 unsigned int msecs; 5235 5236 if (kstrtouint(buf, 0, &msecs)) 5237 return -EINVAL; 5238 5239 WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); 5240 5241 return len; 5242} 5243 5244static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms); 5245 5246static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5247{ 5248 unsigned int caps = 0; 5249 5250 if (get_cap(LRU_GEN_CORE)) 5251 caps |= BIT(LRU_GEN_CORE); 5252 5253 if (should_walk_mmu()) 5254 caps |= BIT(LRU_GEN_MM_WALK); 5255 5256 if (should_clear_pmd_young()) 5257 caps |= BIT(LRU_GEN_NONLEAF_YOUNG); 5258 5259 return sysfs_emit(buf, "0x%04x\n", caps); 5260} 5261 5262/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5263static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, 5264 const char *buf, size_t len) 5265{ 5266 int i; 5267 unsigned int caps; 5268 5269 if (tolower(*buf) == 'n') 5270 caps = 0; 5271 else if (tolower(*buf) == 'y') 5272 caps = -1; 5273 else if (kstrtouint(buf, 0, &caps)) 5274 return -EINVAL; 5275 5276 for (i = 0; i < NR_LRU_GEN_CAPS; i++) { 5277 bool enabled = caps & BIT(i); 5278 5279 if (i == LRU_GEN_CORE) 5280 lru_gen_change_state(enabled); 5281 else if (enabled) 5282 static_branch_enable(&lru_gen_caps[i]); 5283 else 5284 static_branch_disable(&lru_gen_caps[i]); 5285 } 5286 5287 return len; 5288} 5289 5290static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled); 5291 5292static struct attribute *lru_gen_attrs[] = { 5293 &lru_gen_min_ttl_attr.attr, 5294 &lru_gen_enabled_attr.attr, 5295 NULL 5296}; 5297 5298static const struct attribute_group lru_gen_attr_group = { 5299 .name = "lru_gen", 5300 .attrs = lru_gen_attrs, 5301}; 5302 5303/****************************************************************************** 5304 * debugfs interface 5305 ******************************************************************************/ 5306 5307static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) 5308{ 5309 struct mem_cgroup *memcg; 5310 loff_t nr_to_skip = *pos; 5311 5312 m->private = kvmalloc(PATH_MAX, GFP_KERNEL); 5313 if (!m->private) 5314 return ERR_PTR(-ENOMEM); 5315 5316 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5317 do { 5318 int nid; 5319 5320 for_each_node_state(nid, N_MEMORY) { 5321 if (!nr_to_skip--) 5322 return get_lruvec(memcg, nid); 5323 } 5324 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5325 5326 return NULL; 5327} 5328 5329static void lru_gen_seq_stop(struct seq_file *m, void *v) 5330{ 5331 if (!IS_ERR_OR_NULL(v)) 5332 mem_cgroup_iter_break(NULL, lruvec_memcg(v)); 5333 5334 kvfree(m->private); 5335 m->private = NULL; 5336} 5337 5338static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) 5339{ 5340 int nid = lruvec_pgdat(v)->node_id; 5341 struct mem_cgroup *memcg = lruvec_memcg(v); 5342 5343 ++*pos; 5344 5345 nid = next_memory_node(nid); 5346 if (nid == MAX_NUMNODES) { 5347 memcg = mem_cgroup_iter(NULL, memcg, NULL); 5348 if (!memcg) 5349 return NULL; 5350 5351 nid = first_memory_node; 5352 } 5353 5354 return get_lruvec(memcg, nid); 5355} 5356 5357static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, 5358 unsigned long max_seq, unsigned long *min_seq, 5359 unsigned long seq) 5360{ 5361 int i; 5362 int type, tier; 5363 int hist = lru_hist_from_seq(seq); 5364 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5365 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5366 5367 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 5368 seq_printf(m, " %10d", tier); 5369 for (type = 0; type < ANON_AND_FILE; type++) { 5370 const char *s = "xxx"; 5371 unsigned long n[3] = {}; 5372 5373 if (seq == max_seq) { 5374 s = "RTx"; 5375 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); 5376 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); 5377 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { 5378 s = "rep"; 5379 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); 5380 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); 5381 n[2] = READ_ONCE(lrugen->protected[hist][type][tier]); 5382 } 5383 5384 for (i = 0; i < 3; i++) 5385 seq_printf(m, " %10lu%c", n[i], s[i]); 5386 } 5387 seq_putc(m, '\n'); 5388 } 5389 5390 if (!mm_state) 5391 return; 5392 5393 seq_puts(m, " "); 5394 for (i = 0; i < NR_MM_STATS; i++) { 5395 const char *s = "xxxx"; 5396 unsigned long n = 0; 5397 5398 if (seq == max_seq && NR_HIST_GENS == 1) { 5399 s = "TYFA"; 5400 n = READ_ONCE(mm_state->stats[hist][i]); 5401 } else if (seq != max_seq && NR_HIST_GENS > 1) { 5402 s = "tyfa"; 5403 n = READ_ONCE(mm_state->stats[hist][i]); 5404 } 5405 5406 seq_printf(m, " %10lu%c", n, s[i]); 5407 } 5408 seq_putc(m, '\n'); 5409} 5410 5411/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5412static int lru_gen_seq_show(struct seq_file *m, void *v) 5413{ 5414 unsigned long seq; 5415 bool full = debugfs_get_aux_num(m->file); 5416 struct lruvec *lruvec = v; 5417 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5418 int nid = lruvec_pgdat(lruvec)->node_id; 5419 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5420 DEFINE_MAX_SEQ(lruvec); 5421 DEFINE_MIN_SEQ(lruvec); 5422 5423 if (nid == first_memory_node) { 5424 const char *path = memcg ? m->private : ""; 5425 5426#ifdef CONFIG_MEMCG 5427 if (memcg) 5428 cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); 5429#endif 5430 seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); 5431 } 5432 5433 seq_printf(m, " node %5d\n", nid); 5434 5435 if (!full) 5436 seq = evictable_min_seq(min_seq, MAX_SWAPPINESS / 2); 5437 else if (max_seq >= MAX_NR_GENS) 5438 seq = max_seq - MAX_NR_GENS + 1; 5439 else 5440 seq = 0; 5441 5442 for (; seq <= max_seq; seq++) { 5443 int type, zone; 5444 int gen = lru_gen_from_seq(seq); 5445 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 5446 5447 seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); 5448 5449 for (type = 0; type < ANON_AND_FILE; type++) { 5450 unsigned long size = 0; 5451 char mark = full && seq < min_seq[type] ? 'x' : ' '; 5452 5453 for (zone = 0; zone < MAX_NR_ZONES; zone++) 5454 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 5455 5456 seq_printf(m, " %10lu%c", size, mark); 5457 } 5458 5459 seq_putc(m, '\n'); 5460 5461 if (full) 5462 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); 5463 } 5464 5465 return 0; 5466} 5467 5468static const struct seq_operations lru_gen_seq_ops = { 5469 .start = lru_gen_seq_start, 5470 .stop = lru_gen_seq_stop, 5471 .next = lru_gen_seq_next, 5472 .show = lru_gen_seq_show, 5473}; 5474 5475static int run_aging(struct lruvec *lruvec, unsigned long seq, 5476 int swappiness, bool force_scan) 5477{ 5478 DEFINE_MAX_SEQ(lruvec); 5479 5480 if (seq > max_seq) 5481 return -EINVAL; 5482 5483 return try_to_inc_max_seq(lruvec, max_seq, swappiness, force_scan) ? 0 : -EEXIST; 5484} 5485 5486static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5487 int swappiness, unsigned long nr_to_reclaim) 5488{ 5489 DEFINE_MAX_SEQ(lruvec); 5490 5491 if (seq + MIN_NR_GENS > max_seq) 5492 return -EINVAL; 5493 5494 sc->nr_reclaimed = 0; 5495 5496 while (!signal_pending(current)) { 5497 DEFINE_MIN_SEQ(lruvec); 5498 5499 if (seq < evictable_min_seq(min_seq, swappiness)) 5500 return 0; 5501 5502 if (sc->nr_reclaimed >= nr_to_reclaim) 5503 return 0; 5504 5505 if (!evict_folios(nr_to_reclaim - sc->nr_reclaimed, lruvec, sc, 5506 swappiness)) 5507 return 0; 5508 5509 cond_resched(); 5510 } 5511 5512 return -EINTR; 5513} 5514 5515static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, 5516 struct scan_control *sc, int swappiness, unsigned long opt) 5517{ 5518 struct lruvec *lruvec; 5519 int err = -EINVAL; 5520 struct mem_cgroup *memcg = NULL; 5521 5522 if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) 5523 return -EINVAL; 5524 5525 if (!mem_cgroup_disabled()) { 5526 rcu_read_lock(); 5527 5528 memcg = mem_cgroup_from_id(memcg_id); 5529 if (!mem_cgroup_tryget(memcg)) 5530 memcg = NULL; 5531 5532 rcu_read_unlock(); 5533 5534 if (!memcg) 5535 return -EINVAL; 5536 } 5537 5538 if (memcg_id != mem_cgroup_id(memcg)) 5539 goto done; 5540 5541 sc->target_mem_cgroup = memcg; 5542 lruvec = get_lruvec(memcg, nid); 5543 5544 if (swappiness < MIN_SWAPPINESS) 5545 swappiness = get_swappiness(lruvec, sc); 5546 else if (swappiness > SWAPPINESS_ANON_ONLY) 5547 goto done; 5548 5549 switch (cmd) { 5550 case '+': 5551 err = run_aging(lruvec, seq, swappiness, opt); 5552 break; 5553 case '-': 5554 err = run_eviction(lruvec, seq, sc, swappiness, opt); 5555 break; 5556 } 5557done: 5558 mem_cgroup_put(memcg); 5559 5560 return err; 5561} 5562 5563/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5564static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, 5565 size_t len, loff_t *pos) 5566{ 5567 void *buf; 5568 char *cur, *next; 5569 unsigned int flags; 5570 struct blk_plug plug; 5571 int err = -EINVAL; 5572 struct scan_control sc = { 5573 .may_writepage = true, 5574 .may_unmap = true, 5575 .may_swap = true, 5576 .reclaim_idx = MAX_NR_ZONES - 1, 5577 .gfp_mask = GFP_KERNEL, 5578 .proactive = true, 5579 }; 5580 5581 buf = kvmalloc(len + 1, GFP_KERNEL); 5582 if (!buf) 5583 return -ENOMEM; 5584 5585 if (copy_from_user(buf, src, len)) { 5586 kvfree(buf); 5587 return -EFAULT; 5588 } 5589 5590 set_task_reclaim_state(current, &sc.reclaim_state); 5591 flags = memalloc_noreclaim_save(); 5592 blk_start_plug(&plug); 5593 if (!set_mm_walk(NULL, true)) { 5594 err = -ENOMEM; 5595 goto done; 5596 } 5597 5598 next = buf; 5599 next[len] = '\0'; 5600 5601 while ((cur = strsep(&next, ",;\n"))) { 5602 int n; 5603 int end; 5604 char cmd, swap_string[5]; 5605 unsigned int memcg_id; 5606 unsigned int nid; 5607 unsigned long seq; 5608 unsigned int swappiness; 5609 unsigned long opt = -1; 5610 5611 cur = skip_spaces(cur); 5612 if (!*cur) 5613 continue; 5614 5615 n = sscanf(cur, "%c %u %u %lu %n %4s %n %lu %n", &cmd, &memcg_id, &nid, 5616 &seq, &end, swap_string, &end, &opt, &end); 5617 if (n < 4 || cur[end]) { 5618 err = -EINVAL; 5619 break; 5620 } 5621 5622 if (n == 4) { 5623 swappiness = -1; 5624 } else if (!strcmp("max", swap_string)) { 5625 /* set by userspace for anonymous memory only */ 5626 swappiness = SWAPPINESS_ANON_ONLY; 5627 } else { 5628 err = kstrtouint(swap_string, 0, &swappiness); 5629 if (err) 5630 break; 5631 } 5632 5633 err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); 5634 if (err) 5635 break; 5636 } 5637done: 5638 clear_mm_walk(); 5639 blk_finish_plug(&plug); 5640 memalloc_noreclaim_restore(flags); 5641 set_task_reclaim_state(current, NULL); 5642 5643 kvfree(buf); 5644 5645 return err ? : len; 5646} 5647 5648static int lru_gen_seq_open(struct inode *inode, struct file *file) 5649{ 5650 return seq_open(file, &lru_gen_seq_ops); 5651} 5652 5653static const struct file_operations lru_gen_rw_fops = { 5654 .open = lru_gen_seq_open, 5655 .read = seq_read, 5656 .write = lru_gen_seq_write, 5657 .llseek = seq_lseek, 5658 .release = seq_release, 5659}; 5660 5661static const struct file_operations lru_gen_ro_fops = { 5662 .open = lru_gen_seq_open, 5663 .read = seq_read, 5664 .llseek = seq_lseek, 5665 .release = seq_release, 5666}; 5667 5668/****************************************************************************** 5669 * initialization 5670 ******************************************************************************/ 5671 5672void lru_gen_init_pgdat(struct pglist_data *pgdat) 5673{ 5674 int i, j; 5675 5676 spin_lock_init(&pgdat->memcg_lru.lock); 5677 5678 for (i = 0; i < MEMCG_NR_GENS; i++) { 5679 for (j = 0; j < MEMCG_NR_BINS; j++) 5680 INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i); 5681 } 5682} 5683 5684void lru_gen_init_lruvec(struct lruvec *lruvec) 5685{ 5686 int i; 5687 int gen, type, zone; 5688 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5689 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5690 5691 lrugen->max_seq = MIN_NR_GENS + 1; 5692 lrugen->enabled = lru_gen_enabled(); 5693 5694 for (i = 0; i <= MIN_NR_GENS + 1; i++) 5695 lrugen->timestamps[i] = jiffies; 5696 5697 for_each_gen_type_zone(gen, type, zone) 5698 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); 5699 5700 if (mm_state) 5701 mm_state->seq = MIN_NR_GENS; 5702} 5703 5704#ifdef CONFIG_MEMCG 5705 5706void lru_gen_init_memcg(struct mem_cgroup *memcg) 5707{ 5708 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 5709 5710 if (!mm_list) 5711 return; 5712 5713 INIT_LIST_HEAD(&mm_list->fifo); 5714 spin_lock_init(&mm_list->lock); 5715} 5716 5717void lru_gen_exit_memcg(struct mem_cgroup *memcg) 5718{ 5719 int i; 5720 int nid; 5721 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 5722 5723 VM_WARN_ON_ONCE(mm_list && !list_empty(&mm_list->fifo)); 5724 5725 for_each_node(nid) { 5726 struct lruvec *lruvec = get_lruvec(memcg, nid); 5727 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5728 5729 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, 5730 sizeof(lruvec->lrugen.nr_pages))); 5731 5732 lruvec->lrugen.list.next = LIST_POISON1; 5733 5734 if (!mm_state) 5735 continue; 5736 5737 for (i = 0; i < NR_BLOOM_FILTERS; i++) { 5738 bitmap_free(mm_state->filters[i]); 5739 mm_state->filters[i] = NULL; 5740 } 5741 } 5742} 5743 5744#endif /* CONFIG_MEMCG */ 5745 5746static int __init init_lru_gen(void) 5747{ 5748 BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); 5749 BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); 5750 5751 if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) 5752 pr_err("lru_gen: failed to create sysfs group\n"); 5753 5754 debugfs_create_file_aux_num("lru_gen", 0644, NULL, NULL, false, 5755 &lru_gen_rw_fops); 5756 debugfs_create_file_aux_num("lru_gen_full", 0444, NULL, NULL, true, 5757 &lru_gen_ro_fops); 5758 5759 return 0; 5760}; 5761late_initcall(init_lru_gen); 5762 5763#else /* !CONFIG_LRU_GEN */ 5764 5765static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 5766{ 5767 BUILD_BUG(); 5768} 5769 5770static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5771{ 5772 BUILD_BUG(); 5773} 5774 5775static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) 5776{ 5777 BUILD_BUG(); 5778} 5779 5780#endif /* CONFIG_LRU_GEN */ 5781 5782static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5783{ 5784 unsigned long nr[NR_LRU_LISTS]; 5785 unsigned long targets[NR_LRU_LISTS]; 5786 unsigned long nr_to_scan; 5787 enum lru_list lru; 5788 unsigned long nr_reclaimed = 0; 5789 unsigned long nr_to_reclaim = sc->nr_to_reclaim; 5790 bool proportional_reclaim; 5791 struct blk_plug plug; 5792 5793 if (lru_gen_enabled() && !root_reclaim(sc)) { 5794 lru_gen_shrink_lruvec(lruvec, sc); 5795 return; 5796 } 5797 5798 get_scan_count(lruvec, sc, nr); 5799 5800 /* Record the original scan target for proportional adjustments later */ 5801 memcpy(targets, nr, sizeof(nr)); 5802 5803 /* 5804 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal 5805 * event that can occur when there is little memory pressure e.g. 5806 * multiple streaming readers/writers. Hence, we do not abort scanning 5807 * when the requested number of pages are reclaimed when scanning at 5808 * DEF_PRIORITY on the assumption that the fact we are direct 5809 * reclaiming implies that kswapd is not keeping up and it is best to 5810 * do a batch of work at once. For memcg reclaim one check is made to 5811 * abort proportional reclaim if either the file or anon lru has already 5812 * dropped to zero at the first pass. 5813 */ 5814 proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && 5815 sc->priority == DEF_PRIORITY); 5816 5817 blk_start_plug(&plug); 5818 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || 5819 nr[LRU_INACTIVE_FILE]) { 5820 unsigned long nr_anon, nr_file, percentage; 5821 unsigned long nr_scanned; 5822 5823 for_each_evictable_lru(lru) { 5824 if (nr[lru]) { 5825 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); 5826 nr[lru] -= nr_to_scan; 5827 5828 nr_reclaimed += shrink_list(lru, nr_to_scan, 5829 lruvec, sc); 5830 } 5831 } 5832 5833 cond_resched(); 5834 5835 if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) 5836 continue; 5837 5838 /* 5839 * For kswapd and memcg, reclaim at least the number of pages 5840 * requested. Ensure that the anon and file LRUs are scanned 5841 * proportionally what was requested by get_scan_count(). We 5842 * stop reclaiming one LRU and reduce the amount scanning 5843 * proportional to the original scan target. 5844 */ 5845 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; 5846 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; 5847 5848 /* 5849 * It's just vindictive to attack the larger once the smaller 5850 * has gone to zero. And given the way we stop scanning the 5851 * smaller below, this makes sure that we only make one nudge 5852 * towards proportionality once we've got nr_to_reclaim. 5853 */ 5854 if (!nr_file || !nr_anon) 5855 break; 5856 5857 if (nr_file > nr_anon) { 5858 unsigned long scan_target = targets[LRU_INACTIVE_ANON] + 5859 targets[LRU_ACTIVE_ANON] + 1; 5860 lru = LRU_BASE; 5861 percentage = nr_anon * 100 / scan_target; 5862 } else { 5863 unsigned long scan_target = targets[LRU_INACTIVE_FILE] + 5864 targets[LRU_ACTIVE_FILE] + 1; 5865 lru = LRU_FILE; 5866 percentage = nr_file * 100 / scan_target; 5867 } 5868 5869 /* Stop scanning the smaller of the LRU */ 5870 nr[lru] = 0; 5871 nr[lru + LRU_ACTIVE] = 0; 5872 5873 /* 5874 * Recalculate the other LRU scan count based on its original 5875 * scan target and the percentage scanning already complete 5876 */ 5877 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; 5878 nr_scanned = targets[lru] - nr[lru]; 5879 nr[lru] = targets[lru] * (100 - percentage) / 100; 5880 nr[lru] -= min(nr[lru], nr_scanned); 5881 5882 lru += LRU_ACTIVE; 5883 nr_scanned = targets[lru] - nr[lru]; 5884 nr[lru] = targets[lru] * (100 - percentage) / 100; 5885 nr[lru] -= min(nr[lru], nr_scanned); 5886 } 5887 blk_finish_plug(&plug); 5888 sc->nr_reclaimed += nr_reclaimed; 5889 5890 /* 5891 * Even if we did not try to evict anon pages at all, we want to 5892 * rebalance the anon lru active/inactive ratio. 5893 */ 5894 if (can_age_anon_pages(lruvec, sc) && 5895 inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 5896 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 5897 sc, LRU_ACTIVE_ANON); 5898} 5899 5900/* Use reclaim/compaction for costly allocs or under memory pressure */ 5901static bool in_reclaim_compaction(struct scan_control *sc) 5902{ 5903 if (gfp_compaction_allowed(sc->gfp_mask) && sc->order && 5904 (sc->order > PAGE_ALLOC_COSTLY_ORDER || 5905 sc->priority < DEF_PRIORITY - 2)) 5906 return true; 5907 5908 return false; 5909} 5910 5911/* 5912 * Reclaim/compaction is used for high-order allocation requests. It reclaims 5913 * order-0 pages before compacting the zone. should_continue_reclaim() returns 5914 * true if more pages should be reclaimed such that when the page allocator 5915 * calls try_to_compact_pages() that it will have enough free pages to succeed. 5916 * It will give up earlier than that if there is difficulty reclaiming pages. 5917 */ 5918static inline bool should_continue_reclaim(struct pglist_data *pgdat, 5919 unsigned long nr_reclaimed, 5920 struct scan_control *sc) 5921{ 5922 unsigned long pages_for_compaction; 5923 unsigned long inactive_lru_pages; 5924 int z; 5925 struct zone *zone; 5926 5927 /* If not in reclaim/compaction mode, stop */ 5928 if (!in_reclaim_compaction(sc)) 5929 return false; 5930 5931 /* 5932 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX 5933 * number of pages that were scanned. This will return to the caller 5934 * with the risk reclaim/compaction and the resulting allocation attempt 5935 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL 5936 * allocations through requiring that the full LRU list has been scanned 5937 * first, by assuming that zero delta of sc->nr_scanned means full LRU 5938 * scan, but that approximation was wrong, and there were corner cases 5939 * where always a non-zero amount of pages were scanned. 5940 */ 5941 if (!nr_reclaimed) 5942 return false; 5943 5944 /* If compaction would go ahead or the allocation would succeed, stop */ 5945 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) { 5946 unsigned long watermark = min_wmark_pages(zone); 5947 5948 /* Allocation can already succeed, nothing to do */ 5949 if (zone_watermark_ok(zone, sc->order, watermark, 5950 sc->reclaim_idx, 0)) 5951 return false; 5952 5953 if (compaction_suitable(zone, sc->order, watermark, 5954 sc->reclaim_idx)) 5955 return false; 5956 } 5957 5958 /* 5959 * If we have not reclaimed enough pages for compaction and the 5960 * inactive lists are large enough, continue reclaiming 5961 */ 5962 pages_for_compaction = compact_gap(sc->order); 5963 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); 5964 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 5965 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); 5966 5967 return inactive_lru_pages > pages_for_compaction; 5968} 5969 5970static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) 5971{ 5972 struct mem_cgroup *target_memcg = sc->target_mem_cgroup; 5973 struct mem_cgroup_reclaim_cookie reclaim = { 5974 .pgdat = pgdat, 5975 }; 5976 struct mem_cgroup_reclaim_cookie *partial = &reclaim; 5977 struct mem_cgroup *memcg; 5978 5979 /* 5980 * In most cases, direct reclaimers can do partial walks 5981 * through the cgroup tree, using an iterator state that 5982 * persists across invocations. This strikes a balance between 5983 * fairness and allocation latency. 5984 * 5985 * For kswapd, reliable forward progress is more important 5986 * than a quick return to idle. Always do full walks. 5987 */ 5988 if (current_is_kswapd() || sc->memcg_full_walk) 5989 partial = NULL; 5990 5991 memcg = mem_cgroup_iter(target_memcg, NULL, partial); 5992 do { 5993 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 5994 unsigned long reclaimed; 5995 unsigned long scanned; 5996 5997 /* 5998 * This loop can become CPU-bound when target memcgs 5999 * aren't eligible for reclaim - either because they 6000 * don't have any reclaimable pages, or because their 6001 * memory is explicitly protected. Avoid soft lockups. 6002 */ 6003 cond_resched(); 6004 6005 mem_cgroup_calculate_protection(target_memcg, memcg); 6006 6007 if (mem_cgroup_below_min(target_memcg, memcg)) { 6008 /* 6009 * Hard protection. 6010 * If there is no reclaimable memory, OOM. 6011 */ 6012 continue; 6013 } else if (mem_cgroup_below_low(target_memcg, memcg)) { 6014 /* 6015 * Soft protection. 6016 * Respect the protection only as long as 6017 * there is an unprotected supply 6018 * of reclaimable memory from other cgroups. 6019 */ 6020 if (!sc->memcg_low_reclaim) { 6021 sc->memcg_low_skipped = 1; 6022 continue; 6023 } 6024 memcg_memory_event(memcg, MEMCG_LOW); 6025 } 6026 6027 reclaimed = sc->nr_reclaimed; 6028 scanned = sc->nr_scanned; 6029 6030 shrink_lruvec(lruvec, sc); 6031 6032 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, 6033 sc->priority); 6034 6035 /* Record the group's reclaim efficiency */ 6036 if (!sc->proactive) 6037 vmpressure(sc->gfp_mask, memcg, false, 6038 sc->nr_scanned - scanned, 6039 sc->nr_reclaimed - reclaimed); 6040 6041 /* If partial walks are allowed, bail once goal is reached */ 6042 if (partial && sc->nr_reclaimed >= sc->nr_to_reclaim) { 6043 mem_cgroup_iter_break(target_memcg, memcg); 6044 break; 6045 } 6046 } while ((memcg = mem_cgroup_iter(target_memcg, memcg, partial))); 6047} 6048 6049static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) 6050{ 6051 unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed; 6052 struct lruvec *target_lruvec; 6053 bool reclaimable = false; 6054 6055 if (lru_gen_enabled() && root_reclaim(sc)) { 6056 memset(&sc->nr, 0, sizeof(sc->nr)); 6057 lru_gen_shrink_node(pgdat, sc); 6058 return; 6059 } 6060 6061 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 6062 6063again: 6064 memset(&sc->nr, 0, sizeof(sc->nr)); 6065 6066 nr_reclaimed = sc->nr_reclaimed; 6067 nr_scanned = sc->nr_scanned; 6068 6069 prepare_scan_control(pgdat, sc); 6070 6071 shrink_node_memcgs(pgdat, sc); 6072 6073 flush_reclaim_state(sc); 6074 6075 nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed; 6076 6077 /* Record the subtree's reclaim efficiency */ 6078 if (!sc->proactive) 6079 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, 6080 sc->nr_scanned - nr_scanned, nr_node_reclaimed); 6081 6082 if (nr_node_reclaimed) 6083 reclaimable = true; 6084 6085 if (current_is_kswapd()) { 6086 /* 6087 * If reclaim is isolating dirty pages under writeback, 6088 * it implies that the long-lived page allocation rate 6089 * is exceeding the page laundering rate. Either the 6090 * global limits are not being effective at throttling 6091 * processes due to the page distribution throughout 6092 * zones or there is heavy usage of a slow backing 6093 * device. The only option is to throttle from reclaim 6094 * context which is not ideal as there is no guarantee 6095 * the dirtying process is throttled in the same way 6096 * balance_dirty_pages() manages. 6097 * 6098 * Once a node is flagged PGDAT_WRITEBACK, kswapd will 6099 * count the number of pages under pages flagged for 6100 * immediate reclaim and stall if any are encountered 6101 * in the nr_immediate check below. 6102 */ 6103 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) 6104 set_bit(PGDAT_WRITEBACK, &pgdat->flags); 6105 6106 /* 6107 * If kswapd scans pages marked for immediate 6108 * reclaim and under writeback (nr_immediate), it 6109 * implies that pages are cycling through the LRU 6110 * faster than they are written so forcibly stall 6111 * until some pages complete writeback. 6112 */ 6113 if (sc->nr.immediate) 6114 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 6115 } 6116 6117 /* 6118 * Tag a node/memcg as congested if all the dirty pages were marked 6119 * for writeback and immediate reclaim (counted in nr.congested). 6120 * 6121 * Legacy memcg will stall in page writeback so avoid forcibly 6122 * stalling in reclaim_throttle(). 6123 */ 6124 if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested) { 6125 if (cgroup_reclaim(sc) && writeback_throttling_sane(sc)) 6126 set_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags); 6127 6128 if (current_is_kswapd()) 6129 set_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags); 6130 } 6131 6132 /* 6133 * Stall direct reclaim for IO completions if the lruvec is 6134 * node is congested. Allow kswapd to continue until it 6135 * starts encountering unqueued dirty pages or cycling through 6136 * the LRU too quickly. 6137 */ 6138 if (!current_is_kswapd() && current_may_throttle() && 6139 !sc->hibernation_mode && 6140 (test_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags) || 6141 test_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags))) 6142 reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); 6143 6144 if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc)) 6145 goto again; 6146 6147 /* 6148 * Kswapd gives up on balancing particular nodes after too 6149 * many failures to reclaim anything from them and goes to 6150 * sleep. On reclaim progress, reset the failure counter. A 6151 * successful direct reclaim run will revive a dormant kswapd. 6152 */ 6153 if (reclaimable) 6154 atomic_set(&pgdat->kswapd_failures, 0); 6155 else if (sc->cache_trim_mode) 6156 sc->cache_trim_mode_failed = 1; 6157} 6158 6159/* 6160 * Returns true if compaction should go ahead for a costly-order request, or 6161 * the allocation would already succeed without compaction. Return false if we 6162 * should reclaim first. 6163 */ 6164static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) 6165{ 6166 unsigned long watermark; 6167 6168 if (!gfp_compaction_allowed(sc->gfp_mask)) 6169 return false; 6170 6171 /* Allocation can already succeed, nothing to do */ 6172 if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone), 6173 sc->reclaim_idx, 0)) 6174 return true; 6175 6176 /* 6177 * Direct reclaim usually targets the min watermark, but compaction 6178 * takes time to run and there are potentially other callers using the 6179 * pages just freed. So target a higher buffer to give compaction a 6180 * reasonable chance of completing and allocating the pages. 6181 * 6182 * Note that we won't actually reclaim the whole buffer in one attempt 6183 * as the target watermark in should_continue_reclaim() is lower. But if 6184 * we are already above the high+gap watermark, don't reclaim at all. 6185 */ 6186 watermark = high_wmark_pages(zone); 6187 if (compaction_suitable(zone, sc->order, watermark, sc->reclaim_idx)) 6188 return true; 6189 6190 return false; 6191} 6192 6193static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) 6194{ 6195 /* 6196 * If reclaim is making progress greater than 12% efficiency then 6197 * wake all the NOPROGRESS throttled tasks. 6198 */ 6199 if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { 6200 wait_queue_head_t *wqh; 6201 6202 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; 6203 if (waitqueue_active(wqh)) 6204 wake_up(wqh); 6205 6206 return; 6207 } 6208 6209 /* 6210 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will 6211 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages 6212 * under writeback and marked for immediate reclaim at the tail of the 6213 * LRU. 6214 */ 6215 if (current_is_kswapd() || cgroup_reclaim(sc)) 6216 return; 6217 6218 /* Throttle if making no progress at high prioities. */ 6219 if (sc->priority == 1 && !sc->nr_reclaimed) 6220 reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); 6221} 6222 6223/* 6224 * This is the direct reclaim path, for page-allocating processes. We only 6225 * try to reclaim pages from zones which will satisfy the caller's allocation 6226 * request. 6227 * 6228 * If a zone is deemed to be full of pinned pages then just give it a light 6229 * scan then give up on it. 6230 */ 6231static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) 6232{ 6233 struct zoneref *z; 6234 struct zone *zone; 6235 unsigned long nr_soft_reclaimed; 6236 unsigned long nr_soft_scanned; 6237 gfp_t orig_mask; 6238 pg_data_t *last_pgdat = NULL; 6239 pg_data_t *first_pgdat = NULL; 6240 6241 /* 6242 * If the number of buffer_heads in the machine exceeds the maximum 6243 * allowed level, force direct reclaim to scan the highmem zone as 6244 * highmem pages could be pinning lowmem pages storing buffer_heads 6245 */ 6246 orig_mask = sc->gfp_mask; 6247 if (buffer_heads_over_limit) { 6248 sc->gfp_mask |= __GFP_HIGHMEM; 6249 sc->reclaim_idx = gfp_zone(sc->gfp_mask); 6250 } 6251 6252 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6253 sc->reclaim_idx, sc->nodemask) { 6254 /* 6255 * Take care memory controller reclaiming has small influence 6256 * to global LRU. 6257 */ 6258 if (!cgroup_reclaim(sc)) { 6259 if (!cpuset_zone_allowed(zone, 6260 GFP_KERNEL | __GFP_HARDWALL)) 6261 continue; 6262 6263 /* 6264 * If we already have plenty of memory free for 6265 * compaction in this zone, don't free any more. 6266 * Even though compaction is invoked for any 6267 * non-zero order, only frequent costly order 6268 * reclamation is disruptive enough to become a 6269 * noticeable problem, like transparent huge 6270 * page allocations. 6271 */ 6272 if (IS_ENABLED(CONFIG_COMPACTION) && 6273 sc->order > PAGE_ALLOC_COSTLY_ORDER && 6274 compaction_ready(zone, sc)) { 6275 sc->compaction_ready = true; 6276 continue; 6277 } 6278 6279 /* 6280 * Shrink each node in the zonelist once. If the 6281 * zonelist is ordered by zone (not the default) then a 6282 * node may be shrunk multiple times but in that case 6283 * the user prefers lower zones being preserved. 6284 */ 6285 if (zone->zone_pgdat == last_pgdat) 6286 continue; 6287 6288 /* 6289 * This steals pages from memory cgroups over softlimit 6290 * and returns the number of reclaimed pages and 6291 * scanned pages. This works for global memory pressure 6292 * and balancing, not for a memcg's limit. 6293 */ 6294 nr_soft_scanned = 0; 6295 nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat, 6296 sc->order, sc->gfp_mask, 6297 &nr_soft_scanned); 6298 sc->nr_reclaimed += nr_soft_reclaimed; 6299 sc->nr_scanned += nr_soft_scanned; 6300 /* need some check for avoid more shrink_zone() */ 6301 } 6302 6303 if (!first_pgdat) 6304 first_pgdat = zone->zone_pgdat; 6305 6306 /* See comment about same check for global reclaim above */ 6307 if (zone->zone_pgdat == last_pgdat) 6308 continue; 6309 last_pgdat = zone->zone_pgdat; 6310 shrink_node(zone->zone_pgdat, sc); 6311 } 6312 6313 if (first_pgdat) 6314 consider_reclaim_throttle(first_pgdat, sc); 6315 6316 /* 6317 * Restore to original mask to avoid the impact on the caller if we 6318 * promoted it to __GFP_HIGHMEM. 6319 */ 6320 sc->gfp_mask = orig_mask; 6321} 6322 6323static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) 6324{ 6325 struct lruvec *target_lruvec; 6326 unsigned long refaults; 6327 6328 if (lru_gen_enabled()) 6329 return; 6330 6331 target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); 6332 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); 6333 target_lruvec->refaults[WORKINGSET_ANON] = refaults; 6334 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); 6335 target_lruvec->refaults[WORKINGSET_FILE] = refaults; 6336} 6337 6338/* 6339 * This is the main entry point to direct page reclaim. 6340 * 6341 * If a full scan of the inactive list fails to free enough memory then we 6342 * are "out of memory" and something needs to be killed. 6343 * 6344 * If the caller is !__GFP_FS then the probability of a failure is reasonably 6345 * high - the zone may be full of dirty or under-writeback pages, which this 6346 * caller can't do much about. We kick the writeback threads and take explicit 6347 * naps in the hope that some of these pages can be written. But if the 6348 * allocating task holds filesystem locks which prevent writeout this might not 6349 * work, and the allocation attempt will fail. 6350 * 6351 * returns: 0, if no pages reclaimed 6352 * else, the number of pages reclaimed 6353 */ 6354static unsigned long do_try_to_free_pages(struct zonelist *zonelist, 6355 struct scan_control *sc) 6356{ 6357 int initial_priority = sc->priority; 6358 pg_data_t *last_pgdat; 6359 struct zoneref *z; 6360 struct zone *zone; 6361retry: 6362 delayacct_freepages_start(); 6363 6364 if (!cgroup_reclaim(sc)) 6365 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); 6366 6367 do { 6368 if (!sc->proactive) 6369 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, 6370 sc->priority); 6371 sc->nr_scanned = 0; 6372 shrink_zones(zonelist, sc); 6373 6374 if (sc->nr_reclaimed >= sc->nr_to_reclaim) 6375 break; 6376 6377 if (sc->compaction_ready) 6378 break; 6379 6380 /* 6381 * If we're getting trouble reclaiming, start doing 6382 * writepage even in laptop mode. 6383 */ 6384 if (sc->priority < DEF_PRIORITY - 2) 6385 sc->may_writepage = 1; 6386 } while (--sc->priority >= 0); 6387 6388 last_pgdat = NULL; 6389 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, 6390 sc->nodemask) { 6391 if (zone->zone_pgdat == last_pgdat) 6392 continue; 6393 last_pgdat = zone->zone_pgdat; 6394 6395 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); 6396 6397 if (cgroup_reclaim(sc)) { 6398 struct lruvec *lruvec; 6399 6400 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, 6401 zone->zone_pgdat); 6402 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); 6403 } 6404 } 6405 6406 delayacct_freepages_end(); 6407 6408 if (sc->nr_reclaimed) 6409 return sc->nr_reclaimed; 6410 6411 /* Aborted reclaim to try compaction? don't OOM, then */ 6412 if (sc->compaction_ready) 6413 return 1; 6414 6415 /* 6416 * In most cases, direct reclaimers can do partial walks 6417 * through the cgroup tree to meet the reclaim goal while 6418 * keeping latency low. Since the iterator state is shared 6419 * among all direct reclaim invocations (to retain fairness 6420 * among cgroups), though, high concurrency can result in 6421 * individual threads not seeing enough cgroups to make 6422 * meaningful forward progress. Avoid false OOMs in this case. 6423 */ 6424 if (!sc->memcg_full_walk) { 6425 sc->priority = initial_priority; 6426 sc->memcg_full_walk = 1; 6427 goto retry; 6428 } 6429 6430 /* 6431 * We make inactive:active ratio decisions based on the node's 6432 * composition of memory, but a restrictive reclaim_idx or a 6433 * memory.low cgroup setting can exempt large amounts of 6434 * memory from reclaim. Neither of which are very common, so 6435 * instead of doing costly eligibility calculations of the 6436 * entire cgroup subtree up front, we assume the estimates are 6437 * good, and retry with forcible deactivation if that fails. 6438 */ 6439 if (sc->skipped_deactivate) { 6440 sc->priority = initial_priority; 6441 sc->force_deactivate = 1; 6442 sc->skipped_deactivate = 0; 6443 goto retry; 6444 } 6445 6446 /* Untapped cgroup reserves? Don't OOM, retry. */ 6447 if (sc->memcg_low_skipped) { 6448 sc->priority = initial_priority; 6449 sc->force_deactivate = 0; 6450 sc->memcg_low_reclaim = 1; 6451 sc->memcg_low_skipped = 0; 6452 goto retry; 6453 } 6454 6455 return 0; 6456} 6457 6458static bool allow_direct_reclaim(pg_data_t *pgdat) 6459{ 6460 struct zone *zone; 6461 unsigned long pfmemalloc_reserve = 0; 6462 unsigned long free_pages = 0; 6463 int i; 6464 bool wmark_ok; 6465 6466 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES) 6467 return true; 6468 6469 for_each_managed_zone_pgdat(zone, pgdat, i, ZONE_NORMAL) { 6470 if (!zone_reclaimable_pages(zone) && zone_page_state_snapshot(zone, NR_FREE_PAGES)) 6471 continue; 6472 6473 pfmemalloc_reserve += min_wmark_pages(zone); 6474 free_pages += zone_page_state_snapshot(zone, NR_FREE_PAGES); 6475 } 6476 6477 /* If there are no reserves (unexpected config) then do not throttle */ 6478 if (!pfmemalloc_reserve) 6479 return true; 6480 6481 wmark_ok = free_pages > pfmemalloc_reserve / 2; 6482 6483 /* kswapd must be awake if processes are being throttled */ 6484 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { 6485 if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) 6486 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); 6487 6488 wake_up_interruptible(&pgdat->kswapd_wait); 6489 } 6490 6491 return wmark_ok; 6492} 6493 6494/* 6495 * Throttle direct reclaimers if backing storage is backed by the network 6496 * and the PFMEMALLOC reserve for the preferred node is getting dangerously 6497 * depleted. kswapd will continue to make progress and wake the processes 6498 * when the low watermark is reached. 6499 * 6500 * Returns true if a fatal signal was delivered during throttling. If this 6501 * happens, the page allocator should not consider triggering the OOM killer. 6502 */ 6503static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, 6504 nodemask_t *nodemask) 6505{ 6506 struct zoneref *z; 6507 struct zone *zone; 6508 pg_data_t *pgdat = NULL; 6509 6510 /* 6511 * Kernel threads should not be throttled as they may be indirectly 6512 * responsible for cleaning pages necessary for reclaim to make forward 6513 * progress. kjournald for example may enter direct reclaim while 6514 * committing a transaction where throttling it could forcing other 6515 * processes to block on log_wait_commit(). 6516 */ 6517 if (current->flags & PF_KTHREAD) 6518 goto out; 6519 6520 /* 6521 * If a fatal signal is pending, this process should not throttle. 6522 * It should return quickly so it can exit and free its memory 6523 */ 6524 if (fatal_signal_pending(current)) 6525 goto out; 6526 6527 /* 6528 * Check if the pfmemalloc reserves are ok by finding the first node 6529 * with a usable ZONE_NORMAL or lower zone. The expectation is that 6530 * GFP_KERNEL will be required for allocating network buffers when 6531 * swapping over the network so ZONE_HIGHMEM is unusable. 6532 * 6533 * Throttling is based on the first usable node and throttled processes 6534 * wait on a queue until kswapd makes progress and wakes them. There 6535 * is an affinity then between processes waking up and where reclaim 6536 * progress has been made assuming the process wakes on the same node. 6537 * More importantly, processes running on remote nodes will not compete 6538 * for remote pfmemalloc reserves and processes on different nodes 6539 * should make reasonable progress. 6540 */ 6541 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6542 gfp_zone(gfp_mask), nodemask) { 6543 if (zone_idx(zone) > ZONE_NORMAL) 6544 continue; 6545 6546 /* Throttle based on the first usable node */ 6547 pgdat = zone->zone_pgdat; 6548 if (allow_direct_reclaim(pgdat)) 6549 goto out; 6550 break; 6551 } 6552 6553 /* If no zone was usable by the allocation flags then do not throttle */ 6554 if (!pgdat) 6555 goto out; 6556 6557 /* Account for the throttling */ 6558 count_vm_event(PGSCAN_DIRECT_THROTTLE); 6559 6560 /* 6561 * If the caller cannot enter the filesystem, it's possible that it 6562 * is due to the caller holding an FS lock or performing a journal 6563 * transaction in the case of a filesystem like ext[3|4]. In this case, 6564 * it is not safe to block on pfmemalloc_wait as kswapd could be 6565 * blocked waiting on the same lock. Instead, throttle for up to a 6566 * second before continuing. 6567 */ 6568 if (!(gfp_mask & __GFP_FS)) 6569 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, 6570 allow_direct_reclaim(pgdat), HZ); 6571 else 6572 /* Throttle until kswapd wakes the process */ 6573 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, 6574 allow_direct_reclaim(pgdat)); 6575 6576 if (fatal_signal_pending(current)) 6577 return true; 6578 6579out: 6580 return false; 6581} 6582 6583unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 6584 gfp_t gfp_mask, nodemask_t *nodemask) 6585{ 6586 unsigned long nr_reclaimed; 6587 struct scan_control sc = { 6588 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6589 .gfp_mask = current_gfp_context(gfp_mask), 6590 .reclaim_idx = gfp_zone(gfp_mask), 6591 .order = order, 6592 .nodemask = nodemask, 6593 .priority = DEF_PRIORITY, 6594 .may_writepage = !laptop_mode, 6595 .may_unmap = 1, 6596 .may_swap = 1, 6597 }; 6598 6599 /* 6600 * scan_control uses s8 fields for order, priority, and reclaim_idx. 6601 * Confirm they are large enough for max values. 6602 */ 6603 BUILD_BUG_ON(MAX_PAGE_ORDER >= S8_MAX); 6604 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); 6605 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); 6606 6607 /* 6608 * Do not enter reclaim if fatal signal was delivered while throttled. 6609 * 1 is returned so that the page allocator does not OOM kill at this 6610 * point. 6611 */ 6612 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) 6613 return 1; 6614 6615 set_task_reclaim_state(current, &sc.reclaim_state); 6616 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); 6617 6618 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6619 6620 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); 6621 set_task_reclaim_state(current, NULL); 6622 6623 return nr_reclaimed; 6624} 6625 6626#ifdef CONFIG_MEMCG 6627 6628/* Only used by soft limit reclaim. Do not reuse for anything else. */ 6629unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 6630 gfp_t gfp_mask, bool noswap, 6631 pg_data_t *pgdat, 6632 unsigned long *nr_scanned) 6633{ 6634 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6635 struct scan_control sc = { 6636 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6637 .target_mem_cgroup = memcg, 6638 .may_writepage = !laptop_mode, 6639 .may_unmap = 1, 6640 .reclaim_idx = MAX_NR_ZONES - 1, 6641 .may_swap = !noswap, 6642 }; 6643 6644 WARN_ON_ONCE(!current->reclaim_state); 6645 6646 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 6647 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 6648 6649 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, 6650 sc.gfp_mask); 6651 6652 /* 6653 * NOTE: Although we can get the priority field, using it 6654 * here is not a good idea, since it limits the pages we can scan. 6655 * if we don't reclaim here, the shrink_node from balance_pgdat 6656 * will pick up pages from other mem cgroup's as well. We hack 6657 * the priority and make it zero. 6658 */ 6659 shrink_lruvec(lruvec, &sc); 6660 6661 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); 6662 6663 *nr_scanned = sc.nr_scanned; 6664 6665 return sc.nr_reclaimed; 6666} 6667 6668unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6669 unsigned long nr_pages, 6670 gfp_t gfp_mask, 6671 unsigned int reclaim_options, 6672 int *swappiness) 6673{ 6674 unsigned long nr_reclaimed; 6675 unsigned int noreclaim_flag; 6676 struct scan_control sc = { 6677 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 6678 .proactive_swappiness = swappiness, 6679 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | 6680 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), 6681 .reclaim_idx = MAX_NR_ZONES - 1, 6682 .target_mem_cgroup = memcg, 6683 .priority = DEF_PRIORITY, 6684 .may_writepage = !laptop_mode, 6685 .may_unmap = 1, 6686 .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), 6687 .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), 6688 }; 6689 /* 6690 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put 6691 * equal pressure on all the nodes. This is based on the assumption that 6692 * the reclaim does not bail out early. 6693 */ 6694 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 6695 6696 set_task_reclaim_state(current, &sc.reclaim_state); 6697 trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); 6698 noreclaim_flag = memalloc_noreclaim_save(); 6699 6700 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6701 6702 memalloc_noreclaim_restore(noreclaim_flag); 6703 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 6704 set_task_reclaim_state(current, NULL); 6705 6706 return nr_reclaimed; 6707} 6708#else 6709unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6710 unsigned long nr_pages, 6711 gfp_t gfp_mask, 6712 unsigned int reclaim_options, 6713 int *swappiness) 6714{ 6715 return 0; 6716} 6717#endif 6718 6719static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) 6720{ 6721 struct mem_cgroup *memcg; 6722 struct lruvec *lruvec; 6723 6724 if (lru_gen_enabled()) { 6725 lru_gen_age_node(pgdat, sc); 6726 return; 6727 } 6728 6729 lruvec = mem_cgroup_lruvec(NULL, pgdat); 6730 if (!can_age_anon_pages(lruvec, sc)) 6731 return; 6732 6733 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 6734 return; 6735 6736 memcg = mem_cgroup_iter(NULL, NULL, NULL); 6737 do { 6738 lruvec = mem_cgroup_lruvec(memcg, pgdat); 6739 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 6740 sc, LRU_ACTIVE_ANON); 6741 memcg = mem_cgroup_iter(NULL, memcg, NULL); 6742 } while (memcg); 6743} 6744 6745static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) 6746{ 6747 int i; 6748 struct zone *zone; 6749 6750 /* 6751 * Check for watermark boosts top-down as the higher zones 6752 * are more likely to be boosted. Both watermarks and boosts 6753 * should not be checked at the same time as reclaim would 6754 * start prematurely when there is no boosting and a lower 6755 * zone is balanced. 6756 */ 6757 for (i = highest_zoneidx; i >= 0; i--) { 6758 zone = pgdat->node_zones + i; 6759 if (!managed_zone(zone)) 6760 continue; 6761 6762 if (zone->watermark_boost) 6763 return true; 6764 } 6765 6766 return false; 6767} 6768 6769/* 6770 * Returns true if there is an eligible zone balanced for the request order 6771 * and highest_zoneidx 6772 */ 6773static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) 6774{ 6775 int i; 6776 unsigned long mark = -1; 6777 struct zone *zone; 6778 6779 /* 6780 * Check watermarks bottom-up as lower zones are more likely to 6781 * meet watermarks. 6782 */ 6783 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6784 enum zone_stat_item item; 6785 unsigned long free_pages; 6786 6787 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) 6788 mark = promo_wmark_pages(zone); 6789 else 6790 mark = high_wmark_pages(zone); 6791 6792 /* 6793 * In defrag_mode, watermarks must be met in whole 6794 * blocks to avoid polluting allocator fallbacks. 6795 * 6796 * However, kswapd usually cannot accomplish this on 6797 * its own and needs kcompactd support. Once it's 6798 * reclaimed a compaction gap, and kswapd_shrink_node 6799 * has dropped order, simply ensure there are enough 6800 * base pages for compaction, wake kcompactd & sleep. 6801 */ 6802 if (defrag_mode && order) 6803 item = NR_FREE_PAGES_BLOCKS; 6804 else 6805 item = NR_FREE_PAGES; 6806 6807 /* 6808 * When there is a high number of CPUs in the system, 6809 * the cumulative error from the vmstat per-cpu cache 6810 * can blur the line between the watermarks. In that 6811 * case, be safe and get an accurate snapshot. 6812 * 6813 * TODO: NR_FREE_PAGES_BLOCKS moves in steps of 6814 * pageblock_nr_pages, while the vmstat pcp threshold 6815 * is limited to 125. On many configurations that 6816 * counter won't actually be per-cpu cached. But keep 6817 * things simple for now; revisit when somebody cares. 6818 */ 6819 free_pages = zone_page_state(zone, item); 6820 if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark) 6821 free_pages = zone_page_state_snapshot(zone, item); 6822 6823 if (__zone_watermark_ok(zone, order, mark, highest_zoneidx, 6824 0, free_pages)) 6825 return true; 6826 } 6827 6828 /* 6829 * If a node has no managed zone within highest_zoneidx, it does not 6830 * need balancing by definition. This can happen if a zone-restricted 6831 * allocation tries to wake a remote kswapd. 6832 */ 6833 if (mark == -1) 6834 return true; 6835 6836 return false; 6837} 6838 6839/* Clear pgdat state for congested, dirty or under writeback. */ 6840static void clear_pgdat_congested(pg_data_t *pgdat) 6841{ 6842 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); 6843 6844 clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags); 6845 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); 6846 clear_bit(PGDAT_WRITEBACK, &pgdat->flags); 6847} 6848 6849/* 6850 * Prepare kswapd for sleeping. This verifies that there are no processes 6851 * waiting in throttle_direct_reclaim() and that watermarks have been met. 6852 * 6853 * Returns true if kswapd is ready to sleep 6854 */ 6855static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, 6856 int highest_zoneidx) 6857{ 6858 /* 6859 * The throttled processes are normally woken up in balance_pgdat() as 6860 * soon as allow_direct_reclaim() is true. But there is a potential 6861 * race between when kswapd checks the watermarks and a process gets 6862 * throttled. There is also a potential race if processes get 6863 * throttled, kswapd wakes, a large process exits thereby balancing the 6864 * zones, which causes kswapd to exit balance_pgdat() before reaching 6865 * the wake up checks. If kswapd is going to sleep, no process should 6866 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If 6867 * the wake up is premature, processes will wake kswapd and get 6868 * throttled again. The difference from wake ups in balance_pgdat() is 6869 * that here we are under prepare_to_wait(). 6870 */ 6871 if (waitqueue_active(&pgdat->pfmemalloc_wait)) 6872 wake_up_all(&pgdat->pfmemalloc_wait); 6873 6874 /* Hopeless node, leave it to direct reclaim */ 6875 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES) 6876 return true; 6877 6878 if (pgdat_balanced(pgdat, order, highest_zoneidx)) { 6879 clear_pgdat_congested(pgdat); 6880 return true; 6881 } 6882 6883 return false; 6884} 6885 6886/* 6887 * kswapd shrinks a node of pages that are at or below the highest usable 6888 * zone that is currently unbalanced. 6889 * 6890 * Returns true if kswapd scanned at least the requested number of pages to 6891 * reclaim or if the lack of progress was due to pages under writeback. 6892 * This is used to determine if the scanning priority needs to be raised. 6893 */ 6894static bool kswapd_shrink_node(pg_data_t *pgdat, 6895 struct scan_control *sc) 6896{ 6897 struct zone *zone; 6898 int z; 6899 unsigned long nr_reclaimed = sc->nr_reclaimed; 6900 6901 /* Reclaim a number of pages proportional to the number of zones */ 6902 sc->nr_to_reclaim = 0; 6903 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) { 6904 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); 6905 } 6906 6907 /* 6908 * Historically care was taken to put equal pressure on all zones but 6909 * now pressure is applied based on node LRU order. 6910 */ 6911 shrink_node(pgdat, sc); 6912 6913 /* 6914 * Fragmentation may mean that the system cannot be rebalanced for 6915 * high-order allocations. If twice the allocation size has been 6916 * reclaimed then recheck watermarks only at order-0 to prevent 6917 * excessive reclaim. Assume that a process requested a high-order 6918 * can direct reclaim/compact. 6919 */ 6920 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) 6921 sc->order = 0; 6922 6923 /* account for progress from mm_account_reclaimed_pages() */ 6924 return max(sc->nr_scanned, sc->nr_reclaimed - nr_reclaimed) >= sc->nr_to_reclaim; 6925} 6926 6927/* Page allocator PCP high watermark is lowered if reclaim is active. */ 6928static inline void 6929update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) 6930{ 6931 int i; 6932 struct zone *zone; 6933 6934 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6935 if (active) 6936 set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6937 else 6938 clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6939 } 6940} 6941 6942static inline void 6943set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6944{ 6945 update_reclaim_active(pgdat, highest_zoneidx, true); 6946} 6947 6948static inline void 6949clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6950{ 6951 update_reclaim_active(pgdat, highest_zoneidx, false); 6952} 6953 6954/* 6955 * For kswapd, balance_pgdat() will reclaim pages across a node from zones 6956 * that are eligible for use by the caller until at least one zone is 6957 * balanced. 6958 * 6959 * Returns the order kswapd finished reclaiming at. 6960 * 6961 * kswapd scans the zones in the highmem->normal->dma direction. It skips 6962 * zones which have free_pages > high_wmark_pages(zone), but once a zone is 6963 * found to have free_pages <= high_wmark_pages(zone), any page in that zone 6964 * or lower is eligible for reclaim until at least one usable zone is 6965 * balanced. 6966 */ 6967static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) 6968{ 6969 int i; 6970 unsigned long nr_soft_reclaimed; 6971 unsigned long nr_soft_scanned; 6972 unsigned long pflags; 6973 unsigned long nr_boost_reclaim; 6974 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; 6975 bool boosted; 6976 struct zone *zone; 6977 struct scan_control sc = { 6978 .gfp_mask = GFP_KERNEL, 6979 .order = order, 6980 .may_unmap = 1, 6981 }; 6982 6983 set_task_reclaim_state(current, &sc.reclaim_state); 6984 psi_memstall_enter(&pflags); 6985 __fs_reclaim_acquire(_THIS_IP_); 6986 6987 count_vm_event(PAGEOUTRUN); 6988 6989 /* 6990 * Account for the reclaim boost. Note that the zone boost is left in 6991 * place so that parallel allocations that are near the watermark will 6992 * stall or direct reclaim until kswapd is finished. 6993 */ 6994 nr_boost_reclaim = 0; 6995 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6996 nr_boost_reclaim += zone->watermark_boost; 6997 zone_boosts[i] = zone->watermark_boost; 6998 } 6999 boosted = nr_boost_reclaim; 7000 7001restart: 7002 set_reclaim_active(pgdat, highest_zoneidx); 7003 sc.priority = DEF_PRIORITY; 7004 do { 7005 unsigned long nr_reclaimed = sc.nr_reclaimed; 7006 bool raise_priority = true; 7007 bool balanced; 7008 bool ret; 7009 bool was_frozen; 7010 7011 sc.reclaim_idx = highest_zoneidx; 7012 7013 /* 7014 * If the number of buffer_heads exceeds the maximum allowed 7015 * then consider reclaiming from all zones. This has a dual 7016 * purpose -- on 64-bit systems it is expected that 7017 * buffer_heads are stripped during active rotation. On 32-bit 7018 * systems, highmem pages can pin lowmem memory and shrinking 7019 * buffers can relieve lowmem pressure. Reclaim may still not 7020 * go ahead if all eligible zones for the original allocation 7021 * request are balanced to avoid excessive reclaim from kswapd. 7022 */ 7023 if (buffer_heads_over_limit) { 7024 for (i = MAX_NR_ZONES - 1; i >= 0; i--) { 7025 zone = pgdat->node_zones + i; 7026 if (!managed_zone(zone)) 7027 continue; 7028 7029 sc.reclaim_idx = i; 7030 break; 7031 } 7032 } 7033 7034 /* 7035 * If the pgdat is imbalanced then ignore boosting and preserve 7036 * the watermarks for a later time and restart. Note that the 7037 * zone watermarks will be still reset at the end of balancing 7038 * on the grounds that the normal reclaim should be enough to 7039 * re-evaluate if boosting is required when kswapd next wakes. 7040 */ 7041 balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); 7042 if (!balanced && nr_boost_reclaim) { 7043 nr_boost_reclaim = 0; 7044 goto restart; 7045 } 7046 7047 /* 7048 * If boosting is not active then only reclaim if there are no 7049 * eligible zones. Note that sc.reclaim_idx is not used as 7050 * buffer_heads_over_limit may have adjusted it. 7051 */ 7052 if (!nr_boost_reclaim && balanced) 7053 goto out; 7054 7055 /* Limit the priority of boosting to avoid reclaim writeback */ 7056 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) 7057 raise_priority = false; 7058 7059 /* 7060 * Do not writeback or swap pages for boosted reclaim. The 7061 * intent is to relieve pressure not issue sub-optimal IO 7062 * from reclaim context. If no pages are reclaimed, the 7063 * reclaim will be aborted. 7064 */ 7065 sc.may_writepage = !laptop_mode && !nr_boost_reclaim; 7066 sc.may_swap = !nr_boost_reclaim; 7067 7068 /* 7069 * Do some background aging, to give pages a chance to be 7070 * referenced before reclaiming. All pages are rotated 7071 * regardless of classzone as this is about consistent aging. 7072 */ 7073 kswapd_age_node(pgdat, &sc); 7074 7075 /* 7076 * If we're getting trouble reclaiming, start doing writepage 7077 * even in laptop mode. 7078 */ 7079 if (sc.priority < DEF_PRIORITY - 2) 7080 sc.may_writepage = 1; 7081 7082 /* Call soft limit reclaim before calling shrink_node. */ 7083 sc.nr_scanned = 0; 7084 nr_soft_scanned = 0; 7085 nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order, 7086 sc.gfp_mask, &nr_soft_scanned); 7087 sc.nr_reclaimed += nr_soft_reclaimed; 7088 7089 /* 7090 * There should be no need to raise the scanning priority if 7091 * enough pages are already being scanned that that high 7092 * watermark would be met at 100% efficiency. 7093 */ 7094 if (kswapd_shrink_node(pgdat, &sc)) 7095 raise_priority = false; 7096 7097 /* 7098 * If the low watermark is met there is no need for processes 7099 * to be throttled on pfmemalloc_wait as they should not be 7100 * able to safely make forward progress. Wake them 7101 */ 7102 if (waitqueue_active(&pgdat->pfmemalloc_wait) && 7103 allow_direct_reclaim(pgdat)) 7104 wake_up_all(&pgdat->pfmemalloc_wait); 7105 7106 /* Check if kswapd should be suspending */ 7107 __fs_reclaim_release(_THIS_IP_); 7108 ret = kthread_freezable_should_stop(&was_frozen); 7109 __fs_reclaim_acquire(_THIS_IP_); 7110 if (was_frozen || ret) 7111 break; 7112 7113 /* 7114 * Raise priority if scanning rate is too low or there was no 7115 * progress in reclaiming pages 7116 */ 7117 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; 7118 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); 7119 7120 /* 7121 * If reclaim made no progress for a boost, stop reclaim as 7122 * IO cannot be queued and it could be an infinite loop in 7123 * extreme circumstances. 7124 */ 7125 if (nr_boost_reclaim && !nr_reclaimed) 7126 break; 7127 7128 if (raise_priority || !nr_reclaimed) 7129 sc.priority--; 7130 } while (sc.priority >= 1); 7131 7132 /* 7133 * Restart only if it went through the priority loop all the way, 7134 * but cache_trim_mode didn't work. 7135 */ 7136 if (!sc.nr_reclaimed && sc.priority < 1 && 7137 !sc.no_cache_trim_mode && sc.cache_trim_mode_failed) { 7138 sc.no_cache_trim_mode = 1; 7139 goto restart; 7140 } 7141 7142 /* 7143 * If the reclaim was boosted, we might still be far from the 7144 * watermark_high at this point. We need to avoid increasing the 7145 * failure count to prevent the kswapd thread from stopping. 7146 */ 7147 if (!sc.nr_reclaimed && !boosted) 7148 atomic_inc(&pgdat->kswapd_failures); 7149 7150out: 7151 clear_reclaim_active(pgdat, highest_zoneidx); 7152 7153 /* If reclaim was boosted, account for the reclaim done in this pass */ 7154 if (boosted) { 7155 unsigned long flags; 7156 7157 for (i = 0; i <= highest_zoneidx; i++) { 7158 if (!zone_boosts[i]) 7159 continue; 7160 7161 /* Increments are under the zone lock */ 7162 zone = pgdat->node_zones + i; 7163 spin_lock_irqsave(&zone->lock, flags); 7164 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); 7165 spin_unlock_irqrestore(&zone->lock, flags); 7166 } 7167 7168 /* 7169 * As there is now likely space, wakeup kcompact to defragment 7170 * pageblocks. 7171 */ 7172 wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); 7173 } 7174 7175 snapshot_refaults(NULL, pgdat); 7176 __fs_reclaim_release(_THIS_IP_); 7177 psi_memstall_leave(&pflags); 7178 set_task_reclaim_state(current, NULL); 7179 7180 /* 7181 * Return the order kswapd stopped reclaiming at as 7182 * prepare_kswapd_sleep() takes it into account. If another caller 7183 * entered the allocator slow path while kswapd was awake, order will 7184 * remain at the higher level. 7185 */ 7186 return sc.order; 7187} 7188 7189/* 7190 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to 7191 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is 7192 * not a valid index then either kswapd runs for first time or kswapd couldn't 7193 * sleep after previous reclaim attempt (node is still unbalanced). In that 7194 * case return the zone index of the previous kswapd reclaim cycle. 7195 */ 7196static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, 7197 enum zone_type prev_highest_zoneidx) 7198{ 7199 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7200 7201 return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; 7202} 7203 7204static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, 7205 unsigned int highest_zoneidx) 7206{ 7207 long remaining = 0; 7208 DEFINE_WAIT(wait); 7209 7210 if (freezing(current) || kthread_should_stop()) 7211 return; 7212 7213 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7214 7215 /* 7216 * Try to sleep for a short interval. Note that kcompactd will only be 7217 * woken if it is possible to sleep for a short interval. This is 7218 * deliberate on the assumption that if reclaim cannot keep an 7219 * eligible zone balanced that it's also unlikely that compaction will 7220 * succeed. 7221 */ 7222 if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7223 /* 7224 * Compaction records what page blocks it recently failed to 7225 * isolate pages from and skips them in the future scanning. 7226 * When kswapd is going to sleep, it is reasonable to assume 7227 * that pages and compaction may succeed so reset the cache. 7228 */ 7229 reset_isolation_suitable(pgdat); 7230 7231 /* 7232 * We have freed the memory, now we should compact it to make 7233 * allocation of the requested order possible. 7234 */ 7235 wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); 7236 7237 remaining = schedule_timeout(HZ/10); 7238 7239 /* 7240 * If woken prematurely then reset kswapd_highest_zoneidx and 7241 * order. The values will either be from a wakeup request or 7242 * the previous request that slept prematurely. 7243 */ 7244 if (remaining) { 7245 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, 7246 kswapd_highest_zoneidx(pgdat, 7247 highest_zoneidx)); 7248 7249 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) 7250 WRITE_ONCE(pgdat->kswapd_order, reclaim_order); 7251 } 7252 7253 finish_wait(&pgdat->kswapd_wait, &wait); 7254 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7255 } 7256 7257 /* 7258 * After a short sleep, check if it was a premature sleep. If not, then 7259 * go fully to sleep until explicitly woken up. 7260 */ 7261 if (!remaining && 7262 prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7263 trace_mm_vmscan_kswapd_sleep(pgdat->node_id); 7264 7265 /* 7266 * vmstat counters are not perfectly accurate and the estimated 7267 * value for counters such as NR_FREE_PAGES can deviate from the 7268 * true value by nr_online_cpus * threshold. To avoid the zone 7269 * watermarks being breached while under pressure, we reduce the 7270 * per-cpu vmstat threshold while kswapd is awake and restore 7271 * them before going back to sleep. 7272 */ 7273 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); 7274 7275 if (!kthread_should_stop()) 7276 schedule(); 7277 7278 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); 7279 } else { 7280 if (remaining) 7281 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); 7282 else 7283 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); 7284 } 7285 finish_wait(&pgdat->kswapd_wait, &wait); 7286} 7287 7288/* 7289 * The background pageout daemon, started as a kernel thread 7290 * from the init process. 7291 * 7292 * This basically trickles out pages so that we have _some_ 7293 * free memory available even if there is no other activity 7294 * that frees anything up. This is needed for things like routing 7295 * etc, where we otherwise might have all activity going on in 7296 * asynchronous contexts that cannot page things out. 7297 * 7298 * If there are applications that are active memory-allocators 7299 * (most normal use), this basically shouldn't matter. 7300 */ 7301static int kswapd(void *p) 7302{ 7303 unsigned int alloc_order, reclaim_order; 7304 unsigned int highest_zoneidx = MAX_NR_ZONES - 1; 7305 pg_data_t *pgdat = (pg_data_t *)p; 7306 struct task_struct *tsk = current; 7307 7308 /* 7309 * Tell the memory management that we're a "memory allocator", 7310 * and that if we need more memory we should get access to it 7311 * regardless (see "__alloc_pages()"). "kswapd" should 7312 * never get caught in the normal page freeing logic. 7313 * 7314 * (Kswapd normally doesn't need memory anyway, but sometimes 7315 * you need a small amount of memory in order to be able to 7316 * page out something else, and this flag essentially protects 7317 * us from recursively trying to free more memory as we're 7318 * trying to free the first piece of memory in the first place). 7319 */ 7320 tsk->flags |= PF_MEMALLOC | PF_KSWAPD; 7321 set_freezable(); 7322 7323 WRITE_ONCE(pgdat->kswapd_order, 0); 7324 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7325 atomic_set(&pgdat->nr_writeback_throttled, 0); 7326 for ( ; ; ) { 7327 bool was_frozen; 7328 7329 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); 7330 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7331 highest_zoneidx); 7332 7333kswapd_try_sleep: 7334 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, 7335 highest_zoneidx); 7336 7337 /* Read the new order and highest_zoneidx */ 7338 alloc_order = READ_ONCE(pgdat->kswapd_order); 7339 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7340 highest_zoneidx); 7341 WRITE_ONCE(pgdat->kswapd_order, 0); 7342 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7343 7344 if (kthread_freezable_should_stop(&was_frozen)) 7345 break; 7346 7347 /* 7348 * We can speed up thawing tasks if we don't call balance_pgdat 7349 * after returning from the refrigerator 7350 */ 7351 if (was_frozen) 7352 continue; 7353 7354 /* 7355 * Reclaim begins at the requested order but if a high-order 7356 * reclaim fails then kswapd falls back to reclaiming for 7357 * order-0. If that happens, kswapd will consider sleeping 7358 * for the order it finished reclaiming at (reclaim_order) 7359 * but kcompactd is woken to compact for the original 7360 * request (alloc_order). 7361 */ 7362 trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, 7363 alloc_order); 7364 reclaim_order = balance_pgdat(pgdat, alloc_order, 7365 highest_zoneidx); 7366 if (reclaim_order < alloc_order) 7367 goto kswapd_try_sleep; 7368 } 7369 7370 tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); 7371 7372 return 0; 7373} 7374 7375/* 7376 * A zone is low on free memory or too fragmented for high-order memory. If 7377 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's 7378 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim 7379 * has failed or is not needed, still wake up kcompactd if only compaction is 7380 * needed. 7381 */ 7382void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, 7383 enum zone_type highest_zoneidx) 7384{ 7385 pg_data_t *pgdat; 7386 enum zone_type curr_idx; 7387 7388 if (!managed_zone(zone)) 7389 return; 7390 7391 if (!cpuset_zone_allowed(zone, gfp_flags)) 7392 return; 7393 7394 pgdat = zone->zone_pgdat; 7395 curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7396 7397 if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) 7398 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); 7399 7400 if (READ_ONCE(pgdat->kswapd_order) < order) 7401 WRITE_ONCE(pgdat->kswapd_order, order); 7402 7403 if (!waitqueue_active(&pgdat->kswapd_wait)) 7404 return; 7405 7406 /* Hopeless node, leave it to direct reclaim if possible */ 7407 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES || 7408 (pgdat_balanced(pgdat, order, highest_zoneidx) && 7409 !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { 7410 /* 7411 * There may be plenty of free memory available, but it's too 7412 * fragmented for high-order allocations. Wake up kcompactd 7413 * and rely on compaction_suitable() to determine if it's 7414 * needed. If it fails, it will defer subsequent attempts to 7415 * ratelimit its work. 7416 */ 7417 if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) 7418 wakeup_kcompactd(pgdat, order, highest_zoneidx); 7419 return; 7420 } 7421 7422 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, 7423 gfp_flags); 7424 wake_up_interruptible(&pgdat->kswapd_wait); 7425} 7426 7427#ifdef CONFIG_HIBERNATION 7428/* 7429 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of 7430 * freed pages. 7431 * 7432 * Rather than trying to age LRUs the aim is to preserve the overall 7433 * LRU order by reclaiming preferentially 7434 * inactive > active > active referenced > active mapped 7435 */ 7436unsigned long shrink_all_memory(unsigned long nr_to_reclaim) 7437{ 7438 struct scan_control sc = { 7439 .nr_to_reclaim = nr_to_reclaim, 7440 .gfp_mask = GFP_HIGHUSER_MOVABLE, 7441 .reclaim_idx = MAX_NR_ZONES - 1, 7442 .priority = DEF_PRIORITY, 7443 .may_writepage = 1, 7444 .may_unmap = 1, 7445 .may_swap = 1, 7446 .hibernation_mode = 1, 7447 }; 7448 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 7449 unsigned long nr_reclaimed; 7450 unsigned int noreclaim_flag; 7451 7452 fs_reclaim_acquire(sc.gfp_mask); 7453 noreclaim_flag = memalloc_noreclaim_save(); 7454 set_task_reclaim_state(current, &sc.reclaim_state); 7455 7456 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 7457 7458 set_task_reclaim_state(current, NULL); 7459 memalloc_noreclaim_restore(noreclaim_flag); 7460 fs_reclaim_release(sc.gfp_mask); 7461 7462 return nr_reclaimed; 7463} 7464#endif /* CONFIG_HIBERNATION */ 7465 7466/* 7467 * This kswapd start function will be called by init and node-hot-add. 7468 */ 7469void __meminit kswapd_run(int nid) 7470{ 7471 pg_data_t *pgdat = NODE_DATA(nid); 7472 7473 pgdat_kswapd_lock(pgdat); 7474 if (!pgdat->kswapd) { 7475 pgdat->kswapd = kthread_create_on_node(kswapd, pgdat, nid, "kswapd%d", nid); 7476 if (IS_ERR(pgdat->kswapd)) { 7477 /* failure at boot is fatal */ 7478 pr_err("Failed to start kswapd on node %d，ret=%ld\n", 7479 nid, PTR_ERR(pgdat->kswapd)); 7480 BUG_ON(system_state < SYSTEM_RUNNING); 7481 pgdat->kswapd = NULL; 7482 } else { 7483 wake_up_process(pgdat->kswapd); 7484 } 7485 } 7486 pgdat_kswapd_unlock(pgdat); 7487} 7488 7489/* 7490 * Called by memory hotplug when all memory in a node is offlined. Caller must 7491 * be holding mem_hotplug_begin/done(). 7492 */ 7493void __meminit kswapd_stop(int nid) 7494{ 7495 pg_data_t *pgdat = NODE_DATA(nid); 7496 struct task_struct *kswapd; 7497 7498 pgdat_kswapd_lock(pgdat); 7499 kswapd = pgdat->kswapd; 7500 if (kswapd) { 7501 kthread_stop(kswapd); 7502 pgdat->kswapd = NULL; 7503 } 7504 pgdat_kswapd_unlock(pgdat); 7505} 7506 7507static const struct ctl_table vmscan_sysctl_table[] = { 7508 { 7509 .procname = "swappiness", 7510 .data = &vm_swappiness, 7511 .maxlen = sizeof(vm_swappiness), 7512 .mode = 0644, 7513 .proc_handler = proc_dointvec_minmax, 7514 .extra1 = SYSCTL_ZERO, 7515 .extra2 = SYSCTL_TWO_HUNDRED, 7516 }, 7517#ifdef CONFIG_NUMA 7518 { 7519 .procname = "zone_reclaim_mode", 7520 .data = &node_reclaim_mode, 7521 .maxlen = sizeof(node_reclaim_mode), 7522 .mode = 0644, 7523 .proc_handler = proc_dointvec_minmax, 7524 .extra1 = SYSCTL_ZERO, 7525 } 7526#endif 7527}; 7528 7529static int __init kswapd_init(void) 7530{ 7531 int nid; 7532 7533 swap_setup(); 7534 for_each_node_state(nid, N_MEMORY) 7535 kswapd_run(nid); 7536 register_sysctl_init("vm", vmscan_sysctl_table); 7537 return 0; 7538} 7539 7540module_init(kswapd_init) 7541 7542#ifdef CONFIG_NUMA 7543/* 7544 * Node reclaim mode 7545 * 7546 * If non-zero call node_reclaim when the number of free pages falls below 7547 * the watermarks. 7548 */ 7549int node_reclaim_mode __read_mostly; 7550 7551/* 7552 * Priority for NODE_RECLAIM. This determines the fraction of pages 7553 * of a node considered for each zone_reclaim. 4 scans 1/16th of 7554 * a zone. 7555 */ 7556#define NODE_RECLAIM_PRIORITY 4 7557 7558/* 7559 * Percentage of pages in a zone that must be unmapped for node_reclaim to 7560 * occur. 7561 */ 7562int sysctl_min_unmapped_ratio = 1; 7563 7564/* 7565 * If the number of slab pages in a zone grows beyond this percentage then 7566 * slab reclaim needs to occur. 7567 */ 7568int sysctl_min_slab_ratio = 5; 7569 7570static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) 7571{ 7572 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); 7573 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + 7574 node_page_state(pgdat, NR_ACTIVE_FILE); 7575 7576 /* 7577 * It's possible for there to be more file mapped pages than 7578 * accounted for by the pages on the file LRU lists because 7579 * tmpfs pages accounted for as ANON can also be FILE_MAPPED 7580 */ 7581 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; 7582} 7583 7584/* Work out how many page cache pages we can reclaim in this reclaim_mode */ 7585static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) 7586{ 7587 unsigned long nr_pagecache_reclaimable; 7588 unsigned long delta = 0; 7589 7590 /* 7591 * If RECLAIM_UNMAP is set, then all file pages are considered 7592 * potentially reclaimable. Otherwise, we have to worry about 7593 * pages like swapcache and node_unmapped_file_pages() provides 7594 * a better estimate 7595 */ 7596 if (node_reclaim_mode & RECLAIM_UNMAP) 7597 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); 7598 else 7599 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); 7600 7601 /* 7602 * Since we can't clean folios through reclaim, remove dirty file 7603 * folios from consideration. 7604 */ 7605 delta += node_page_state(pgdat, NR_FILE_DIRTY); 7606 7607 /* Watch for any possible underflows due to delta */ 7608 if (unlikely(delta > nr_pagecache_reclaimable)) 7609 delta = nr_pagecache_reclaimable; 7610 7611 return nr_pagecache_reclaimable - delta; 7612} 7613 7614/* 7615 * Try to free up some pages from this node through reclaim. 7616 */ 7617static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, 7618 unsigned long nr_pages, 7619 struct scan_control *sc) 7620{ 7621 struct task_struct *p = current; 7622 unsigned int noreclaim_flag; 7623 unsigned long pflags; 7624 7625 trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, sc->order, 7626 sc->gfp_mask); 7627 7628 cond_resched(); 7629 psi_memstall_enter(&pflags); 7630 delayacct_freepages_start(); 7631 fs_reclaim_acquire(sc->gfp_mask); 7632 /* 7633 * We need to be able to allocate from the reserves for RECLAIM_UNMAP 7634 */ 7635 noreclaim_flag = memalloc_noreclaim_save(); 7636 set_task_reclaim_state(p, &sc->reclaim_state); 7637 7638 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || 7639 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { 7640 /* 7641 * Free memory by calling shrink node with increasing 7642 * priorities until we have enough memory freed. 7643 */ 7644 do { 7645 shrink_node(pgdat, sc); 7646 } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0); 7647 } 7648 7649 set_task_reclaim_state(p, NULL); 7650 memalloc_noreclaim_restore(noreclaim_flag); 7651 fs_reclaim_release(sc->gfp_mask); 7652 delayacct_freepages_end(); 7653 psi_memstall_leave(&pflags); 7654 7655 trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed); 7656 7657 return sc->nr_reclaimed; 7658} 7659 7660int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7661{ 7662 int ret; 7663 /* Minimum pages needed in order to stay on node */ 7664 const unsigned long nr_pages = 1 << order; 7665 struct scan_control sc = { 7666 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 7667 .gfp_mask = current_gfp_context(gfp_mask), 7668 .order = order, 7669 .priority = NODE_RECLAIM_PRIORITY, 7670 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), 7671 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), 7672 .may_swap = 1, 7673 .reclaim_idx = gfp_zone(gfp_mask), 7674 }; 7675 7676 /* 7677 * Node reclaim reclaims unmapped file backed pages and 7678 * slab pages if we are over the defined limits. 7679 * 7680 * A small portion of unmapped file backed pages is needed for 7681 * file I/O otherwise pages read by file I/O will be immediately 7682 * thrown out if the node is overallocated. So we do not reclaim 7683 * if less than a specified percentage of the node is used by 7684 * unmapped file backed pages. 7685 */ 7686 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && 7687 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= 7688 pgdat->min_slab_pages) 7689 return NODE_RECLAIM_FULL; 7690 7691 /* 7692 * Do not scan if the allocation should not be delayed. 7693 */ 7694 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) 7695 return NODE_RECLAIM_NOSCAN; 7696 7697 /* 7698 * Only run node reclaim on the local node or on nodes that do not 7699 * have associated processors. This will favor the local processor 7700 * over remote processors and spread off node memory allocations 7701 * as wide as possible. 7702 */ 7703 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) 7704 return NODE_RECLAIM_NOSCAN; 7705 7706 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) 7707 return NODE_RECLAIM_NOSCAN; 7708 7709 ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc) >= nr_pages; 7710 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7711 7712 if (ret) 7713 count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS); 7714 else 7715 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); 7716 7717 return ret; 7718} 7719 7720#else 7721 7722static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, 7723 unsigned long nr_pages, 7724 struct scan_control *sc) 7725{ 7726 return 0; 7727} 7728 7729#endif 7730 7731enum { 7732 MEMORY_RECLAIM_SWAPPINESS = 0, 7733 MEMORY_RECLAIM_SWAPPINESS_MAX, 7734 MEMORY_RECLAIM_NULL, 7735}; 7736static const match_table_t tokens = { 7737 { MEMORY_RECLAIM_SWAPPINESS, "swappiness=%d"}, 7738 { MEMORY_RECLAIM_SWAPPINESS_MAX, "swappiness=max"}, 7739 { MEMORY_RECLAIM_NULL, NULL }, 7740}; 7741 7742int user_proactive_reclaim(char *buf, 7743 struct mem_cgroup *memcg, pg_data_t *pgdat) 7744{ 7745 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 7746 unsigned long nr_to_reclaim, nr_reclaimed = 0; 7747 int swappiness = -1; 7748 char *old_buf, *start; 7749 substring_t args[MAX_OPT_ARGS]; 7750 gfp_t gfp_mask = GFP_KERNEL; 7751 7752 if (!buf || (!memcg && !pgdat) || (memcg && pgdat)) 7753 return -EINVAL; 7754 7755 buf = strstrip(buf); 7756 7757 old_buf = buf; 7758 nr_to_reclaim = memparse(buf, &buf) / PAGE_SIZE; 7759 if (buf == old_buf) 7760 return -EINVAL; 7761 7762 buf = strstrip(buf); 7763 7764 while ((start = strsep(&buf, " ")) != NULL) { 7765 if (!strlen(start)) 7766 continue; 7767 switch (match_token(start, tokens, args)) { 7768 case MEMORY_RECLAIM_SWAPPINESS: 7769 if (match_int(&args[0], &swappiness)) 7770 return -EINVAL; 7771 if (swappiness < MIN_SWAPPINESS || 7772 swappiness > MAX_SWAPPINESS) 7773 return -EINVAL; 7774 break; 7775 case MEMORY_RECLAIM_SWAPPINESS_MAX: 7776 swappiness = SWAPPINESS_ANON_ONLY; 7777 break; 7778 default: 7779 return -EINVAL; 7780 } 7781 } 7782 7783 while (nr_reclaimed < nr_to_reclaim) { 7784 /* Will converge on zero, but reclaim enforces a minimum */ 7785 unsigned long batch_size = (nr_to_reclaim - nr_reclaimed) / 4; 7786 unsigned long reclaimed; 7787 7788 if (signal_pending(current)) 7789 return -EINTR; 7790 7791 /* 7792 * This is the final attempt, drain percpu lru caches in the 7793 * hope of introducing more evictable pages. 7794 */ 7795 if (!nr_retries) 7796 lru_add_drain_all(); 7797 7798 if (memcg) { 7799 unsigned int reclaim_options; 7800 7801 reclaim_options = MEMCG_RECLAIM_MAY_SWAP | 7802 MEMCG_RECLAIM_PROACTIVE; 7803 reclaimed = try_to_free_mem_cgroup_pages(memcg, 7804 batch_size, gfp_mask, 7805 reclaim_options, 7806 swappiness == -1 ? NULL : &swappiness); 7807 } else { 7808 struct scan_control sc = { 7809 .gfp_mask = current_gfp_context(gfp_mask), 7810 .reclaim_idx = gfp_zone(gfp_mask), 7811 .proactive_swappiness = swappiness == -1 ? NULL : &swappiness, 7812 .priority = DEF_PRIORITY, 7813 .may_writepage = !laptop_mode, 7814 .nr_to_reclaim = max(batch_size, SWAP_CLUSTER_MAX), 7815 .may_unmap = 1, 7816 .may_swap = 1, 7817 .proactive = 1, 7818 }; 7819 7820 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, 7821 &pgdat->flags)) 7822 return -EBUSY; 7823 7824 reclaimed = __node_reclaim(pgdat, gfp_mask, 7825 batch_size, &sc); 7826 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7827 } 7828 7829 if (!reclaimed && !nr_retries--) 7830 return -EAGAIN; 7831 7832 nr_reclaimed += reclaimed; 7833 } 7834 7835 return 0; 7836} 7837 7838/** 7839 * check_move_unevictable_folios - Move evictable folios to appropriate zone 7840 * lru list 7841 * @fbatch: Batch of lru folios to check. 7842 * 7843 * Checks folios for evictability, if an evictable folio is in the unevictable 7844 * lru list, moves it to the appropriate evictable lru list. This function 7845 * should be only used for lru folios. 7846 */ 7847void check_move_unevictable_folios(struct folio_batch *fbatch) 7848{ 7849 struct lruvec *lruvec = NULL; 7850 int pgscanned = 0; 7851 int pgrescued = 0; 7852 int i; 7853 7854 for (i = 0; i < fbatch->nr; i++) { 7855 struct folio *folio = fbatch->folios[i]; 7856 int nr_pages = folio_nr_pages(folio); 7857 7858 pgscanned += nr_pages; 7859 7860 /* block memcg migration while the folio moves between lrus */ 7861 if (!folio_test_clear_lru(folio)) 7862 continue; 7863 7864 lruvec = folio_lruvec_relock_irq(folio, lruvec); 7865 if (folio_evictable(folio) && folio_test_unevictable(folio)) { 7866 lruvec_del_folio(lruvec, folio); 7867 folio_clear_unevictable(folio); 7868 lruvec_add_folio(lruvec, folio); 7869 pgrescued += nr_pages; 7870 } 7871 folio_set_lru(folio); 7872 } 7873 7874 if (lruvec) { 7875 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); 7876 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7877 unlock_page_lruvec_irq(lruvec); 7878 } else if (pgscanned) { 7879 count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7880 } 7881} 7882EXPORT_SYMBOL_GPL(check_move_unevictable_folios); 7883 7884#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 7885static ssize_t reclaim_store(struct device *dev, 7886 struct device_attribute *attr, 7887 const char *buf, size_t count) 7888{ 7889 int ret, nid = dev->id; 7890 7891 ret = user_proactive_reclaim((char *)buf, NULL, NODE_DATA(nid)); 7892 return ret ? -EAGAIN : count; 7893} 7894 7895static DEVICE_ATTR_WO(reclaim); 7896int reclaim_register_node(struct node *node) 7897{ 7898 return device_create_file(&node->dev, &dev_attr_reclaim); 7899} 7900 7901void reclaim_unregister_node(struct node *node) 7902{ 7903 return device_remove_file(&node->dev, &dev_attr_reclaim); 7904} 7905#endif
~/
Read-only file preview.