FILE
vmstat.c
File preview with syntax highlighting.
linux-6.19.14/mm/vmstat.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * linux/mm/vmstat.c 4 * 5 * Manages VM statistics 6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 7 * 8 * zoned VM statistics 9 * Copyright (C) 2006 Silicon Graphics, Inc., 10 * Christoph Lameter <cl@gentwo.org> 11 * Copyright (C) 2008-2014 Christoph Lameter 12 */ 13#include <linux/fs.h> 14#include <linux/mm.h> 15#include <linux/err.h> 16#include <linux/module.h> 17#include <linux/slab.h> 18#include <linux/cpu.h> 19#include <linux/cpumask.h> 20#include <linux/vmstat.h> 21#include <linux/proc_fs.h> 22#include <linux/seq_file.h> 23#include <linux/debugfs.h> 24#include <linux/sched.h> 25#include <linux/math64.h> 26#include <linux/writeback.h> 27#include <linux/compaction.h> 28#include <linux/mm_inline.h> 29#include <linux/page_owner.h> 30#include <linux/sched/isolation.h> 31 32#include "internal.h" 33 34#ifdef CONFIG_PROC_FS 35#ifdef CONFIG_NUMA 36#define ENABLE_NUMA_STAT 1 37static int sysctl_vm_numa_stat = ENABLE_NUMA_STAT; 38 39/* zero numa counters within a zone */ 40static void zero_zone_numa_counters(struct zone *zone) 41{ 42 int item, cpu; 43 44 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) { 45 atomic_long_set(&zone->vm_numa_event[item], 0); 46 for_each_online_cpu(cpu) { 47 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item] 48 = 0; 49 } 50 } 51} 52 53/* zero numa counters of all the populated zones */ 54static void zero_zones_numa_counters(void) 55{ 56 struct zone *zone; 57 58 for_each_populated_zone(zone) 59 zero_zone_numa_counters(zone); 60} 61 62/* zero global numa counters */ 63static void zero_global_numa_counters(void) 64{ 65 int item; 66 67 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 68 atomic_long_set(&vm_numa_event[item], 0); 69} 70 71static void invalid_numa_statistics(void) 72{ 73 zero_zones_numa_counters(); 74 zero_global_numa_counters(); 75} 76 77static DEFINE_MUTEX(vm_numa_stat_lock); 78 79static int sysctl_vm_numa_stat_handler(const struct ctl_table *table, int write, 80 void *buffer, size_t *length, loff_t *ppos) 81{ 82 int ret, oldval; 83 84 mutex_lock(&vm_numa_stat_lock); 85 if (write) 86 oldval = sysctl_vm_numa_stat; 87 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 88 if (ret || !write) 89 goto out; 90 91 if (oldval == sysctl_vm_numa_stat) 92 goto out; 93 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) { 94 static_branch_enable(&vm_numa_stat_key); 95 pr_info("enable numa statistics\n"); 96 } else { 97 static_branch_disable(&vm_numa_stat_key); 98 invalid_numa_statistics(); 99 pr_info("disable numa statistics, and clear numa counters\n"); 100 } 101 102out: 103 mutex_unlock(&vm_numa_stat_lock); 104 return ret; 105} 106#endif 107#endif /* CONFIG_PROC_FS */ 108 109#ifdef CONFIG_VM_EVENT_COUNTERS 110DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 111EXPORT_PER_CPU_SYMBOL(vm_event_states); 112 113static void sum_vm_events(unsigned long *ret) 114{ 115 int cpu; 116 int i; 117 118 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 119 120 for_each_online_cpu(cpu) { 121 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 122 123 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 124 ret[i] += this->event[i]; 125 } 126} 127 128/* 129 * Accumulate the vm event counters across all CPUs. 130 * The result is unavoidably approximate - it can change 131 * during and after execution of this function. 132*/ 133void all_vm_events(unsigned long *ret) 134{ 135 cpus_read_lock(); 136 sum_vm_events(ret); 137 cpus_read_unlock(); 138} 139EXPORT_SYMBOL_GPL(all_vm_events); 140 141/* 142 * Fold the foreign cpu events into our own. 143 * 144 * This is adding to the events on one processor 145 * but keeps the global counts constant. 146 */ 147void vm_events_fold_cpu(int cpu) 148{ 149 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 150 int i; 151 152 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 153 count_vm_events(i, fold_state->event[i]); 154 fold_state->event[i] = 0; 155 } 156} 157 158#endif /* CONFIG_VM_EVENT_COUNTERS */ 159 160/* 161 * Manage combined zone based / global counters 162 * 163 * vm_stat contains the global counters 164 */ 165atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 166atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp; 167atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS] __cacheline_aligned_in_smp; 168EXPORT_SYMBOL(vm_zone_stat); 169EXPORT_SYMBOL(vm_node_stat); 170 171#ifdef CONFIG_NUMA 172static void fold_vm_zone_numa_events(struct zone *zone) 173{ 174 unsigned long zone_numa_events[NR_VM_NUMA_EVENT_ITEMS] = { 0, }; 175 int cpu; 176 enum numa_stat_item item; 177 178 for_each_online_cpu(cpu) { 179 struct per_cpu_zonestat *pzstats; 180 181 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 182 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 183 zone_numa_events[item] += xchg(&pzstats->vm_numa_event[item], 0); 184 } 185 186 for (item = 0; item < NR_VM_NUMA_EVENT_ITEMS; item++) 187 zone_numa_event_add(zone_numa_events[item], zone, item); 188} 189 190void fold_vm_numa_events(void) 191{ 192 struct zone *zone; 193 194 for_each_populated_zone(zone) 195 fold_vm_zone_numa_events(zone); 196} 197#endif 198 199#ifdef CONFIG_SMP 200 201int calculate_pressure_threshold(struct zone *zone) 202{ 203 int threshold; 204 int watermark_distance; 205 206 /* 207 * As vmstats are not up to date, there is drift between the estimated 208 * and real values. For high thresholds and a high number of CPUs, it 209 * is possible for the min watermark to be breached while the estimated 210 * value looks fine. The pressure threshold is a reduced value such 211 * that even the maximum amount of drift will not accidentally breach 212 * the min watermark 213 */ 214 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 215 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 216 217 /* 218 * Maximum threshold is 125 219 */ 220 threshold = min(125, threshold); 221 222 return threshold; 223} 224 225int calculate_normal_threshold(struct zone *zone) 226{ 227 int threshold; 228 int mem; /* memory in 128 MB units */ 229 230 /* 231 * The threshold scales with the number of processors and the amount 232 * of memory per zone. More memory means that we can defer updates for 233 * longer, more processors could lead to more contention. 234 * fls() is used to have a cheap way of logarithmic scaling. 235 * 236 * Some sample thresholds: 237 * 238 * Threshold Processors (fls) Zonesize fls(mem)+1 239 * ------------------------------------------------------------------ 240 * 8 1 1 0.9-1 GB 4 241 * 16 2 2 0.9-1 GB 4 242 * 20 2 2 1-2 GB 5 243 * 24 2 2 2-4 GB 6 244 * 28 2 2 4-8 GB 7 245 * 32 2 2 8-16 GB 8 246 * 4 2 2 <128M 1 247 * 30 4 3 2-4 GB 5 248 * 48 4 3 8-16 GB 8 249 * 32 8 4 1-2 GB 4 250 * 32 8 4 0.9-1GB 4 251 * 10 16 5 <128M 1 252 * 40 16 5 900M 4 253 * 70 64 7 2-4 GB 5 254 * 84 64 7 4-8 GB 6 255 * 108 512 9 4-8 GB 6 256 * 125 1024 10 8-16 GB 8 257 * 125 1024 10 16-32 GB 9 258 */ 259 260 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT); 261 262 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 263 264 /* 265 * Maximum threshold is 125 266 */ 267 threshold = min(125, threshold); 268 269 return threshold; 270} 271 272/* 273 * Refresh the thresholds for each zone. 274 */ 275void refresh_zone_stat_thresholds(void) 276{ 277 struct pglist_data *pgdat; 278 struct zone *zone; 279 int cpu; 280 int threshold; 281 282 /* Zero current pgdat thresholds */ 283 for_each_online_pgdat(pgdat) { 284 for_each_online_cpu(cpu) { 285 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0; 286 } 287 } 288 289 for_each_populated_zone(zone) { 290 struct pglist_data *pgdat = zone->zone_pgdat; 291 unsigned long max_drift, tolerate_drift; 292 293 threshold = calculate_normal_threshold(zone); 294 295 for_each_online_cpu(cpu) { 296 int pgdat_threshold; 297 298 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 299 = threshold; 300 301 /* Base nodestat threshold on the largest populated zone. */ 302 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold; 303 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold 304 = max(threshold, pgdat_threshold); 305 } 306 307 /* 308 * Only set percpu_drift_mark if there is a danger that 309 * NR_FREE_PAGES reports the low watermark is ok when in fact 310 * the min watermark could be breached by an allocation 311 */ 312 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 313 max_drift = num_online_cpus() * threshold; 314 if (max_drift > tolerate_drift) 315 zone->percpu_drift_mark = high_wmark_pages(zone) + 316 max_drift; 317 } 318} 319 320void set_pgdat_percpu_threshold(pg_data_t *pgdat, 321 int (*calculate_pressure)(struct zone *)) 322{ 323 struct zone *zone; 324 int cpu; 325 int threshold; 326 int i; 327 328 for (i = 0; i < pgdat->nr_zones; i++) { 329 zone = &pgdat->node_zones[i]; 330 if (!zone->percpu_drift_mark) 331 continue; 332 333 threshold = (*calculate_pressure)(zone); 334 for_each_online_cpu(cpu) 335 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold 336 = threshold; 337 } 338} 339 340/* 341 * For use when we know that interrupts are disabled, 342 * or when we know that preemption is disabled and that 343 * particular counter cannot be updated from interrupt context. 344 */ 345void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 346 long delta) 347{ 348 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 349 s8 __percpu *p = pcp->vm_stat_diff + item; 350 long x; 351 long t; 352 353 /* 354 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels, 355 * atomicity is provided by IRQs being disabled -- either explicitly 356 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables 357 * CPU migrations and preemption potentially corrupts a counter so 358 * disable preemption. 359 */ 360 preempt_disable_nested(); 361 362 x = delta + __this_cpu_read(*p); 363 364 t = __this_cpu_read(pcp->stat_threshold); 365 366 if (unlikely(abs(x) > t)) { 367 zone_page_state_add(x, zone, item); 368 x = 0; 369 } 370 __this_cpu_write(*p, x); 371 372 preempt_enable_nested(); 373} 374EXPORT_SYMBOL(__mod_zone_page_state); 375 376void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 377 long delta) 378{ 379 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 380 s8 __percpu *p = pcp->vm_node_stat_diff + item; 381 long x; 382 long t; 383 384 if (vmstat_item_in_bytes(item)) { 385 /* 386 * Only cgroups use subpage accounting right now; at 387 * the global level, these items still change in 388 * multiples of whole pages. Store them as pages 389 * internally to keep the per-cpu counters compact. 390 */ 391 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 392 delta >>= PAGE_SHIFT; 393 } 394 395 /* See __mod_zone_page_state() */ 396 preempt_disable_nested(); 397 398 x = delta + __this_cpu_read(*p); 399 400 t = __this_cpu_read(pcp->stat_threshold); 401 402 if (unlikely(abs(x) > t)) { 403 node_page_state_add(x, pgdat, item); 404 x = 0; 405 } 406 __this_cpu_write(*p, x); 407 408 preempt_enable_nested(); 409} 410EXPORT_SYMBOL(__mod_node_page_state); 411 412/* 413 * Optimized increment and decrement functions. 414 * 415 * These are only for a single page and therefore can take a struct page * 416 * argument instead of struct zone *. This allows the inclusion of the code 417 * generated for page_zone(page) into the optimized functions. 418 * 419 * No overflow check is necessary and therefore the differential can be 420 * incremented or decremented in place which may allow the compilers to 421 * generate better code. 422 * The increment or decrement is known and therefore one boundary check can 423 * be omitted. 424 * 425 * NOTE: These functions are very performance sensitive. Change only 426 * with care. 427 * 428 * Some processors have inc/dec instructions that are atomic vs an interrupt. 429 * However, the code must first determine the differential location in a zone 430 * based on the processor number and then inc/dec the counter. There is no 431 * guarantee without disabling preemption that the processor will not change 432 * in between and therefore the atomicity vs. interrupt cannot be exploited 433 * in a useful way here. 434 */ 435void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 436{ 437 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 438 s8 __percpu *p = pcp->vm_stat_diff + item; 439 s8 v, t; 440 441 /* See __mod_zone_page_state() */ 442 preempt_disable_nested(); 443 444 v = __this_cpu_inc_return(*p); 445 t = __this_cpu_read(pcp->stat_threshold); 446 if (unlikely(v > t)) { 447 s8 overstep = t >> 1; 448 449 zone_page_state_add(v + overstep, zone, item); 450 __this_cpu_write(*p, -overstep); 451 } 452 453 preempt_enable_nested(); 454} 455 456void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 457{ 458 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 459 s8 __percpu *p = pcp->vm_node_stat_diff + item; 460 s8 v, t; 461 462 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 463 464 /* See __mod_zone_page_state() */ 465 preempt_disable_nested(); 466 467 v = __this_cpu_inc_return(*p); 468 t = __this_cpu_read(pcp->stat_threshold); 469 if (unlikely(v > t)) { 470 s8 overstep = t >> 1; 471 472 node_page_state_add(v + overstep, pgdat, item); 473 __this_cpu_write(*p, -overstep); 474 } 475 476 preempt_enable_nested(); 477} 478 479void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 480{ 481 __inc_zone_state(page_zone(page), item); 482} 483EXPORT_SYMBOL(__inc_zone_page_state); 484 485void __inc_node_page_state(struct page *page, enum node_stat_item item) 486{ 487 __inc_node_state(page_pgdat(page), item); 488} 489EXPORT_SYMBOL(__inc_node_page_state); 490 491void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 492{ 493 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 494 s8 __percpu *p = pcp->vm_stat_diff + item; 495 s8 v, t; 496 497 /* See __mod_zone_page_state() */ 498 preempt_disable_nested(); 499 500 v = __this_cpu_dec_return(*p); 501 t = __this_cpu_read(pcp->stat_threshold); 502 if (unlikely(v < - t)) { 503 s8 overstep = t >> 1; 504 505 zone_page_state_add(v - overstep, zone, item); 506 __this_cpu_write(*p, overstep); 507 } 508 509 preempt_enable_nested(); 510} 511 512void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 513{ 514 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 515 s8 __percpu *p = pcp->vm_node_stat_diff + item; 516 s8 v, t; 517 518 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 519 520 /* See __mod_zone_page_state() */ 521 preempt_disable_nested(); 522 523 v = __this_cpu_dec_return(*p); 524 t = __this_cpu_read(pcp->stat_threshold); 525 if (unlikely(v < - t)) { 526 s8 overstep = t >> 1; 527 528 node_page_state_add(v - overstep, pgdat, item); 529 __this_cpu_write(*p, overstep); 530 } 531 532 preempt_enable_nested(); 533} 534 535void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 536{ 537 __dec_zone_state(page_zone(page), item); 538} 539EXPORT_SYMBOL(__dec_zone_page_state); 540 541void __dec_node_page_state(struct page *page, enum node_stat_item item) 542{ 543 __dec_node_state(page_pgdat(page), item); 544} 545EXPORT_SYMBOL(__dec_node_page_state); 546 547#ifdef CONFIG_HAVE_CMPXCHG_LOCAL 548/* 549 * If we have cmpxchg_local support then we do not need to incur the overhead 550 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 551 * 552 * mod_state() modifies the zone counter state through atomic per cpu 553 * operations. 554 * 555 * Overstep mode specifies how overstep should handled: 556 * 0 No overstepping 557 * 1 Overstepping half of threshold 558 * -1 Overstepping minus half of threshold 559*/ 560static inline void mod_zone_state(struct zone *zone, 561 enum zone_stat_item item, long delta, int overstep_mode) 562{ 563 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats; 564 s8 __percpu *p = pcp->vm_stat_diff + item; 565 long n, t, z; 566 s8 o; 567 568 o = this_cpu_read(*p); 569 do { 570 z = 0; /* overflow to zone counters */ 571 572 /* 573 * The fetching of the stat_threshold is racy. We may apply 574 * a counter threshold to the wrong the cpu if we get 575 * rescheduled while executing here. However, the next 576 * counter update will apply the threshold again and 577 * therefore bring the counter under the threshold again. 578 * 579 * Most of the time the thresholds are the same anyways 580 * for all cpus in a zone. 581 */ 582 t = this_cpu_read(pcp->stat_threshold); 583 584 n = delta + (long)o; 585 586 if (abs(n) > t) { 587 int os = overstep_mode * (t >> 1) ; 588 589 /* Overflow must be added to zone counters */ 590 z = n + os; 591 n = -os; 592 } 593 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 594 595 if (z) 596 zone_page_state_add(z, zone, item); 597} 598 599void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 600 long delta) 601{ 602 mod_zone_state(zone, item, delta, 0); 603} 604EXPORT_SYMBOL(mod_zone_page_state); 605 606void inc_zone_page_state(struct page *page, enum zone_stat_item item) 607{ 608 mod_zone_state(page_zone(page), item, 1, 1); 609} 610EXPORT_SYMBOL(inc_zone_page_state); 611 612void dec_zone_page_state(struct page *page, enum zone_stat_item item) 613{ 614 mod_zone_state(page_zone(page), item, -1, -1); 615} 616EXPORT_SYMBOL(dec_zone_page_state); 617 618static inline void mod_node_state(struct pglist_data *pgdat, 619 enum node_stat_item item, int delta, int overstep_mode) 620{ 621 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats; 622 s8 __percpu *p = pcp->vm_node_stat_diff + item; 623 long n, t, z; 624 s8 o; 625 626 if (vmstat_item_in_bytes(item)) { 627 /* 628 * Only cgroups use subpage accounting right now; at 629 * the global level, these items still change in 630 * multiples of whole pages. Store them as pages 631 * internally to keep the per-cpu counters compact. 632 */ 633 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 634 delta >>= PAGE_SHIFT; 635 } 636 637 o = this_cpu_read(*p); 638 do { 639 z = 0; /* overflow to node counters */ 640 641 /* 642 * The fetching of the stat_threshold is racy. We may apply 643 * a counter threshold to the wrong the cpu if we get 644 * rescheduled while executing here. However, the next 645 * counter update will apply the threshold again and 646 * therefore bring the counter under the threshold again. 647 * 648 * Most of the time the thresholds are the same anyways 649 * for all cpus in a node. 650 */ 651 t = this_cpu_read(pcp->stat_threshold); 652 653 n = delta + (long)o; 654 655 if (abs(n) > t) { 656 int os = overstep_mode * (t >> 1) ; 657 658 /* Overflow must be added to node counters */ 659 z = n + os; 660 n = -os; 661 } 662 } while (!this_cpu_try_cmpxchg(*p, &o, n)); 663 664 if (z) 665 node_page_state_add(z, pgdat, item); 666} 667 668void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 669 long delta) 670{ 671 mod_node_state(pgdat, item, delta, 0); 672} 673EXPORT_SYMBOL(mod_node_page_state); 674 675void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 676{ 677 mod_node_state(pgdat, item, 1, 1); 678} 679 680void inc_node_page_state(struct page *page, enum node_stat_item item) 681{ 682 mod_node_state(page_pgdat(page), item, 1, 1); 683} 684EXPORT_SYMBOL(inc_node_page_state); 685 686void dec_node_page_state(struct page *page, enum node_stat_item item) 687{ 688 mod_node_state(page_pgdat(page), item, -1, -1); 689} 690EXPORT_SYMBOL(dec_node_page_state); 691#else 692/* 693 * Use interrupt disable to serialize counter updates 694 */ 695void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 696 long delta) 697{ 698 unsigned long flags; 699 700 local_irq_save(flags); 701 __mod_zone_page_state(zone, item, delta); 702 local_irq_restore(flags); 703} 704EXPORT_SYMBOL(mod_zone_page_state); 705 706void inc_zone_page_state(struct page *page, enum zone_stat_item item) 707{ 708 unsigned long flags; 709 struct zone *zone; 710 711 zone = page_zone(page); 712 local_irq_save(flags); 713 __inc_zone_state(zone, item); 714 local_irq_restore(flags); 715} 716EXPORT_SYMBOL(inc_zone_page_state); 717 718void dec_zone_page_state(struct page *page, enum zone_stat_item item) 719{ 720 unsigned long flags; 721 722 local_irq_save(flags); 723 __dec_zone_page_state(page, item); 724 local_irq_restore(flags); 725} 726EXPORT_SYMBOL(dec_zone_page_state); 727 728void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 729{ 730 unsigned long flags; 731 732 local_irq_save(flags); 733 __inc_node_state(pgdat, item); 734 local_irq_restore(flags); 735} 736EXPORT_SYMBOL(inc_node_state); 737 738void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, 739 long delta) 740{ 741 unsigned long flags; 742 743 local_irq_save(flags); 744 __mod_node_page_state(pgdat, item, delta); 745 local_irq_restore(flags); 746} 747EXPORT_SYMBOL(mod_node_page_state); 748 749void inc_node_page_state(struct page *page, enum node_stat_item item) 750{ 751 unsigned long flags; 752 struct pglist_data *pgdat; 753 754 pgdat = page_pgdat(page); 755 local_irq_save(flags); 756 __inc_node_state(pgdat, item); 757 local_irq_restore(flags); 758} 759EXPORT_SYMBOL(inc_node_page_state); 760 761void dec_node_page_state(struct page *page, enum node_stat_item item) 762{ 763 unsigned long flags; 764 765 local_irq_save(flags); 766 __dec_node_page_state(page, item); 767 local_irq_restore(flags); 768} 769EXPORT_SYMBOL(dec_node_page_state); 770#endif 771 772/* 773 * Fold a differential into the global counters. 774 * Returns whether counters were updated. 775 */ 776static int fold_diff(int *zone_diff, int *node_diff) 777{ 778 int i; 779 bool changed = false; 780 781 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 782 if (zone_diff[i]) { 783 atomic_long_add(zone_diff[i], &vm_zone_stat[i]); 784 changed = true; 785 } 786 } 787 788 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 789 if (node_diff[i]) { 790 atomic_long_add(node_diff[i], &vm_node_stat[i]); 791 changed = true; 792 } 793 } 794 795 return changed; 796} 797 798/* 799 * Update the zone counters for the current cpu. 800 * 801 * Note that refresh_cpu_vm_stats strives to only access 802 * node local memory. The per cpu pagesets on remote zones are placed 803 * in the memory local to the processor using that pageset. So the 804 * loop over all zones will access a series of cachelines local to 805 * the processor. 806 * 807 * The call to zone_page_state_add updates the cachelines with the 808 * statistics in the remote zone struct as well as the global cachelines 809 * with the global counters. These could cause remote node cache line 810 * bouncing and will have to be only done when necessary. 811 * 812 * The function returns whether global counters were updated. 813 */ 814static bool refresh_cpu_vm_stats(bool do_pagesets) 815{ 816 struct pglist_data *pgdat; 817 struct zone *zone; 818 int i; 819 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 820 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 821 bool changed = false; 822 823 for_each_populated_zone(zone) { 824 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 825 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset; 826 827 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 828 int v; 829 830 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0); 831 if (v) { 832 833 atomic_long_add(v, &zone->vm_stat[i]); 834 global_zone_diff[i] += v; 835#ifdef CONFIG_NUMA 836 /* 3 seconds idle till flush */ 837 __this_cpu_write(pcp->expire, 3); 838#endif 839 } 840 } 841 842 if (do_pagesets) { 843 cond_resched(); 844 845 if (decay_pcp_high(zone, this_cpu_ptr(pcp))) 846 changed = true; 847#ifdef CONFIG_NUMA 848 /* 849 * Deal with draining the remote pageset of this 850 * processor 851 * 852 * Check if there are pages remaining in this pageset 853 * if not then there is nothing to expire. 854 */ 855 if (!__this_cpu_read(pcp->expire) || 856 !__this_cpu_read(pcp->count)) 857 continue; 858 859 /* 860 * We never drain zones local to this processor. 861 */ 862 if (zone_to_nid(zone) == numa_node_id()) { 863 __this_cpu_write(pcp->expire, 0); 864 continue; 865 } 866 867 if (__this_cpu_dec_return(pcp->expire)) { 868 changed = true; 869 continue; 870 } 871 872 if (__this_cpu_read(pcp->count)) { 873 drain_zone_pages(zone, this_cpu_ptr(pcp)); 874 changed = true; 875 } 876#endif 877 } 878 } 879 880 for_each_online_pgdat(pgdat) { 881 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats; 882 883 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 884 int v; 885 886 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0); 887 if (v) { 888 atomic_long_add(v, &pgdat->vm_stat[i]); 889 global_node_diff[i] += v; 890 } 891 } 892 } 893 894 if (fold_diff(global_zone_diff, global_node_diff)) 895 changed = true; 896 return changed; 897} 898 899/* 900 * Fold the data for an offline cpu into the global array. 901 * There cannot be any access by the offline cpu and therefore 902 * synchronization is simplified. 903 */ 904void cpu_vm_stats_fold(int cpu) 905{ 906 struct pglist_data *pgdat; 907 struct zone *zone; 908 int i; 909 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 910 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, }; 911 912 for_each_populated_zone(zone) { 913 struct per_cpu_zonestat *pzstats; 914 915 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 916 917 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 918 if (pzstats->vm_stat_diff[i]) { 919 int v; 920 921 v = pzstats->vm_stat_diff[i]; 922 pzstats->vm_stat_diff[i] = 0; 923 atomic_long_add(v, &zone->vm_stat[i]); 924 global_zone_diff[i] += v; 925 } 926 } 927#ifdef CONFIG_NUMA 928 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 929 if (pzstats->vm_numa_event[i]) { 930 unsigned long v; 931 932 v = pzstats->vm_numa_event[i]; 933 pzstats->vm_numa_event[i] = 0; 934 zone_numa_event_add(v, zone, i); 935 } 936 } 937#endif 938 } 939 940 for_each_online_pgdat(pgdat) { 941 struct per_cpu_nodestat *p; 942 943 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 944 945 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) 946 if (p->vm_node_stat_diff[i]) { 947 int v; 948 949 v = p->vm_node_stat_diff[i]; 950 p->vm_node_stat_diff[i] = 0; 951 atomic_long_add(v, &pgdat->vm_stat[i]); 952 global_node_diff[i] += v; 953 } 954 } 955 956 fold_diff(global_zone_diff, global_node_diff); 957} 958 959/* 960 * this is only called if !populated_zone(zone), which implies no other users of 961 * pset->vm_stat_diff[] exist. 962 */ 963void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats) 964{ 965 unsigned long v; 966 int i; 967 968 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 969 if (pzstats->vm_stat_diff[i]) { 970 v = pzstats->vm_stat_diff[i]; 971 pzstats->vm_stat_diff[i] = 0; 972 zone_page_state_add(v, zone, i); 973 } 974 } 975 976#ifdef CONFIG_NUMA 977 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) { 978 if (pzstats->vm_numa_event[i]) { 979 v = pzstats->vm_numa_event[i]; 980 pzstats->vm_numa_event[i] = 0; 981 zone_numa_event_add(v, zone, i); 982 } 983 } 984#endif 985} 986#endif 987 988#ifdef CONFIG_NUMA 989/* 990 * Determine the per node value of a stat item. This function 991 * is called frequently in a NUMA machine, so try to be as 992 * frugal as possible. 993 */ 994unsigned long sum_zone_node_page_state(int node, 995 enum zone_stat_item item) 996{ 997 struct zone *zones = NODE_DATA(node)->node_zones; 998 int i; 999 unsigned long count = 0; 1000 1001 for (i = 0; i < MAX_NR_ZONES; i++) 1002 count += zone_page_state(zones + i, item); 1003 1004 return count; 1005} 1006 1007/* Determine the per node value of a numa stat item. */ 1008unsigned long sum_zone_numa_event_state(int node, 1009 enum numa_stat_item item) 1010{ 1011 struct zone *zones = NODE_DATA(node)->node_zones; 1012 unsigned long count = 0; 1013 int i; 1014 1015 for (i = 0; i < MAX_NR_ZONES; i++) 1016 count += zone_numa_event_state(zones + i, item); 1017 1018 return count; 1019} 1020 1021/* 1022 * Determine the per node value of a stat item. 1023 */ 1024unsigned long node_page_state_pages(struct pglist_data *pgdat, 1025 enum node_stat_item item) 1026{ 1027 long x = atomic_long_read(&pgdat->vm_stat[item]); 1028#ifdef CONFIG_SMP 1029 if (x < 0) 1030 x = 0; 1031#endif 1032 return x; 1033} 1034 1035unsigned long node_page_state(struct pglist_data *pgdat, 1036 enum node_stat_item item) 1037{ 1038 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 1039 1040 return node_page_state_pages(pgdat, item); 1041} 1042#endif 1043 1044/* 1045 * Count number of pages "struct page" and "struct page_ext" consume. 1046 * nr_memmap_boot_pages: # of pages allocated by boot allocator 1047 * nr_memmap_pages: # of pages that were allocated by buddy allocator 1048 */ 1049static atomic_long_t nr_memmap_boot_pages = ATOMIC_LONG_INIT(0); 1050static atomic_long_t nr_memmap_pages = ATOMIC_LONG_INIT(0); 1051 1052void memmap_boot_pages_add(long delta) 1053{ 1054 atomic_long_add(delta, &nr_memmap_boot_pages); 1055} 1056 1057void memmap_pages_add(long delta) 1058{ 1059 atomic_long_add(delta, &nr_memmap_pages); 1060} 1061 1062#ifdef CONFIG_COMPACTION 1063 1064struct contig_page_info { 1065 unsigned long free_pages; 1066 unsigned long free_blocks_total; 1067 unsigned long free_blocks_suitable; 1068}; 1069 1070/* 1071 * Calculate the number of free pages in a zone, how many contiguous 1072 * pages are free and how many are large enough to satisfy an allocation of 1073 * the target size. Note that this function makes no attempt to estimate 1074 * how many suitable free blocks there *might* be if MOVABLE pages were 1075 * migrated. Calculating that is possible, but expensive and can be 1076 * figured out from userspace 1077 */ 1078static void fill_contig_page_info(struct zone *zone, 1079 unsigned int suitable_order, 1080 struct contig_page_info *info) 1081{ 1082 unsigned int order; 1083 1084 info->free_pages = 0; 1085 info->free_blocks_total = 0; 1086 info->free_blocks_suitable = 0; 1087 1088 for (order = 0; order < NR_PAGE_ORDERS; order++) { 1089 unsigned long blocks; 1090 1091 /* 1092 * Count number of free blocks. 1093 * 1094 * Access to nr_free is lockless as nr_free is used only for 1095 * diagnostic purposes. Use data_race to avoid KCSAN warning. 1096 */ 1097 blocks = data_race(zone->free_area[order].nr_free); 1098 info->free_blocks_total += blocks; 1099 1100 /* Count free base pages */ 1101 info->free_pages += blocks << order; 1102 1103 /* Count the suitable free blocks */ 1104 if (order >= suitable_order) 1105 info->free_blocks_suitable += blocks << 1106 (order - suitable_order); 1107 } 1108} 1109 1110/* 1111 * A fragmentation index only makes sense if an allocation of a requested 1112 * size would fail. If that is true, the fragmentation index indicates 1113 * whether external fragmentation or a lack of memory was the problem. 1114 * The value can be used to determine if page reclaim or compaction 1115 * should be used 1116 */ 1117static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 1118{ 1119 unsigned long requested = 1UL << order; 1120 1121 if (WARN_ON_ONCE(order > MAX_PAGE_ORDER)) 1122 return 0; 1123 1124 if (!info->free_blocks_total) 1125 return 0; 1126 1127 /* Fragmentation index only makes sense when a request would fail */ 1128 if (info->free_blocks_suitable) 1129 return -1000; 1130 1131 /* 1132 * Index is between 0 and 1 so return within 3 decimal places 1133 * 1134 * 0 => allocation would fail due to lack of memory 1135 * 1 => allocation would fail due to fragmentation 1136 */ 1137 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 1138} 1139 1140/* 1141 * Calculates external fragmentation within a zone wrt the given order. 1142 * It is defined as the percentage of pages found in blocks of size 1143 * less than 1 << order. It returns values in range [0, 100]. 1144 */ 1145unsigned int extfrag_for_order(struct zone *zone, unsigned int order) 1146{ 1147 struct contig_page_info info; 1148 1149 fill_contig_page_info(zone, order, &info); 1150 if (info.free_pages == 0) 1151 return 0; 1152 1153 return div_u64((info.free_pages - 1154 (info.free_blocks_suitable << order)) * 100, 1155 info.free_pages); 1156} 1157 1158/* Same as __fragmentation index but allocs contig_page_info on stack */ 1159int fragmentation_index(struct zone *zone, unsigned int order) 1160{ 1161 struct contig_page_info info; 1162 1163 fill_contig_page_info(zone, order, &info); 1164 return __fragmentation_index(order, &info); 1165} 1166#endif 1167 1168#if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \ 1169 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG) 1170#ifdef CONFIG_ZONE_DMA 1171#define TEXT_FOR_DMA(xx, yy) [xx##_DMA] = yy "_dma", 1172#else 1173#define TEXT_FOR_DMA(xx, yy) 1174#endif 1175 1176#ifdef CONFIG_ZONE_DMA32 1177#define TEXT_FOR_DMA32(xx, yy) [xx##_DMA32] = yy "_dma32", 1178#else 1179#define TEXT_FOR_DMA32(xx, yy) 1180#endif 1181 1182#ifdef CONFIG_HIGHMEM 1183#define TEXT_FOR_HIGHMEM(xx, yy) [xx##_HIGH] = yy "_high", 1184#else 1185#define TEXT_FOR_HIGHMEM(xx, yy) 1186#endif 1187 1188#ifdef CONFIG_ZONE_DEVICE 1189#define TEXT_FOR_DEVICE(xx, yy) [xx##_DEVICE] = yy "_device", 1190#else 1191#define TEXT_FOR_DEVICE(xx, yy) 1192#endif 1193 1194#define TEXTS_FOR_ZONES(xx, yy) \ 1195 TEXT_FOR_DMA(xx, yy) \ 1196 TEXT_FOR_DMA32(xx, yy) \ 1197 [xx##_NORMAL] = yy "_normal", \ 1198 TEXT_FOR_HIGHMEM(xx, yy) \ 1199 [xx##_MOVABLE] = yy "_movable", \ 1200 TEXT_FOR_DEVICE(xx, yy) 1201 1202const char * const vmstat_text[] = { 1203 /* enum zone_stat_item counters */ 1204#define I(x) (x) 1205 [I(NR_FREE_PAGES)] = "nr_free_pages", 1206 [I(NR_FREE_PAGES_BLOCKS)] = "nr_free_pages_blocks", 1207 [I(NR_ZONE_INACTIVE_ANON)] = "nr_zone_inactive_anon", 1208 [I(NR_ZONE_ACTIVE_ANON)] = "nr_zone_active_anon", 1209 [I(NR_ZONE_INACTIVE_FILE)] = "nr_zone_inactive_file", 1210 [I(NR_ZONE_ACTIVE_FILE)] = "nr_zone_active_file", 1211 [I(NR_ZONE_UNEVICTABLE)] = "nr_zone_unevictable", 1212 [I(NR_ZONE_WRITE_PENDING)] = "nr_zone_write_pending", 1213 [I(NR_MLOCK)] = "nr_mlock", 1214#if IS_ENABLED(CONFIG_ZSMALLOC) 1215 [I(NR_ZSPAGES)] = "nr_zspages", 1216#endif 1217 [I(NR_FREE_CMA_PAGES)] = "nr_free_cma", 1218#ifdef CONFIG_UNACCEPTED_MEMORY 1219 [I(NR_UNACCEPTED)] = "nr_unaccepted", 1220#endif 1221#undef I 1222 1223 /* enum numa_stat_item counters */ 1224#define I(x) (NR_VM_ZONE_STAT_ITEMS + x) 1225#ifdef CONFIG_NUMA 1226 [I(NUMA_HIT)] = "numa_hit", 1227 [I(NUMA_MISS)] = "numa_miss", 1228 [I(NUMA_FOREIGN)] = "numa_foreign", 1229 [I(NUMA_INTERLEAVE_HIT)] = "numa_interleave", 1230 [I(NUMA_LOCAL)] = "numa_local", 1231 [I(NUMA_OTHER)] = "numa_other", 1232#endif 1233#undef I 1234 1235 /* enum node_stat_item counters */ 1236#define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + x) 1237 [I(NR_INACTIVE_ANON)] = "nr_inactive_anon", 1238 [I(NR_ACTIVE_ANON)] = "nr_active_anon", 1239 [I(NR_INACTIVE_FILE)] = "nr_inactive_file", 1240 [I(NR_ACTIVE_FILE)] = "nr_active_file", 1241 [I(NR_UNEVICTABLE)] = "nr_unevictable", 1242 [I(NR_SLAB_RECLAIMABLE_B)] = "nr_slab_reclaimable", 1243 [I(NR_SLAB_UNRECLAIMABLE_B)] = "nr_slab_unreclaimable", 1244 [I(NR_ISOLATED_ANON)] = "nr_isolated_anon", 1245 [I(NR_ISOLATED_FILE)] = "nr_isolated_file", 1246 [I(WORKINGSET_NODES)] = "workingset_nodes", 1247 [I(WORKINGSET_REFAULT_ANON)] = "workingset_refault_anon", 1248 [I(WORKINGSET_REFAULT_FILE)] = "workingset_refault_file", 1249 [I(WORKINGSET_ACTIVATE_ANON)] = "workingset_activate_anon", 1250 [I(WORKINGSET_ACTIVATE_FILE)] = "workingset_activate_file", 1251 [I(WORKINGSET_RESTORE_ANON)] = "workingset_restore_anon", 1252 [I(WORKINGSET_RESTORE_FILE)] = "workingset_restore_file", 1253 [I(WORKINGSET_NODERECLAIM)] = "workingset_nodereclaim", 1254 [I(NR_ANON_MAPPED)] = "nr_anon_pages", 1255 [I(NR_FILE_MAPPED)] = "nr_mapped", 1256 [I(NR_FILE_PAGES)] = "nr_file_pages", 1257 [I(NR_FILE_DIRTY)] = "nr_dirty", 1258 [I(NR_WRITEBACK)] = "nr_writeback", 1259 [I(NR_SHMEM)] = "nr_shmem", 1260 [I(NR_SHMEM_THPS)] = "nr_shmem_hugepages", 1261 [I(NR_SHMEM_PMDMAPPED)] = "nr_shmem_pmdmapped", 1262 [I(NR_FILE_THPS)] = "nr_file_hugepages", 1263 [I(NR_FILE_PMDMAPPED)] = "nr_file_pmdmapped", 1264 [I(NR_ANON_THPS)] = "nr_anon_transparent_hugepages", 1265 [I(NR_VMSCAN_WRITE)] = "nr_vmscan_write", 1266 [I(NR_VMSCAN_IMMEDIATE)] = "nr_vmscan_immediate_reclaim", 1267 [I(NR_DIRTIED)] = "nr_dirtied", 1268 [I(NR_WRITTEN)] = "nr_written", 1269 [I(NR_THROTTLED_WRITTEN)] = "nr_throttled_written", 1270 [I(NR_KERNEL_MISC_RECLAIMABLE)] = "nr_kernel_misc_reclaimable", 1271 [I(NR_FOLL_PIN_ACQUIRED)] = "nr_foll_pin_acquired", 1272 [I(NR_FOLL_PIN_RELEASED)] = "nr_foll_pin_released", 1273 [I(NR_KERNEL_STACK_KB)] = "nr_kernel_stack", 1274#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) 1275 [I(NR_KERNEL_SCS_KB)] = "nr_shadow_call_stack", 1276#endif 1277 [I(NR_PAGETABLE)] = "nr_page_table_pages", 1278 [I(NR_SECONDARY_PAGETABLE)] = "nr_sec_page_table_pages", 1279#ifdef CONFIG_IOMMU_SUPPORT 1280 [I(NR_IOMMU_PAGES)] = "nr_iommu_pages", 1281#endif 1282#ifdef CONFIG_SWAP 1283 [I(NR_SWAPCACHE)] = "nr_swapcached", 1284#endif 1285#ifdef CONFIG_NUMA_BALANCING 1286 [I(PGPROMOTE_SUCCESS)] = "pgpromote_success", 1287 [I(PGPROMOTE_CANDIDATE)] = "pgpromote_candidate", 1288 [I(PGPROMOTE_CANDIDATE_NRL)] = "pgpromote_candidate_nrl", 1289#endif 1290 [I(PGDEMOTE_KSWAPD)] = "pgdemote_kswapd", 1291 [I(PGDEMOTE_DIRECT)] = "pgdemote_direct", 1292 [I(PGDEMOTE_KHUGEPAGED)] = "pgdemote_khugepaged", 1293 [I(PGDEMOTE_PROACTIVE)] = "pgdemote_proactive", 1294#ifdef CONFIG_HUGETLB_PAGE 1295 [I(NR_HUGETLB)] = "nr_hugetlb", 1296#endif 1297 [I(NR_BALLOON_PAGES)] = "nr_balloon_pages", 1298 [I(NR_KERNEL_FILE_PAGES)] = "nr_kernel_file_pages", 1299#undef I 1300 1301 /* system-wide enum vm_stat_item counters */ 1302#define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1303 NR_VM_NODE_STAT_ITEMS + x) 1304 [I(NR_DIRTY_THRESHOLD)] = "nr_dirty_threshold", 1305 [I(NR_DIRTY_BG_THRESHOLD)] = "nr_dirty_background_threshold", 1306 [I(NR_MEMMAP_PAGES)] = "nr_memmap_pages", 1307 [I(NR_MEMMAP_BOOT_PAGES)] = "nr_memmap_boot_pages", 1308#undef I 1309 1310#if defined(CONFIG_VM_EVENT_COUNTERS) 1311 /* enum vm_event_item counters */ 1312#define I(x) (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1313 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS + x) 1314 1315 [I(PGPGIN)] = "pgpgin", 1316 [I(PGPGOUT)] = "pgpgout", 1317 [I(PSWPIN)] = "pswpin", 1318 [I(PSWPOUT)] = "pswpout", 1319 1320#define OFF (NR_VM_ZONE_STAT_ITEMS + NR_VM_NUMA_EVENT_ITEMS + \ 1321 NR_VM_NODE_STAT_ITEMS + NR_VM_STAT_ITEMS) 1322 TEXTS_FOR_ZONES(OFF+PGALLOC, "pgalloc") 1323 TEXTS_FOR_ZONES(OFF+ALLOCSTALL, "allocstall") 1324 TEXTS_FOR_ZONES(OFF+PGSCAN_SKIP, "pgskip") 1325#undef OFF 1326 1327 [I(PGFREE)] = "pgfree", 1328 [I(PGACTIVATE)] = "pgactivate", 1329 [I(PGDEACTIVATE)] = "pgdeactivate", 1330 [I(PGLAZYFREE)] = "pglazyfree", 1331 1332 [I(PGFAULT)] = "pgfault", 1333 [I(PGMAJFAULT)] = "pgmajfault", 1334 [I(PGLAZYFREED)] = "pglazyfreed", 1335 1336 [I(PGREFILL)] = "pgrefill", 1337 [I(PGREUSE)] = "pgreuse", 1338 [I(PGSTEAL_KSWAPD)] = "pgsteal_kswapd", 1339 [I(PGSTEAL_DIRECT)] = "pgsteal_direct", 1340 [I(PGSTEAL_KHUGEPAGED)] = "pgsteal_khugepaged", 1341 [I(PGSTEAL_PROACTIVE)] = "pgsteal_proactive", 1342 [I(PGSCAN_KSWAPD)] = "pgscan_kswapd", 1343 [I(PGSCAN_DIRECT)] = "pgscan_direct", 1344 [I(PGSCAN_KHUGEPAGED)] = "pgscan_khugepaged", 1345 [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", 1346 [I(PGSCAN_DIRECT_THROTTLE)] = "pgscan_direct_throttle", 1347 [I(PGSCAN_ANON)] = "pgscan_anon", 1348 [I(PGSCAN_FILE)] = "pgscan_file", 1349 [I(PGSTEAL_ANON)] = "pgsteal_anon", 1350 [I(PGSTEAL_FILE)] = "pgsteal_file", 1351 1352#ifdef CONFIG_NUMA 1353 [I(PGSCAN_ZONE_RECLAIM_SUCCESS)] = "zone_reclaim_success", 1354 [I(PGSCAN_ZONE_RECLAIM_FAILED)] = "zone_reclaim_failed", 1355#endif 1356 [I(PGINODESTEAL)] = "pginodesteal", 1357 [I(SLABS_SCANNED)] = "slabs_scanned", 1358 [I(KSWAPD_INODESTEAL)] = "kswapd_inodesteal", 1359 [I(KSWAPD_LOW_WMARK_HIT_QUICKLY)] = "kswapd_low_wmark_hit_quickly", 1360 [I(KSWAPD_HIGH_WMARK_HIT_QUICKLY)] = "kswapd_high_wmark_hit_quickly", 1361 [I(PAGEOUTRUN)] = "pageoutrun", 1362 1363 [I(PGROTATED)] = "pgrotated", 1364 1365 [I(DROP_PAGECACHE)] = "drop_pagecache", 1366 [I(DROP_SLAB)] = "drop_slab", 1367 [I(OOM_KILL)] = "oom_kill", 1368 1369#ifdef CONFIG_NUMA_BALANCING 1370 [I(NUMA_PTE_UPDATES)] = "numa_pte_updates", 1371 [I(NUMA_HUGE_PTE_UPDATES)] = "numa_huge_pte_updates", 1372 [I(NUMA_HINT_FAULTS)] = "numa_hint_faults", 1373 [I(NUMA_HINT_FAULTS_LOCAL)] = "numa_hint_faults_local", 1374 [I(NUMA_PAGE_MIGRATE)] = "numa_pages_migrated", 1375#endif 1376#ifdef CONFIG_MIGRATION 1377 [I(PGMIGRATE_SUCCESS)] = "pgmigrate_success", 1378 [I(PGMIGRATE_FAIL)] = "pgmigrate_fail", 1379 [I(THP_MIGRATION_SUCCESS)] = "thp_migration_success", 1380 [I(THP_MIGRATION_FAIL)] = "thp_migration_fail", 1381 [I(THP_MIGRATION_SPLIT)] = "thp_migration_split", 1382#endif 1383#ifdef CONFIG_COMPACTION 1384 [I(COMPACTMIGRATE_SCANNED)] = "compact_migrate_scanned", 1385 [I(COMPACTFREE_SCANNED)] = "compact_free_scanned", 1386 [I(COMPACTISOLATED)] = "compact_isolated", 1387 [I(COMPACTSTALL)] = "compact_stall", 1388 [I(COMPACTFAIL)] = "compact_fail", 1389 [I(COMPACTSUCCESS)] = "compact_success", 1390 [I(KCOMPACTD_WAKE)] = "compact_daemon_wake", 1391 [I(KCOMPACTD_MIGRATE_SCANNED)] = "compact_daemon_migrate_scanned", 1392 [I(KCOMPACTD_FREE_SCANNED)] = "compact_daemon_free_scanned", 1393#endif 1394 1395#ifdef CONFIG_HUGETLB_PAGE 1396 [I(HTLB_BUDDY_PGALLOC)] = "htlb_buddy_alloc_success", 1397 [I(HTLB_BUDDY_PGALLOC_FAIL)] = "htlb_buddy_alloc_fail", 1398#endif 1399#ifdef CONFIG_CMA 1400 [I(CMA_ALLOC_SUCCESS)] = "cma_alloc_success", 1401 [I(CMA_ALLOC_FAIL)] = "cma_alloc_fail", 1402#endif 1403 [I(UNEVICTABLE_PGCULLED)] = "unevictable_pgs_culled", 1404 [I(UNEVICTABLE_PGSCANNED)] = "unevictable_pgs_scanned", 1405 [I(UNEVICTABLE_PGRESCUED)] = "unevictable_pgs_rescued", 1406 [I(UNEVICTABLE_PGMLOCKED)] = "unevictable_pgs_mlocked", 1407 [I(UNEVICTABLE_PGMUNLOCKED)] = "unevictable_pgs_munlocked", 1408 [I(UNEVICTABLE_PGCLEARED)] = "unevictable_pgs_cleared", 1409 [I(UNEVICTABLE_PGSTRANDED)] = "unevictable_pgs_stranded", 1410 1411#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1412 [I(THP_FAULT_ALLOC)] = "thp_fault_alloc", 1413 [I(THP_FAULT_FALLBACK)] = "thp_fault_fallback", 1414 [I(THP_FAULT_FALLBACK_CHARGE)] = "thp_fault_fallback_charge", 1415 [I(THP_COLLAPSE_ALLOC)] = "thp_collapse_alloc", 1416 [I(THP_COLLAPSE_ALLOC_FAILED)] = "thp_collapse_alloc_failed", 1417 [I(THP_FILE_ALLOC)] = "thp_file_alloc", 1418 [I(THP_FILE_FALLBACK)] = "thp_file_fallback", 1419 [I(THP_FILE_FALLBACK_CHARGE)] = "thp_file_fallback_charge", 1420 [I(THP_FILE_MAPPED)] = "thp_file_mapped", 1421 [I(THP_SPLIT_PAGE)] = "thp_split_page", 1422 [I(THP_SPLIT_PAGE_FAILED)] = "thp_split_page_failed", 1423 [I(THP_DEFERRED_SPLIT_PAGE)] = "thp_deferred_split_page", 1424 [I(THP_UNDERUSED_SPLIT_PAGE)] = "thp_underused_split_page", 1425 [I(THP_SPLIT_PMD)] = "thp_split_pmd", 1426 [I(THP_SCAN_EXCEED_NONE_PTE)] = "thp_scan_exceed_none_pte", 1427 [I(THP_SCAN_EXCEED_SWAP_PTE)] = "thp_scan_exceed_swap_pte", 1428 [I(THP_SCAN_EXCEED_SHARED_PTE)] = "thp_scan_exceed_share_pte", 1429#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1430 [I(THP_SPLIT_PUD)] = "thp_split_pud", 1431#endif 1432 [I(THP_ZERO_PAGE_ALLOC)] = "thp_zero_page_alloc", 1433 [I(THP_ZERO_PAGE_ALLOC_FAILED)] = "thp_zero_page_alloc_failed", 1434 [I(THP_SWPOUT)] = "thp_swpout", 1435 [I(THP_SWPOUT_FALLBACK)] = "thp_swpout_fallback", 1436#endif 1437#ifdef CONFIG_MEMORY_BALLOON 1438 [I(BALLOON_INFLATE)] = "balloon_inflate", 1439 [I(BALLOON_DEFLATE)] = "balloon_deflate", 1440#ifdef CONFIG_BALLOON_COMPACTION 1441 [I(BALLOON_MIGRATE)] = "balloon_migrate", 1442#endif 1443#endif /* CONFIG_MEMORY_BALLOON */ 1444#ifdef CONFIG_DEBUG_TLBFLUSH 1445 [I(NR_TLB_REMOTE_FLUSH)] = "nr_tlb_remote_flush", 1446 [I(NR_TLB_REMOTE_FLUSH_RECEIVED)] = "nr_tlb_remote_flush_received", 1447 [I(NR_TLB_LOCAL_FLUSH_ALL)] = "nr_tlb_local_flush_all", 1448 [I(NR_TLB_LOCAL_FLUSH_ONE)] = "nr_tlb_local_flush_one", 1449#endif /* CONFIG_DEBUG_TLBFLUSH */ 1450 1451#ifdef CONFIG_SWAP 1452 [I(SWAP_RA)] = "swap_ra", 1453 [I(SWAP_RA_HIT)] = "swap_ra_hit", 1454 [I(SWPIN_ZERO)] = "swpin_zero", 1455 [I(SWPOUT_ZERO)] = "swpout_zero", 1456#ifdef CONFIG_KSM 1457 [I(KSM_SWPIN_COPY)] = "ksm_swpin_copy", 1458#endif 1459#endif 1460#ifdef CONFIG_KSM 1461 [I(COW_KSM)] = "cow_ksm", 1462#endif 1463#ifdef CONFIG_ZSWAP 1464 [I(ZSWPIN)] = "zswpin", 1465 [I(ZSWPOUT)] = "zswpout", 1466 [I(ZSWPWB)] = "zswpwb", 1467#endif 1468#ifdef CONFIG_X86 1469 [I(DIRECT_MAP_LEVEL2_SPLIT)] = "direct_map_level2_splits", 1470 [I(DIRECT_MAP_LEVEL3_SPLIT)] = "direct_map_level3_splits", 1471 [I(DIRECT_MAP_LEVEL2_COLLAPSE)] = "direct_map_level2_collapses", 1472 [I(DIRECT_MAP_LEVEL3_COLLAPSE)] = "direct_map_level3_collapses", 1473#endif 1474#ifdef CONFIG_PER_VMA_LOCK_STATS 1475 [I(VMA_LOCK_SUCCESS)] = "vma_lock_success", 1476 [I(VMA_LOCK_ABORT)] = "vma_lock_abort", 1477 [I(VMA_LOCK_RETRY)] = "vma_lock_retry", 1478 [I(VMA_LOCK_MISS)] = "vma_lock_miss", 1479#endif 1480#ifdef CONFIG_DEBUG_STACK_USAGE 1481 [I(KSTACK_1K)] = "kstack_1k", 1482#if THREAD_SIZE > 1024 1483 [I(KSTACK_2K)] = "kstack_2k", 1484#endif 1485#if THREAD_SIZE > 2048 1486 [I(KSTACK_4K)] = "kstack_4k", 1487#endif 1488#if THREAD_SIZE > 4096 1489 [I(KSTACK_8K)] = "kstack_8k", 1490#endif 1491#if THREAD_SIZE > 8192 1492 [I(KSTACK_16K)] = "kstack_16k", 1493#endif 1494#if THREAD_SIZE > 16384 1495 [I(KSTACK_32K)] = "kstack_32k", 1496#endif 1497#if THREAD_SIZE > 32768 1498 [I(KSTACK_64K)] = "kstack_64k", 1499#endif 1500#if THREAD_SIZE > 65536 1501 [I(KSTACK_REST)] = "kstack_rest", 1502#endif 1503#endif 1504#undef I 1505#endif /* CONFIG_VM_EVENT_COUNTERS */ 1506}; 1507#endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */ 1508 1509#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \ 1510 defined(CONFIG_PROC_FS) 1511static void *frag_start(struct seq_file *m, loff_t *pos) 1512{ 1513 pg_data_t *pgdat; 1514 loff_t node = *pos; 1515 1516 for (pgdat = first_online_pgdat(); 1517 pgdat && node; 1518 pgdat = next_online_pgdat(pgdat)) 1519 --node; 1520 1521 return pgdat; 1522} 1523 1524static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 1525{ 1526 pg_data_t *pgdat = (pg_data_t *)arg; 1527 1528 (*pos)++; 1529 return next_online_pgdat(pgdat); 1530} 1531 1532static void frag_stop(struct seq_file *m, void *arg) 1533{ 1534} 1535 1536/* 1537 * Walk zones in a node and print using a callback. 1538 * If @assert_populated is true, only use callback for zones that are populated. 1539 */ 1540static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 1541 bool assert_populated, bool nolock, 1542 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 1543{ 1544 struct zone *zone; 1545 struct zone *node_zones = pgdat->node_zones; 1546 unsigned long flags; 1547 1548 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 1549 if (assert_populated && !populated_zone(zone)) 1550 continue; 1551 1552 if (!nolock) 1553 spin_lock_irqsave(&zone->lock, flags); 1554 print(m, pgdat, zone); 1555 if (!nolock) 1556 spin_unlock_irqrestore(&zone->lock, flags); 1557 } 1558} 1559#endif 1560 1561#ifdef CONFIG_PROC_FS 1562static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 1563 struct zone *zone) 1564{ 1565 int order; 1566 1567 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1568 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1569 /* 1570 * Access to nr_free is lockless as nr_free is used only for 1571 * printing purposes. Use data_race to avoid KCSAN warning. 1572 */ 1573 seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free)); 1574 seq_putc(m, '\n'); 1575} 1576 1577/* 1578 * This walks the free areas for each zone. 1579 */ 1580static int frag_show(struct seq_file *m, void *arg) 1581{ 1582 pg_data_t *pgdat = (pg_data_t *)arg; 1583 walk_zones_in_node(m, pgdat, true, false, frag_show_print); 1584 return 0; 1585} 1586 1587static void pagetypeinfo_showfree_print(struct seq_file *m, 1588 pg_data_t *pgdat, struct zone *zone) 1589{ 1590 int order, mtype; 1591 1592 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 1593 seq_printf(m, "Node %4d, zone %8s, type %12s ", 1594 pgdat->node_id, 1595 zone->name, 1596 migratetype_names[mtype]); 1597 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 1598 unsigned long freecount = 0; 1599 struct free_area *area; 1600 struct list_head *curr; 1601 bool overflow = false; 1602 1603 area = &(zone->free_area[order]); 1604 1605 list_for_each(curr, &area->free_list[mtype]) { 1606 /* 1607 * Cap the free_list iteration because it might 1608 * be really large and we are under a spinlock 1609 * so a long time spent here could trigger a 1610 * hard lockup detector. Anyway this is a 1611 * debugging tool so knowing there is a handful 1612 * of pages of this order should be more than 1613 * sufficient. 1614 */ 1615 if (++freecount >= 100000) { 1616 overflow = true; 1617 break; 1618 } 1619 } 1620 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount); 1621 spin_unlock_irq(&zone->lock); 1622 cond_resched(); 1623 spin_lock_irq(&zone->lock); 1624 } 1625 seq_putc(m, '\n'); 1626 } 1627} 1628 1629/* Print out the free pages at each order for each migatetype */ 1630static void pagetypeinfo_showfree(struct seq_file *m, void *arg) 1631{ 1632 int order; 1633 pg_data_t *pgdat = (pg_data_t *)arg; 1634 1635 /* Print header */ 1636 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 1637 for (order = 0; order < NR_PAGE_ORDERS; ++order) 1638 seq_printf(m, "%6d ", order); 1639 seq_putc(m, '\n'); 1640 1641 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print); 1642} 1643 1644static void pagetypeinfo_showblockcount_print(struct seq_file *m, 1645 pg_data_t *pgdat, struct zone *zone) 1646{ 1647 int mtype; 1648 unsigned long pfn; 1649 unsigned long start_pfn = zone->zone_start_pfn; 1650 unsigned long end_pfn = zone_end_pfn(zone); 1651 unsigned long count[MIGRATE_TYPES] = { 0, }; 1652 1653 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 1654 struct page *page; 1655 1656 page = pfn_to_online_page(pfn); 1657 if (!page) 1658 continue; 1659 1660 if (page_zone(page) != zone) 1661 continue; 1662 1663 mtype = get_pageblock_migratetype(page); 1664 1665 if (mtype < MIGRATE_TYPES) 1666 count[mtype]++; 1667 } 1668 1669 /* Print counts */ 1670 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 1671 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1672 seq_printf(m, "%12lu ", count[mtype]); 1673 seq_putc(m, '\n'); 1674} 1675 1676/* Print out the number of pageblocks for each migratetype */ 1677static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 1678{ 1679 int mtype; 1680 pg_data_t *pgdat = (pg_data_t *)arg; 1681 1682 seq_printf(m, "\n%-23s", "Number of blocks type "); 1683 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1684 seq_printf(m, "%12s ", migratetype_names[mtype]); 1685 seq_putc(m, '\n'); 1686 walk_zones_in_node(m, pgdat, true, false, 1687 pagetypeinfo_showblockcount_print); 1688} 1689 1690/* 1691 * Print out the number of pageblocks for each migratetype that contain pages 1692 * of other types. This gives an indication of how well fallbacks are being 1693 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER 1694 * to determine what is going on 1695 */ 1696static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) 1697{ 1698#ifdef CONFIG_PAGE_OWNER 1699 int mtype; 1700 1701 if (!static_branch_unlikely(&page_owner_inited)) 1702 return; 1703 1704 drain_all_pages(NULL); 1705 1706 seq_printf(m, "\n%-23s", "Number of mixed blocks "); 1707 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 1708 seq_printf(m, "%12s ", migratetype_names[mtype]); 1709 seq_putc(m, '\n'); 1710 1711 walk_zones_in_node(m, pgdat, true, true, 1712 pagetypeinfo_showmixedcount_print); 1713#endif /* CONFIG_PAGE_OWNER */ 1714} 1715 1716/* 1717 * This prints out statistics in relation to grouping pages by mobility. 1718 * It is expensive to collect so do not constantly read the file. 1719 */ 1720static int pagetypeinfo_show(struct seq_file *m, void *arg) 1721{ 1722 pg_data_t *pgdat = (pg_data_t *)arg; 1723 1724 /* check memoryless node */ 1725 if (!node_state(pgdat->node_id, N_MEMORY)) 1726 return 0; 1727 1728 seq_printf(m, "Page block order: %d\n", pageblock_order); 1729 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 1730 seq_putc(m, '\n'); 1731 pagetypeinfo_showfree(m, pgdat); 1732 pagetypeinfo_showblockcount(m, pgdat); 1733 pagetypeinfo_showmixedcount(m, pgdat); 1734 1735 return 0; 1736} 1737 1738static const struct seq_operations fragmentation_op = { 1739 .start = frag_start, 1740 .next = frag_next, 1741 .stop = frag_stop, 1742 .show = frag_show, 1743}; 1744 1745static const struct seq_operations pagetypeinfo_op = { 1746 .start = frag_start, 1747 .next = frag_next, 1748 .stop = frag_stop, 1749 .show = pagetypeinfo_show, 1750}; 1751 1752static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone) 1753{ 1754 int zid; 1755 1756 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1757 struct zone *compare = &pgdat->node_zones[zid]; 1758 1759 if (populated_zone(compare)) 1760 return zone == compare; 1761 } 1762 1763 return false; 1764} 1765 1766static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 1767 struct zone *zone) 1768{ 1769 int i; 1770 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 1771 if (is_zone_first_populated(pgdat, zone)) { 1772 seq_printf(m, "\n per-node stats"); 1773 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1774 unsigned long pages = node_page_state_pages(pgdat, i); 1775 1776 if (vmstat_item_print_in_thp(i)) 1777 pages /= HPAGE_PMD_NR; 1778 seq_printf(m, "\n %-12s %lu", node_stat_name(i), 1779 pages); 1780 } 1781 } 1782 seq_printf(m, 1783 "\n pages free %lu" 1784 "\n boost %lu" 1785 "\n min %lu" 1786 "\n low %lu" 1787 "\n high %lu" 1788 "\n promo %lu" 1789 "\n spanned %lu" 1790 "\n present %lu" 1791 "\n managed %lu" 1792 "\n cma %lu", 1793 zone_page_state(zone, NR_FREE_PAGES), 1794 zone->watermark_boost, 1795 min_wmark_pages(zone), 1796 low_wmark_pages(zone), 1797 high_wmark_pages(zone), 1798 promo_wmark_pages(zone), 1799 zone->spanned_pages, 1800 zone->present_pages, 1801 zone_managed_pages(zone), 1802 zone_cma_pages(zone)); 1803 1804 seq_printf(m, 1805 "\n protection: (%ld", 1806 zone->lowmem_reserve[0]); 1807 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 1808 seq_printf(m, ", %ld", zone->lowmem_reserve[i]); 1809 seq_putc(m, ')'); 1810 1811 /* If unpopulated, no other information is useful */ 1812 if (!populated_zone(zone)) { 1813 seq_putc(m, '\n'); 1814 return; 1815 } 1816 1817 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1818 seq_printf(m, "\n %-12s %lu", zone_stat_name(i), 1819 zone_page_state(zone, i)); 1820 1821#ifdef CONFIG_NUMA 1822 fold_vm_zone_numa_events(zone); 1823 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1824 seq_printf(m, "\n %-12s %lu", numa_stat_name(i), 1825 zone_numa_event_state(zone, i)); 1826#endif 1827 1828 seq_printf(m, "\n pagesets"); 1829 for_each_online_cpu(i) { 1830 struct per_cpu_pages *pcp; 1831 struct per_cpu_zonestat __maybe_unused *pzstats; 1832 1833 pcp = per_cpu_ptr(zone->per_cpu_pageset, i); 1834 seq_printf(m, 1835 "\n cpu: %i" 1836 "\n count: %i" 1837 "\n high: %i" 1838 "\n batch: %i" 1839 "\n high_min: %i" 1840 "\n high_max: %i", 1841 i, 1842 pcp->count, 1843 pcp->high, 1844 pcp->batch, 1845 pcp->high_min, 1846 pcp->high_max); 1847#ifdef CONFIG_SMP 1848 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i); 1849 seq_printf(m, "\n vm stats threshold: %d", 1850 pzstats->stat_threshold); 1851#endif 1852 } 1853 seq_printf(m, 1854 "\n node_unreclaimable: %u" 1855 "\n start_pfn: %lu" 1856 "\n reserved_highatomic: %lu" 1857 "\n free_highatomic: %lu", 1858 atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES, 1859 zone->zone_start_pfn, 1860 zone->nr_reserved_highatomic, 1861 zone->nr_free_highatomic); 1862 seq_putc(m, '\n'); 1863} 1864 1865/* 1866 * Output information about zones in @pgdat. All zones are printed regardless 1867 * of whether they are populated or not: lowmem_reserve_ratio operates on the 1868 * set of all zones and userspace would not be aware of such zones if they are 1869 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio). 1870 */ 1871static int zoneinfo_show(struct seq_file *m, void *arg) 1872{ 1873 pg_data_t *pgdat = (pg_data_t *)arg; 1874 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print); 1875 return 0; 1876} 1877 1878static const struct seq_operations zoneinfo_op = { 1879 .start = frag_start, /* iterate over all zones. The same as in 1880 * fragmentation. */ 1881 .next = frag_next, 1882 .stop = frag_stop, 1883 .show = zoneinfo_show, 1884}; 1885 1886#define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \ 1887 NR_VM_NUMA_EVENT_ITEMS + \ 1888 NR_VM_NODE_STAT_ITEMS + \ 1889 NR_VM_STAT_ITEMS + \ 1890 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \ 1891 NR_VM_EVENT_ITEMS : 0)) 1892 1893static void *vmstat_start(struct seq_file *m, loff_t *pos) 1894{ 1895 unsigned long *v; 1896 int i; 1897 1898 if (*pos >= NR_VMSTAT_ITEMS) 1899 return NULL; 1900 1901 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) != NR_VMSTAT_ITEMS); 1902 fold_vm_numa_events(); 1903 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL); 1904 m->private = v; 1905 if (!v) 1906 return ERR_PTR(-ENOMEM); 1907 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1908 v[i] = global_zone_page_state(i); 1909 v += NR_VM_ZONE_STAT_ITEMS; 1910 1911#ifdef CONFIG_NUMA 1912 for (i = 0; i < NR_VM_NUMA_EVENT_ITEMS; i++) 1913 v[i] = global_numa_event_state(i); 1914 v += NR_VM_NUMA_EVENT_ITEMS; 1915#endif 1916 1917 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 1918 v[i] = global_node_page_state_pages(i); 1919 if (vmstat_item_print_in_thp(i)) 1920 v[i] /= HPAGE_PMD_NR; 1921 } 1922 v += NR_VM_NODE_STAT_ITEMS; 1923 1924 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1925 v + NR_DIRTY_THRESHOLD); 1926 v[NR_MEMMAP_PAGES] = atomic_long_read(&nr_memmap_pages); 1927 v[NR_MEMMAP_BOOT_PAGES] = atomic_long_read(&nr_memmap_boot_pages); 1928 v += NR_VM_STAT_ITEMS; 1929 1930#ifdef CONFIG_VM_EVENT_COUNTERS 1931 all_vm_events(v); 1932 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1933 v[PGPGOUT] /= 2; 1934#endif 1935 return (unsigned long *)m->private + *pos; 1936} 1937 1938static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1939{ 1940 (*pos)++; 1941 if (*pos >= NR_VMSTAT_ITEMS) 1942 return NULL; 1943 return (unsigned long *)m->private + *pos; 1944} 1945 1946static int vmstat_show(struct seq_file *m, void *arg) 1947{ 1948 unsigned long *l = arg; 1949 unsigned long off = l - (unsigned long *)m->private; 1950 1951 seq_puts(m, vmstat_text[off]); 1952 seq_put_decimal_ull(m, " ", *l); 1953 seq_putc(m, '\n'); 1954 1955 if (off == NR_VMSTAT_ITEMS - 1) { 1956 /* 1957 * We've come to the end - add any deprecated counters to avoid 1958 * breaking userspace which might depend on them being present. 1959 */ 1960 seq_puts(m, "nr_unstable 0\n"); 1961 } 1962 return 0; 1963} 1964 1965static void vmstat_stop(struct seq_file *m, void *arg) 1966{ 1967 kfree(m->private); 1968 m->private = NULL; 1969} 1970 1971static const struct seq_operations vmstat_op = { 1972 .start = vmstat_start, 1973 .next = vmstat_next, 1974 .stop = vmstat_stop, 1975 .show = vmstat_show, 1976}; 1977#endif /* CONFIG_PROC_FS */ 1978 1979#ifdef CONFIG_SMP 1980static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1981static int sysctl_stat_interval __read_mostly = HZ; 1982static int vmstat_late_init_done; 1983 1984#ifdef CONFIG_PROC_FS 1985static void refresh_vm_stats(struct work_struct *work) 1986{ 1987 refresh_cpu_vm_stats(true); 1988} 1989 1990static int vmstat_refresh(const struct ctl_table *table, int write, 1991 void *buffer, size_t *lenp, loff_t *ppos) 1992{ 1993 long val; 1994 int err; 1995 int i; 1996 1997 /* 1998 * The regular update, every sysctl_stat_interval, may come later 1999 * than expected: leaving a significant amount in per_cpu buckets. 2000 * This is particularly misleading when checking a quantity of HUGE 2001 * pages, immediately after running a test. /proc/sys/vm/stat_refresh, 2002 * which can equally be echo'ed to or cat'ted from (by root), 2003 * can be used to update the stats just before reading them. 2004 * 2005 * Oh, and since global_zone_page_state() etc. are so careful to hide 2006 * transiently negative values, report an error here if any of 2007 * the stats is negative, so we know to go looking for imbalance. 2008 */ 2009 err = schedule_on_each_cpu(refresh_vm_stats); 2010 if (err) 2011 return err; 2012 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) { 2013 /* 2014 * Skip checking stats known to go negative occasionally. 2015 */ 2016 switch (i) { 2017 case NR_ZONE_WRITE_PENDING: 2018 case NR_FREE_CMA_PAGES: 2019 continue; 2020 } 2021 val = atomic_long_read(&vm_zone_stat[i]); 2022 if (val < 0) { 2023 pr_warn("%s: %s %ld\n", 2024 __func__, zone_stat_name(i), val); 2025 } 2026 } 2027 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { 2028 /* 2029 * Skip checking stats known to go negative occasionally. 2030 */ 2031 switch (i) { 2032 case NR_WRITEBACK: 2033 continue; 2034 } 2035 val = atomic_long_read(&vm_node_stat[i]); 2036 if (val < 0) { 2037 pr_warn("%s: %s %ld\n", 2038 __func__, node_stat_name(i), val); 2039 } 2040 } 2041 if (write) 2042 *ppos += *lenp; 2043 else 2044 *lenp = 0; 2045 return 0; 2046} 2047#endif /* CONFIG_PROC_FS */ 2048 2049static void vmstat_update(struct work_struct *w) 2050{ 2051 if (refresh_cpu_vm_stats(true)) { 2052 /* 2053 * Counters were updated so we expect more updates 2054 * to occur in the future. Keep on running the 2055 * update worker thread. 2056 */ 2057 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq, 2058 this_cpu_ptr(&vmstat_work), 2059 round_jiffies_relative(sysctl_stat_interval)); 2060 } 2061} 2062 2063/* 2064 * Check if the diffs for a certain cpu indicate that 2065 * an update is needed. 2066 */ 2067static bool need_update(int cpu) 2068{ 2069 pg_data_t *last_pgdat = NULL; 2070 struct zone *zone; 2071 2072 for_each_populated_zone(zone) { 2073 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu); 2074 struct per_cpu_nodestat *n; 2075 2076 /* 2077 * The fast way of checking if there are any vmstat diffs. 2078 */ 2079 if (memchr_inv(pzstats->vm_stat_diff, 0, sizeof(pzstats->vm_stat_diff))) 2080 return true; 2081 2082 if (last_pgdat == zone->zone_pgdat) 2083 continue; 2084 last_pgdat = zone->zone_pgdat; 2085 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu); 2086 if (memchr_inv(n->vm_node_stat_diff, 0, sizeof(n->vm_node_stat_diff))) 2087 return true; 2088 } 2089 return false; 2090} 2091 2092/* 2093 * Switch off vmstat processing and then fold all the remaining differentials 2094 * until the diffs stay at zero. The function is used by NOHZ and can only be 2095 * invoked when tick processing is not active. 2096 */ 2097void quiet_vmstat(void) 2098{ 2099 if (system_state != SYSTEM_RUNNING) 2100 return; 2101 2102 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work))) 2103 return; 2104 2105 if (!need_update(smp_processor_id())) 2106 return; 2107 2108 /* 2109 * Just refresh counters and do not care about the pending delayed 2110 * vmstat_update. It doesn't fire that often to matter and canceling 2111 * it would be too expensive from this path. 2112 * vmstat_shepherd will take care about that for us. 2113 */ 2114 refresh_cpu_vm_stats(false); 2115} 2116 2117/* 2118 * Shepherd worker thread that checks the 2119 * differentials of processors that have their worker 2120 * threads for vm statistics updates disabled because of 2121 * inactivity. 2122 */ 2123static void vmstat_shepherd(struct work_struct *w); 2124 2125static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd); 2126 2127static void vmstat_shepherd(struct work_struct *w) 2128{ 2129 int cpu; 2130 2131 cpus_read_lock(); 2132 /* Check processors whose vmstat worker threads have been disabled */ 2133 for_each_online_cpu(cpu) { 2134 struct delayed_work *dw = &per_cpu(vmstat_work, cpu); 2135 2136 /* 2137 * In kernel users of vmstat counters either require the precise value and 2138 * they are using zone_page_state_snapshot interface or they can live with 2139 * an imprecision as the regular flushing can happen at arbitrary time and 2140 * cumulative error can grow (see calculate_normal_threshold). 2141 * 2142 * From that POV the regular flushing can be postponed for CPUs that have 2143 * been isolated from the kernel interference without critical 2144 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd 2145 * for all isolated CPUs to avoid interference with the isolated workload. 2146 */ 2147 if (cpu_is_isolated(cpu)) 2148 continue; 2149 2150 if (!delayed_work_pending(dw) && need_update(cpu)) 2151 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); 2152 2153 cond_resched(); 2154 } 2155 cpus_read_unlock(); 2156 2157 schedule_delayed_work(&shepherd, 2158 round_jiffies_relative(sysctl_stat_interval)); 2159} 2160 2161static void __init start_shepherd_timer(void) 2162{ 2163 int cpu; 2164 2165 for_each_possible_cpu(cpu) { 2166 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu), 2167 vmstat_update); 2168 2169 /* 2170 * For secondary CPUs during CPU hotplug scenarios, 2171 * vmstat_cpu_online() will enable the work. 2172 * mm/vmstat:online enables and disables vmstat_work 2173 * symmetrically during CPU hotplug events. 2174 */ 2175 if (!cpu_online(cpu)) 2176 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2177 } 2178 2179 schedule_delayed_work(&shepherd, 2180 round_jiffies_relative(sysctl_stat_interval)); 2181} 2182 2183static void __init init_cpu_node_state(void) 2184{ 2185 int node; 2186 2187 for_each_online_node(node) { 2188 if (!cpumask_empty(cpumask_of_node(node))) 2189 node_set_state(node, N_CPU); 2190 } 2191} 2192 2193static int vmstat_cpu_online(unsigned int cpu) 2194{ 2195 if (vmstat_late_init_done) 2196 refresh_zone_stat_thresholds(); 2197 2198 if (!node_state(cpu_to_node(cpu), N_CPU)) { 2199 node_set_state(cpu_to_node(cpu), N_CPU); 2200 } 2201 enable_delayed_work(&per_cpu(vmstat_work, cpu)); 2202 2203 return 0; 2204} 2205 2206static int vmstat_cpu_down_prep(unsigned int cpu) 2207{ 2208 disable_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 2209 return 0; 2210} 2211 2212static int vmstat_cpu_dead(unsigned int cpu) 2213{ 2214 const struct cpumask *node_cpus; 2215 int node; 2216 2217 node = cpu_to_node(cpu); 2218 2219 refresh_zone_stat_thresholds(); 2220 node_cpus = cpumask_of_node(node); 2221 if (!cpumask_empty(node_cpus)) 2222 return 0; 2223 2224 node_clear_state(node, N_CPU); 2225 2226 return 0; 2227} 2228 2229static int __init vmstat_late_init(void) 2230{ 2231 refresh_zone_stat_thresholds(); 2232 vmstat_late_init_done = 1; 2233 2234 return 0; 2235} 2236late_initcall(vmstat_late_init); 2237#endif 2238 2239#ifdef CONFIG_PROC_FS 2240static const struct ctl_table vmstat_table[] = { 2241#ifdef CONFIG_SMP 2242 { 2243 .procname = "stat_interval", 2244 .data = &sysctl_stat_interval, 2245 .maxlen = sizeof(sysctl_stat_interval), 2246 .mode = 0644, 2247 .proc_handler = proc_dointvec_jiffies, 2248 }, 2249 { 2250 .procname = "stat_refresh", 2251 .data = NULL, 2252 .maxlen = 0, 2253 .mode = 0600, 2254 .proc_handler = vmstat_refresh, 2255 }, 2256#endif 2257#ifdef CONFIG_NUMA 2258 { 2259 .procname = "numa_stat", 2260 .data = &sysctl_vm_numa_stat, 2261 .maxlen = sizeof(int), 2262 .mode = 0644, 2263 .proc_handler = sysctl_vm_numa_stat_handler, 2264 .extra1 = SYSCTL_ZERO, 2265 .extra2 = SYSCTL_ONE, 2266 }, 2267#endif 2268}; 2269#endif 2270 2271struct workqueue_struct *mm_percpu_wq; 2272 2273void __init init_mm_internals(void) 2274{ 2275 int ret __maybe_unused; 2276 2277 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0); 2278 2279#ifdef CONFIG_SMP 2280 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", 2281 NULL, vmstat_cpu_dead); 2282 if (ret < 0) 2283 pr_err("vmstat: failed to register 'dead' hotplug state\n"); 2284 2285 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", 2286 vmstat_cpu_online, 2287 vmstat_cpu_down_prep); 2288 if (ret < 0) 2289 pr_err("vmstat: failed to register 'online' hotplug state\n"); 2290 2291 cpus_read_lock(); 2292 init_cpu_node_state(); 2293 cpus_read_unlock(); 2294 2295 start_shepherd_timer(); 2296#endif 2297#ifdef CONFIG_PROC_FS 2298 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); 2299 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); 2300 proc_create_seq("vmstat", 0444, NULL, &vmstat_op); 2301 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op); 2302 register_sysctl_init("vm", vmstat_table); 2303#endif 2304} 2305 2306#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 2307 2308/* 2309 * Return an index indicating how much of the available free memory is 2310 * unusable for an allocation of the requested size. 2311 */ 2312static int unusable_free_index(unsigned int order, 2313 struct contig_page_info *info) 2314{ 2315 /* No free memory is interpreted as all free memory is unusable */ 2316 if (info->free_pages == 0) 2317 return 1000; 2318 2319 /* 2320 * Index should be a value between 0 and 1. Return a value to 3 2321 * decimal places. 2322 * 2323 * 0 => no fragmentation 2324 * 1 => high fragmentation 2325 */ 2326 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 2327 2328} 2329 2330static void unusable_show_print(struct seq_file *m, 2331 pg_data_t *pgdat, struct zone *zone) 2332{ 2333 unsigned int order; 2334 int index; 2335 struct contig_page_info info; 2336 2337 seq_printf(m, "Node %d, zone %8s ", 2338 pgdat->node_id, 2339 zone->name); 2340 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2341 fill_contig_page_info(zone, order, &info); 2342 index = unusable_free_index(order, &info); 2343 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 2344 } 2345 2346 seq_putc(m, '\n'); 2347} 2348 2349/* 2350 * Display unusable free space index 2351 * 2352 * The unusable free space index measures how much of the available free 2353 * memory cannot be used to satisfy an allocation of a given size and is a 2354 * value between 0 and 1. The higher the value, the more of free memory is 2355 * unusable and by implication, the worse the external fragmentation is. This 2356 * can be expressed as a percentage by multiplying by 100. 2357 */ 2358static int unusable_show(struct seq_file *m, void *arg) 2359{ 2360 pg_data_t *pgdat = (pg_data_t *)arg; 2361 2362 /* check memoryless node */ 2363 if (!node_state(pgdat->node_id, N_MEMORY)) 2364 return 0; 2365 2366 walk_zones_in_node(m, pgdat, true, false, unusable_show_print); 2367 2368 return 0; 2369} 2370 2371static const struct seq_operations unusable_sops = { 2372 .start = frag_start, 2373 .next = frag_next, 2374 .stop = frag_stop, 2375 .show = unusable_show, 2376}; 2377 2378DEFINE_SEQ_ATTRIBUTE(unusable); 2379 2380static void extfrag_show_print(struct seq_file *m, 2381 pg_data_t *pgdat, struct zone *zone) 2382{ 2383 unsigned int order; 2384 int index; 2385 2386 /* Alloc on stack as interrupts are disabled for zone walk */ 2387 struct contig_page_info info; 2388 2389 seq_printf(m, "Node %d, zone %8s ", 2390 pgdat->node_id, 2391 zone->name); 2392 for (order = 0; order < NR_PAGE_ORDERS; ++order) { 2393 fill_contig_page_info(zone, order, &info); 2394 index = __fragmentation_index(order, &info); 2395 seq_printf(m, "%2d.%03d ", index / 1000, index % 1000); 2396 } 2397 2398 seq_putc(m, '\n'); 2399} 2400 2401/* 2402 * Display fragmentation index for orders that allocations would fail for 2403 */ 2404static int extfrag_show(struct seq_file *m, void *arg) 2405{ 2406 pg_data_t *pgdat = (pg_data_t *)arg; 2407 2408 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print); 2409 2410 return 0; 2411} 2412 2413static const struct seq_operations extfrag_sops = { 2414 .start = frag_start, 2415 .next = frag_next, 2416 .stop = frag_stop, 2417 .show = extfrag_show, 2418}; 2419 2420DEFINE_SEQ_ATTRIBUTE(extfrag); 2421 2422static int __init extfrag_debug_init(void) 2423{ 2424 struct dentry *extfrag_debug_root; 2425 2426 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 2427 2428 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL, 2429 &unusable_fops); 2430 2431 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL, 2432 &extfrag_fops); 2433 2434 return 0; 2435} 2436 2437module_init(extfrag_debug_init); 2438 2439#endif
~/
Read-only file preview.