FILE
page-writeback.c
File preview with syntax highlighting.
linux-6.19.14/mm/page-writeback.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * mm/page-writeback.c 4 * 5 * Copyright (C) 2002, Linus Torvalds. 6 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 7 * 8 * Contains functions related to writing back dirty pages at the 9 * address_space level. 10 * 11 * 10Apr2002 Andrew Morton 12 * Initial version 13 */ 14 15#include <linux/kernel.h> 16#include <linux/math64.h> 17#include <linux/export.h> 18#include <linux/spinlock.h> 19#include <linux/fs.h> 20#include <linux/mm.h> 21#include <linux/swap.h> 22#include <linux/slab.h> 23#include <linux/pagemap.h> 24#include <linux/writeback.h> 25#include <linux/init.h> 26#include <linux/backing-dev.h> 27#include <linux/task_io_accounting_ops.h> 28#include <linux/blkdev.h> 29#include <linux/mpage.h> 30#include <linux/rmap.h> 31#include <linux/percpu.h> 32#include <linux/smp.h> 33#include <linux/sysctl.h> 34#include <linux/cpu.h> 35#include <linux/syscalls.h> 36#include <linux/pagevec.h> 37#include <linux/timer.h> 38#include <linux/sched/rt.h> 39#include <linux/sched/signal.h> 40#include <linux/mm_inline.h> 41#include <linux/shmem_fs.h> 42#include <trace/events/writeback.h> 43 44#include "internal.h" 45 46/* 47 * Sleep at most 200ms at a time in balance_dirty_pages(). 48 */ 49#define MAX_PAUSE max(HZ/5, 1) 50 51/* 52 * Try to keep balance_dirty_pages() call intervals higher than this many pages 53 * by raising pause time to max_pause when falls below it. 54 */ 55#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10)) 56 57/* 58 * Estimate write bandwidth or update dirty limit at 200ms intervals. 59 */ 60#define BANDWIDTH_INTERVAL max(HZ/5, 1) 61 62#define RATELIMIT_CALC_SHIFT 10 63 64/* 65 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited 66 * will look to see if it needs to force writeback or throttling. 67 */ 68static long ratelimit_pages = 32; 69 70/* The following parameters are exported via /proc/sys/vm */ 71 72/* 73 * Start background writeback (via writeback threads) at this percentage 74 */ 75static int dirty_background_ratio = 10; 76 77/* 78 * dirty_background_bytes starts at 0 (disabled) so that it is a function of 79 * dirty_background_ratio * the amount of dirtyable memory 80 */ 81static unsigned long dirty_background_bytes; 82 83/* 84 * free highmem will not be subtracted from the total free memory 85 * for calculating free ratios if vm_highmem_is_dirtyable is true 86 */ 87static int vm_highmem_is_dirtyable; 88 89/* 90 * The generator of dirty data starts writeback at this percentage 91 */ 92static int vm_dirty_ratio = 20; 93 94/* 95 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of 96 * vm_dirty_ratio * the amount of dirtyable memory 97 */ 98static unsigned long vm_dirty_bytes; 99 100/* 101 * The interval between `kupdate'-style writebacks 102 */ 103unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ 104 105EXPORT_SYMBOL_GPL(dirty_writeback_interval); 106 107/* 108 * The longest time for which data is allowed to remain dirty 109 */ 110unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */ 111 112/* 113 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies: 114 * a full sync is triggered after this time elapses without any disk activity. 115 */ 116int laptop_mode; 117 118EXPORT_SYMBOL(laptop_mode); 119 120/* End of sysctl-exported parameters */ 121 122struct wb_domain global_wb_domain; 123 124/* 125 * Length of period for aging writeout fractions of bdis. This is an 126 * arbitrarily chosen number. The longer the period, the slower fractions will 127 * reflect changes in current writeout rate. 128 */ 129#define VM_COMPLETIONS_PERIOD_LEN (3*HZ) 130 131#ifdef CONFIG_CGROUP_WRITEBACK 132 133#define GDTC_INIT(__wb) .wb = (__wb), \ 134 .dom = &global_wb_domain, \ 135 .wb_completions = &(__wb)->completions 136 137#define GDTC_INIT_NO_WB .dom = &global_wb_domain 138 139#define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \ 140 .dom = mem_cgroup_wb_domain(__wb), \ 141 .wb_completions = &(__wb)->memcg_completions, \ 142 .gdtc = __gdtc 143 144static bool mdtc_valid(struct dirty_throttle_control *dtc) 145{ 146 return dtc->dom; 147} 148 149static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 150{ 151 return dtc->dom; 152} 153 154static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 155{ 156 return mdtc->gdtc; 157} 158 159static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 160{ 161 return &wb->memcg_completions; 162} 163 164static void wb_min_max_ratio(struct bdi_writeback *wb, 165 unsigned long *minp, unsigned long *maxp) 166{ 167 unsigned long this_bw = READ_ONCE(wb->avg_write_bandwidth); 168 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); 169 unsigned long long min = wb->bdi->min_ratio; 170 unsigned long long max = wb->bdi->max_ratio; 171 172 /* 173 * @wb may already be clean by the time control reaches here and 174 * the total may not include its bw. 175 */ 176 if (this_bw < tot_bw) { 177 if (min) { 178 min *= this_bw; 179 min = div64_ul(min, tot_bw); 180 } 181 if (max < 100 * BDI_RATIO_SCALE) { 182 max *= this_bw; 183 max = div64_ul(max, tot_bw); 184 } 185 } 186 187 *minp = min; 188 *maxp = max; 189} 190 191#else /* CONFIG_CGROUP_WRITEBACK */ 192 193#define GDTC_INIT(__wb) .wb = (__wb), \ 194 .wb_completions = &(__wb)->completions 195#define GDTC_INIT_NO_WB 196#define MDTC_INIT(__wb, __gdtc) 197 198static bool mdtc_valid(struct dirty_throttle_control *dtc) 199{ 200 return false; 201} 202 203static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 204{ 205 return &global_wb_domain; 206} 207 208static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 209{ 210 return NULL; 211} 212 213static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 214{ 215 return NULL; 216} 217 218static void wb_min_max_ratio(struct bdi_writeback *wb, 219 unsigned long *minp, unsigned long *maxp) 220{ 221 *minp = wb->bdi->min_ratio; 222 *maxp = wb->bdi->max_ratio; 223} 224 225#endif /* CONFIG_CGROUP_WRITEBACK */ 226 227/* 228 * In a memory zone, there is a certain amount of pages we consider 229 * available for the page cache, which is essentially the number of 230 * free and reclaimable pages, minus some zone reserves to protect 231 * lowmem and the ability to uphold the zone's watermarks without 232 * requiring writeback. 233 * 234 * This number of dirtyable pages is the base value of which the 235 * user-configurable dirty ratio is the effective number of pages that 236 * are allowed to be actually dirtied. Per individual zone, or 237 * globally by using the sum of dirtyable pages over all zones. 238 * 239 * Because the user is allowed to specify the dirty limit globally as 240 * absolute number of bytes, calculating the per-zone dirty limit can 241 * require translating the configured limit into a percentage of 242 * global dirtyable memory first. 243 */ 244 245/** 246 * node_dirtyable_memory - number of dirtyable pages in a node 247 * @pgdat: the node 248 * 249 * Return: the node's number of pages potentially available for dirty 250 * page cache. This is the base value for the per-node dirty limits. 251 */ 252static unsigned long node_dirtyable_memory(struct pglist_data *pgdat) 253{ 254 unsigned long nr_pages = 0; 255 int z; 256 257 for (z = 0; z < MAX_NR_ZONES; z++) { 258 struct zone *zone = pgdat->node_zones + z; 259 260 if (!populated_zone(zone)) 261 continue; 262 263 nr_pages += zone_page_state(zone, NR_FREE_PAGES); 264 } 265 266 /* 267 * Pages reserved for the kernel should not be considered 268 * dirtyable, to prevent a situation where reclaim has to 269 * clean pages in order to balance the zones. 270 */ 271 nr_pages -= min(nr_pages, pgdat->totalreserve_pages); 272 273 nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE); 274 nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE); 275 276 return nr_pages; 277} 278 279static unsigned long highmem_dirtyable_memory(unsigned long total) 280{ 281#ifdef CONFIG_HIGHMEM 282 int node; 283 unsigned long x = 0; 284 int i; 285 286 for_each_node_state(node, N_HIGH_MEMORY) { 287 for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) { 288 struct zone *z; 289 unsigned long nr_pages; 290 291 if (!is_highmem_idx(i)) 292 continue; 293 294 z = &NODE_DATA(node)->node_zones[i]; 295 if (!populated_zone(z)) 296 continue; 297 298 nr_pages = zone_page_state(z, NR_FREE_PAGES); 299 /* watch for underflows */ 300 nr_pages -= min(nr_pages, high_wmark_pages(z)); 301 nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE); 302 nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE); 303 x += nr_pages; 304 } 305 } 306 307 /* 308 * Make sure that the number of highmem pages is never larger 309 * than the number of the total dirtyable memory. This can only 310 * occur in very strange VM situations but we want to make sure 311 * that this does not occur. 312 */ 313 return min(x, total); 314#else 315 return 0; 316#endif 317} 318 319/** 320 * global_dirtyable_memory - number of globally dirtyable pages 321 * 322 * Return: the global number of pages potentially available for dirty 323 * page cache. This is the base value for the global dirty limits. 324 */ 325static unsigned long global_dirtyable_memory(void) 326{ 327 unsigned long x; 328 329 x = global_zone_page_state(NR_FREE_PAGES); 330 /* 331 * Pages reserved for the kernel should not be considered 332 * dirtyable, to prevent a situation where reclaim has to 333 * clean pages in order to balance the zones. 334 */ 335 x -= min(x, totalreserve_pages); 336 337 x += global_node_page_state(NR_INACTIVE_FILE); 338 x += global_node_page_state(NR_ACTIVE_FILE); 339 340 if (!vm_highmem_is_dirtyable) 341 x -= highmem_dirtyable_memory(x); 342 343 return x + 1; /* Ensure that we never return 0 */ 344} 345 346/** 347 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain 348 * @dtc: dirty_throttle_control of interest 349 * 350 * Calculate @dtc->thresh and ->bg_thresh considering 351 * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}. The caller 352 * must ensure that @dtc->avail is set before calling this function. The 353 * dirty limits will be lifted by 1/4 for real-time tasks. 354 */ 355static void domain_dirty_limits(struct dirty_throttle_control *dtc) 356{ 357 const unsigned long available_memory = dtc->avail; 358 struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc); 359 unsigned long bytes = vm_dirty_bytes; 360 unsigned long bg_bytes = dirty_background_bytes; 361 /* convert ratios to per-PAGE_SIZE for higher precision */ 362 unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100; 363 unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100; 364 unsigned long thresh; 365 unsigned long bg_thresh; 366 struct task_struct *tsk; 367 368 /* gdtc is !NULL iff @dtc is for memcg domain */ 369 if (gdtc) { 370 unsigned long global_avail = gdtc->avail; 371 372 /* 373 * The byte settings can't be applied directly to memcg 374 * domains. Convert them to ratios by scaling against 375 * globally available memory. As the ratios are in 376 * per-PAGE_SIZE, they can be obtained by dividing bytes by 377 * number of pages. 378 */ 379 if (bytes) 380 ratio = min(DIV_ROUND_UP(bytes, global_avail), 381 PAGE_SIZE); 382 if (bg_bytes) 383 bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail), 384 PAGE_SIZE); 385 bytes = bg_bytes = 0; 386 } 387 388 if (bytes) 389 thresh = DIV_ROUND_UP(bytes, PAGE_SIZE); 390 else 391 thresh = (ratio * available_memory) / PAGE_SIZE; 392 393 if (bg_bytes) 394 bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE); 395 else 396 bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE; 397 398 tsk = current; 399 if (rt_or_dl_task(tsk)) { 400 bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32; 401 thresh += thresh / 4 + global_wb_domain.dirty_limit / 32; 402 } 403 /* 404 * Dirty throttling logic assumes the limits in page units fit into 405 * 32-bits. This gives 16TB dirty limits max which is hopefully enough. 406 */ 407 if (thresh > UINT_MAX) 408 thresh = UINT_MAX; 409 /* This makes sure bg_thresh is within 32-bits as well */ 410 if (bg_thresh >= thresh) 411 bg_thresh = thresh / 2; 412 dtc->thresh = thresh; 413 dtc->bg_thresh = bg_thresh; 414 415 /* we should eventually report the domain in the TP */ 416 if (!gdtc) 417 trace_global_dirty_state(bg_thresh, thresh); 418} 419 420/** 421 * global_dirty_limits - background-writeback and dirty-throttling thresholds 422 * @pbackground: out parameter for bg_thresh 423 * @pdirty: out parameter for thresh 424 * 425 * Calculate bg_thresh and thresh for global_wb_domain. See 426 * domain_dirty_limits() for details. 427 */ 428void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) 429{ 430 struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB }; 431 432 gdtc.avail = global_dirtyable_memory(); 433 domain_dirty_limits(&gdtc); 434 435 *pbackground = gdtc.bg_thresh; 436 *pdirty = gdtc.thresh; 437} 438 439/** 440 * node_dirty_limit - maximum number of dirty pages allowed in a node 441 * @pgdat: the node 442 * 443 * Return: the maximum number of dirty pages allowed in a node, based 444 * on the node's dirtyable memory. 445 */ 446static unsigned long node_dirty_limit(struct pglist_data *pgdat) 447{ 448 unsigned long node_memory = node_dirtyable_memory(pgdat); 449 struct task_struct *tsk = current; 450 unsigned long dirty; 451 452 if (vm_dirty_bytes) 453 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) * 454 node_memory / global_dirtyable_memory(); 455 else 456 dirty = vm_dirty_ratio * node_memory / 100; 457 458 if (rt_or_dl_task(tsk)) 459 dirty += dirty / 4; 460 461 /* 462 * Dirty throttling logic assumes the limits in page units fit into 463 * 32-bits. This gives 16TB dirty limits max which is hopefully enough. 464 */ 465 return min_t(unsigned long, dirty, UINT_MAX); 466} 467 468/** 469 * node_dirty_ok - tells whether a node is within its dirty limits 470 * @pgdat: the node to check 471 * 472 * Return: %true when the dirty pages in @pgdat are within the node's 473 * dirty limit, %false if the limit is exceeded. 474 */ 475bool node_dirty_ok(struct pglist_data *pgdat) 476{ 477 unsigned long limit = node_dirty_limit(pgdat); 478 unsigned long nr_pages = 0; 479 480 nr_pages += node_page_state(pgdat, NR_FILE_DIRTY); 481 nr_pages += node_page_state(pgdat, NR_WRITEBACK); 482 483 return nr_pages <= limit; 484} 485 486#ifdef CONFIG_SYSCTL 487static int dirty_background_ratio_handler(const struct ctl_table *table, int write, 488 void *buffer, size_t *lenp, loff_t *ppos) 489{ 490 int ret; 491 492 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 493 if (ret == 0 && write) 494 dirty_background_bytes = 0; 495 return ret; 496} 497 498static int dirty_background_bytes_handler(const struct ctl_table *table, int write, 499 void *buffer, size_t *lenp, loff_t *ppos) 500{ 501 int ret; 502 unsigned long old_bytes = dirty_background_bytes; 503 504 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 505 if (ret == 0 && write) { 506 if (DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE) > 507 UINT_MAX) { 508 dirty_background_bytes = old_bytes; 509 return -ERANGE; 510 } 511 dirty_background_ratio = 0; 512 } 513 return ret; 514} 515 516static int dirty_ratio_handler(const struct ctl_table *table, int write, void *buffer, 517 size_t *lenp, loff_t *ppos) 518{ 519 int old_ratio = vm_dirty_ratio; 520 int ret; 521 522 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 523 if (ret == 0 && write && vm_dirty_ratio != old_ratio) { 524 vm_dirty_bytes = 0; 525 writeback_set_ratelimit(); 526 } 527 return ret; 528} 529 530static int dirty_bytes_handler(const struct ctl_table *table, int write, 531 void *buffer, size_t *lenp, loff_t *ppos) 532{ 533 unsigned long old_bytes = vm_dirty_bytes; 534 int ret; 535 536 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 537 if (ret == 0 && write && vm_dirty_bytes != old_bytes) { 538 if (DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) > UINT_MAX) { 539 vm_dirty_bytes = old_bytes; 540 return -ERANGE; 541 } 542 writeback_set_ratelimit(); 543 vm_dirty_ratio = 0; 544 } 545 return ret; 546} 547#endif 548 549static unsigned long wp_next_time(unsigned long cur_time) 550{ 551 cur_time += VM_COMPLETIONS_PERIOD_LEN; 552 /* 0 has a special meaning... */ 553 if (!cur_time) 554 return 1; 555 return cur_time; 556} 557 558static void wb_domain_writeout_add(struct wb_domain *dom, 559 struct fprop_local_percpu *completions, 560 unsigned int max_prop_frac, long nr) 561{ 562 __fprop_add_percpu_max(&dom->completions, completions, 563 max_prop_frac, nr); 564 /* First event after period switching was turned off? */ 565 if (unlikely(!dom->period_time)) { 566 /* 567 * We can race with other wb_domain_writeout_add calls here but 568 * it does not cause any harm since the resulting time when 569 * timer will fire and what is in writeout_period_time will be 570 * roughly the same. 571 */ 572 dom->period_time = wp_next_time(jiffies); 573 mod_timer(&dom->period_timer, dom->period_time); 574 } 575} 576 577/* 578 * Increment @wb's writeout completion count and the global writeout 579 * completion count. Called from __folio_end_writeback(). 580 */ 581static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr) 582{ 583 struct wb_domain *cgdom; 584 585 wb_stat_mod(wb, WB_WRITTEN, nr); 586 wb_domain_writeout_add(&global_wb_domain, &wb->completions, 587 wb->bdi->max_prop_frac, nr); 588 589 cgdom = mem_cgroup_wb_domain(wb); 590 if (cgdom) 591 wb_domain_writeout_add(cgdom, wb_memcg_completions(wb), 592 wb->bdi->max_prop_frac, nr); 593} 594 595void wb_writeout_inc(struct bdi_writeback *wb) 596{ 597 unsigned long flags; 598 599 local_irq_save(flags); 600 __wb_writeout_add(wb, 1); 601 local_irq_restore(flags); 602} 603EXPORT_SYMBOL_GPL(wb_writeout_inc); 604 605/* 606 * On idle system, we can be called long after we scheduled because we use 607 * deferred timers so count with missed periods. 608 */ 609static void writeout_period(struct timer_list *t) 610{ 611 struct wb_domain *dom = timer_container_of(dom, t, period_timer); 612 int miss_periods = (jiffies - dom->period_time) / 613 VM_COMPLETIONS_PERIOD_LEN; 614 615 if (fprop_new_period(&dom->completions, miss_periods + 1)) { 616 dom->period_time = wp_next_time(dom->period_time + 617 miss_periods * VM_COMPLETIONS_PERIOD_LEN); 618 mod_timer(&dom->period_timer, dom->period_time); 619 } else { 620 /* 621 * Aging has zeroed all fractions. Stop wasting CPU on period 622 * updates. 623 */ 624 dom->period_time = 0; 625 } 626} 627 628int wb_domain_init(struct wb_domain *dom, gfp_t gfp) 629{ 630 memset(dom, 0, sizeof(*dom)); 631 632 spin_lock_init(&dom->lock); 633 634 timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE); 635 636 dom->dirty_limit_tstamp = jiffies; 637 638 return fprop_global_init(&dom->completions, gfp); 639} 640 641#ifdef CONFIG_CGROUP_WRITEBACK 642void wb_domain_exit(struct wb_domain *dom) 643{ 644 timer_delete_sync(&dom->period_timer); 645 fprop_global_destroy(&dom->completions); 646} 647#endif 648 649/* 650 * bdi_min_ratio keeps the sum of the minimum dirty shares of all 651 * registered backing devices, which, for obvious reasons, can not 652 * exceed 100%. 653 */ 654static unsigned int bdi_min_ratio; 655 656static int bdi_check_pages_limit(unsigned long pages) 657{ 658 unsigned long max_dirty_pages = global_dirtyable_memory(); 659 660 if (pages > max_dirty_pages) 661 return -EINVAL; 662 663 return 0; 664} 665 666static unsigned long bdi_ratio_from_pages(unsigned long pages) 667{ 668 unsigned long background_thresh; 669 unsigned long dirty_thresh; 670 unsigned long ratio; 671 672 global_dirty_limits(&background_thresh, &dirty_thresh); 673 if (!dirty_thresh) 674 return -EINVAL; 675 ratio = div64_u64(pages * 100ULL * BDI_RATIO_SCALE, dirty_thresh); 676 677 return ratio; 678} 679 680static u64 bdi_get_bytes(unsigned int ratio) 681{ 682 unsigned long background_thresh; 683 unsigned long dirty_thresh; 684 u64 bytes; 685 686 global_dirty_limits(&background_thresh, &dirty_thresh); 687 bytes = (dirty_thresh * PAGE_SIZE * ratio) / BDI_RATIO_SCALE / 100; 688 689 return bytes; 690} 691 692static int __bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio) 693{ 694 unsigned int delta; 695 int ret = 0; 696 697 if (min_ratio > 100 * BDI_RATIO_SCALE) 698 return -EINVAL; 699 700 spin_lock_bh(&bdi_lock); 701 if (min_ratio > bdi->max_ratio) { 702 ret = -EINVAL; 703 } else { 704 if (min_ratio < bdi->min_ratio) { 705 delta = bdi->min_ratio - min_ratio; 706 bdi_min_ratio -= delta; 707 bdi->min_ratio = min_ratio; 708 } else { 709 delta = min_ratio - bdi->min_ratio; 710 if (bdi_min_ratio + delta < 100 * BDI_RATIO_SCALE) { 711 bdi_min_ratio += delta; 712 bdi->min_ratio = min_ratio; 713 } else { 714 ret = -EINVAL; 715 } 716 } 717 } 718 spin_unlock_bh(&bdi_lock); 719 720 return ret; 721} 722 723static int __bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio) 724{ 725 int ret = 0; 726 727 if (max_ratio > 100 * BDI_RATIO_SCALE) 728 return -EINVAL; 729 730 spin_lock_bh(&bdi_lock); 731 if (bdi->min_ratio > max_ratio) { 732 ret = -EINVAL; 733 } else { 734 bdi->max_ratio = max_ratio; 735 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 736 (100 * BDI_RATIO_SCALE); 737 } 738 spin_unlock_bh(&bdi_lock); 739 740 return ret; 741} 742 743int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio) 744{ 745 return __bdi_set_min_ratio(bdi, min_ratio); 746} 747 748int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio) 749{ 750 return __bdi_set_max_ratio(bdi, max_ratio); 751} 752 753int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio) 754{ 755 return __bdi_set_min_ratio(bdi, min_ratio * BDI_RATIO_SCALE); 756} 757 758int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio) 759{ 760 return __bdi_set_max_ratio(bdi, max_ratio * BDI_RATIO_SCALE); 761} 762EXPORT_SYMBOL(bdi_set_max_ratio); 763 764u64 bdi_get_min_bytes(struct backing_dev_info *bdi) 765{ 766 return bdi_get_bytes(bdi->min_ratio); 767} 768 769int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes) 770{ 771 int ret; 772 unsigned long pages = min_bytes >> PAGE_SHIFT; 773 long min_ratio; 774 775 ret = bdi_check_pages_limit(pages); 776 if (ret) 777 return ret; 778 779 min_ratio = bdi_ratio_from_pages(pages); 780 if (min_ratio < 0) 781 return min_ratio; 782 return __bdi_set_min_ratio(bdi, min_ratio); 783} 784 785u64 bdi_get_max_bytes(struct backing_dev_info *bdi) 786{ 787 return bdi_get_bytes(bdi->max_ratio); 788} 789 790int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes) 791{ 792 int ret; 793 unsigned long pages = max_bytes >> PAGE_SHIFT; 794 long max_ratio; 795 796 ret = bdi_check_pages_limit(pages); 797 if (ret) 798 return ret; 799 800 max_ratio = bdi_ratio_from_pages(pages); 801 if (max_ratio < 0) 802 return max_ratio; 803 return __bdi_set_max_ratio(bdi, max_ratio); 804} 805 806int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit) 807{ 808 if (strict_limit > 1) 809 return -EINVAL; 810 811 spin_lock_bh(&bdi_lock); 812 if (strict_limit) 813 bdi->capabilities |= BDI_CAP_STRICTLIMIT; 814 else 815 bdi->capabilities &= ~BDI_CAP_STRICTLIMIT; 816 spin_unlock_bh(&bdi_lock); 817 818 return 0; 819} 820 821static unsigned long dirty_freerun_ceiling(unsigned long thresh, 822 unsigned long bg_thresh) 823{ 824 return (thresh + bg_thresh) / 2; 825} 826 827static unsigned long hard_dirty_limit(struct wb_domain *dom, 828 unsigned long thresh) 829{ 830 return max(thresh, dom->dirty_limit); 831} 832 833/* 834 * Memory which can be further allocated to a memcg domain is capped by 835 * system-wide clean memory excluding the amount being used in the domain. 836 */ 837static void mdtc_calc_avail(struct dirty_throttle_control *mdtc, 838 unsigned long filepages, unsigned long headroom) 839{ 840 struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc); 841 unsigned long clean = filepages - min(filepages, mdtc->dirty); 842 unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty); 843 unsigned long other_clean = global_clean - min(global_clean, clean); 844 845 mdtc->avail = filepages + min(headroom, other_clean); 846} 847 848static inline bool dtc_is_global(struct dirty_throttle_control *dtc) 849{ 850 return mdtc_gdtc(dtc) == NULL; 851} 852 853/* 854 * Dirty background will ignore pages being written as we're trying to 855 * decide whether to put more under writeback. 856 */ 857static void domain_dirty_avail(struct dirty_throttle_control *dtc, 858 bool include_writeback) 859{ 860 if (dtc_is_global(dtc)) { 861 dtc->avail = global_dirtyable_memory(); 862 dtc->dirty = global_node_page_state(NR_FILE_DIRTY); 863 if (include_writeback) 864 dtc->dirty += global_node_page_state(NR_WRITEBACK); 865 } else { 866 unsigned long filepages = 0, headroom = 0, writeback = 0; 867 868 mem_cgroup_wb_stats(dtc->wb, &filepages, &headroom, &dtc->dirty, 869 &writeback); 870 if (include_writeback) 871 dtc->dirty += writeback; 872 mdtc_calc_avail(dtc, filepages, headroom); 873 } 874} 875 876/** 877 * __wb_calc_thresh - @wb's share of dirty threshold 878 * @dtc: dirty_throttle_context of interest 879 * @thresh: dirty throttling or dirty background threshold of wb_domain in @dtc 880 * 881 * Note that balance_dirty_pages() will only seriously take dirty throttling 882 * threshold as a hard limit when sleeping max_pause per page is not enough 883 * to keep the dirty pages under control. For example, when the device is 884 * completely stalled due to some error conditions, or when there are 1000 885 * dd tasks writing to a slow 10MB/s USB key. 886 * In the other normal situations, it acts more gently by throttling the tasks 887 * more (rather than completely block them) when the wb dirty pages go high. 888 * 889 * It allocates high/low dirty limits to fast/slow devices, in order to prevent 890 * - starving fast devices 891 * - piling up dirty pages (that will take long time to sync) on slow devices 892 * 893 * The wb's share of dirty limit will be adapting to its throughput and 894 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set. 895 * 896 * Return: @wb's dirty limit in pages. For dirty throttling limit, the term 897 * "dirty" in the context of dirty balancing includes all PG_dirty and 898 * PG_writeback pages. 899 */ 900static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc, 901 unsigned long thresh) 902{ 903 struct wb_domain *dom = dtc_dom(dtc); 904 struct bdi_writeback *wb = dtc->wb; 905 u64 wb_thresh; 906 u64 wb_max_thresh; 907 unsigned long numerator, denominator; 908 unsigned long wb_min_ratio, wb_max_ratio; 909 910 /* 911 * Calculate this wb's share of the thresh ratio. 912 */ 913 fprop_fraction_percpu(&dom->completions, dtc->wb_completions, 914 &numerator, &denominator); 915 916 wb_thresh = (thresh * (100 * BDI_RATIO_SCALE - bdi_min_ratio)) / (100 * BDI_RATIO_SCALE); 917 wb_thresh *= numerator; 918 wb_thresh = div64_ul(wb_thresh, denominator); 919 920 wb_min_max_ratio(wb, &wb_min_ratio, &wb_max_ratio); 921 922 wb_thresh += (thresh * wb_min_ratio) / (100 * BDI_RATIO_SCALE); 923 924 /* 925 * It's very possible that wb_thresh is close to 0 not because the 926 * device is slow, but that it has remained inactive for long time. 927 * Honour such devices a reasonable good (hopefully IO efficient) 928 * threshold, so that the occasional writes won't be blocked and active 929 * writes can rampup the threshold quickly. 930 */ 931 if (thresh > dtc->dirty) { 932 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) 933 wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 100); 934 else 935 wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 8); 936 } 937 938 wb_max_thresh = thresh * wb_max_ratio / (100 * BDI_RATIO_SCALE); 939 if (wb_thresh > wb_max_thresh) 940 wb_thresh = wb_max_thresh; 941 942 return wb_thresh; 943} 944 945unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh) 946{ 947 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; 948 949 domain_dirty_avail(&gdtc, true); 950 return __wb_calc_thresh(&gdtc, thresh); 951} 952 953unsigned long cgwb_calc_thresh(struct bdi_writeback *wb) 954{ 955 struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB }; 956 struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) }; 957 958 domain_dirty_avail(&gdtc, true); 959 domain_dirty_avail(&mdtc, true); 960 domain_dirty_limits(&mdtc); 961 962 return __wb_calc_thresh(&mdtc, mdtc.thresh); 963} 964 965/* 966 * setpoint - dirty 3 967 * f(dirty) := 1.0 + (----------------) 968 * limit - setpoint 969 * 970 * it's a 3rd order polynomial that subjects to 971 * 972 * (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast 973 * (2) f(setpoint) = 1.0 => the balance point 974 * (3) f(limit) = 0 => the hard limit 975 * (4) df/dx <= 0 => negative feedback control 976 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse) 977 * => fast response on large errors; small oscillation near setpoint 978 */ 979static long long pos_ratio_polynom(unsigned long setpoint, 980 unsigned long dirty, 981 unsigned long limit) 982{ 983 long long pos_ratio; 984 long x; 985 986 x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT, 987 (limit - setpoint) | 1); 988 pos_ratio = x; 989 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 990 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 991 pos_ratio += 1 << RATELIMIT_CALC_SHIFT; 992 993 return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT); 994} 995 996/* 997 * Dirty position control. 998 * 999 * (o) global/bdi setpoints 1000 * 1001 * We want the dirty pages be balanced around the global/wb setpoints. 1002 * When the number of dirty pages is higher/lower than the setpoint, the 1003 * dirty position control ratio (and hence task dirty ratelimit) will be 1004 * decreased/increased to bring the dirty pages back to the setpoint. 1005 * 1006 * pos_ratio = 1 << RATELIMIT_CALC_SHIFT 1007 * 1008 * if (dirty < setpoint) scale up pos_ratio 1009 * if (dirty > setpoint) scale down pos_ratio 1010 * 1011 * if (wb_dirty < wb_setpoint) scale up pos_ratio 1012 * if (wb_dirty > wb_setpoint) scale down pos_ratio 1013 * 1014 * task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT 1015 * 1016 * (o) global control line 1017 * 1018 * ^ pos_ratio 1019 * | 1020 * | |<===== global dirty control scope ======>| 1021 * 2.0 * * * * * * * 1022 * | .* 1023 * | . * 1024 * | . * 1025 * | . * 1026 * | . * 1027 * | . * 1028 * 1.0 ................................* 1029 * | . . * 1030 * | . . * 1031 * | . . * 1032 * | . . * 1033 * | . . * 1034 * 0 +------------.------------------.----------------------*-------------> 1035 * freerun^ setpoint^ limit^ dirty pages 1036 * 1037 * (o) wb control line 1038 * 1039 * ^ pos_ratio 1040 * | 1041 * | * 1042 * | * 1043 * | * 1044 * | * 1045 * | * |<=========== span ============>| 1046 * 1.0 .......................* 1047 * | . * 1048 * | . * 1049 * | . * 1050 * | . * 1051 * | . * 1052 * | . * 1053 * | . * 1054 * | . * 1055 * | . * 1056 * | . * 1057 * | . * 1058 * 1/4 ...............................................* * * * * * * * * * * * 1059 * | . . 1060 * | . . 1061 * | . . 1062 * 0 +----------------------.-------------------------------.-------------> 1063 * wb_setpoint^ x_intercept^ 1064 * 1065 * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can 1066 * be smoothly throttled down to normal if it starts high in situations like 1067 * - start writing to a slow SD card and a fast disk at the same time. The SD 1068 * card's wb_dirty may rush to many times higher than wb_setpoint. 1069 * - the wb dirty thresh drops quickly due to change of JBOD workload 1070 */ 1071static void wb_position_ratio(struct dirty_throttle_control *dtc) 1072{ 1073 struct bdi_writeback *wb = dtc->wb; 1074 unsigned long write_bw = READ_ONCE(wb->avg_write_bandwidth); 1075 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 1076 unsigned long limit = dtc->limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 1077 unsigned long wb_thresh = dtc->wb_thresh; 1078 unsigned long x_intercept; 1079 unsigned long setpoint; /* dirty pages' target balance point */ 1080 unsigned long wb_setpoint; 1081 unsigned long span; 1082 long long pos_ratio; /* for scaling up/down the rate limit */ 1083 long x; 1084 1085 dtc->pos_ratio = 0; 1086 1087 if (unlikely(dtc->dirty >= limit)) 1088 return; 1089 1090 /* 1091 * global setpoint 1092 * 1093 * See comment for pos_ratio_polynom(). 1094 */ 1095 setpoint = (freerun + limit) / 2; 1096 pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit); 1097 1098 /* 1099 * The strictlimit feature is a tool preventing mistrusted filesystems 1100 * from growing a large number of dirty pages before throttling. For 1101 * such filesystems balance_dirty_pages always checks wb counters 1102 * against wb limits. Even if global "nr_dirty" is under "freerun". 1103 * This is especially important for fuse which sets bdi->max_ratio to 1104 * 1% by default. 1105 * 1106 * Here, in wb_position_ratio(), we calculate pos_ratio based on 1107 * two values: wb_dirty and wb_thresh. Let's consider an example: 1108 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global 1109 * limits are set by default to 10% and 20% (background and throttle). 1110 * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages. 1111 * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is 1112 * about ~6K pages (as the average of background and throttle wb 1113 * limits). The 3rd order polynomial will provide positive feedback if 1114 * wb_dirty is under wb_setpoint and vice versa. 1115 * 1116 * Note, that we cannot use global counters in these calculations 1117 * because we want to throttle process writing to a strictlimit wb 1118 * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB 1119 * in the example above). 1120 */ 1121 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 1122 long long wb_pos_ratio; 1123 1124 if (dtc->wb_dirty >= wb_thresh) 1125 return; 1126 1127 wb_setpoint = dirty_freerun_ceiling(wb_thresh, 1128 dtc->wb_bg_thresh); 1129 1130 if (wb_setpoint == 0 || wb_setpoint == wb_thresh) 1131 return; 1132 1133 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty, 1134 wb_thresh); 1135 1136 /* 1137 * Typically, for strictlimit case, wb_setpoint << setpoint 1138 * and pos_ratio >> wb_pos_ratio. In the other words global 1139 * state ("dirty") is not limiting factor and we have to 1140 * make decision based on wb counters. But there is an 1141 * important case when global pos_ratio should get precedence: 1142 * global limits are exceeded (e.g. due to activities on other 1143 * wb's) while given strictlimit wb is below limit. 1144 * 1145 * "pos_ratio * wb_pos_ratio" would work for the case above, 1146 * but it would look too non-natural for the case of all 1147 * activity in the system coming from a single strictlimit wb 1148 * with bdi->max_ratio == 100%. 1149 * 1150 * Note that min() below somewhat changes the dynamics of the 1151 * control system. Normally, pos_ratio value can be well over 3 1152 * (when globally we are at freerun and wb is well below wb 1153 * setpoint). Now the maximum pos_ratio in the same situation 1154 * is 2. We might want to tweak this if we observe the control 1155 * system is too slow to adapt. 1156 */ 1157 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio); 1158 return; 1159 } 1160 1161 /* 1162 * We have computed basic pos_ratio above based on global situation. If 1163 * the wb is over/under its share of dirty pages, we want to scale 1164 * pos_ratio further down/up. That is done by the following mechanism. 1165 */ 1166 1167 /* 1168 * wb setpoint 1169 * 1170 * f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint) 1171 * 1172 * x_intercept - wb_dirty 1173 * := -------------------------- 1174 * x_intercept - wb_setpoint 1175 * 1176 * The main wb control line is a linear function that subjects to 1177 * 1178 * (1) f(wb_setpoint) = 1.0 1179 * (2) k = - 1 / (8 * write_bw) (in single wb case) 1180 * or equally: x_intercept = wb_setpoint + 8 * write_bw 1181 * 1182 * For single wb case, the dirty pages are observed to fluctuate 1183 * regularly within range 1184 * [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2] 1185 * for various filesystems, where (2) can yield in a reasonable 12.5% 1186 * fluctuation range for pos_ratio. 1187 * 1188 * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its 1189 * own size, so move the slope over accordingly and choose a slope that 1190 * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh. 1191 */ 1192 if (unlikely(wb_thresh > dtc->thresh)) 1193 wb_thresh = dtc->thresh; 1194 /* 1195 * scale global setpoint to wb's: 1196 * wb_setpoint = setpoint * wb_thresh / thresh 1197 */ 1198 x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1); 1199 wb_setpoint = setpoint * (u64)x >> 16; 1200 /* 1201 * Use span=(8*write_bw) in single wb case as indicated by 1202 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case. 1203 * 1204 * wb_thresh thresh - wb_thresh 1205 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh 1206 * thresh thresh 1207 */ 1208 span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16; 1209 x_intercept = wb_setpoint + span; 1210 1211 if (dtc->wb_dirty < x_intercept - span / 4) { 1212 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty), 1213 (x_intercept - wb_setpoint) | 1); 1214 } else 1215 pos_ratio /= 4; 1216 1217 /* 1218 * wb reserve area, safeguard against dirty pool underrun and disk idle 1219 * It may push the desired control point of global dirty pages higher 1220 * than setpoint. 1221 */ 1222 x_intercept = wb_thresh / 2; 1223 if (dtc->wb_dirty < x_intercept) { 1224 if (dtc->wb_dirty > x_intercept / 8) 1225 pos_ratio = div_u64(pos_ratio * x_intercept, 1226 dtc->wb_dirty); 1227 else 1228 pos_ratio *= 8; 1229 } 1230 1231 dtc->pos_ratio = pos_ratio; 1232} 1233 1234static void wb_update_write_bandwidth(struct bdi_writeback *wb, 1235 unsigned long elapsed, 1236 unsigned long written) 1237{ 1238 const unsigned long period = roundup_pow_of_two(3 * HZ); 1239 unsigned long avg = wb->avg_write_bandwidth; 1240 unsigned long old = wb->write_bandwidth; 1241 u64 bw; 1242 1243 /* 1244 * bw = written * HZ / elapsed 1245 * 1246 * bw * elapsed + write_bandwidth * (period - elapsed) 1247 * write_bandwidth = --------------------------------------------------- 1248 * period 1249 * 1250 * @written may have decreased due to folio_redirty_for_writepage(). 1251 * Avoid underflowing @bw calculation. 1252 */ 1253 bw = written - min(written, wb->written_stamp); 1254 bw *= HZ; 1255 if (unlikely(elapsed > period)) { 1256 bw = div64_ul(bw, elapsed); 1257 avg = bw; 1258 goto out; 1259 } 1260 bw += (u64)wb->write_bandwidth * (period - elapsed); 1261 bw >>= ilog2(period); 1262 1263 /* 1264 * one more level of smoothing, for filtering out sudden spikes 1265 */ 1266 if (avg > old && old >= (unsigned long)bw) 1267 avg -= (avg - old) >> 3; 1268 1269 if (avg < old && old <= (unsigned long)bw) 1270 avg += (old - avg) >> 3; 1271 1272out: 1273 /* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */ 1274 avg = max(avg, 1LU); 1275 if (wb_has_dirty_io(wb)) { 1276 long delta = avg - wb->avg_write_bandwidth; 1277 WARN_ON_ONCE(atomic_long_add_return(delta, 1278 &wb->bdi->tot_write_bandwidth) <= 0); 1279 } 1280 wb->write_bandwidth = bw; 1281 WRITE_ONCE(wb->avg_write_bandwidth, avg); 1282} 1283 1284static void update_dirty_limit(struct dirty_throttle_control *dtc) 1285{ 1286 struct wb_domain *dom = dtc_dom(dtc); 1287 unsigned long thresh = dtc->thresh; 1288 unsigned long limit = dom->dirty_limit; 1289 1290 /* 1291 * Follow up in one step. 1292 */ 1293 if (limit < thresh) { 1294 limit = thresh; 1295 goto update; 1296 } 1297 1298 /* 1299 * Follow down slowly. Use the higher one as the target, because thresh 1300 * may drop below dirty. This is exactly the reason to introduce 1301 * dom->dirty_limit which is guaranteed to lie above the dirty pages. 1302 */ 1303 thresh = max(thresh, dtc->dirty); 1304 if (limit > thresh) { 1305 limit -= (limit - thresh) >> 5; 1306 goto update; 1307 } 1308 return; 1309update: 1310 dom->dirty_limit = limit; 1311} 1312 1313static void domain_update_dirty_limit(struct dirty_throttle_control *dtc, 1314 unsigned long now) 1315{ 1316 struct wb_domain *dom = dtc_dom(dtc); 1317 1318 /* 1319 * check locklessly first to optimize away locking for the most time 1320 */ 1321 if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) 1322 return; 1323 1324 spin_lock(&dom->lock); 1325 if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) { 1326 update_dirty_limit(dtc); 1327 dom->dirty_limit_tstamp = now; 1328 } 1329 spin_unlock(&dom->lock); 1330} 1331 1332/* 1333 * Maintain wb->dirty_ratelimit, the base dirty throttle rate. 1334 * 1335 * Normal wb tasks will be curbed at or below it in long term. 1336 * Obviously it should be around (write_bw / N) when there are N dd tasks. 1337 */ 1338static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc, 1339 unsigned long dirtied, 1340 unsigned long elapsed) 1341{ 1342 struct bdi_writeback *wb = dtc->wb; 1343 unsigned long dirty = dtc->dirty; 1344 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 1345 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 1346 unsigned long setpoint = (freerun + limit) / 2; 1347 unsigned long write_bw = wb->avg_write_bandwidth; 1348 unsigned long dirty_ratelimit = wb->dirty_ratelimit; 1349 unsigned long dirty_rate; 1350 unsigned long task_ratelimit; 1351 unsigned long balanced_dirty_ratelimit; 1352 unsigned long step; 1353 unsigned long x; 1354 unsigned long shift; 1355 1356 /* 1357 * The dirty rate will match the writeout rate in long term, except 1358 * when dirty pages are truncated by userspace or re-dirtied by FS. 1359 */ 1360 dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed; 1361 1362 /* 1363 * task_ratelimit reflects each dd's dirty rate for the past 200ms. 1364 */ 1365 task_ratelimit = (u64)dirty_ratelimit * 1366 dtc->pos_ratio >> RATELIMIT_CALC_SHIFT; 1367 task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */ 1368 1369 /* 1370 * A linear estimation of the "balanced" throttle rate. The theory is, 1371 * if there are N dd tasks, each throttled at task_ratelimit, the wb's 1372 * dirty_rate will be measured to be (N * task_ratelimit). So the below 1373 * formula will yield the balanced rate limit (write_bw / N). 1374 * 1375 * Note that the expanded form is not a pure rate feedback: 1376 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1) 1377 * but also takes pos_ratio into account: 1378 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2) 1379 * 1380 * (1) is not realistic because pos_ratio also takes part in balancing 1381 * the dirty rate. Consider the state 1382 * pos_ratio = 0.5 (3) 1383 * rate = 2 * (write_bw / N) (4) 1384 * If (1) is used, it will stuck in that state! Because each dd will 1385 * be throttled at 1386 * task_ratelimit = pos_ratio * rate = (write_bw / N) (5) 1387 * yielding 1388 * dirty_rate = N * task_ratelimit = write_bw (6) 1389 * put (6) into (1) we get 1390 * rate_(i+1) = rate_(i) (7) 1391 * 1392 * So we end up using (2) to always keep 1393 * rate_(i+1) ~= (write_bw / N) (8) 1394 * regardless of the value of pos_ratio. As long as (8) is satisfied, 1395 * pos_ratio is able to drive itself to 1.0, which is not only where 1396 * the dirty count meet the setpoint, but also where the slope of 1397 * pos_ratio is most flat and hence task_ratelimit is least fluctuated. 1398 */ 1399 balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw, 1400 dirty_rate | 1); 1401 /* 1402 * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw 1403 */ 1404 if (unlikely(balanced_dirty_ratelimit > write_bw)) 1405 balanced_dirty_ratelimit = write_bw; 1406 1407 /* 1408 * We could safely do this and return immediately: 1409 * 1410 * wb->dirty_ratelimit = balanced_dirty_ratelimit; 1411 * 1412 * However to get a more stable dirty_ratelimit, the below elaborated 1413 * code makes use of task_ratelimit to filter out singular points and 1414 * limit the step size. 1415 * 1416 * The below code essentially only uses the relative value of 1417 * 1418 * task_ratelimit - dirty_ratelimit 1419 * = (pos_ratio - 1) * dirty_ratelimit 1420 * 1421 * which reflects the direction and size of dirty position error. 1422 */ 1423 1424 /* 1425 * dirty_ratelimit will follow balanced_dirty_ratelimit iff 1426 * task_ratelimit is on the same side of dirty_ratelimit, too. 1427 * For example, when 1428 * - dirty_ratelimit > balanced_dirty_ratelimit 1429 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint) 1430 * lowering dirty_ratelimit will help meet both the position and rate 1431 * control targets. Otherwise, don't update dirty_ratelimit if it will 1432 * only help meet the rate target. After all, what the users ultimately 1433 * feel and care are stable dirty rate and small position error. 1434 * 1435 * |task_ratelimit - dirty_ratelimit| is used to limit the step size 1436 * and filter out the singular points of balanced_dirty_ratelimit. Which 1437 * keeps jumping around randomly and can even leap far away at times 1438 * due to the small 200ms estimation period of dirty_rate (we want to 1439 * keep that period small to reduce time lags). 1440 */ 1441 step = 0; 1442 1443 /* 1444 * For strictlimit case, calculations above were based on wb counters 1445 * and limits (starting from pos_ratio = wb_position_ratio() and up to 1446 * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate). 1447 * Hence, to calculate "step" properly, we have to use wb_dirty as 1448 * "dirty" and wb_setpoint as "setpoint". 1449 */ 1450 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 1451 dirty = dtc->wb_dirty; 1452 setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2; 1453 } 1454 1455 if (dirty < setpoint) { 1456 x = min3(wb->balanced_dirty_ratelimit, 1457 balanced_dirty_ratelimit, task_ratelimit); 1458 if (dirty_ratelimit < x) 1459 step = x - dirty_ratelimit; 1460 } else { 1461 x = max3(wb->balanced_dirty_ratelimit, 1462 balanced_dirty_ratelimit, task_ratelimit); 1463 if (dirty_ratelimit > x) 1464 step = dirty_ratelimit - x; 1465 } 1466 1467 /* 1468 * Don't pursue 100% rate matching. It's impossible since the balanced 1469 * rate itself is constantly fluctuating. So decrease the track speed 1470 * when it gets close to the target. Helps eliminate pointless tremors. 1471 */ 1472 shift = dirty_ratelimit / (2 * step + 1); 1473 if (shift < BITS_PER_LONG) 1474 step = DIV_ROUND_UP(step >> shift, 8); 1475 else 1476 step = 0; 1477 1478 if (dirty_ratelimit < balanced_dirty_ratelimit) 1479 dirty_ratelimit += step; 1480 else 1481 dirty_ratelimit -= step; 1482 1483 WRITE_ONCE(wb->dirty_ratelimit, max(dirty_ratelimit, 1UL)); 1484 wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit; 1485 1486 trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit); 1487} 1488 1489static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc, 1490 struct dirty_throttle_control *mdtc, 1491 bool update_ratelimit) 1492{ 1493 struct bdi_writeback *wb = gdtc->wb; 1494 unsigned long now = jiffies; 1495 unsigned long elapsed; 1496 unsigned long dirtied; 1497 unsigned long written; 1498 1499 spin_lock(&wb->list_lock); 1500 1501 /* 1502 * Lockless checks for elapsed time are racy and delayed update after 1503 * IO completion doesn't do it at all (to make sure written pages are 1504 * accounted reasonably quickly). Make sure elapsed >= 1 to avoid 1505 * division errors. 1506 */ 1507 elapsed = max(now - wb->bw_time_stamp, 1UL); 1508 dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]); 1509 written = percpu_counter_read(&wb->stat[WB_WRITTEN]); 1510 1511 if (update_ratelimit) { 1512 domain_update_dirty_limit(gdtc, now); 1513 wb_update_dirty_ratelimit(gdtc, dirtied, elapsed); 1514 1515 /* 1516 * @mdtc is always NULL if !CGROUP_WRITEBACK but the 1517 * compiler has no way to figure that out. Help it. 1518 */ 1519 if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) { 1520 domain_update_dirty_limit(mdtc, now); 1521 wb_update_dirty_ratelimit(mdtc, dirtied, elapsed); 1522 } 1523 } 1524 wb_update_write_bandwidth(wb, elapsed, written); 1525 1526 wb->dirtied_stamp = dirtied; 1527 wb->written_stamp = written; 1528 WRITE_ONCE(wb->bw_time_stamp, now); 1529 spin_unlock(&wb->list_lock); 1530} 1531 1532void wb_update_bandwidth(struct bdi_writeback *wb) 1533{ 1534 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; 1535 1536 __wb_update_bandwidth(&gdtc, NULL, false); 1537} 1538 1539/* Interval after which we consider wb idle and don't estimate bandwidth */ 1540#define WB_BANDWIDTH_IDLE_JIF (HZ) 1541 1542static void wb_bandwidth_estimate_start(struct bdi_writeback *wb) 1543{ 1544 unsigned long now = jiffies; 1545 unsigned long elapsed = now - READ_ONCE(wb->bw_time_stamp); 1546 1547 if (elapsed > WB_BANDWIDTH_IDLE_JIF && 1548 !atomic_read(&wb->writeback_inodes)) { 1549 spin_lock(&wb->list_lock); 1550 wb->dirtied_stamp = wb_stat(wb, WB_DIRTIED); 1551 wb->written_stamp = wb_stat(wb, WB_WRITTEN); 1552 WRITE_ONCE(wb->bw_time_stamp, now); 1553 spin_unlock(&wb->list_lock); 1554 } 1555} 1556 1557/* 1558 * After a task dirtied this many pages, balance_dirty_pages_ratelimited() 1559 * will look to see if it needs to start dirty throttling. 1560 * 1561 * If dirty_poll_interval is too low, big NUMA machines will call the expensive 1562 * global_zone_page_state() too often. So scale it near-sqrt to the safety margin 1563 * (the number of pages we may dirty without exceeding the dirty limits). 1564 */ 1565static unsigned long dirty_poll_interval(unsigned long dirty, 1566 unsigned long thresh) 1567{ 1568 if (thresh > dirty) 1569 return 1UL << (ilog2(thresh - dirty) >> 1); 1570 1571 return 1; 1572} 1573 1574static unsigned long wb_max_pause(struct bdi_writeback *wb, 1575 unsigned long wb_dirty) 1576{ 1577 unsigned long bw = READ_ONCE(wb->avg_write_bandwidth); 1578 unsigned long t; 1579 1580 /* 1581 * Limit pause time for small memory systems. If sleeping for too long 1582 * time, a small pool of dirty/writeback pages may go empty and disk go 1583 * idle. 1584 * 1585 * 8 serves as the safety ratio. 1586 */ 1587 t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8)); 1588 t++; 1589 1590 return min_t(unsigned long, t, MAX_PAUSE); 1591} 1592 1593static long wb_min_pause(struct bdi_writeback *wb, 1594 long max_pause, 1595 unsigned long task_ratelimit, 1596 unsigned long dirty_ratelimit, 1597 int *nr_dirtied_pause) 1598{ 1599 long hi = ilog2(READ_ONCE(wb->avg_write_bandwidth)); 1600 long lo = ilog2(READ_ONCE(wb->dirty_ratelimit)); 1601 long t; /* target pause */ 1602 long pause; /* estimated next pause */ 1603 int pages; /* target nr_dirtied_pause */ 1604 1605 /* target for 10ms pause on 1-dd case */ 1606 t = max(1, HZ / 100); 1607 1608 /* 1609 * Scale up pause time for concurrent dirtiers in order to reduce CPU 1610 * overheads. 1611 * 1612 * (N * 10ms) on 2^N concurrent tasks. 1613 */ 1614 if (hi > lo) 1615 t += (hi - lo) * (10 * HZ) / 1024; 1616 1617 /* 1618 * This is a bit convoluted. We try to base the next nr_dirtied_pause 1619 * on the much more stable dirty_ratelimit. However the next pause time 1620 * will be computed based on task_ratelimit and the two rate limits may 1621 * depart considerably at some time. Especially if task_ratelimit goes 1622 * below dirty_ratelimit/2 and the target pause is max_pause, the next 1623 * pause time will be max_pause*2 _trimmed down_ to max_pause. As a 1624 * result task_ratelimit won't be executed faithfully, which could 1625 * eventually bring down dirty_ratelimit. 1626 * 1627 * We apply two rules to fix it up: 1628 * 1) try to estimate the next pause time and if necessary, use a lower 1629 * nr_dirtied_pause so as not to exceed max_pause. When this happens, 1630 * nr_dirtied_pause will be "dancing" with task_ratelimit. 1631 * 2) limit the target pause time to max_pause/2, so that the normal 1632 * small fluctuations of task_ratelimit won't trigger rule (1) and 1633 * nr_dirtied_pause will remain as stable as dirty_ratelimit. 1634 */ 1635 t = min(t, 1 + max_pause / 2); 1636 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1637 1638 /* 1639 * Tiny nr_dirtied_pause is found to hurt I/O performance in the test 1640 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}. 1641 * When the 16 consecutive reads are often interrupted by some dirty 1642 * throttling pause during the async writes, cfq will go into idles 1643 * (deadline is fine). So push nr_dirtied_pause as high as possible 1644 * until reaches DIRTY_POLL_THRESH=32 pages. 1645 */ 1646 if (pages < DIRTY_POLL_THRESH) { 1647 t = max_pause; 1648 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1649 if (pages > DIRTY_POLL_THRESH) { 1650 pages = DIRTY_POLL_THRESH; 1651 t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit; 1652 } 1653 } 1654 1655 pause = HZ * pages / (task_ratelimit + 1); 1656 if (pause > max_pause) { 1657 t = max_pause; 1658 pages = task_ratelimit * t / roundup_pow_of_two(HZ); 1659 } 1660 1661 *nr_dirtied_pause = pages; 1662 /* 1663 * The minimal pause time will normally be half the target pause time. 1664 */ 1665 return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t; 1666} 1667 1668static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) 1669{ 1670 struct bdi_writeback *wb = dtc->wb; 1671 unsigned long wb_reclaimable; 1672 1673 /* 1674 * wb_thresh is not treated as some limiting factor as 1675 * dirty_thresh, due to reasons 1676 * - in JBOD setup, wb_thresh can fluctuate a lot 1677 * - in a system with HDD and USB key, the USB key may somehow 1678 * go into state (wb_dirty >> wb_thresh) either because 1679 * wb_dirty starts high, or because wb_thresh drops low. 1680 * In this case we don't want to hard throttle the USB key 1681 * dirtiers for 100 seconds until wb_dirty drops under 1682 * wb_thresh. Instead the auxiliary wb control line in 1683 * wb_position_ratio() will let the dirtier task progress 1684 * at some rate <= (write_bw / 2) for bringing down wb_dirty. 1685 */ 1686 dtc->wb_thresh = __wb_calc_thresh(dtc, dtc->thresh); 1687 dtc->wb_bg_thresh = dtc->thresh ? 1688 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0; 1689 1690 /* 1691 * In order to avoid the stacked BDI deadlock we need 1692 * to ensure we accurately count the 'dirty' pages when 1693 * the threshold is low. 1694 * 1695 * Otherwise it would be possible to get thresh+n pages 1696 * reported dirty, even though there are thresh-m pages 1697 * actually dirty; with m+n sitting in the percpu 1698 * deltas. 1699 */ 1700 if (dtc->wb_thresh < 2 * wb_stat_error()) { 1701 wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE); 1702 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK); 1703 } else { 1704 wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE); 1705 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK); 1706 } 1707} 1708 1709static unsigned long domain_poll_intv(struct dirty_throttle_control *dtc, 1710 bool strictlimit) 1711{ 1712 unsigned long dirty, thresh; 1713 1714 if (strictlimit) { 1715 dirty = dtc->wb_dirty; 1716 thresh = dtc->wb_thresh; 1717 } else { 1718 dirty = dtc->dirty; 1719 thresh = dtc->thresh; 1720 } 1721 1722 return dirty_poll_interval(dirty, thresh); 1723} 1724 1725/* 1726 * Throttle it only when the background writeback cannot catch-up. This avoids 1727 * (excessively) small writeouts when the wb limits are ramping up in case of 1728 * !strictlimit. 1729 * 1730 * In strictlimit case make decision based on the wb counters and limits. Small 1731 * writeouts when the wb limits are ramping up are the price we consciously pay 1732 * for strictlimit-ing. 1733 */ 1734static void domain_dirty_freerun(struct dirty_throttle_control *dtc, 1735 bool strictlimit) 1736{ 1737 unsigned long dirty, thresh, bg_thresh; 1738 1739 if (unlikely(strictlimit)) { 1740 wb_dirty_limits(dtc); 1741 dirty = dtc->wb_dirty; 1742 thresh = dtc->wb_thresh; 1743 bg_thresh = dtc->wb_bg_thresh; 1744 } else { 1745 dirty = dtc->dirty; 1746 thresh = dtc->thresh; 1747 bg_thresh = dtc->bg_thresh; 1748 } 1749 dtc->freerun = dirty <= dirty_freerun_ceiling(thresh, bg_thresh); 1750} 1751 1752static void balance_domain_limits(struct dirty_throttle_control *dtc, 1753 bool strictlimit) 1754{ 1755 domain_dirty_avail(dtc, true); 1756 domain_dirty_limits(dtc); 1757 domain_dirty_freerun(dtc, strictlimit); 1758} 1759 1760static void wb_dirty_freerun(struct dirty_throttle_control *dtc, 1761 bool strictlimit) 1762{ 1763 dtc->freerun = false; 1764 1765 /* was already handled in domain_dirty_freerun */ 1766 if (strictlimit) 1767 return; 1768 1769 wb_dirty_limits(dtc); 1770 /* 1771 * LOCAL_THROTTLE tasks must not be throttled when below the per-wb 1772 * freerun ceiling. 1773 */ 1774 if (!(current->flags & PF_LOCAL_THROTTLE)) 1775 return; 1776 1777 dtc->freerun = dtc->wb_dirty < 1778 dirty_freerun_ceiling(dtc->wb_thresh, dtc->wb_bg_thresh); 1779} 1780 1781static inline void wb_dirty_exceeded(struct dirty_throttle_control *dtc, 1782 bool strictlimit) 1783{ 1784 dtc->dirty_exceeded = (dtc->wb_dirty > dtc->wb_thresh) && 1785 ((dtc->dirty > dtc->thresh) || strictlimit); 1786} 1787 1788/* 1789 * The limits fields dirty_exceeded and pos_ratio won't be updated if wb is 1790 * in freerun state. Please don't use these invalid fields in freerun case. 1791 */ 1792static void balance_wb_limits(struct dirty_throttle_control *dtc, 1793 bool strictlimit) 1794{ 1795 wb_dirty_freerun(dtc, strictlimit); 1796 if (dtc->freerun) 1797 return; 1798 1799 wb_dirty_exceeded(dtc, strictlimit); 1800 wb_position_ratio(dtc); 1801} 1802 1803/* 1804 * balance_dirty_pages() must be called by processes which are generating dirty 1805 * data. It looks at the number of dirty pages in the machine and will force 1806 * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2. 1807 * If we're over `background_thresh' then the writeback threads are woken to 1808 * perform some writeout. 1809 */ 1810static int balance_dirty_pages(struct bdi_writeback *wb, 1811 unsigned long pages_dirtied, unsigned int flags) 1812{ 1813 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1814 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1815 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1816 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1817 &mdtc_stor : NULL; 1818 struct dirty_throttle_control *sdtc; 1819 unsigned long nr_dirty; 1820 long period; 1821 long pause; 1822 long max_pause; 1823 long min_pause; 1824 int nr_dirtied_pause; 1825 unsigned long task_ratelimit; 1826 unsigned long dirty_ratelimit; 1827 struct backing_dev_info *bdi = wb->bdi; 1828 bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT; 1829 unsigned long start_time = jiffies; 1830 int ret = 0; 1831 1832 for (;;) { 1833 unsigned long now = jiffies; 1834 1835 nr_dirty = global_node_page_state(NR_FILE_DIRTY); 1836 1837 balance_domain_limits(gdtc, strictlimit); 1838 if (mdtc) { 1839 /* 1840 * If @wb belongs to !root memcg, repeat the same 1841 * basic calculations for the memcg domain. 1842 */ 1843 balance_domain_limits(mdtc, strictlimit); 1844 } 1845 1846 /* 1847 * In laptop mode, we wait until hitting the higher threshold 1848 * before starting background writeout, and then write out all 1849 * the way down to the lower threshold. So slow writers cause 1850 * minimal disk activity. 1851 * 1852 * In normal mode, we start background writeout at the lower 1853 * background_thresh, to keep the amount of dirty memory low. 1854 */ 1855 if (!laptop_mode && nr_dirty > gdtc->bg_thresh && 1856 !writeback_in_progress(wb)) 1857 wb_start_background_writeback(wb); 1858 1859 /* 1860 * If memcg domain is in effect, @dirty should be under 1861 * both global and memcg freerun ceilings. 1862 */ 1863 if (gdtc->freerun && (!mdtc || mdtc->freerun)) { 1864 unsigned long intv; 1865 unsigned long m_intv; 1866 1867free_running: 1868 intv = domain_poll_intv(gdtc, strictlimit); 1869 m_intv = ULONG_MAX; 1870 1871 current->dirty_paused_when = now; 1872 current->nr_dirtied = 0; 1873 if (mdtc) 1874 m_intv = domain_poll_intv(mdtc, strictlimit); 1875 current->nr_dirtied_pause = min(intv, m_intv); 1876 break; 1877 } 1878 1879 /* Start writeback even when in laptop mode */ 1880 if (unlikely(!writeback_in_progress(wb))) 1881 wb_start_background_writeback(wb); 1882 1883 mem_cgroup_flush_foreign(wb); 1884 1885 /* 1886 * Calculate global domain's pos_ratio and select the 1887 * global dtc by default. 1888 */ 1889 balance_wb_limits(gdtc, strictlimit); 1890 if (gdtc->freerun) 1891 goto free_running; 1892 sdtc = gdtc; 1893 1894 if (mdtc) { 1895 /* 1896 * If memcg domain is in effect, calculate its 1897 * pos_ratio. @wb should satisfy constraints from 1898 * both global and memcg domains. Choose the one 1899 * w/ lower pos_ratio. 1900 */ 1901 balance_wb_limits(mdtc, strictlimit); 1902 if (mdtc->freerun) 1903 goto free_running; 1904 if (mdtc->pos_ratio < gdtc->pos_ratio) 1905 sdtc = mdtc; 1906 } 1907 1908 wb->dirty_exceeded = gdtc->dirty_exceeded || 1909 (mdtc && mdtc->dirty_exceeded); 1910 if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) + 1911 BANDWIDTH_INTERVAL)) 1912 __wb_update_bandwidth(gdtc, mdtc, true); 1913 1914 /* throttle according to the chosen dtc */ 1915 dirty_ratelimit = READ_ONCE(wb->dirty_ratelimit); 1916 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >> 1917 RATELIMIT_CALC_SHIFT; 1918 max_pause = wb_max_pause(wb, sdtc->wb_dirty); 1919 min_pause = wb_min_pause(wb, max_pause, 1920 task_ratelimit, dirty_ratelimit, 1921 &nr_dirtied_pause); 1922 1923 if (unlikely(task_ratelimit == 0)) { 1924 period = max_pause; 1925 pause = max_pause; 1926 goto pause; 1927 } 1928 period = HZ * pages_dirtied / task_ratelimit; 1929 pause = period; 1930 if (current->dirty_paused_when) 1931 pause -= now - current->dirty_paused_when; 1932 /* 1933 * For less than 1s think time (ext3/4 may block the dirtier 1934 * for up to 800ms from time to time on 1-HDD; so does xfs, 1935 * however at much less frequency), try to compensate it in 1936 * future periods by updating the virtual time; otherwise just 1937 * do a reset, as it may be a light dirtier. 1938 */ 1939 if (pause < min_pause) { 1940 trace_balance_dirty_pages(wb, 1941 sdtc, 1942 dirty_ratelimit, 1943 task_ratelimit, 1944 pages_dirtied, 1945 period, 1946 min(pause, 0L), 1947 start_time); 1948 if (pause < -HZ) { 1949 current->dirty_paused_when = now; 1950 current->nr_dirtied = 0; 1951 } else if (period) { 1952 current->dirty_paused_when += period; 1953 current->nr_dirtied = 0; 1954 } else if (current->nr_dirtied_pause <= pages_dirtied) 1955 current->nr_dirtied_pause += pages_dirtied; 1956 break; 1957 } 1958 if (unlikely(pause > max_pause)) { 1959 /* for occasional dropped task_ratelimit */ 1960 now += min(pause - max_pause, max_pause); 1961 pause = max_pause; 1962 } 1963 1964pause: 1965 trace_balance_dirty_pages(wb, 1966 sdtc, 1967 dirty_ratelimit, 1968 task_ratelimit, 1969 pages_dirtied, 1970 period, 1971 pause, 1972 start_time); 1973 if (flags & BDP_ASYNC) { 1974 ret = -EAGAIN; 1975 break; 1976 } 1977 __set_current_state(TASK_KILLABLE); 1978 bdi->last_bdp_sleep = jiffies; 1979 io_schedule_timeout(pause); 1980 1981 current->dirty_paused_when = now + pause; 1982 current->nr_dirtied = 0; 1983 current->nr_dirtied_pause = nr_dirtied_pause; 1984 1985 /* 1986 * This is typically equal to (dirty < thresh) and can also 1987 * keep "1000+ dd on a slow USB stick" under control. 1988 */ 1989 if (task_ratelimit) 1990 break; 1991 1992 /* 1993 * In the case of an unresponsive NFS server and the NFS dirty 1994 * pages exceeds dirty_thresh, give the other good wb's a pipe 1995 * to go through, so that tasks on them still remain responsive. 1996 * 1997 * In theory 1 page is enough to keep the consumer-producer 1998 * pipe going: the flusher cleans 1 page => the task dirties 1 1999 * more page. However wb_dirty has accounting errors. So use 2000 * the larger and more IO friendly wb_stat_error. 2001 */ 2002 if (sdtc->wb_dirty <= wb_stat_error()) 2003 break; 2004 2005 if (fatal_signal_pending(current)) 2006 break; 2007 } 2008 return ret; 2009} 2010 2011static DEFINE_PER_CPU(int, bdp_ratelimits); 2012 2013/* 2014 * Normal tasks are throttled by 2015 * loop { 2016 * dirty tsk->nr_dirtied_pause pages; 2017 * take a snap in balance_dirty_pages(); 2018 * } 2019 * However there is a worst case. If every task exit immediately when dirtied 2020 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be 2021 * called to throttle the page dirties. The solution is to save the not yet 2022 * throttled page dirties in dirty_throttle_leaks on task exit and charge them 2023 * randomly into the running tasks. This works well for the above worst case, 2024 * as the new task will pick up and accumulate the old task's leaked dirty 2025 * count and eventually get throttled. 2026 */ 2027DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0; 2028 2029/** 2030 * balance_dirty_pages_ratelimited_flags - Balance dirty memory state. 2031 * @mapping: address_space which was dirtied. 2032 * @flags: BDP flags. 2033 * 2034 * Processes which are dirtying memory should call in here once for each page 2035 * which was newly dirtied. The function will periodically check the system's 2036 * dirty state and will initiate writeback if needed. 2037 * 2038 * See balance_dirty_pages_ratelimited() for details. 2039 * 2040 * Return: If @flags contains BDP_ASYNC, it may return -EAGAIN to 2041 * indicate that memory is out of balance and the caller must wait 2042 * for I/O to complete. Otherwise, it will return 0 to indicate 2043 * that either memory was already in balance, or it was able to sleep 2044 * until the amount of dirty memory returned to balance. 2045 */ 2046int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, 2047 unsigned int flags) 2048{ 2049 struct inode *inode = mapping->host; 2050 struct backing_dev_info *bdi = inode_to_bdi(inode); 2051 struct bdi_writeback *wb = NULL; 2052 int ratelimit; 2053 int ret = 0; 2054 int *p; 2055 2056 if (!(bdi->capabilities & BDI_CAP_WRITEBACK)) 2057 return ret; 2058 2059 if (inode_cgwb_enabled(inode)) 2060 wb = wb_get_create_current(bdi, GFP_KERNEL); 2061 if (!wb) 2062 wb = &bdi->wb; 2063 2064 ratelimit = current->nr_dirtied_pause; 2065 if (wb->dirty_exceeded) 2066 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); 2067 2068 preempt_disable(); 2069 /* 2070 * This prevents one CPU to accumulate too many dirtied pages without 2071 * calling into balance_dirty_pages(), which can happen when there are 2072 * 1000+ tasks, all of them start dirtying pages at exactly the same 2073 * time, hence all honoured too large initial task->nr_dirtied_pause. 2074 */ 2075 p = this_cpu_ptr(&bdp_ratelimits); 2076 if (unlikely(current->nr_dirtied >= ratelimit)) 2077 *p = 0; 2078 else if (unlikely(*p >= ratelimit_pages)) { 2079 *p = 0; 2080 ratelimit = 0; 2081 } 2082 /* 2083 * Pick up the dirtied pages by the exited tasks. This avoids lots of 2084 * short-lived tasks (eg. gcc invocations in a kernel build) escaping 2085 * the dirty throttling and livelock other long-run dirtiers. 2086 */ 2087 p = this_cpu_ptr(&dirty_throttle_leaks); 2088 if (*p > 0 && current->nr_dirtied < ratelimit) { 2089 unsigned long nr_pages_dirtied; 2090 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied); 2091 *p -= nr_pages_dirtied; 2092 current->nr_dirtied += nr_pages_dirtied; 2093 } 2094 preempt_enable(); 2095 2096 if (unlikely(current->nr_dirtied >= ratelimit)) 2097 ret = balance_dirty_pages(wb, current->nr_dirtied, flags); 2098 2099 wb_put(wb); 2100 return ret; 2101} 2102EXPORT_SYMBOL_GPL(balance_dirty_pages_ratelimited_flags); 2103 2104/** 2105 * balance_dirty_pages_ratelimited - balance dirty memory state. 2106 * @mapping: address_space which was dirtied. 2107 * 2108 * Processes which are dirtying memory should call in here once for each page 2109 * which was newly dirtied. The function will periodically check the system's 2110 * dirty state and will initiate writeback if needed. 2111 * 2112 * Once we're over the dirty memory limit we decrease the ratelimiting 2113 * by a lot, to prevent individual processes from overshooting the limit 2114 * by (ratelimit_pages) each. 2115 */ 2116void balance_dirty_pages_ratelimited(struct address_space *mapping) 2117{ 2118 balance_dirty_pages_ratelimited_flags(mapping, 0); 2119} 2120EXPORT_SYMBOL(balance_dirty_pages_ratelimited); 2121 2122/* 2123 * Similar to wb_dirty_limits, wb_bg_dirty_limits also calculates dirty 2124 * and thresh, but it's for background writeback. 2125 */ 2126static void wb_bg_dirty_limits(struct dirty_throttle_control *dtc) 2127{ 2128 struct bdi_writeback *wb = dtc->wb; 2129 2130 dtc->wb_bg_thresh = __wb_calc_thresh(dtc, dtc->bg_thresh); 2131 if (dtc->wb_bg_thresh < 2 * wb_stat_error()) 2132 dtc->wb_dirty = wb_stat_sum(wb, WB_RECLAIMABLE); 2133 else 2134 dtc->wb_dirty = wb_stat(wb, WB_RECLAIMABLE); 2135} 2136 2137static bool domain_over_bg_thresh(struct dirty_throttle_control *dtc) 2138{ 2139 domain_dirty_avail(dtc, false); 2140 domain_dirty_limits(dtc); 2141 if (dtc->dirty > dtc->bg_thresh) 2142 return true; 2143 2144 wb_bg_dirty_limits(dtc); 2145 if (dtc->wb_dirty > dtc->wb_bg_thresh) 2146 return true; 2147 2148 return false; 2149} 2150 2151/** 2152 * wb_over_bg_thresh - does @wb need to be written back? 2153 * @wb: bdi_writeback of interest 2154 * 2155 * Determines whether background writeback should keep writing @wb or it's 2156 * clean enough. 2157 * 2158 * Return: %true if writeback should continue. 2159 */ 2160bool wb_over_bg_thresh(struct bdi_writeback *wb) 2161{ 2162 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; 2163 struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) }; 2164 2165 if (domain_over_bg_thresh(&gdtc)) 2166 return true; 2167 2168 if (mdtc_valid(&mdtc)) 2169 return domain_over_bg_thresh(&mdtc); 2170 2171 return false; 2172} 2173 2174#ifdef CONFIG_SYSCTL 2175/* 2176 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 2177 */ 2178static int dirty_writeback_centisecs_handler(const struct ctl_table *table, int write, 2179 void *buffer, size_t *length, loff_t *ppos) 2180{ 2181 unsigned int old_interval = dirty_writeback_interval; 2182 int ret; 2183 2184 ret = proc_dointvec(table, write, buffer, length, ppos); 2185 2186 /* 2187 * Writing 0 to dirty_writeback_interval will disable periodic writeback 2188 * and a different non-zero value will wakeup the writeback threads. 2189 * wb_wakeup_delayed() would be more appropriate, but it's a pain to 2190 * iterate over all bdis and wbs. 2191 * The reason we do this is to make the change take effect immediately. 2192 */ 2193 if (!ret && write && dirty_writeback_interval && 2194 dirty_writeback_interval != old_interval) 2195 wakeup_flusher_threads(WB_REASON_PERIODIC); 2196 2197 return ret; 2198} 2199#endif 2200 2201void laptop_mode_timer_fn(struct timer_list *t) 2202{ 2203 struct backing_dev_info *backing_dev_info = 2204 timer_container_of(backing_dev_info, t, laptop_mode_wb_timer); 2205 2206 wakeup_flusher_threads_bdi(backing_dev_info, WB_REASON_LAPTOP_TIMER); 2207} 2208 2209/* 2210 * We've spun up the disk and we're in laptop mode: schedule writeback 2211 * of all dirty data a few seconds from now. If the flush is already scheduled 2212 * then push it back - the user is still using the disk. 2213 */ 2214void laptop_io_completion(struct backing_dev_info *info) 2215{ 2216 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode); 2217} 2218 2219/* 2220 * We're in laptop mode and we've just synced. The sync's writes will have 2221 * caused another writeback to be scheduled by laptop_io_completion. 2222 * Nothing needs to be written back anymore, so we unschedule the writeback. 2223 */ 2224void laptop_sync_completion(void) 2225{ 2226 struct backing_dev_info *bdi; 2227 2228 rcu_read_lock(); 2229 2230 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) 2231 timer_delete(&bdi->laptop_mode_wb_timer); 2232 2233 rcu_read_unlock(); 2234} 2235 2236/* 2237 * If ratelimit_pages is too high then we can get into dirty-data overload 2238 * if a large number of processes all perform writes at the same time. 2239 * 2240 * Here we set ratelimit_pages to a level which ensures that when all CPUs are 2241 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory 2242 * thresholds. 2243 */ 2244 2245void writeback_set_ratelimit(void) 2246{ 2247 struct wb_domain *dom = &global_wb_domain; 2248 unsigned long background_thresh; 2249 unsigned long dirty_thresh; 2250 2251 global_dirty_limits(&background_thresh, &dirty_thresh); 2252 dom->dirty_limit = dirty_thresh; 2253 ratelimit_pages = dirty_thresh / (num_online_cpus() * 32); 2254 if (ratelimit_pages < 16) 2255 ratelimit_pages = 16; 2256} 2257 2258static int page_writeback_cpu_online(unsigned int cpu) 2259{ 2260 writeback_set_ratelimit(); 2261 return 0; 2262} 2263 2264#ifdef CONFIG_SYSCTL 2265 2266/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ 2267static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE; 2268 2269static const struct ctl_table vm_page_writeback_sysctls[] = { 2270 { 2271 .procname = "dirty_background_ratio", 2272 .data = &dirty_background_ratio, 2273 .maxlen = sizeof(dirty_background_ratio), 2274 .mode = 0644, 2275 .proc_handler = dirty_background_ratio_handler, 2276 .extra1 = SYSCTL_ZERO, 2277 .extra2 = SYSCTL_ONE_HUNDRED, 2278 }, 2279 { 2280 .procname = "dirty_background_bytes", 2281 .data = &dirty_background_bytes, 2282 .maxlen = sizeof(dirty_background_bytes), 2283 .mode = 0644, 2284 .proc_handler = dirty_background_bytes_handler, 2285 .extra1 = SYSCTL_LONG_ONE, 2286 }, 2287 { 2288 .procname = "dirty_ratio", 2289 .data = &vm_dirty_ratio, 2290 .maxlen = sizeof(vm_dirty_ratio), 2291 .mode = 0644, 2292 .proc_handler = dirty_ratio_handler, 2293 .extra1 = SYSCTL_ZERO, 2294 .extra2 = SYSCTL_ONE_HUNDRED, 2295 }, 2296 { 2297 .procname = "dirty_bytes", 2298 .data = &vm_dirty_bytes, 2299 .maxlen = sizeof(vm_dirty_bytes), 2300 .mode = 0644, 2301 .proc_handler = dirty_bytes_handler, 2302 .extra1 = (void *)&dirty_bytes_min, 2303 }, 2304 { 2305 .procname = "dirty_writeback_centisecs", 2306 .data = &dirty_writeback_interval, 2307 .maxlen = sizeof(dirty_writeback_interval), 2308 .mode = 0644, 2309 .proc_handler = dirty_writeback_centisecs_handler, 2310 }, 2311 { 2312 .procname = "dirty_expire_centisecs", 2313 .data = &dirty_expire_interval, 2314 .maxlen = sizeof(dirty_expire_interval), 2315 .mode = 0644, 2316 .proc_handler = proc_dointvec_minmax, 2317 .extra1 = SYSCTL_ZERO, 2318 }, 2319#ifdef CONFIG_HIGHMEM 2320 { 2321 .procname = "highmem_is_dirtyable", 2322 .data = &vm_highmem_is_dirtyable, 2323 .maxlen = sizeof(vm_highmem_is_dirtyable), 2324 .mode = 0644, 2325 .proc_handler = proc_dointvec_minmax, 2326 .extra1 = SYSCTL_ZERO, 2327 .extra2 = SYSCTL_ONE, 2328 }, 2329#endif 2330 { 2331 .procname = "laptop_mode", 2332 .data = &laptop_mode, 2333 .maxlen = sizeof(laptop_mode), 2334 .mode = 0644, 2335 .proc_handler = proc_dointvec_jiffies, 2336 }, 2337}; 2338#endif 2339 2340/* 2341 * Called early on to tune the page writeback dirty limits. 2342 * 2343 * We used to scale dirty pages according to how total memory 2344 * related to pages that could be allocated for buffers. 2345 * 2346 * However, that was when we used "dirty_ratio" to scale with 2347 * all memory, and we don't do that any more. "dirty_ratio" 2348 * is now applied to total non-HIGHPAGE memory, and as such we can't 2349 * get into the old insane situation any more where we had 2350 * large amounts of dirty pages compared to a small amount of 2351 * non-HIGHMEM memory. 2352 * 2353 * But we might still want to scale the dirty_ratio by how 2354 * much memory the box has.. 2355 */ 2356void __init page_writeback_init(void) 2357{ 2358 BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL)); 2359 2360 cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/writeback:online", 2361 page_writeback_cpu_online, NULL); 2362 cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL, 2363 page_writeback_cpu_online); 2364#ifdef CONFIG_SYSCTL 2365 register_sysctl_init("vm", vm_page_writeback_sysctls); 2366#endif 2367} 2368 2369/** 2370 * tag_pages_for_writeback - tag pages to be written by writeback 2371 * @mapping: address space structure to write 2372 * @start: starting page index 2373 * @end: ending page index (inclusive) 2374 * 2375 * This function scans the page range from @start to @end (inclusive) and tags 2376 * all pages that have DIRTY tag set with a special TOWRITE tag. The caller 2377 * can then use the TOWRITE tag to identify pages eligible for writeback. 2378 * This mechanism is used to avoid livelocking of writeback by a process 2379 * steadily creating new dirty pages in the file (thus it is important for this 2380 * function to be quick so that it can tag pages faster than a dirtying process 2381 * can create them). 2382 */ 2383void tag_pages_for_writeback(struct address_space *mapping, 2384 pgoff_t start, pgoff_t end) 2385{ 2386 XA_STATE(xas, &mapping->i_pages, start); 2387 unsigned int tagged = 0; 2388 void *page; 2389 2390 xas_lock_irq(&xas); 2391 xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) { 2392 xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE); 2393 if (++tagged % XA_CHECK_SCHED) 2394 continue; 2395 2396 xas_pause(&xas); 2397 xas_unlock_irq(&xas); 2398 cond_resched(); 2399 xas_lock_irq(&xas); 2400 } 2401 xas_unlock_irq(&xas); 2402} 2403EXPORT_SYMBOL(tag_pages_for_writeback); 2404 2405static bool folio_prepare_writeback(struct address_space *mapping, 2406 struct writeback_control *wbc, struct folio *folio) 2407{ 2408 /* 2409 * Folio truncated or invalidated. We can freely skip it then, 2410 * even for data integrity operations: the folio has disappeared 2411 * concurrently, so there could be no real expectation of this 2412 * data integrity operation even if there is now a new, dirty 2413 * folio at the same pagecache index. 2414 */ 2415 if (unlikely(folio->mapping != mapping)) 2416 return false; 2417 2418 /* 2419 * Did somebody else write it for us? 2420 */ 2421 if (!folio_test_dirty(folio)) 2422 return false; 2423 2424 if (folio_test_writeback(folio)) { 2425 if (wbc->sync_mode == WB_SYNC_NONE) 2426 return false; 2427 folio_wait_writeback(folio); 2428 } 2429 BUG_ON(folio_test_writeback(folio)); 2430 2431 if (!folio_clear_dirty_for_io(folio)) 2432 return false; 2433 2434 return true; 2435} 2436 2437 2438static pgoff_t wbc_end(struct writeback_control *wbc) 2439{ 2440 if (wbc->range_cyclic) 2441 return -1; 2442 return wbc->range_end >> PAGE_SHIFT; 2443} 2444 2445static struct folio *writeback_get_folio(struct address_space *mapping, 2446 struct writeback_control *wbc) 2447{ 2448 struct folio *folio; 2449 2450retry: 2451 folio = folio_batch_next(&wbc->fbatch); 2452 if (!folio) { 2453 folio_batch_release(&wbc->fbatch); 2454 cond_resched(); 2455 filemap_get_folios_tag(mapping, &wbc->index, wbc_end(wbc), 2456 wbc_to_tag(wbc), &wbc->fbatch); 2457 folio = folio_batch_next(&wbc->fbatch); 2458 if (!folio) 2459 return NULL; 2460 } 2461 2462 folio_lock(folio); 2463 if (unlikely(!folio_prepare_writeback(mapping, wbc, folio))) { 2464 folio_unlock(folio); 2465 goto retry; 2466 } 2467 2468 trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); 2469 return folio; 2470} 2471 2472/** 2473 * writeback_iter - iterate folio of a mapping for writeback 2474 * @mapping: address space structure to write 2475 * @wbc: writeback context 2476 * @folio: previously iterated folio (%NULL to start) 2477 * @error: in-out pointer for writeback errors (see below) 2478 * 2479 * This function returns the next folio for the writeback operation described by 2480 * @wbc on @mapping and should be called in a while loop in the ->writepages 2481 * implementation. 2482 * 2483 * To start the writeback operation, %NULL is passed in the @folio argument, and 2484 * for every subsequent iteration the folio returned previously should be passed 2485 * back in. 2486 * 2487 * If there was an error in the per-folio writeback inside the writeback_iter() 2488 * loop, @error should be set to the error value. 2489 * 2490 * Once the writeback described in @wbc has finished, this function will return 2491 * %NULL and if there was an error in any iteration restore it to @error. 2492 * 2493 * Note: callers should not manually break out of the loop using break or goto 2494 * but must keep calling writeback_iter() until it returns %NULL. 2495 * 2496 * Return: the folio to write or %NULL if the loop is done. 2497 */ 2498struct folio *writeback_iter(struct address_space *mapping, 2499 struct writeback_control *wbc, struct folio *folio, int *error) 2500{ 2501 if (!folio) { 2502 folio_batch_init(&wbc->fbatch); 2503 wbc->saved_err = *error = 0; 2504 2505 /* 2506 * For range cyclic writeback we remember where we stopped so 2507 * that we can continue where we stopped. 2508 * 2509 * For non-cyclic writeback we always start at the beginning of 2510 * the passed in range. 2511 */ 2512 if (wbc->range_cyclic) 2513 wbc->index = mapping->writeback_index; 2514 else 2515 wbc->index = wbc->range_start >> PAGE_SHIFT; 2516 2517 /* 2518 * To avoid livelocks when other processes dirty new pages, we 2519 * first tag pages which should be written back and only then 2520 * start writing them. 2521 * 2522 * For data-integrity writeback we have to be careful so that we 2523 * do not miss some pages (e.g., because some other process has 2524 * cleared the TOWRITE tag we set). The rule we follow is that 2525 * TOWRITE tag can be cleared only by the process clearing the 2526 * DIRTY tag (and submitting the page for I/O). 2527 */ 2528 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2529 tag_pages_for_writeback(mapping, wbc->index, 2530 wbc_end(wbc)); 2531 } else { 2532 wbc->nr_to_write -= folio_nr_pages(folio); 2533 2534 WARN_ON_ONCE(*error > 0); 2535 2536 /* 2537 * For integrity writeback we have to keep going until we have 2538 * written all the folios we tagged for writeback above, even if 2539 * we run past wbc->nr_to_write or encounter errors. 2540 * We stash away the first error we encounter in wbc->saved_err 2541 * so that it can be retrieved when we're done. This is because 2542 * the file system may still have state to clear for each folio. 2543 * 2544 * For background writeback we exit as soon as we run past 2545 * wbc->nr_to_write or encounter the first error. 2546 */ 2547 if (wbc->sync_mode == WB_SYNC_ALL) { 2548 if (*error && !wbc->saved_err) 2549 wbc->saved_err = *error; 2550 } else { 2551 if (*error || wbc->nr_to_write <= 0) 2552 goto done; 2553 } 2554 } 2555 2556 folio = writeback_get_folio(mapping, wbc); 2557 if (!folio) { 2558 /* 2559 * To avoid deadlocks between range_cyclic writeback and callers 2560 * that hold folios in writeback to aggregate I/O until 2561 * the writeback iteration finishes, we do not loop back to the 2562 * start of the file. Doing so causes a folio lock/folio 2563 * writeback access order inversion - we should only ever lock 2564 * multiple folios in ascending folio->index order, and looping 2565 * back to the start of the file violates that rule and causes 2566 * deadlocks. 2567 */ 2568 if (wbc->range_cyclic) 2569 mapping->writeback_index = 0; 2570 2571 /* 2572 * Return the first error we encountered (if there was any) to 2573 * the caller. 2574 */ 2575 *error = wbc->saved_err; 2576 } 2577 return folio; 2578 2579done: 2580 if (wbc->range_cyclic) 2581 mapping->writeback_index = folio_next_index(folio); 2582 folio_batch_release(&wbc->fbatch); 2583 return NULL; 2584} 2585EXPORT_SYMBOL_GPL(writeback_iter); 2586 2587int do_writepages(struct address_space *mapping, struct writeback_control *wbc) 2588{ 2589 int ret; 2590 struct bdi_writeback *wb; 2591 2592 if (wbc->nr_to_write <= 0) 2593 return 0; 2594 wb = inode_to_wb_wbc(mapping->host, wbc); 2595 wb_bandwidth_estimate_start(wb); 2596 while (1) { 2597 if (mapping->a_ops->writepages) 2598 ret = mapping->a_ops->writepages(mapping, wbc); 2599 else 2600 /* deal with chardevs and other special files */ 2601 ret = 0; 2602 if (ret != -ENOMEM || wbc->sync_mode != WB_SYNC_ALL) 2603 break; 2604 2605 /* 2606 * Lacking an allocation context or the locality or writeback 2607 * state of any of the inode's pages, throttle based on 2608 * writeback activity on the local node. It's as good a 2609 * guess as any. 2610 */ 2611 reclaim_throttle(NODE_DATA(numa_node_id()), 2612 VMSCAN_THROTTLE_WRITEBACK); 2613 } 2614 /* 2615 * Usually few pages are written by now from those we've just submitted 2616 * but if there's constant writeback being submitted, this makes sure 2617 * writeback bandwidth is updated once in a while. 2618 */ 2619 if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) + 2620 BANDWIDTH_INTERVAL)) 2621 wb_update_bandwidth(wb); 2622 return ret; 2623} 2624 2625/* 2626 * For address_spaces which do not use buffers nor write back. 2627 */ 2628bool noop_dirty_folio(struct address_space *mapping, struct folio *folio) 2629{ 2630 if (!folio_test_dirty(folio)) 2631 return !folio_test_set_dirty(folio); 2632 return false; 2633} 2634EXPORT_SYMBOL(noop_dirty_folio); 2635 2636/* 2637 * Helper function for set_page_dirty family. 2638 * 2639 * NOTE: This relies on being atomic wrt interrupts. 2640 */ 2641static void folio_account_dirtied(struct folio *folio, 2642 struct address_space *mapping) 2643{ 2644 struct inode *inode = mapping->host; 2645 2646 trace_writeback_dirty_folio(folio, mapping); 2647 2648 if (mapping_can_writeback(mapping)) { 2649 struct bdi_writeback *wb; 2650 long nr = folio_nr_pages(folio); 2651 2652 inode_attach_wb(inode, folio); 2653 wb = inode_to_wb(inode); 2654 2655 lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, nr); 2656 __zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr); 2657 __node_stat_mod_folio(folio, NR_DIRTIED, nr); 2658 wb_stat_mod(wb, WB_RECLAIMABLE, nr); 2659 wb_stat_mod(wb, WB_DIRTIED, nr); 2660 task_io_account_write(nr * PAGE_SIZE); 2661 current->nr_dirtied += nr; 2662 __this_cpu_add(bdp_ratelimits, nr); 2663 2664 mem_cgroup_track_foreign_dirty(folio, wb); 2665 } 2666} 2667 2668/* 2669 * Helper function for deaccounting dirty page without writeback. 2670 * 2671 */ 2672void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb) 2673{ 2674 long nr = folio_nr_pages(folio); 2675 2676 lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr); 2677 zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr); 2678 wb_stat_mod(wb, WB_RECLAIMABLE, -nr); 2679 task_io_account_cancelled_write(nr * PAGE_SIZE); 2680} 2681 2682/* 2683 * Mark the folio dirty, and set it dirty in the page cache. 2684 * 2685 * If warn is true, then emit a warning if the folio is not uptodate and has 2686 * not been truncated. 2687 * 2688 * It is the caller's responsibility to prevent the folio from being truncated 2689 * while this function is in progress, although it may have been truncated 2690 * before this function is called. Most callers have the folio locked. 2691 * A few have the folio blocked from truncation through other means (e.g. 2692 * zap_vma_pages() has it mapped and is holding the page table lock). 2693 * When called from mark_buffer_dirty(), the filesystem should hold a 2694 * reference to the buffer_head that is being marked dirty, which causes 2695 * try_to_free_buffers() to fail. 2696 */ 2697void __folio_mark_dirty(struct folio *folio, struct address_space *mapping, 2698 int warn) 2699{ 2700 unsigned long flags; 2701 2702 /* 2703 * Shmem writeback relies on swap, and swap writeback is LRU based, 2704 * not using the dirty mark. 2705 */ 2706 VM_WARN_ON_ONCE(folio_test_swapcache(folio) || shmem_mapping(mapping)); 2707 2708 xa_lock_irqsave(&mapping->i_pages, flags); 2709 if (folio->mapping) { /* Race with truncate? */ 2710 WARN_ON_ONCE(warn && !folio_test_uptodate(folio)); 2711 folio_account_dirtied(folio, mapping); 2712 __xa_set_mark(&mapping->i_pages, folio->index, 2713 PAGECACHE_TAG_DIRTY); 2714 } 2715 xa_unlock_irqrestore(&mapping->i_pages, flags); 2716} 2717 2718/** 2719 * filemap_dirty_folio - Mark a folio dirty for filesystems which do not use buffer_heads. 2720 * @mapping: Address space this folio belongs to. 2721 * @folio: Folio to be marked as dirty. 2722 * 2723 * Filesystems which do not use buffer heads should call this function 2724 * from their dirty_folio address space operation. It ignores the 2725 * contents of folio_get_private(), so if the filesystem marks individual 2726 * blocks as dirty, the filesystem should handle that itself. 2727 * 2728 * This is also sometimes used by filesystems which use buffer_heads when 2729 * a single buffer is being dirtied: we want to set the folio dirty in 2730 * that case, but not all the buffers. This is a "bottom-up" dirtying, 2731 * whereas block_dirty_folio() is a "top-down" dirtying. 2732 * 2733 * The caller must ensure this doesn't race with truncation. Most will 2734 * simply hold the folio lock, but e.g. zap_pte_range() calls with the 2735 * folio mapped and the pte lock held, which also locks out truncation. 2736 */ 2737bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio) 2738{ 2739 if (folio_test_set_dirty(folio)) 2740 return false; 2741 2742 __folio_mark_dirty(folio, mapping, !folio_test_private(folio)); 2743 2744 if (mapping->host) { 2745 /* !PageAnon && !swapper_space */ 2746 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 2747 } 2748 return true; 2749} 2750EXPORT_SYMBOL(filemap_dirty_folio); 2751 2752/** 2753 * folio_redirty_for_writepage - Decline to write a dirty folio. 2754 * @wbc: The writeback control. 2755 * @folio: The folio. 2756 * 2757 * When a writepage implementation decides that it doesn't want to write 2758 * @folio for some reason, it should call this function, unlock @folio and 2759 * return 0. 2760 * 2761 * Return: True if we redirtied the folio. False if someone else dirtied 2762 * it first. 2763 */ 2764bool folio_redirty_for_writepage(struct writeback_control *wbc, 2765 struct folio *folio) 2766{ 2767 struct address_space *mapping = folio->mapping; 2768 long nr = folio_nr_pages(folio); 2769 bool ret; 2770 2771 wbc->pages_skipped += nr; 2772 ret = filemap_dirty_folio(mapping, folio); 2773 if (mapping && mapping_can_writeback(mapping)) { 2774 struct inode *inode = mapping->host; 2775 struct bdi_writeback *wb; 2776 struct wb_lock_cookie cookie = {}; 2777 2778 wb = unlocked_inode_to_wb_begin(inode, &cookie); 2779 current->nr_dirtied -= nr; 2780 node_stat_mod_folio(folio, NR_DIRTIED, -nr); 2781 wb_stat_mod(wb, WB_DIRTIED, -nr); 2782 unlocked_inode_to_wb_end(inode, &cookie); 2783 } 2784 return ret; 2785} 2786EXPORT_SYMBOL(folio_redirty_for_writepage); 2787 2788/** 2789 * folio_mark_dirty - Mark a folio as being modified. 2790 * @folio: The folio. 2791 * 2792 * The folio may not be truncated while this function is running. 2793 * Holding the folio lock is sufficient to prevent truncation, but some 2794 * callers cannot acquire a sleeping lock. These callers instead hold 2795 * the page table lock for a page table which contains at least one page 2796 * in this folio. Truncation will block on the page table lock as it 2797 * unmaps pages before removing the folio from its mapping. 2798 * 2799 * Return: True if the folio was newly dirtied, false if it was already dirty. 2800 */ 2801bool folio_mark_dirty(struct folio *folio) 2802{ 2803 struct address_space *mapping = folio_mapping(folio); 2804 2805 if (likely(mapping)) { 2806 /* 2807 * readahead/folio_deactivate could remain 2808 * PG_readahead/PG_reclaim due to race with folio_end_writeback 2809 * About readahead, if the folio is written, the flags would be 2810 * reset. So no problem. 2811 * About folio_deactivate, if the folio is redirtied, 2812 * the flag will be reset. So no problem. but if the 2813 * folio is used by readahead it will confuse readahead 2814 * and make it restart the size rampup process. But it's 2815 * a trivial problem. 2816 */ 2817 if (folio_test_reclaim(folio)) 2818 folio_clear_reclaim(folio); 2819 return mapping->a_ops->dirty_folio(mapping, folio); 2820 } 2821 2822 return noop_dirty_folio(mapping, folio); 2823} 2824EXPORT_SYMBOL(folio_mark_dirty); 2825 2826/* 2827 * folio_mark_dirty() is racy if the caller has no reference against 2828 * folio->mapping->host, and if the folio is unlocked. This is because another 2829 * CPU could truncate the folio off the mapping and then free the mapping. 2830 * 2831 * Usually, the folio _is_ locked, or the caller is a user-space process which 2832 * holds a reference on the inode by having an open file. 2833 * 2834 * In other cases, the folio should be locked before running folio_mark_dirty(). 2835 */ 2836bool folio_mark_dirty_lock(struct folio *folio) 2837{ 2838 bool ret; 2839 2840 folio_lock(folio); 2841 ret = folio_mark_dirty(folio); 2842 folio_unlock(folio); 2843 return ret; 2844} 2845EXPORT_SYMBOL(folio_mark_dirty_lock); 2846 2847/* 2848 * This cancels just the dirty bit on the kernel page itself, it does NOT 2849 * actually remove dirty bits on any mmap's that may be around. It also 2850 * leaves the page tagged dirty, so any sync activity will still find it on 2851 * the dirty lists, and in particular, clear_page_dirty_for_io() will still 2852 * look at the dirty bits in the VM. 2853 * 2854 * Doing this should *normally* only ever be done when a page is truncated, 2855 * and is not actually mapped anywhere at all. However, fs/buffer.c does 2856 * this when it notices that somebody has cleaned out all the buffers on a 2857 * page without actually doing it through the VM. Can you say "ext3 is 2858 * horribly ugly"? Thought you could. 2859 */ 2860void __folio_cancel_dirty(struct folio *folio) 2861{ 2862 struct address_space *mapping = folio_mapping(folio); 2863 2864 if (mapping_can_writeback(mapping)) { 2865 struct inode *inode = mapping->host; 2866 struct bdi_writeback *wb; 2867 struct wb_lock_cookie cookie = {}; 2868 2869 wb = unlocked_inode_to_wb_begin(inode, &cookie); 2870 2871 if (folio_test_clear_dirty(folio)) 2872 folio_account_cleaned(folio, wb); 2873 2874 unlocked_inode_to_wb_end(inode, &cookie); 2875 } else { 2876 folio_clear_dirty(folio); 2877 } 2878} 2879EXPORT_SYMBOL(__folio_cancel_dirty); 2880 2881/* 2882 * Clear a folio's dirty flag, while caring for dirty memory accounting. 2883 * Returns true if the folio was previously dirty. 2884 * 2885 * This is for preparing to put the folio under writeout. We leave 2886 * the folio tagged as dirty in the xarray so that a concurrent 2887 * write-for-sync can discover it via a PAGECACHE_TAG_DIRTY walk. 2888 * The ->writepage implementation will run either folio_start_writeback() 2889 * or folio_mark_dirty(), at which stage we bring the folio's dirty flag 2890 * and xarray dirty tag back into sync. 2891 * 2892 * This incoherency between the folio's dirty flag and xarray tag is 2893 * unfortunate, but it only exists while the folio is locked. 2894 */ 2895bool folio_clear_dirty_for_io(struct folio *folio) 2896{ 2897 struct address_space *mapping = folio_mapping(folio); 2898 bool ret = false; 2899 2900 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 2901 2902 if (mapping && mapping_can_writeback(mapping)) { 2903 struct inode *inode = mapping->host; 2904 struct bdi_writeback *wb; 2905 struct wb_lock_cookie cookie = {}; 2906 2907 /* 2908 * Yes, Virginia, this is indeed insane. 2909 * 2910 * We use this sequence to make sure that 2911 * (a) we account for dirty stats properly 2912 * (b) we tell the low-level filesystem to 2913 * mark the whole folio dirty if it was 2914 * dirty in a pagetable. Only to then 2915 * (c) clean the folio again and return 1 to 2916 * cause the writeback. 2917 * 2918 * This way we avoid all nasty races with the 2919 * dirty bit in multiple places and clearing 2920 * them concurrently from different threads. 2921 * 2922 * Note! Normally the "folio_mark_dirty(folio)" 2923 * has no effect on the actual dirty bit - since 2924 * that will already usually be set. But we 2925 * need the side effects, and it can help us 2926 * avoid races. 2927 * 2928 * We basically use the folio "master dirty bit" 2929 * as a serialization point for all the different 2930 * threads doing their things. 2931 */ 2932 if (folio_mkclean(folio)) 2933 folio_mark_dirty(folio); 2934 /* 2935 * We carefully synchronise fault handlers against 2936 * installing a dirty pte and marking the folio dirty 2937 * at this point. We do this by having them hold the 2938 * page lock while dirtying the folio, and folios are 2939 * always locked coming in here, so we get the desired 2940 * exclusion. 2941 */ 2942 wb = unlocked_inode_to_wb_begin(inode, &cookie); 2943 if (folio_test_clear_dirty(folio)) { 2944 long nr = folio_nr_pages(folio); 2945 lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr); 2946 zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr); 2947 wb_stat_mod(wb, WB_RECLAIMABLE, -nr); 2948 ret = true; 2949 } 2950 unlocked_inode_to_wb_end(inode, &cookie); 2951 return ret; 2952 } 2953 return folio_test_clear_dirty(folio); 2954} 2955EXPORT_SYMBOL(folio_clear_dirty_for_io); 2956 2957static void wb_inode_writeback_start(struct bdi_writeback *wb) 2958{ 2959 atomic_inc(&wb->writeback_inodes); 2960} 2961 2962static void wb_inode_writeback_end(struct bdi_writeback *wb) 2963{ 2964 unsigned long flags; 2965 atomic_dec(&wb->writeback_inodes); 2966 /* 2967 * Make sure estimate of writeback throughput gets updated after 2968 * writeback completed. We delay the update by BANDWIDTH_INTERVAL 2969 * (which is the interval other bandwidth updates use for batching) so 2970 * that if multiple inodes end writeback at a similar time, they get 2971 * batched into one bandwidth update. 2972 */ 2973 spin_lock_irqsave(&wb->work_lock, flags); 2974 if (test_bit(WB_registered, &wb->state)) 2975 queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL); 2976 spin_unlock_irqrestore(&wb->work_lock, flags); 2977} 2978 2979bool __folio_end_writeback(struct folio *folio) 2980{ 2981 long nr = folio_nr_pages(folio); 2982 struct address_space *mapping = folio_mapping(folio); 2983 bool ret; 2984 2985 if (mapping && mapping_use_writeback_tags(mapping)) { 2986 struct inode *inode = mapping->host; 2987 struct bdi_writeback *wb; 2988 unsigned long flags; 2989 2990 xa_lock_irqsave(&mapping->i_pages, flags); 2991 ret = folio_xor_flags_has_waiters(folio, 1 << PG_writeback); 2992 __xa_clear_mark(&mapping->i_pages, folio->index, 2993 PAGECACHE_TAG_WRITEBACK); 2994 2995 wb = inode_to_wb(inode); 2996 wb_stat_mod(wb, WB_WRITEBACK, -nr); 2997 __wb_writeout_add(wb, nr); 2998 if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) { 2999 wb_inode_writeback_end(wb); 3000 if (mapping->host) 3001 sb_clear_inode_writeback(mapping->host); 3002 } 3003 3004 xa_unlock_irqrestore(&mapping->i_pages, flags); 3005 } else { 3006 ret = folio_xor_flags_has_waiters(folio, 1 << PG_writeback); 3007 } 3008 3009 lruvec_stat_mod_folio(folio, NR_WRITEBACK, -nr); 3010 zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr); 3011 node_stat_mod_folio(folio, NR_WRITTEN, nr); 3012 3013 return ret; 3014} 3015 3016void __folio_start_writeback(struct folio *folio, bool keep_write) 3017{ 3018 long nr = folio_nr_pages(folio); 3019 struct address_space *mapping = folio_mapping(folio); 3020 int access_ret; 3021 3022 VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); 3023 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 3024 3025 if (mapping && mapping_use_writeback_tags(mapping)) { 3026 XA_STATE(xas, &mapping->i_pages, folio->index); 3027 struct inode *inode = mapping->host; 3028 struct bdi_writeback *wb; 3029 unsigned long flags; 3030 bool on_wblist; 3031 3032 xas_lock_irqsave(&xas, flags); 3033 xas_load(&xas); 3034 folio_test_set_writeback(folio); 3035 3036 on_wblist = mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK); 3037 3038 xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK); 3039 wb = inode_to_wb(inode); 3040 wb_stat_mod(wb, WB_WRITEBACK, nr); 3041 if (!on_wblist) { 3042 wb_inode_writeback_start(wb); 3043 /* 3044 * We can come through here when swapping anonymous 3045 * folios, so we don't necessarily have an inode to 3046 * track for sync. 3047 */ 3048 if (mapping->host) 3049 sb_mark_inode_writeback(mapping->host); 3050 } 3051 3052 if (!folio_test_dirty(folio)) 3053 xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); 3054 if (!keep_write) 3055 xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); 3056 xas_unlock_irqrestore(&xas, flags); 3057 } else { 3058 folio_test_set_writeback(folio); 3059 } 3060 3061 lruvec_stat_mod_folio(folio, NR_WRITEBACK, nr); 3062 zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr); 3063 3064 access_ret = arch_make_folio_accessible(folio); 3065 /* 3066 * If writeback has been triggered on a page that cannot be made 3067 * accessible, it is too late to recover here. 3068 */ 3069 VM_BUG_ON_FOLIO(access_ret != 0, folio); 3070} 3071EXPORT_SYMBOL(__folio_start_writeback); 3072 3073/** 3074 * folio_wait_writeback - Wait for a folio to finish writeback. 3075 * @folio: The folio to wait for. 3076 * 3077 * If the folio is currently being written back to storage, wait for the 3078 * I/O to complete. 3079 * 3080 * Context: Sleeps. Must be called in process context and with 3081 * no spinlocks held. Caller should hold a reference on the folio. 3082 * If the folio is not locked, writeback may start again after writeback 3083 * has finished. 3084 */ 3085void folio_wait_writeback(struct folio *folio) 3086{ 3087 while (folio_test_writeback(folio)) { 3088 trace_folio_wait_writeback(folio, folio_mapping(folio)); 3089 folio_wait_bit(folio, PG_writeback); 3090 } 3091} 3092EXPORT_SYMBOL_GPL(folio_wait_writeback); 3093 3094/** 3095 * folio_wait_writeback_killable - Wait for a folio to finish writeback. 3096 * @folio: The folio to wait for. 3097 * 3098 * If the folio is currently being written back to storage, wait for the 3099 * I/O to complete or a fatal signal to arrive. 3100 * 3101 * Context: Sleeps. Must be called in process context and with 3102 * no spinlocks held. Caller should hold a reference on the folio. 3103 * If the folio is not locked, writeback may start again after writeback 3104 * has finished. 3105 * Return: 0 on success, -EINTR if we get a fatal signal while waiting. 3106 */ 3107int folio_wait_writeback_killable(struct folio *folio) 3108{ 3109 while (folio_test_writeback(folio)) { 3110 trace_folio_wait_writeback(folio, folio_mapping(folio)); 3111 if (folio_wait_bit_killable(folio, PG_writeback)) 3112 return -EINTR; 3113 } 3114 3115 return 0; 3116} 3117EXPORT_SYMBOL_GPL(folio_wait_writeback_killable); 3118 3119/** 3120 * folio_wait_stable() - wait for writeback to finish, if necessary. 3121 * @folio: The folio to wait on. 3122 * 3123 * This function determines if the given folio is related to a backing 3124 * device that requires folio contents to be held stable during writeback. 3125 * If so, then it will wait for any pending writeback to complete. 3126 * 3127 * Context: Sleeps. Must be called in process context and with 3128 * no spinlocks held. Caller should hold a reference on the folio. 3129 * If the folio is not locked, writeback may start again after writeback 3130 * has finished. 3131 */ 3132void folio_wait_stable(struct folio *folio) 3133{ 3134 if (mapping_stable_writes(folio_mapping(folio))) 3135 folio_wait_writeback(folio); 3136} 3137EXPORT_SYMBOL_GPL(folio_wait_stable);
~/
Read-only file preview.