FILE
memory-failure.c
File preview with syntax highlighting.
linux-6.19.14/mm/memory-failure.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2008, 2009 Intel Corporation 4 * Authors: Andi Kleen, Fengguang Wu 5 * 6 * High level machine check handler. Handles pages reported by the 7 * hardware as being corrupted usually due to a multi-bit ECC memory or cache 8 * failure. 9 * 10 * In addition there is a "soft offline" entry point that allows stop using 11 * not-yet-corrupted-by-suspicious pages without killing anything. 12 * 13 * Handles page cache pages in various states. The tricky part 14 * here is that we can access any page asynchronously in respect to 15 * other VM users, because memory failures could happen anytime and 16 * anywhere. This could violate some of their assumptions. This is why 17 * this code has to be extremely careful. Generally it tries to use 18 * normal locking rules, as in get the standard locks, even if that means 19 * the error handling takes potentially a long time. 20 * 21 * It can be very tempting to add handling for obscure cases here. 22 * In general any code for handling new cases should only be added iff: 23 * - You know how to test it. 24 * - You have a test that can be added to mce-test 25 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/ 26 * - The case actually shows up as a frequent (top 10) page state in 27 * tools/mm/page-types when running a real workload. 28 * 29 * There are several operations here with exponential complexity because 30 * of unsuitable VM data structures. For example the operation to map back 31 * from RMAP chains to processes has to walk the complete process list and 32 * has non linear complexity with the number. But since memory corruptions 33 * are rare we hope to get away with this. This avoids impacting the core 34 * VM. 35 */ 36 37#define pr_fmt(fmt) "Memory failure: " fmt 38 39#include <linux/kernel.h> 40#include <linux/mm.h> 41#include <linux/memory-failure.h> 42#include <linux/page-flags.h> 43#include <linux/sched/signal.h> 44#include <linux/sched/task.h> 45#include <linux/dax.h> 46#include <linux/ksm.h> 47#include <linux/rmap.h> 48#include <linux/export.h> 49#include <linux/pagemap.h> 50#include <linux/swap.h> 51#include <linux/backing-dev.h> 52#include <linux/migrate.h> 53#include <linux/slab.h> 54#include <linux/leafops.h> 55#include <linux/hugetlb.h> 56#include <linux/memory_hotplug.h> 57#include <linux/mm_inline.h> 58#include <linux/memremap.h> 59#include <linux/kfifo.h> 60#include <linux/ratelimit.h> 61#include <linux/pagewalk.h> 62#include <linux/shmem_fs.h> 63#include <linux/sysctl.h> 64 65#define CREATE_TRACE_POINTS 66#include <trace/events/memory-failure.h> 67 68#include "swap.h" 69#include "internal.h" 70 71static int sysctl_memory_failure_early_kill __read_mostly; 72 73static int sysctl_memory_failure_recovery __read_mostly = 1; 74 75static int sysctl_enable_soft_offline __read_mostly = 1; 76 77atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0); 78 79static bool hw_memory_failure __read_mostly = false; 80 81static DEFINE_MUTEX(mf_mutex); 82 83void num_poisoned_pages_inc(unsigned long pfn) 84{ 85 atomic_long_inc(&num_poisoned_pages); 86 memblk_nr_poison_inc(pfn); 87} 88 89void num_poisoned_pages_sub(unsigned long pfn, long i) 90{ 91 atomic_long_sub(i, &num_poisoned_pages); 92 if (pfn != -1UL) 93 memblk_nr_poison_sub(pfn, i); 94} 95 96/** 97 * MF_ATTR_RO - Create sysfs entry for each memory failure statistics. 98 * @_name: name of the file in the per NUMA sysfs directory. 99 */ 100#define MF_ATTR_RO(_name) \ 101static ssize_t _name##_show(struct device *dev, \ 102 struct device_attribute *attr, \ 103 char *buf) \ 104{ \ 105 struct memory_failure_stats *mf_stats = \ 106 &NODE_DATA(dev->id)->mf_stats; \ 107 return sysfs_emit(buf, "%lu\n", mf_stats->_name); \ 108} \ 109static DEVICE_ATTR_RO(_name) 110 111MF_ATTR_RO(total); 112MF_ATTR_RO(ignored); 113MF_ATTR_RO(failed); 114MF_ATTR_RO(delayed); 115MF_ATTR_RO(recovered); 116 117static struct attribute *memory_failure_attr[] = { 118 &dev_attr_total.attr, 119 &dev_attr_ignored.attr, 120 &dev_attr_failed.attr, 121 &dev_attr_delayed.attr, 122 &dev_attr_recovered.attr, 123 NULL, 124}; 125 126const struct attribute_group memory_failure_attr_group = { 127 .name = "memory_failure", 128 .attrs = memory_failure_attr, 129}; 130 131static const struct ctl_table memory_failure_table[] = { 132 { 133 .procname = "memory_failure_early_kill", 134 .data = &sysctl_memory_failure_early_kill, 135 .maxlen = sizeof(sysctl_memory_failure_early_kill), 136 .mode = 0644, 137 .proc_handler = proc_dointvec_minmax, 138 .extra1 = SYSCTL_ZERO, 139 .extra2 = SYSCTL_ONE, 140 }, 141 { 142 .procname = "memory_failure_recovery", 143 .data = &sysctl_memory_failure_recovery, 144 .maxlen = sizeof(sysctl_memory_failure_recovery), 145 .mode = 0644, 146 .proc_handler = proc_dointvec_minmax, 147 .extra1 = SYSCTL_ZERO, 148 .extra2 = SYSCTL_ONE, 149 }, 150 { 151 .procname = "enable_soft_offline", 152 .data = &sysctl_enable_soft_offline, 153 .maxlen = sizeof(sysctl_enable_soft_offline), 154 .mode = 0644, 155 .proc_handler = proc_dointvec_minmax, 156 .extra1 = SYSCTL_ZERO, 157 .extra2 = SYSCTL_ONE, 158 } 159}; 160 161static struct rb_root_cached pfn_space_itree = RB_ROOT_CACHED; 162 163static DEFINE_MUTEX(pfn_space_lock); 164 165/* 166 * Return values: 167 * 1: the page is dissolved (if needed) and taken off from buddy, 168 * 0: the page is dissolved (if needed) and not taken off from buddy, 169 * < 0: failed to dissolve. 170 */ 171static int __page_handle_poison(struct page *page) 172{ 173 int ret; 174 175 /* 176 * zone_pcp_disable() can't be used here. It will 177 * hold pcp_batch_high_lock and dissolve_free_hugetlb_folio() might hold 178 * cpu_hotplug_lock via static_key_slow_dec() when hugetlb vmemmap 179 * optimization is enabled. This will break current lock dependency 180 * chain and leads to deadlock. 181 * Disabling pcp before dissolving the page was a deterministic 182 * approach because we made sure that those pages cannot end up in any 183 * PCP list. Draining PCP lists expels those pages to the buddy system, 184 * but nothing guarantees that those pages do not get back to a PCP 185 * queue if we need to refill those. 186 */ 187 ret = dissolve_free_hugetlb_folio(page_folio(page)); 188 if (!ret) { 189 drain_all_pages(page_zone(page)); 190 ret = take_page_off_buddy(page); 191 } 192 193 return ret; 194} 195 196static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release) 197{ 198 if (hugepage_or_freepage) { 199 /* 200 * Doing this check for free pages is also fine since 201 * dissolve_free_hugetlb_folio() returns 0 for non-hugetlb folios as well. 202 */ 203 if (__page_handle_poison(page) <= 0) 204 /* 205 * We could fail to take off the target page from buddy 206 * for example due to racy page allocation, but that's 207 * acceptable because soft-offlined page is not broken 208 * and if someone really want to use it, they should 209 * take it. 210 */ 211 return false; 212 } 213 214 SetPageHWPoison(page); 215 if (release) 216 put_page(page); 217 page_ref_inc(page); 218 num_poisoned_pages_inc(page_to_pfn(page)); 219 220 return true; 221} 222 223static hwpoison_filter_func_t __rcu *hwpoison_filter_func __read_mostly; 224 225void hwpoison_filter_register(hwpoison_filter_func_t *filter) 226{ 227 rcu_assign_pointer(hwpoison_filter_func, filter); 228} 229EXPORT_SYMBOL_GPL(hwpoison_filter_register); 230 231void hwpoison_filter_unregister(void) 232{ 233 RCU_INIT_POINTER(hwpoison_filter_func, NULL); 234 synchronize_rcu(); 235} 236EXPORT_SYMBOL_GPL(hwpoison_filter_unregister); 237 238static int hwpoison_filter(struct page *p) 239{ 240 int ret = 0; 241 hwpoison_filter_func_t *filter; 242 243 rcu_read_lock(); 244 filter = rcu_dereference(hwpoison_filter_func); 245 if (filter) 246 ret = filter(p); 247 rcu_read_unlock(); 248 249 return ret; 250} 251 252/* 253 * Kill all processes that have a poisoned page mapped and then isolate 254 * the page. 255 * 256 * General strategy: 257 * Find all processes having the page mapped and kill them. 258 * But we keep a page reference around so that the page is not 259 * actually freed yet. 260 * Then stash the page away 261 * 262 * There's no convenient way to get back to mapped processes 263 * from the VMAs. So do a brute-force search over all 264 * running processes. 265 * 266 * Remember that machine checks are not common (or rather 267 * if they are common you have other problems), so this shouldn't 268 * be a performance issue. 269 * 270 * Also there are some races possible while we get from the 271 * error detection to actually handle it. 272 */ 273 274struct to_kill { 275 struct list_head nd; 276 struct task_struct *tsk; 277 unsigned long addr; 278 short size_shift; 279}; 280 281/* 282 * Send all the processes who have the page mapped a signal. 283 * ``action optional'' if they are not immediately affected by the error 284 * ``action required'' if error happened in current execution context 285 */ 286static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) 287{ 288 struct task_struct *t = tk->tsk; 289 short addr_lsb = tk->size_shift; 290 int ret = 0; 291 292 pr_err("%#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n", 293 pfn, t->comm, task_pid_nr(t)); 294 295 if ((flags & MF_ACTION_REQUIRED) && (t == current)) 296 ret = force_sig_mceerr(BUS_MCEERR_AR, 297 (void __user *)tk->addr, addr_lsb); 298 else 299 /* 300 * Signal other processes sharing the page if they have 301 * PF_MCE_EARLY set. 302 * Don't use force here, it's convenient if the signal 303 * can be temporarily blocked. 304 */ 305 ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, 306 addr_lsb, t); 307 if (ret < 0) 308 pr_info("Error sending signal to %s:%d: %d\n", 309 t->comm, task_pid_nr(t), ret); 310 return ret; 311} 312 313/* 314 * Unknown page type encountered. Try to check whether it can turn PageLRU by 315 * lru_add_drain_all. 316 */ 317void shake_folio(struct folio *folio) 318{ 319 if (folio_test_hugetlb(folio)) 320 return; 321 /* 322 * TODO: Could shrink slab caches here if a lightweight range-based 323 * shrinker will be available. 324 */ 325 if (folio_test_slab(folio)) 326 return; 327 328 lru_add_drain_all(); 329} 330EXPORT_SYMBOL_GPL(shake_folio); 331 332static void shake_page(struct page *page) 333{ 334 shake_folio(page_folio(page)); 335} 336 337static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, 338 unsigned long address) 339{ 340 unsigned long ret = 0; 341 pgd_t *pgd; 342 p4d_t *p4d; 343 pud_t *pud; 344 pmd_t *pmd; 345 pte_t *pte; 346 pte_t ptent; 347 348 VM_BUG_ON_VMA(address == -EFAULT, vma); 349 pgd = pgd_offset(vma->vm_mm, address); 350 if (!pgd_present(*pgd)) 351 return 0; 352 p4d = p4d_offset(pgd, address); 353 if (!p4d_present(*p4d)) 354 return 0; 355 pud = pud_offset(p4d, address); 356 if (!pud_present(*pud)) 357 return 0; 358 if (pud_trans_huge(*pud)) 359 return PUD_SHIFT; 360 pmd = pmd_offset(pud, address); 361 if (!pmd_present(*pmd)) 362 return 0; 363 if (pmd_trans_huge(*pmd)) 364 return PMD_SHIFT; 365 pte = pte_offset_map(pmd, address); 366 if (!pte) 367 return 0; 368 ptent = ptep_get(pte); 369 if (pte_present(ptent)) 370 ret = PAGE_SHIFT; 371 pte_unmap(pte); 372 return ret; 373} 374 375/* 376 * Failure handling: if we can't find or can't kill a process there's 377 * not much we can do. We just print a message and ignore otherwise. 378 */ 379 380/* 381 * Schedule a process for later kill. 382 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. 383 */ 384static void __add_to_kill(struct task_struct *tsk, const struct page *p, 385 struct vm_area_struct *vma, struct list_head *to_kill, 386 unsigned long addr) 387{ 388 struct to_kill *tk; 389 390 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC); 391 if (!tk) { 392 pr_err("Out of memory while machine check handling\n"); 393 return; 394 } 395 396 tk->addr = addr; 397 if (is_zone_device_page(p)) 398 tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); 399 else 400 tk->size_shift = folio_shift(page_folio(p)); 401 402 /* 403 * Send SIGKILL if "tk->addr == -EFAULT". Also, as 404 * "tk->size_shift" is always non-zero for !is_zone_device_page(), 405 * so "tk->size_shift == 0" effectively checks no mapping on 406 * ZONE_DEVICE. Indeed, when a devdax page is mmapped N times 407 * to a process' address space, it's possible not all N VMAs 408 * contain mappings for the page, but at least one VMA does. 409 * Only deliver SIGBUS with payload derived from the VMA that 410 * has a mapping for the page. 411 */ 412 if (tk->addr == -EFAULT) { 413 pr_info("Unable to find user space address %lx in %s\n", 414 page_to_pfn(p), tsk->comm); 415 } else if (tk->size_shift == 0) { 416 kfree(tk); 417 return; 418 } 419 420 get_task_struct(tsk); 421 tk->tsk = tsk; 422 list_add_tail(&tk->nd, to_kill); 423} 424 425static void add_to_kill_anon_file(struct task_struct *tsk, const struct page *p, 426 struct vm_area_struct *vma, struct list_head *to_kill, 427 unsigned long addr) 428{ 429 if (addr == -EFAULT) 430 return; 431 __add_to_kill(tsk, p, vma, to_kill, addr); 432} 433 434#ifdef CONFIG_KSM 435static bool task_in_to_kill_list(struct list_head *to_kill, 436 struct task_struct *tsk) 437{ 438 struct to_kill *tk, *next; 439 440 list_for_each_entry_safe(tk, next, to_kill, nd) { 441 if (tk->tsk == tsk) 442 return true; 443 } 444 445 return false; 446} 447 448void add_to_kill_ksm(struct task_struct *tsk, const struct page *p, 449 struct vm_area_struct *vma, struct list_head *to_kill, 450 unsigned long addr) 451{ 452 if (!task_in_to_kill_list(to_kill, tsk)) 453 __add_to_kill(tsk, p, vma, to_kill, addr); 454} 455#endif 456/* 457 * Kill the processes that have been collected earlier. 458 * 459 * Only do anything when FORCEKILL is set, otherwise just free the 460 * list (this is used for clean pages which do not need killing) 461 */ 462static void kill_procs(struct list_head *to_kill, int forcekill, 463 unsigned long pfn, int flags) 464{ 465 struct to_kill *tk, *next; 466 467 list_for_each_entry_safe(tk, next, to_kill, nd) { 468 if (forcekill) { 469 if (tk->addr == -EFAULT) { 470 pr_err("%#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n", 471 pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); 472 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, 473 tk->tsk, PIDTYPE_PID); 474 } 475 476 /* 477 * In theory the process could have mapped 478 * something else on the address in-between. We could 479 * check for that, but we need to tell the 480 * process anyways. 481 */ 482 else if (kill_proc(tk, pfn, flags) < 0) 483 pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n", 484 pfn, tk->tsk->comm, task_pid_nr(tk->tsk)); 485 } 486 list_del(&tk->nd); 487 put_task_struct(tk->tsk); 488 kfree(tk); 489 } 490} 491 492/* 493 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO) 494 * on behalf of the thread group. Return task_struct of the (first found) 495 * dedicated thread if found, and return NULL otherwise. 496 * 497 * We already hold rcu lock in the caller, so we don't have to call 498 * rcu_read_lock/unlock() in this function. 499 */ 500static struct task_struct *find_early_kill_thread(struct task_struct *tsk) 501{ 502 struct task_struct *t; 503 504 for_each_thread(tsk, t) { 505 if (t->flags & PF_MCE_PROCESS) { 506 if (t->flags & PF_MCE_EARLY) 507 return t; 508 } else { 509 if (sysctl_memory_failure_early_kill) 510 return t; 511 } 512 } 513 return NULL; 514} 515 516/* 517 * Determine whether a given process is "early kill" process which expects 518 * to be signaled when some page under the process is hwpoisoned. 519 * Return task_struct of the dedicated thread (main thread unless explicitly 520 * specified) if the process is "early kill" and otherwise returns NULL. 521 * 522 * Note that the above is true for Action Optional case. For Action Required 523 * case, it's only meaningful to the current thread which need to be signaled 524 * with SIGBUS, this error is Action Optional for other non current 525 * processes sharing the same error page,if the process is "early kill", the 526 * task_struct of the dedicated thread will also be returned. 527 */ 528struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) 529{ 530 if (!tsk->mm) 531 return NULL; 532 /* 533 * Comparing ->mm here because current task might represent 534 * a subthread, while tsk always points to the main thread. 535 */ 536 if (force_early && tsk->mm == current->mm) 537 return current; 538 539 return find_early_kill_thread(tsk); 540} 541 542/* 543 * Collect processes when the error hit an anonymous page. 544 */ 545static void collect_procs_anon(const struct folio *folio, 546 const struct page *page, struct list_head *to_kill, 547 int force_early) 548{ 549 struct task_struct *tsk; 550 struct anon_vma *av; 551 pgoff_t pgoff; 552 553 av = folio_lock_anon_vma_read(folio, NULL); 554 if (av == NULL) /* Not actually mapped anymore */ 555 return; 556 557 pgoff = page_pgoff(folio, page); 558 rcu_read_lock(); 559 for_each_process(tsk) { 560 struct vm_area_struct *vma; 561 struct anon_vma_chain *vmac; 562 struct task_struct *t = task_early_kill(tsk, force_early); 563 unsigned long addr; 564 565 if (!t) 566 continue; 567 anon_vma_interval_tree_foreach(vmac, &av->rb_root, 568 pgoff, pgoff) { 569 vma = vmac->vma; 570 if (vma->vm_mm != t->mm) 571 continue; 572 addr = page_mapped_in_vma(page, vma); 573 add_to_kill_anon_file(t, page, vma, to_kill, addr); 574 } 575 } 576 rcu_read_unlock(); 577 anon_vma_unlock_read(av); 578} 579 580/* 581 * Collect processes when the error hit a file mapped page. 582 */ 583static void collect_procs_file(const struct folio *folio, 584 const struct page *page, struct list_head *to_kill, 585 int force_early) 586{ 587 struct vm_area_struct *vma; 588 struct task_struct *tsk; 589 struct address_space *mapping = folio->mapping; 590 pgoff_t pgoff; 591 592 i_mmap_lock_read(mapping); 593 rcu_read_lock(); 594 pgoff = page_pgoff(folio, page); 595 for_each_process(tsk) { 596 struct task_struct *t = task_early_kill(tsk, force_early); 597 unsigned long addr; 598 599 if (!t) 600 continue; 601 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, 602 pgoff) { 603 /* 604 * Send early kill signal to tasks where a vma covers 605 * the page but the corrupted page is not necessarily 606 * mapped in its pte. 607 * Assume applications who requested early kill want 608 * to be informed of all such data corruptions. 609 */ 610 if (vma->vm_mm != t->mm) 611 continue; 612 addr = page_address_in_vma(folio, page, vma); 613 add_to_kill_anon_file(t, page, vma, to_kill, addr); 614 } 615 } 616 rcu_read_unlock(); 617 i_mmap_unlock_read(mapping); 618} 619 620#ifdef CONFIG_FS_DAX 621static void add_to_kill_fsdax(struct task_struct *tsk, const struct page *p, 622 struct vm_area_struct *vma, 623 struct list_head *to_kill, pgoff_t pgoff) 624{ 625 unsigned long addr = vma_address(vma, pgoff, 1); 626 __add_to_kill(tsk, p, vma, to_kill, addr); 627} 628 629/* 630 * Collect processes when the error hit a fsdax page. 631 */ 632static void collect_procs_fsdax(const struct page *page, 633 struct address_space *mapping, pgoff_t pgoff, 634 struct list_head *to_kill, bool pre_remove) 635{ 636 struct vm_area_struct *vma; 637 struct task_struct *tsk; 638 639 i_mmap_lock_read(mapping); 640 rcu_read_lock(); 641 for_each_process(tsk) { 642 struct task_struct *t = tsk; 643 644 /* 645 * Search for all tasks while MF_MEM_PRE_REMOVE is set, because 646 * the current may not be the one accessing the fsdax page. 647 * Otherwise, search for the current task. 648 */ 649 if (!pre_remove) 650 t = task_early_kill(tsk, true); 651 if (!t) 652 continue; 653 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { 654 if (vma->vm_mm == t->mm) 655 add_to_kill_fsdax(t, page, vma, to_kill, pgoff); 656 } 657 } 658 rcu_read_unlock(); 659 i_mmap_unlock_read(mapping); 660} 661#endif /* CONFIG_FS_DAX */ 662 663/* 664 * Collect the processes who have the corrupted page mapped to kill. 665 */ 666static void collect_procs(const struct folio *folio, const struct page *page, 667 struct list_head *tokill, int force_early) 668{ 669 if (!folio->mapping) 670 return; 671 if (unlikely(folio_test_ksm(folio))) 672 collect_procs_ksm(folio, page, tokill, force_early); 673 else if (folio_test_anon(folio)) 674 collect_procs_anon(folio, page, tokill, force_early); 675 else 676 collect_procs_file(folio, page, tokill, force_early); 677} 678 679struct hwpoison_walk { 680 struct to_kill tk; 681 unsigned long pfn; 682 int flags; 683}; 684 685static void set_to_kill(struct to_kill *tk, unsigned long addr, short shift) 686{ 687 tk->addr = addr; 688 tk->size_shift = shift; 689} 690 691static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, 692 unsigned long poisoned_pfn, struct to_kill *tk) 693{ 694 unsigned long pfn = 0; 695 unsigned long hwpoison_vaddr; 696 unsigned long mask; 697 698 if (pte_present(pte)) { 699 pfn = pte_pfn(pte); 700 } else { 701 const softleaf_t entry = softleaf_from_pte(pte); 702 703 if (softleaf_is_hwpoison(entry)) 704 pfn = softleaf_to_pfn(entry); 705 } 706 707 mask = ~((1UL << (shift - PAGE_SHIFT)) - 1); 708 if (!pfn || pfn != (poisoned_pfn & mask)) 709 return 0; 710 711 hwpoison_vaddr = addr + ((poisoned_pfn - pfn) << PAGE_SHIFT); 712 set_to_kill(tk, hwpoison_vaddr, shift); 713 return 1; 714} 715 716#ifdef CONFIG_TRANSPARENT_HUGEPAGE 717static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, 718 struct hwpoison_walk *hwp) 719{ 720 pmd_t pmd = *pmdp; 721 unsigned long pfn; 722 unsigned long hwpoison_vaddr; 723 724 if (!pmd_present(pmd)) 725 return 0; 726 pfn = pmd_pfn(pmd); 727 if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) { 728 hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT); 729 set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT); 730 return 1; 731 } 732 return 0; 733} 734#else 735static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, 736 struct hwpoison_walk *hwp) 737{ 738 return 0; 739} 740#endif 741 742static int hwpoison_pte_range(pmd_t *pmdp, unsigned long addr, 743 unsigned long end, struct mm_walk *walk) 744{ 745 struct hwpoison_walk *hwp = walk->private; 746 int ret = 0; 747 pte_t *ptep, *mapped_pte; 748 spinlock_t *ptl; 749 750 ptl = pmd_trans_huge_lock(pmdp, walk->vma); 751 if (ptl) { 752 ret = check_hwpoisoned_pmd_entry(pmdp, addr, hwp); 753 spin_unlock(ptl); 754 goto out; 755 } 756 757 mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp, 758 addr, &ptl); 759 if (!ptep) 760 goto out; 761 762 for (; addr != end; ptep++, addr += PAGE_SIZE) { 763 ret = check_hwpoisoned_entry(ptep_get(ptep), addr, PAGE_SHIFT, 764 hwp->pfn, &hwp->tk); 765 if (ret == 1) 766 break; 767 } 768 pte_unmap_unlock(mapped_pte, ptl); 769out: 770 cond_resched(); 771 return ret; 772} 773 774#ifdef CONFIG_HUGETLB_PAGE 775static int hwpoison_hugetlb_range(pte_t *ptep, unsigned long hmask, 776 unsigned long addr, unsigned long end, 777 struct mm_walk *walk) 778{ 779 struct hwpoison_walk *hwp = walk->private; 780 struct hstate *h = hstate_vma(walk->vma); 781 spinlock_t *ptl; 782 pte_t pte; 783 int ret; 784 785 ptl = huge_pte_lock(h, walk->mm, ptep); 786 pte = huge_ptep_get(walk->mm, addr, ptep); 787 ret = check_hwpoisoned_entry(pte, addr, huge_page_shift(h), 788 hwp->pfn, &hwp->tk); 789 spin_unlock(ptl); 790 return ret; 791} 792#else 793#define hwpoison_hugetlb_range NULL 794#endif 795 796static int hwpoison_test_walk(unsigned long start, unsigned long end, 797 struct mm_walk *walk) 798{ 799 /* We also want to consider pages mapped into VM_PFNMAP. */ 800 return 0; 801} 802 803static const struct mm_walk_ops hwpoison_walk_ops = { 804 .pmd_entry = hwpoison_pte_range, 805 .hugetlb_entry = hwpoison_hugetlb_range, 806 .test_walk = hwpoison_test_walk, 807 .walk_lock = PGWALK_RDLOCK, 808}; 809 810/* 811 * Sends SIGBUS to the current process with error info. 812 * 813 * This function is intended to handle "Action Required" MCEs on already 814 * hardware poisoned pages. They could happen, for example, when 815 * memory_failure() failed to unmap the error page at the first call, or 816 * when multiple local machine checks happened on different CPUs. 817 * 818 * MCE handler currently has no easy access to the error virtual address, 819 * so this function walks page table to find it. The returned virtual address 820 * is proper in most cases, but it could be wrong when the application 821 * process has multiple entries mapping the error page. 822 */ 823static int kill_accessing_process(struct task_struct *p, unsigned long pfn, 824 int flags) 825{ 826 int ret; 827 struct hwpoison_walk priv = { 828 .pfn = pfn, 829 }; 830 priv.tk.tsk = p; 831 832 if (!p->mm) 833 return -EFAULT; 834 835 mmap_read_lock(p->mm); 836 ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwpoison_walk_ops, 837 (void *)&priv); 838 /* 839 * ret = 1 when CMCI wins, regardless of whether try_to_unmap() 840 * succeeds or fails, then kill the process with SIGBUS. 841 * ret = 0 when poison page is a clean page and it's dropped, no 842 * SIGBUS is needed. 843 */ 844 if (ret == 1 && priv.tk.addr) 845 kill_proc(&priv.tk, pfn, flags); 846 mmap_read_unlock(p->mm); 847 848 return ret > 0 ? -EHWPOISON : 0; 849} 850 851/* 852 * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed. 853 * But it could not do more to isolate the page from being accessed again, 854 * nor does it kill the process. This is extremely rare and one of the 855 * potential causes is that the page state has been changed due to 856 * underlying race condition. This is the most severe outcomes. 857 * 858 * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed. 859 * It should have killed the process, but it can't isolate the page, 860 * due to conditions such as extra pin, unmap failure, etc. Accessing 861 * the page again may trigger another MCE and the process will be killed 862 * by the m-f() handler immediately. 863 * 864 * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed. 865 * The page is unmapped, and is removed from the LRU or file mapping. 866 * An attempt to access the page again will trigger page fault and the 867 * PF handler will kill the process. 868 * 869 * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed. 870 * The page has been completely isolated, that is, unmapped, taken out of 871 * the buddy system, or hole-punnched out of the file mapping. 872 */ 873static const char *action_name[] = { 874 [MF_IGNORED] = "Ignored", 875 [MF_FAILED] = "Failed", 876 [MF_DELAYED] = "Delayed", 877 [MF_RECOVERED] = "Recovered", 878}; 879 880static const char * const action_page_types[] = { 881 [MF_MSG_KERNEL] = "reserved kernel page", 882 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page", 883 [MF_MSG_HUGE] = "huge page", 884 [MF_MSG_FREE_HUGE] = "free huge page", 885 [MF_MSG_GET_HWPOISON] = "get hwpoison page", 886 [MF_MSG_UNMAP_FAILED] = "unmapping failed page", 887 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page", 888 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page", 889 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page", 890 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page", 891 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page", 892 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page", 893 [MF_MSG_DIRTY_LRU] = "dirty LRU page", 894 [MF_MSG_CLEAN_LRU] = "clean LRU page", 895 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page", 896 [MF_MSG_BUDDY] = "free buddy page", 897 [MF_MSG_DAX] = "dax page", 898 [MF_MSG_UNSPLIT_THP] = "unsplit thp", 899 [MF_MSG_ALREADY_POISONED] = "already poisoned page", 900 [MF_MSG_PFN_MAP] = "non struct page pfn", 901 [MF_MSG_UNKNOWN] = "unknown page", 902}; 903 904/* 905 * XXX: It is possible that a page is isolated from LRU cache, 906 * and then kept in swap cache or failed to remove from page cache. 907 * The page count will stop it from being freed by unpoison. 908 * Stress tests should be aware of this memory leak problem. 909 */ 910static int delete_from_lru_cache(struct folio *folio) 911{ 912 if (folio_isolate_lru(folio)) { 913 /* 914 * Clear sensible page flags, so that the buddy system won't 915 * complain when the folio is unpoison-and-freed. 916 */ 917 folio_clear_active(folio); 918 folio_clear_unevictable(folio); 919 920 /* 921 * Poisoned page might never drop its ref count to 0 so we have 922 * to uncharge it manually from its memcg. 923 */ 924 mem_cgroup_uncharge(folio); 925 926 /* 927 * drop the refcount elevated by folio_isolate_lru() 928 */ 929 folio_put(folio); 930 return 0; 931 } 932 return -EIO; 933} 934 935static int truncate_error_folio(struct folio *folio, unsigned long pfn, 936 struct address_space *mapping) 937{ 938 int ret = MF_FAILED; 939 940 if (mapping->a_ops->error_remove_folio) { 941 int err = mapping->a_ops->error_remove_folio(mapping, folio); 942 943 if (err != 0) 944 pr_info("%#lx: Failed to punch page: %d\n", pfn, err); 945 else if (!filemap_release_folio(folio, GFP_NOIO)) 946 pr_info("%#lx: failed to release buffers\n", pfn); 947 else 948 ret = MF_RECOVERED; 949 } else { 950 /* 951 * If the file system doesn't support it just invalidate 952 * This fails on dirty or anything with private pages 953 */ 954 if (mapping_evict_folio(mapping, folio)) 955 ret = MF_RECOVERED; 956 else 957 pr_info("%#lx: Failed to invalidate\n", pfn); 958 } 959 960 return ret; 961} 962 963struct page_state { 964 unsigned long mask; 965 unsigned long res; 966 enum mf_action_page_type type; 967 968 /* Callback ->action() has to unlock the relevant page inside it. */ 969 int (*action)(struct page_state *ps, struct page *p); 970}; 971 972/* 973 * Return true if page is still referenced by others, otherwise return 974 * false. 975 * 976 * The extra_pins is true when one extra refcount is expected. 977 */ 978static bool has_extra_refcount(struct page_state *ps, struct page *p, 979 bool extra_pins) 980{ 981 int count = page_count(p) - 1; 982 983 if (extra_pins) 984 count -= folio_nr_pages(page_folio(p)); 985 986 if (count > 0) { 987 pr_err("%#lx: %s still referenced by %d users\n", 988 page_to_pfn(p), action_page_types[ps->type], count); 989 return true; 990 } 991 992 return false; 993} 994 995/* 996 * Error hit kernel page. 997 * Do nothing, try to be lucky and not touch this instead. For a few cases we 998 * could be more sophisticated. 999 */ 1000static int me_kernel(struct page_state *ps, struct page *p) 1001{ 1002 unlock_page(p); 1003 return MF_IGNORED; 1004} 1005 1006/* 1007 * Page in unknown state. Do nothing. 1008 * This is a catch-all in case we fail to make sense of the page state. 1009 */ 1010static int me_unknown(struct page_state *ps, struct page *p) 1011{ 1012 pr_err("%#lx: Unknown page state\n", page_to_pfn(p)); 1013 unlock_page(p); 1014 return MF_IGNORED; 1015} 1016 1017/* 1018 * Clean (or cleaned) page cache page. 1019 */ 1020static int me_pagecache_clean(struct page_state *ps, struct page *p) 1021{ 1022 struct folio *folio = page_folio(p); 1023 int ret; 1024 struct address_space *mapping; 1025 bool extra_pins; 1026 1027 delete_from_lru_cache(folio); 1028 1029 /* 1030 * For anonymous folios the only reference left 1031 * should be the one m_f() holds. 1032 */ 1033 if (folio_test_anon(folio)) { 1034 ret = MF_RECOVERED; 1035 goto out; 1036 } 1037 1038 /* 1039 * Now truncate the page in the page cache. This is really 1040 * more like a "temporary hole punch" 1041 * Don't do this for block devices when someone else 1042 * has a reference, because it could be file system metadata 1043 * and that's not safe to truncate. 1044 */ 1045 mapping = folio_mapping(folio); 1046 if (!mapping) { 1047 /* Folio has been torn down in the meantime */ 1048 ret = MF_FAILED; 1049 goto out; 1050 } 1051 1052 /* 1053 * The shmem page is kept in page cache instead of truncating 1054 * so is expected to have an extra refcount after error-handling. 1055 */ 1056 extra_pins = shmem_mapping(mapping); 1057 1058 /* 1059 * Truncation is a bit tricky. Enable it per file system for now. 1060 * 1061 * Open: to take i_rwsem or not for this? Right now we don't. 1062 */ 1063 ret = truncate_error_folio(folio, page_to_pfn(p), mapping); 1064 if (has_extra_refcount(ps, p, extra_pins)) 1065 ret = MF_FAILED; 1066 1067out: 1068 folio_unlock(folio); 1069 1070 return ret; 1071} 1072 1073/* 1074 * Dirty pagecache page 1075 * Issues: when the error hit a hole page the error is not properly 1076 * propagated. 1077 */ 1078static int me_pagecache_dirty(struct page_state *ps, struct page *p) 1079{ 1080 struct folio *folio = page_folio(p); 1081 struct address_space *mapping = folio_mapping(folio); 1082 1083 /* TBD: print more information about the file. */ 1084 if (mapping) { 1085 /* 1086 * IO error will be reported by write(), fsync(), etc. 1087 * who check the mapping. 1088 * This way the application knows that something went 1089 * wrong with its dirty file data. 1090 */ 1091 mapping_set_error(mapping, -EIO); 1092 } 1093 1094 return me_pagecache_clean(ps, p); 1095} 1096 1097/* 1098 * Clean and dirty swap cache. 1099 * 1100 * Dirty swap cache page is tricky to handle. The page could live both in page 1101 * table and swap cache(ie. page is freshly swapped in). So it could be 1102 * referenced concurrently by 2 types of PTEs: 1103 * normal PTEs and swap PTEs. We try to handle them consistently by calling 1104 * try_to_unmap(!TTU_HWPOISON) to convert the normal PTEs to swap PTEs, 1105 * and then 1106 * - clear dirty bit to prevent IO 1107 * - remove from LRU 1108 * - but keep in the swap cache, so that when we return to it on 1109 * a later page fault, we know the application is accessing 1110 * corrupted data and shall be killed (we installed simple 1111 * interception code in do_swap_page to catch it). 1112 * 1113 * Clean swap cache pages can be directly isolated. A later page fault will 1114 * bring in the known good data from disk. 1115 */ 1116static int me_swapcache_dirty(struct page_state *ps, struct page *p) 1117{ 1118 struct folio *folio = page_folio(p); 1119 int ret; 1120 bool extra_pins = false; 1121 1122 folio_clear_dirty(folio); 1123 /* Trigger EIO in shmem: */ 1124 folio_clear_uptodate(folio); 1125 1126 ret = delete_from_lru_cache(folio) ? MF_FAILED : MF_DELAYED; 1127 folio_unlock(folio); 1128 1129 if (ret == MF_DELAYED) 1130 extra_pins = true; 1131 1132 if (has_extra_refcount(ps, p, extra_pins)) 1133 ret = MF_FAILED; 1134 1135 return ret; 1136} 1137 1138static int me_swapcache_clean(struct page_state *ps, struct page *p) 1139{ 1140 struct folio *folio = page_folio(p); 1141 int ret; 1142 1143 swap_cache_del_folio(folio); 1144 1145 ret = delete_from_lru_cache(folio) ? MF_FAILED : MF_RECOVERED; 1146 folio_unlock(folio); 1147 1148 if (has_extra_refcount(ps, p, false)) 1149 ret = MF_FAILED; 1150 1151 return ret; 1152} 1153 1154/* 1155 * Huge pages. Needs work. 1156 * Issues: 1157 * - Error on hugepage is contained in hugepage unit (not in raw page unit.) 1158 * To narrow down kill region to one page, we need to break up pmd. 1159 */ 1160static int me_huge_page(struct page_state *ps, struct page *p) 1161{ 1162 struct folio *folio = page_folio(p); 1163 int res; 1164 struct address_space *mapping; 1165 bool extra_pins = false; 1166 1167 mapping = folio_mapping(folio); 1168 if (mapping) { 1169 res = truncate_error_folio(folio, page_to_pfn(p), mapping); 1170 /* The page is kept in page cache. */ 1171 extra_pins = true; 1172 folio_unlock(folio); 1173 } else { 1174 folio_unlock(folio); 1175 /* 1176 * migration entry prevents later access on error hugepage, 1177 * so we can free and dissolve it into buddy to save healthy 1178 * subpages. 1179 */ 1180 folio_put(folio); 1181 if (__page_handle_poison(p) > 0) { 1182 page_ref_inc(p); 1183 res = MF_RECOVERED; 1184 } else { 1185 res = MF_FAILED; 1186 } 1187 } 1188 1189 if (has_extra_refcount(ps, p, extra_pins)) 1190 res = MF_FAILED; 1191 1192 return res; 1193} 1194 1195/* 1196 * Various page states we can handle. 1197 * 1198 * A page state is defined by its current page->flags bits. 1199 * The table matches them in order and calls the right handler. 1200 * 1201 * This is quite tricky because we can access page at any time 1202 * in its live cycle, so all accesses have to be extremely careful. 1203 * 1204 * This is not complete. More states could be added. 1205 * For any missing state don't attempt recovery. 1206 */ 1207 1208#define dirty (1UL << PG_dirty) 1209#define sc ((1UL << PG_swapcache) | (1UL << PG_swapbacked)) 1210#define unevict (1UL << PG_unevictable) 1211#define mlock (1UL << PG_mlocked) 1212#define lru (1UL << PG_lru) 1213#define head (1UL << PG_head) 1214#define reserved (1UL << PG_reserved) 1215 1216static struct page_state error_states[] = { 1217 { reserved, reserved, MF_MSG_KERNEL, me_kernel }, 1218 /* 1219 * free pages are specially detected outside this table: 1220 * PG_buddy pages only make a small fraction of all free pages. 1221 */ 1222 1223 { head, head, MF_MSG_HUGE, me_huge_page }, 1224 1225 { sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty }, 1226 { sc|dirty, sc, MF_MSG_CLEAN_SWAPCACHE, me_swapcache_clean }, 1227 1228 { mlock|dirty, mlock|dirty, MF_MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty }, 1229 { mlock|dirty, mlock, MF_MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean }, 1230 1231 { unevict|dirty, unevict|dirty, MF_MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty }, 1232 { unevict|dirty, unevict, MF_MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean }, 1233 1234 { lru|dirty, lru|dirty, MF_MSG_DIRTY_LRU, me_pagecache_dirty }, 1235 { lru|dirty, lru, MF_MSG_CLEAN_LRU, me_pagecache_clean }, 1236 1237 /* 1238 * Catchall entry: must be at end. 1239 */ 1240 { 0, 0, MF_MSG_UNKNOWN, me_unknown }, 1241}; 1242 1243#undef dirty 1244#undef sc 1245#undef unevict 1246#undef mlock 1247#undef lru 1248#undef head 1249#undef reserved 1250 1251static void update_per_node_mf_stats(unsigned long pfn, 1252 enum mf_result result) 1253{ 1254 int nid = MAX_NUMNODES; 1255 struct memory_failure_stats *mf_stats = NULL; 1256 1257 nid = pfn_to_nid(pfn); 1258 if (unlikely(nid < 0 || nid >= MAX_NUMNODES)) { 1259 WARN_ONCE(1, "Memory failure: pfn=%#lx, invalid nid=%d", pfn, nid); 1260 return; 1261 } 1262 1263 mf_stats = &NODE_DATA(nid)->mf_stats; 1264 switch (result) { 1265 case MF_IGNORED: 1266 ++mf_stats->ignored; 1267 break; 1268 case MF_FAILED: 1269 ++mf_stats->failed; 1270 break; 1271 case MF_DELAYED: 1272 ++mf_stats->delayed; 1273 break; 1274 case MF_RECOVERED: 1275 ++mf_stats->recovered; 1276 break; 1277 default: 1278 WARN_ONCE(1, "Memory failure: mf_result=%d is not properly handled", result); 1279 break; 1280 } 1281 ++mf_stats->total; 1282} 1283 1284/* 1285 * "Dirty/Clean" indication is not 100% accurate due to the possibility of 1286 * setting PG_dirty outside page lock. See also comment above set_page_dirty(). 1287 */ 1288static int action_result(unsigned long pfn, enum mf_action_page_type type, 1289 enum mf_result result) 1290{ 1291 trace_memory_failure_event(pfn, type, result); 1292 1293 if (type != MF_MSG_ALREADY_POISONED && type != MF_MSG_PFN_MAP) { 1294 num_poisoned_pages_inc(pfn); 1295 update_per_node_mf_stats(pfn, result); 1296 } 1297 1298 pr_err("%#lx: recovery action for %s: %s\n", 1299 pfn, action_page_types[type], action_name[result]); 1300 1301 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; 1302} 1303 1304static int page_action(struct page_state *ps, struct page *p, 1305 unsigned long pfn) 1306{ 1307 int result; 1308 1309 /* page p should be unlocked after returning from ps->action(). */ 1310 result = ps->action(ps, p); 1311 1312 /* Could do more checks here if page looks ok */ 1313 /* 1314 * Could adjust zone counters here to correct for the missing page. 1315 */ 1316 1317 return action_result(pfn, ps->type, result); 1318} 1319 1320static inline bool PageHWPoisonTakenOff(struct page *page) 1321{ 1322 return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON; 1323} 1324 1325void SetPageHWPoisonTakenOff(struct page *page) 1326{ 1327 set_page_private(page, MAGIC_HWPOISON); 1328} 1329 1330void ClearPageHWPoisonTakenOff(struct page *page) 1331{ 1332 if (PageHWPoison(page)) 1333 set_page_private(page, 0); 1334} 1335 1336/* 1337 * Return true if a page type of a given page is supported by hwpoison 1338 * mechanism (while handling could fail), otherwise false. This function 1339 * does not return true for hugetlb or device memory pages, so it's assumed 1340 * to be called only in the context where we never have such pages. 1341 */ 1342static inline bool HWPoisonHandlable(struct page *page, unsigned long flags) 1343{ 1344 if (PageSlab(page)) 1345 return false; 1346 1347 /* Soft offline could migrate movable_ops pages */ 1348 if ((flags & MF_SOFT_OFFLINE) && page_has_movable_ops(page)) 1349 return true; 1350 1351 return PageLRU(page) || is_free_buddy_page(page); 1352} 1353 1354static int __get_hwpoison_page(struct page *page, unsigned long flags) 1355{ 1356 struct folio *folio = page_folio(page); 1357 int ret = 0; 1358 bool hugetlb = false; 1359 1360 ret = get_hwpoison_hugetlb_folio(folio, &hugetlb, false); 1361 if (hugetlb) { 1362 /* Make sure hugetlb demotion did not happen from under us. */ 1363 if (folio == page_folio(page)) 1364 return ret; 1365 if (ret > 0) { 1366 folio_put(folio); 1367 folio = page_folio(page); 1368 } 1369 } 1370 1371 /* 1372 * This check prevents from calling folio_try_get() for any 1373 * unsupported type of folio in order to reduce the risk of unexpected 1374 * races caused by taking a folio refcount. 1375 */ 1376 if (!HWPoisonHandlable(&folio->page, flags)) 1377 return -EBUSY; 1378 1379 if (folio_try_get(folio)) { 1380 if (folio == page_folio(page)) 1381 return 1; 1382 1383 pr_info("%#lx cannot catch tail\n", page_to_pfn(page)); 1384 folio_put(folio); 1385 } 1386 1387 return 0; 1388} 1389 1390#define GET_PAGE_MAX_RETRY_NUM 3 1391 1392static int get_any_page(struct page *p, unsigned long flags) 1393{ 1394 int ret = 0, pass = 0; 1395 bool count_increased = false; 1396 1397 if (flags & MF_COUNT_INCREASED) 1398 count_increased = true; 1399 1400try_again: 1401 if (!count_increased) { 1402 ret = __get_hwpoison_page(p, flags); 1403 if (!ret) { 1404 if (page_count(p)) { 1405 /* We raced with an allocation, retry. */ 1406 if (pass++ < GET_PAGE_MAX_RETRY_NUM) 1407 goto try_again; 1408 ret = -EBUSY; 1409 } else if (!PageHuge(p) && !is_free_buddy_page(p)) { 1410 /* We raced with put_page, retry. */ 1411 if (pass++ < GET_PAGE_MAX_RETRY_NUM) 1412 goto try_again; 1413 ret = -EIO; 1414 } 1415 goto out; 1416 } else if (ret == -EBUSY) { 1417 /* 1418 * We raced with (possibly temporary) unhandlable 1419 * page, retry. 1420 */ 1421 if (pass++ < 3) { 1422 shake_page(p); 1423 goto try_again; 1424 } 1425 ret = -EIO; 1426 goto out; 1427 } 1428 } 1429 1430 if (PageHuge(p) || HWPoisonHandlable(p, flags)) { 1431 ret = 1; 1432 } else { 1433 /* 1434 * A page we cannot handle. Check whether we can turn 1435 * it into something we can handle. 1436 */ 1437 if (pass++ < GET_PAGE_MAX_RETRY_NUM) { 1438 put_page(p); 1439 shake_page(p); 1440 count_increased = false; 1441 goto try_again; 1442 } 1443 put_page(p); 1444 ret = -EIO; 1445 } 1446out: 1447 if (ret == -EIO) 1448 pr_err("%#lx: unhandlable page.\n", page_to_pfn(p)); 1449 1450 return ret; 1451} 1452 1453static int __get_unpoison_page(struct page *page) 1454{ 1455 struct folio *folio = page_folio(page); 1456 int ret = 0; 1457 bool hugetlb = false; 1458 1459 ret = get_hwpoison_hugetlb_folio(folio, &hugetlb, true); 1460 if (hugetlb) { 1461 /* Make sure hugetlb demotion did not happen from under us. */ 1462 if (folio == page_folio(page)) 1463 return ret; 1464 if (ret > 0) 1465 folio_put(folio); 1466 } 1467 1468 /* 1469 * PageHWPoisonTakenOff pages are not only marked as PG_hwpoison, 1470 * but also isolated from buddy freelist, so need to identify the 1471 * state and have to cancel both operations to unpoison. 1472 */ 1473 if (PageHWPoisonTakenOff(page)) 1474 return -EHWPOISON; 1475 1476 return get_page_unless_zero(page) ? 1 : 0; 1477} 1478 1479/** 1480 * get_hwpoison_page() - Get refcount for memory error handling 1481 * @p: Raw error page (hit by memory error) 1482 * @flags: Flags controlling behavior of error handling 1483 * 1484 * get_hwpoison_page() takes a page refcount of an error page to handle memory 1485 * error on it, after checking that the error page is in a well-defined state 1486 * (defined as a page-type we can successfully handle the memory error on it, 1487 * such as LRU page and hugetlb page). 1488 * 1489 * Memory error handling could be triggered at any time on any type of page, 1490 * so it's prone to race with typical memory management lifecycle (like 1491 * allocation and free). So to avoid such races, get_hwpoison_page() takes 1492 * extra care for the error page's state (as done in __get_hwpoison_page()), 1493 * and has some retry logic in get_any_page(). 1494 * 1495 * When called from unpoison_memory(), the caller should already ensure that 1496 * the given page has PG_hwpoison. So it's never reused for other page 1497 * allocations, and __get_unpoison_page() never races with them. 1498 * 1499 * Return: 0 on failure or free buddy (hugetlb) page, 1500 * 1 on success for in-use pages in a well-defined state, 1501 * -EIO for pages on which we can not handle memory errors, 1502 * -EBUSY when get_hwpoison_page() has raced with page lifecycle 1503 * operations like allocation and free, 1504 * -EHWPOISON when the page is hwpoisoned and taken off from buddy. 1505 */ 1506static int get_hwpoison_page(struct page *p, unsigned long flags) 1507{ 1508 int ret; 1509 1510 zone_pcp_disable(page_zone(p)); 1511 if (flags & MF_UNPOISON) 1512 ret = __get_unpoison_page(p); 1513 else 1514 ret = get_any_page(p, flags); 1515 zone_pcp_enable(page_zone(p)); 1516 1517 return ret; 1518} 1519 1520/* 1521 * The caller must guarantee the folio isn't large folio, except hugetlb. 1522 * try_to_unmap() can't handle it. 1523 */ 1524int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill) 1525{ 1526 enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC | TTU_HWPOISON; 1527 struct address_space *mapping; 1528 1529 if (folio_test_swapcache(folio)) { 1530 pr_err("%#lx: keeping poisoned page in swap cache\n", pfn); 1531 ttu &= ~TTU_HWPOISON; 1532 } 1533 1534 /* 1535 * Propagate the dirty bit from PTEs to struct page first, because we 1536 * need this to decide if we should kill or just drop the page. 1537 * XXX: the dirty test could be racy: set_page_dirty() may not always 1538 * be called inside page lock (it's recommended but not enforced). 1539 */ 1540 mapping = folio_mapping(folio); 1541 if (!must_kill && !folio_test_dirty(folio) && mapping && 1542 mapping_can_writeback(mapping)) { 1543 if (folio_mkclean(folio)) { 1544 folio_set_dirty(folio); 1545 } else { 1546 ttu &= ~TTU_HWPOISON; 1547 pr_info("%#lx: corrupted page was clean: dropped without side effects\n", 1548 pfn); 1549 } 1550 } 1551 1552 if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) { 1553 /* 1554 * For hugetlb folios in shared mappings, try_to_unmap 1555 * could potentially call huge_pmd_unshare. Because of 1556 * this, take semaphore in write mode here and set 1557 * TTU_RMAP_LOCKED to indicate we have taken the lock 1558 * at this higher level. 1559 */ 1560 mapping = hugetlb_folio_mapping_lock_write(folio); 1561 if (!mapping) { 1562 pr_info("%#lx: could not lock mapping for mapped hugetlb folio\n", 1563 folio_pfn(folio)); 1564 return -EBUSY; 1565 } 1566 1567 try_to_unmap(folio, ttu|TTU_RMAP_LOCKED); 1568 i_mmap_unlock_write(mapping); 1569 } else { 1570 try_to_unmap(folio, ttu); 1571 } 1572 1573 return folio_mapped(folio) ? -EBUSY : 0; 1574} 1575 1576/* 1577 * Do all that is necessary to remove user space mappings. Unmap 1578 * the pages and send SIGBUS to the processes if the data was dirty. 1579 */ 1580static bool hwpoison_user_mappings(struct folio *folio, struct page *p, 1581 unsigned long pfn, int flags) 1582{ 1583 LIST_HEAD(tokill); 1584 bool unmap_success; 1585 int forcekill; 1586 bool mlocked = folio_test_mlocked(folio); 1587 1588 /* 1589 * Here we are interested only in user-mapped pages, so skip any 1590 * other types of pages. 1591 */ 1592 if (folio_test_reserved(folio) || folio_test_slab(folio) || 1593 folio_test_pgtable(folio) || folio_test_offline(folio)) 1594 return true; 1595 if (!(folio_test_lru(folio) || folio_test_hugetlb(folio))) 1596 return true; 1597 1598 /* 1599 * This check implies we don't kill processes if their pages 1600 * are in the swap cache early. Those are always late kills. 1601 */ 1602 if (!folio_mapped(folio)) 1603 return true; 1604 1605 /* 1606 * First collect all the processes that have the page 1607 * mapped in dirty form. This has to be done before try_to_unmap, 1608 * because ttu takes the rmap data structures down. 1609 */ 1610 collect_procs(folio, p, &tokill, flags & MF_ACTION_REQUIRED); 1611 1612 unmap_success = !unmap_poisoned_folio(folio, pfn, flags & MF_MUST_KILL); 1613 if (!unmap_success) 1614 pr_err("%#lx: failed to unmap page (folio mapcount=%d)\n", 1615 pfn, folio_mapcount(folio)); 1616 1617 /* 1618 * try_to_unmap() might put mlocked page in lru cache, so call 1619 * shake_page() again to ensure that it's flushed. 1620 */ 1621 if (mlocked) 1622 shake_folio(folio); 1623 1624 /* 1625 * Now that the dirty bit has been propagated to the 1626 * struct page and all unmaps done we can decide if 1627 * killing is needed or not. Only kill when the page 1628 * was dirty or the process is not restartable, 1629 * otherwise the tokill list is merely 1630 * freed. When there was a problem unmapping earlier 1631 * use a more force-full uncatchable kill to prevent 1632 * any accesses to the poisoned memory. 1633 */ 1634 forcekill = folio_test_dirty(folio) || (flags & MF_MUST_KILL) || 1635 !unmap_success; 1636 kill_procs(&tokill, forcekill, pfn, flags); 1637 1638 return unmap_success; 1639} 1640 1641static int identify_page_state(unsigned long pfn, struct page *p, 1642 unsigned long page_flags) 1643{ 1644 struct page_state *ps; 1645 1646 /* 1647 * The first check uses the current page flags which may not have any 1648 * relevant information. The second check with the saved page flags is 1649 * carried out only if the first check can't determine the page status. 1650 */ 1651 for (ps = error_states;; ps++) 1652 if ((p->flags.f & ps->mask) == ps->res) 1653 break; 1654 1655 page_flags |= (p->flags.f & (1UL << PG_dirty)); 1656 1657 if (!ps->mask) 1658 for (ps = error_states;; ps++) 1659 if ((page_flags & ps->mask) == ps->res) 1660 break; 1661 return page_action(ps, p, pfn); 1662} 1663 1664/* 1665 * When 'release' is 'false', it means that if thp split has failed, 1666 * there is still more to do, hence the page refcount we took earlier 1667 * is still needed. 1668 */ 1669static int try_to_split_thp_page(struct page *page, unsigned int new_order, 1670 bool release) 1671{ 1672 int ret; 1673 1674 lock_page(page); 1675 ret = split_huge_page_to_order(page, new_order); 1676 unlock_page(page); 1677 1678 if (ret && release) 1679 put_page(page); 1680 1681 return ret; 1682} 1683 1684static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn, 1685 struct address_space *mapping, pgoff_t index, int flags) 1686{ 1687 struct to_kill *tk; 1688 unsigned long size = 0; 1689 1690 list_for_each_entry(tk, to_kill, nd) 1691 if (tk->size_shift) 1692 size = max(size, 1UL << tk->size_shift); 1693 1694 if (size) { 1695 /* 1696 * Unmap the largest mapping to avoid breaking up device-dax 1697 * mappings which are constant size. The actual size of the 1698 * mapping being torn down is communicated in siginfo, see 1699 * kill_proc() 1700 */ 1701 loff_t start = ((loff_t)index << PAGE_SHIFT) & ~(size - 1); 1702 1703 unmap_mapping_range(mapping, start, size, 0); 1704 } 1705 1706 kill_procs(to_kill, flags & MF_MUST_KILL, pfn, flags); 1707} 1708 1709/* 1710 * Only dev_pagemap pages get here, such as fsdax when the filesystem 1711 * either do not claim or fails to claim a hwpoison event, or devdax. 1712 * The fsdax pages are initialized per base page, and the devdax pages 1713 * could be initialized either as base pages, or as compound pages with 1714 * vmemmap optimization enabled. Devdax is simplistic in its dealing with 1715 * hwpoison, such that, if a subpage of a compound page is poisoned, 1716 * simply mark the compound head page is by far sufficient. 1717 */ 1718static int mf_generic_kill_procs(unsigned long long pfn, int flags, 1719 struct dev_pagemap *pgmap) 1720{ 1721 struct folio *folio = pfn_folio(pfn); 1722 LIST_HEAD(to_kill); 1723 dax_entry_t cookie; 1724 int rc = 0; 1725 1726 /* 1727 * Prevent the inode from being freed while we are interrogating 1728 * the address_space, typically this would be handled by 1729 * lock_page(), but dax pages do not use the page lock. This 1730 * also prevents changes to the mapping of this pfn until 1731 * poison signaling is complete. 1732 */ 1733 cookie = dax_lock_folio(folio); 1734 if (!cookie) 1735 return -EBUSY; 1736 1737 if (hwpoison_filter(&folio->page)) { 1738 rc = -EOPNOTSUPP; 1739 goto unlock; 1740 } 1741 1742 switch (pgmap->type) { 1743 case MEMORY_DEVICE_PRIVATE: 1744 case MEMORY_DEVICE_COHERENT: 1745 /* 1746 * TODO: Handle device pages which may need coordination 1747 * with device-side memory. 1748 */ 1749 rc = -ENXIO; 1750 goto unlock; 1751 default: 1752 break; 1753 } 1754 1755 /* 1756 * Use this flag as an indication that the dax page has been 1757 * remapped UC to prevent speculative consumption of poison. 1758 */ 1759 SetPageHWPoison(&folio->page); 1760 1761 /* 1762 * Unlike System-RAM there is no possibility to swap in a 1763 * different physical page at a given virtual address, so all 1764 * userspace consumption of ZONE_DEVICE memory necessitates 1765 * SIGBUS (i.e. MF_MUST_KILL) 1766 */ 1767 flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; 1768 collect_procs(folio, &folio->page, &to_kill, true); 1769 1770 unmap_and_kill(&to_kill, pfn, folio->mapping, folio->index, flags); 1771unlock: 1772 dax_unlock_folio(folio, cookie); 1773 return rc; 1774} 1775 1776#ifdef CONFIG_FS_DAX 1777/** 1778 * mf_dax_kill_procs - Collect and kill processes who are using this file range 1779 * @mapping: address_space of the file in use 1780 * @index: start pgoff of the range within the file 1781 * @count: length of the range, in unit of PAGE_SIZE 1782 * @mf_flags: memory failure flags 1783 */ 1784int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index, 1785 unsigned long count, int mf_flags) 1786{ 1787 LIST_HEAD(to_kill); 1788 dax_entry_t cookie; 1789 struct page *page; 1790 size_t end = index + count; 1791 bool pre_remove = mf_flags & MF_MEM_PRE_REMOVE; 1792 1793 mf_flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; 1794 1795 for (; index < end; index++) { 1796 page = NULL; 1797 cookie = dax_lock_mapping_entry(mapping, index, &page); 1798 if (!cookie) 1799 return -EBUSY; 1800 if (!page) 1801 goto unlock; 1802 1803 if (!pre_remove) 1804 SetPageHWPoison(page); 1805 1806 /* 1807 * The pre_remove case is revoking access, the memory is still 1808 * good and could theoretically be put back into service. 1809 */ 1810 collect_procs_fsdax(page, mapping, index, &to_kill, pre_remove); 1811 unmap_and_kill(&to_kill, page_to_pfn(page), mapping, 1812 index, mf_flags); 1813unlock: 1814 dax_unlock_mapping_entry(mapping, index, cookie); 1815 } 1816 return 0; 1817} 1818EXPORT_SYMBOL_GPL(mf_dax_kill_procs); 1819#endif /* CONFIG_FS_DAX */ 1820 1821#ifdef CONFIG_HUGETLB_PAGE 1822 1823/* 1824 * Struct raw_hwp_page represents information about "raw error page", 1825 * constructing singly linked list from ->_hugetlb_hwpoison field of folio. 1826 */ 1827struct raw_hwp_page { 1828 struct llist_node node; 1829 struct page *page; 1830}; 1831 1832static inline struct llist_head *raw_hwp_list_head(struct folio *folio) 1833{ 1834 return (struct llist_head *)&folio->_hugetlb_hwpoison; 1835} 1836 1837bool is_raw_hwpoison_page_in_hugepage(struct page *page) 1838{ 1839 struct llist_head *raw_hwp_head; 1840 struct raw_hwp_page *p; 1841 struct folio *folio = page_folio(page); 1842 bool ret = false; 1843 1844 if (!folio_test_hwpoison(folio)) 1845 return false; 1846 1847 if (!folio_test_hugetlb(folio)) 1848 return PageHWPoison(page); 1849 1850 /* 1851 * When RawHwpUnreliable is set, kernel lost track of which subpages 1852 * are HWPOISON. So return as if ALL subpages are HWPOISONed. 1853 */ 1854 if (folio_test_hugetlb_raw_hwp_unreliable(folio)) 1855 return true; 1856 1857 mutex_lock(&mf_mutex); 1858 1859 raw_hwp_head = raw_hwp_list_head(folio); 1860 llist_for_each_entry(p, raw_hwp_head->first, node) { 1861 if (page == p->page) { 1862 ret = true; 1863 break; 1864 } 1865 } 1866 1867 mutex_unlock(&mf_mutex); 1868 1869 return ret; 1870} 1871 1872static unsigned long __folio_free_raw_hwp(struct folio *folio, bool move_flag) 1873{ 1874 struct llist_node *head; 1875 struct raw_hwp_page *p, *next; 1876 unsigned long count = 0; 1877 1878 head = llist_del_all(raw_hwp_list_head(folio)); 1879 llist_for_each_entry_safe(p, next, head, node) { 1880 if (move_flag) 1881 SetPageHWPoison(p->page); 1882 else 1883 num_poisoned_pages_sub(page_to_pfn(p->page), 1); 1884 kfree(p); 1885 count++; 1886 } 1887 return count; 1888} 1889 1890#define MF_HUGETLB_FREED 0 /* freed hugepage */ 1891#define MF_HUGETLB_IN_USED 1 /* in-use hugepage */ 1892#define MF_HUGETLB_NON_HUGEPAGE 2 /* not a hugepage */ 1893#define MF_HUGETLB_FOLIO_PRE_POISONED 3 /* folio already poisoned */ 1894#define MF_HUGETLB_PAGE_PRE_POISONED 4 /* exact page already poisoned */ 1895#define MF_HUGETLB_RETRY 5 /* hugepage is busy, retry */ 1896/* 1897 * Set hugetlb folio as hwpoisoned, update folio private raw hwpoison list 1898 * to keep track of the poisoned pages. 1899 */ 1900static int hugetlb_update_hwpoison(struct folio *folio, struct page *page) 1901{ 1902 struct llist_head *head; 1903 struct raw_hwp_page *raw_hwp; 1904 struct raw_hwp_page *p; 1905 int ret = folio_test_set_hwpoison(folio) ? MF_HUGETLB_FOLIO_PRE_POISONED : 0; 1906 1907 /* 1908 * Once the hwpoison hugepage has lost reliable raw error info, 1909 * there is little meaning to keep additional error info precisely, 1910 * so skip to add additional raw error info. 1911 */ 1912 if (folio_test_hugetlb_raw_hwp_unreliable(folio)) 1913 return MF_HUGETLB_FOLIO_PRE_POISONED; 1914 head = raw_hwp_list_head(folio); 1915 llist_for_each_entry(p, head->first, node) { 1916 if (p->page == page) 1917 return MF_HUGETLB_PAGE_PRE_POISONED; 1918 } 1919 1920 raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC); 1921 if (raw_hwp) { 1922 raw_hwp->page = page; 1923 llist_add(&raw_hwp->node, head); 1924 } else { 1925 /* 1926 * Failed to save raw error info. We no longer trace all 1927 * hwpoisoned subpages, and we need refuse to free/dissolve 1928 * this hwpoisoned hugepage. 1929 */ 1930 folio_set_hugetlb_raw_hwp_unreliable(folio); 1931 /* 1932 * Once hugetlb_raw_hwp_unreliable is set, raw_hwp_page is not 1933 * used any more, so free it. 1934 */ 1935 __folio_free_raw_hwp(folio, false); 1936 } 1937 return ret; 1938} 1939 1940static unsigned long folio_free_raw_hwp(struct folio *folio, bool move_flag) 1941{ 1942 /* 1943 * hugetlb_vmemmap_optimized hugepages can't be freed because struct 1944 * pages for tail pages are required but they don't exist. 1945 */ 1946 if (move_flag && folio_test_hugetlb_vmemmap_optimized(folio)) 1947 return 0; 1948 1949 /* 1950 * hugetlb_raw_hwp_unreliable hugepages shouldn't be unpoisoned by 1951 * definition. 1952 */ 1953 if (folio_test_hugetlb_raw_hwp_unreliable(folio)) 1954 return 0; 1955 1956 return __folio_free_raw_hwp(folio, move_flag); 1957} 1958 1959void folio_clear_hugetlb_hwpoison(struct folio *folio) 1960{ 1961 if (folio_test_hugetlb_raw_hwp_unreliable(folio)) 1962 return; 1963 if (folio_test_hugetlb_vmemmap_optimized(folio)) 1964 return; 1965 folio_clear_hwpoison(folio); 1966 folio_free_raw_hwp(folio, true); 1967} 1968 1969/* 1970 * Called from hugetlb code with hugetlb_lock held. 1971 */ 1972int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, 1973 bool *migratable_cleared) 1974{ 1975 struct page *page = pfn_to_page(pfn); 1976 struct folio *folio = page_folio(page); 1977 bool count_increased = false; 1978 int ret, rc; 1979 1980 if (!folio_test_hugetlb(folio)) { 1981 ret = MF_HUGETLB_NON_HUGEPAGE; 1982 goto out; 1983 } else if (flags & MF_COUNT_INCREASED) { 1984 ret = MF_HUGETLB_IN_USED; 1985 count_increased = true; 1986 } else if (folio_test_hugetlb_freed(folio)) { 1987 ret = MF_HUGETLB_FREED; 1988 } else if (folio_test_hugetlb_migratable(folio)) { 1989 if (folio_try_get(folio)) { 1990 ret = MF_HUGETLB_IN_USED; 1991 count_increased = true; 1992 } else { 1993 ret = MF_HUGETLB_FREED; 1994 } 1995 } else { 1996 ret = MF_HUGETLB_RETRY; 1997 if (!(flags & MF_NO_RETRY)) 1998 goto out; 1999 } 2000 2001 rc = hugetlb_update_hwpoison(folio, page); 2002 if (rc >= MF_HUGETLB_FOLIO_PRE_POISONED) { 2003 ret = rc; 2004 goto out; 2005 } 2006 2007 /* 2008 * Clearing hugetlb_migratable for hwpoisoned hugepages to prevent them 2009 * from being migrated by memory hotremove. 2010 */ 2011 if (count_increased && folio_test_hugetlb_migratable(folio)) { 2012 folio_clear_hugetlb_migratable(folio); 2013 *migratable_cleared = true; 2014 } 2015 2016 return ret; 2017out: 2018 if (count_increased) 2019 folio_put(folio); 2020 return ret; 2021} 2022 2023/* 2024 * Taking refcount of hugetlb pages needs extra care about race conditions 2025 * with basic operations like hugepage allocation/free/demotion. 2026 * So some of prechecks for hwpoison (pinning, and testing/setting 2027 * PageHWPoison) should be done in single hugetlb_lock range. 2028 * Returns: 2029 * 0 - not hugetlb, or recovered 2030 * -EBUSY - not recovered 2031 * -EOPNOTSUPP - hwpoison_filter'ed 2032 * -EHWPOISON - folio or exact page already poisoned 2033 * -EFAULT - kill_accessing_process finds current->mm null 2034 */ 2035static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb) 2036{ 2037 int res, rv; 2038 struct page *p = pfn_to_page(pfn); 2039 struct folio *folio; 2040 unsigned long page_flags; 2041 bool migratable_cleared = false; 2042 2043 *hugetlb = 1; 2044retry: 2045 res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared); 2046 switch (res) { 2047 case MF_HUGETLB_NON_HUGEPAGE: /* fallback to normal page handling */ 2048 *hugetlb = 0; 2049 return 0; 2050 case MF_HUGETLB_RETRY: 2051 if (!(flags & MF_NO_RETRY)) { 2052 flags |= MF_NO_RETRY; 2053 goto retry; 2054 } 2055 return action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); 2056 case MF_HUGETLB_FOLIO_PRE_POISONED: 2057 case MF_HUGETLB_PAGE_PRE_POISONED: 2058 rv = -EHWPOISON; 2059 if (flags & MF_ACTION_REQUIRED) 2060 rv = kill_accessing_process(current, pfn, flags); 2061 if (res == MF_HUGETLB_PAGE_PRE_POISONED) 2062 action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED); 2063 else 2064 action_result(pfn, MF_MSG_HUGE, MF_FAILED); 2065 return rv; 2066 default: 2067 WARN_ON((res != MF_HUGETLB_FREED) && (res != MF_HUGETLB_IN_USED)); 2068 break; 2069 } 2070 2071 folio = page_folio(p); 2072 folio_lock(folio); 2073 2074 if (hwpoison_filter(p)) { 2075 folio_clear_hugetlb_hwpoison(folio); 2076 if (migratable_cleared) 2077 folio_set_hugetlb_migratable(folio); 2078 folio_unlock(folio); 2079 if (res == MF_HUGETLB_IN_USED) 2080 folio_put(folio); 2081 return -EOPNOTSUPP; 2082 } 2083 2084 /* 2085 * Handling free hugepage. The possible race with hugepage allocation 2086 * or demotion can be prevented by PageHWPoison flag. 2087 */ 2088 if (res == MF_HUGETLB_FREED) { 2089 folio_unlock(folio); 2090 if (__page_handle_poison(p) > 0) { 2091 page_ref_inc(p); 2092 res = MF_RECOVERED; 2093 } else { 2094 res = MF_FAILED; 2095 } 2096 return action_result(pfn, MF_MSG_FREE_HUGE, res); 2097 } 2098 2099 page_flags = folio->flags.f; 2100 2101 if (!hwpoison_user_mappings(folio, p, pfn, flags)) { 2102 folio_unlock(folio); 2103 return action_result(pfn, MF_MSG_UNMAP_FAILED, MF_FAILED); 2104 } 2105 2106 return identify_page_state(pfn, p, page_flags); 2107} 2108 2109#else 2110static inline int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb) 2111{ 2112 return 0; 2113} 2114 2115static inline unsigned long folio_free_raw_hwp(struct folio *folio, bool flag) 2116{ 2117 return 0; 2118} 2119#endif /* CONFIG_HUGETLB_PAGE */ 2120 2121/* Drop the extra refcount in case we come from madvise() */ 2122static void put_ref_page(unsigned long pfn, int flags) 2123{ 2124 if (!(flags & MF_COUNT_INCREASED)) 2125 return; 2126 2127 put_page(pfn_to_page(pfn)); 2128} 2129 2130static int memory_failure_dev_pagemap(unsigned long pfn, int flags, 2131 struct dev_pagemap *pgmap) 2132{ 2133 int rc = -ENXIO; 2134 2135 /* device metadata space is not recoverable */ 2136 if (!pgmap_pfn_valid(pgmap, pfn)) 2137 goto out; 2138 2139 /* 2140 * Call driver's implementation to handle the memory failure, otherwise 2141 * fall back to generic handler. 2142 */ 2143 if (pgmap_has_memory_failure(pgmap)) { 2144 rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); 2145 /* 2146 * Fall back to generic handler too if operation is not 2147 * supported inside the driver/device/filesystem. 2148 */ 2149 if (rc != -EOPNOTSUPP) 2150 goto out; 2151 } 2152 2153 rc = mf_generic_kill_procs(pfn, flags, pgmap); 2154out: 2155 /* drop pgmap ref acquired in caller */ 2156 put_dev_pagemap(pgmap); 2157 if (rc != -EOPNOTSUPP) 2158 action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED); 2159 return rc; 2160} 2161 2162/* 2163 * The calling condition is as such: thp split failed, page might have 2164 * been RDMA pinned, not much can be done for recovery. 2165 * But a SIGBUS should be delivered with vaddr provided so that the user 2166 * application has a chance to recover. Also, application processes' 2167 * election for MCE early killed will be honored. 2168 */ 2169static void kill_procs_now(struct page *p, unsigned long pfn, int flags, 2170 struct folio *folio) 2171{ 2172 LIST_HEAD(tokill); 2173 2174 folio_lock(folio); 2175 collect_procs(folio, p, &tokill, flags & MF_ACTION_REQUIRED); 2176 folio_unlock(folio); 2177 2178 kill_procs(&tokill, true, pfn, flags); 2179} 2180 2181int register_pfn_address_space(struct pfn_address_space *pfn_space) 2182{ 2183 guard(mutex)(&pfn_space_lock); 2184 2185 if (!pfn_space->pfn_to_vma_pgoff) 2186 return -EINVAL; 2187 2188 if (interval_tree_iter_first(&pfn_space_itree, 2189 pfn_space->node.start, 2190 pfn_space->node.last)) 2191 return -EBUSY; 2192 2193 interval_tree_insert(&pfn_space->node, &pfn_space_itree); 2194 2195 return 0; 2196} 2197EXPORT_SYMBOL_GPL(register_pfn_address_space); 2198 2199void unregister_pfn_address_space(struct pfn_address_space *pfn_space) 2200{ 2201 guard(mutex)(&pfn_space_lock); 2202 2203 if (interval_tree_iter_first(&pfn_space_itree, 2204 pfn_space->node.start, 2205 pfn_space->node.last)) 2206 interval_tree_remove(&pfn_space->node, &pfn_space_itree); 2207} 2208EXPORT_SYMBOL_GPL(unregister_pfn_address_space); 2209 2210static void add_to_kill_pgoff(struct task_struct *tsk, 2211 struct vm_area_struct *vma, 2212 struct list_head *to_kill, 2213 pgoff_t pgoff) 2214{ 2215 struct to_kill *tk; 2216 2217 tk = kmalloc(sizeof(*tk), GFP_ATOMIC); 2218 if (!tk) { 2219 pr_info("Unable to kill proc %d\n", tsk->pid); 2220 return; 2221 } 2222 2223 /* Check for pgoff not backed by struct page */ 2224 tk->addr = vma_address(vma, pgoff, 1); 2225 tk->size_shift = PAGE_SHIFT; 2226 2227 if (tk->addr == -EFAULT) 2228 pr_info("Unable to find address %lx in %s\n", 2229 pgoff, tsk->comm); 2230 2231 get_task_struct(tsk); 2232 tk->tsk = tsk; 2233 list_add_tail(&tk->nd, to_kill); 2234} 2235 2236/* 2237 * Collect processes when the error hit a PFN not backed by struct page. 2238 */ 2239static void collect_procs_pfn(struct pfn_address_space *pfn_space, 2240 unsigned long pfn, struct list_head *to_kill) 2241{ 2242 struct vm_area_struct *vma; 2243 struct task_struct *tsk; 2244 struct address_space *mapping = pfn_space->mapping; 2245 2246 i_mmap_lock_read(mapping); 2247 rcu_read_lock(); 2248 for_each_process(tsk) { 2249 struct task_struct *t = tsk; 2250 2251 t = task_early_kill(tsk, true); 2252 if (!t) 2253 continue; 2254 vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) { 2255 pgoff_t pgoff; 2256 2257 if (vma->vm_mm == t->mm && 2258 !pfn_space->pfn_to_vma_pgoff(vma, pfn, &pgoff)) 2259 add_to_kill_pgoff(t, vma, to_kill, pgoff); 2260 } 2261 } 2262 rcu_read_unlock(); 2263 i_mmap_unlock_read(mapping); 2264} 2265 2266/** 2267 * memory_failure_pfn - Handle memory failure on a page not backed by 2268 * struct page. 2269 * @pfn: Page Number of the corrupted page 2270 * @flags: fine tune action taken 2271 * 2272 * Return: 2273 * 0 - success, 2274 * -EBUSY - Page PFN does not belong to any address space mapping. 2275 */ 2276static int memory_failure_pfn(unsigned long pfn, int flags) 2277{ 2278 struct interval_tree_node *node; 2279 LIST_HEAD(tokill); 2280 2281 scoped_guard(mutex, &pfn_space_lock) { 2282 bool mf_handled = false; 2283 2284 /* 2285 * Modules registers with MM the address space mapping to 2286 * the device memory they manage. Iterate to identify 2287 * exactly which address space has mapped to this failing 2288 * PFN. 2289 */ 2290 for (node = interval_tree_iter_first(&pfn_space_itree, pfn, pfn); node; 2291 node = interval_tree_iter_next(node, pfn, pfn)) { 2292 struct pfn_address_space *pfn_space = 2293 container_of(node, struct pfn_address_space, node); 2294 2295 collect_procs_pfn(pfn_space, pfn, &tokill); 2296 2297 mf_handled = true; 2298 } 2299 2300 if (!mf_handled) 2301 return action_result(pfn, MF_MSG_PFN_MAP, MF_IGNORED); 2302 } 2303 2304 /* 2305 * Unlike System-RAM there is no possibility to swap in a different 2306 * physical page at a given virtual address, so all userspace 2307 * consumption of direct PFN memory necessitates SIGBUS (i.e. 2308 * MF_MUST_KILL) 2309 */ 2310 flags |= MF_ACTION_REQUIRED | MF_MUST_KILL; 2311 2312 kill_procs(&tokill, true, pfn, flags); 2313 2314 return action_result(pfn, MF_MSG_PFN_MAP, MF_RECOVERED); 2315} 2316 2317/** 2318 * memory_failure - Handle memory failure of a page. 2319 * @pfn: Page Number of the corrupted page 2320 * @flags: fine tune action taken 2321 * 2322 * This function is called by the low level machine check code 2323 * of an architecture when it detects hardware memory corruption 2324 * of a page. It tries its best to recover, which includes 2325 * dropping pages, killing processes etc. 2326 * 2327 * The function is primarily of use for corruptions that 2328 * happen outside the current execution context (e.g. when 2329 * detected by a background scrubber) 2330 * 2331 * Must run in process context (e.g. a work queue) with interrupts 2332 * enabled and no spinlocks held. 2333 * 2334 * Return: 2335 * 0 - success, 2336 * -ENXIO - memory not managed by the kernel 2337 * -EOPNOTSUPP - hwpoison_filter() filtered the error event, 2338 * -EHWPOISON - the page was already poisoned, potentially 2339 * kill process, 2340 * other negative values - failure. 2341 */ 2342int memory_failure(unsigned long pfn, int flags) 2343{ 2344 struct page *p; 2345 struct folio *folio; 2346 struct dev_pagemap *pgmap; 2347 int res = 0; 2348 unsigned long page_flags; 2349 bool retry = true; 2350 int hugetlb = 0; 2351 2352 if (!sysctl_memory_failure_recovery) 2353 panic("Memory failure on page %lx", pfn); 2354 2355 mutex_lock(&mf_mutex); 2356 2357 if (!(flags & MF_SW_SIMULATED)) 2358 hw_memory_failure = true; 2359 2360 p = pfn_to_online_page(pfn); 2361 if (!p) { 2362 res = arch_memory_failure(pfn, flags); 2363 if (res == 0) 2364 goto unlock_mutex; 2365 2366 if (!pfn_valid(pfn) && !arch_is_platform_page(PFN_PHYS(pfn))) { 2367 /* 2368 * The PFN is not backed by struct page. 2369 */ 2370 res = memory_failure_pfn(pfn, flags); 2371 goto unlock_mutex; 2372 } 2373 2374 if (pfn_valid(pfn)) { 2375 pgmap = get_dev_pagemap(pfn); 2376 put_ref_page(pfn, flags); 2377 if (pgmap) { 2378 res = memory_failure_dev_pagemap(pfn, flags, 2379 pgmap); 2380 goto unlock_mutex; 2381 } 2382 } 2383 pr_err("%#lx: memory outside kernel control\n", pfn); 2384 res = -ENXIO; 2385 goto unlock_mutex; 2386 } 2387 2388try_again: 2389 res = try_memory_failure_hugetlb(pfn, flags, &hugetlb); 2390 if (hugetlb) 2391 goto unlock_mutex; 2392 2393 if (TestSetPageHWPoison(p)) { 2394 res = -EHWPOISON; 2395 if (flags & MF_ACTION_REQUIRED) 2396 res = kill_accessing_process(current, pfn, flags); 2397 if (flags & MF_COUNT_INCREASED) 2398 put_page(p); 2399 action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED); 2400 goto unlock_mutex; 2401 } 2402 2403 /* 2404 * We need/can do nothing about count=0 pages. 2405 * 1) it's a free page, and therefore in safe hand: 2406 * check_new_page() will be the gate keeper. 2407 * 2) it's part of a non-compound high order page. 2408 * Implies some kernel user: cannot stop them from 2409 * R/W the page; let's pray that the page has been 2410 * used and will be freed some time later. 2411 * In fact it's dangerous to directly bump up page count from 0, 2412 * that may make page_ref_freeze()/page_ref_unfreeze() mismatch. 2413 */ 2414 res = get_hwpoison_page(p, flags); 2415 if (!res) { 2416 if (is_free_buddy_page(p)) { 2417 if (take_page_off_buddy(p)) { 2418 page_ref_inc(p); 2419 res = MF_RECOVERED; 2420 } else { 2421 /* We lost the race, try again */ 2422 if (retry) { 2423 ClearPageHWPoison(p); 2424 retry = false; 2425 goto try_again; 2426 } 2427 res = MF_FAILED; 2428 } 2429 res = action_result(pfn, MF_MSG_BUDDY, res); 2430 } else { 2431 res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED); 2432 } 2433 goto unlock_mutex; 2434 } else if (res < 0) { 2435 res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED); 2436 goto unlock_mutex; 2437 } 2438 2439 folio = page_folio(p); 2440 2441 /* filter pages that are protected from hwpoison test by users */ 2442 folio_lock(folio); 2443 if (hwpoison_filter(p)) { 2444 ClearPageHWPoison(p); 2445 folio_unlock(folio); 2446 folio_put(folio); 2447 res = -EOPNOTSUPP; 2448 goto unlock_mutex; 2449 } 2450 folio_unlock(folio); 2451 2452 if (folio_test_large(folio)) { 2453 const int new_order = min_order_for_split(folio); 2454 int err; 2455 2456 /* 2457 * The flag must be set after the refcount is bumped 2458 * otherwise it may race with THP split. 2459 * And the flag can't be set in get_hwpoison_page() since 2460 * it is called by soft offline too and it is just called 2461 * for !MF_COUNT_INCREASED. So here seems to be the best 2462 * place. 2463 * 2464 * Don't need care about the above error handling paths for 2465 * get_hwpoison_page() since they handle either free page 2466 * or unhandlable page. The refcount is bumped iff the 2467 * page is a valid handlable page. 2468 */ 2469 folio_set_has_hwpoisoned(folio); 2470 err = try_to_split_thp_page(p, new_order, /* release= */ false); 2471 /* 2472 * If splitting a folio to order-0 fails, kill the process. 2473 * Split the folio regardless to minimize unusable pages. 2474 * Because the memory failure code cannot handle large 2475 * folios, this split is always treated as if it failed. 2476 */ 2477 if (err || new_order) { 2478 /* get folio again in case the original one is split */ 2479 folio = page_folio(p); 2480 res = -EHWPOISON; 2481 kill_procs_now(p, pfn, flags, folio); 2482 put_page(p); 2483 action_result(pfn, MF_MSG_UNSPLIT_THP, MF_FAILED); 2484 goto unlock_mutex; 2485 } 2486 VM_BUG_ON_PAGE(!page_count(p), p); 2487 folio = page_folio(p); 2488 } 2489 2490 /* 2491 * We ignore non-LRU pages for good reasons. 2492 * - PG_locked is only well defined for LRU pages and a few others 2493 * - to avoid races with __SetPageLocked() 2494 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) 2495 * The check (unnecessarily) ignores LRU pages being isolated and 2496 * walked by the page reclaim code, however that's not a big loss. 2497 */ 2498 shake_folio(folio); 2499 2500 folio_lock(folio); 2501 2502 /* 2503 * We're only intended to deal with the non-Compound page here. 2504 * The page cannot become compound pages again as folio has been 2505 * splited and extra refcnt is held. 2506 */ 2507 WARN_ON(folio_test_large(folio)); 2508 2509 /* 2510 * We use page flags to determine what action should be taken, but 2511 * the flags can be modified by the error containment action. One 2512 * example is an mlocked page, where PG_mlocked is cleared by 2513 * folio_remove_rmap_*() in try_to_unmap_one(). So to determine page 2514 * status correctly, we save a copy of the page flags at this time. 2515 */ 2516 page_flags = folio->flags.f; 2517 2518 /* 2519 * __munlock_folio() may clear a writeback folio's LRU flag without 2520 * the folio lock. We need to wait for writeback completion for this 2521 * folio or it may trigger a vfs BUG while evicting inode. 2522 */ 2523 if (!folio_test_lru(folio) && !folio_test_writeback(folio)) 2524 goto identify_page_state; 2525 2526 /* 2527 * It's very difficult to mess with pages currently under IO 2528 * and in many cases impossible, so we just avoid it here. 2529 */ 2530 folio_wait_writeback(folio); 2531 2532 /* 2533 * Now take care of user space mappings. 2534 * Abort on fail: __filemap_remove_folio() assumes unmapped page. 2535 */ 2536 if (!hwpoison_user_mappings(folio, p, pfn, flags)) { 2537 res = action_result(pfn, MF_MSG_UNMAP_FAILED, MF_FAILED); 2538 goto unlock_page; 2539 } 2540 2541 /* 2542 * Torn down by someone else? 2543 */ 2544 if (folio_test_lru(folio) && !folio_test_swapcache(folio) && 2545 folio->mapping == NULL) { 2546 res = action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED); 2547 goto unlock_page; 2548 } 2549 2550identify_page_state: 2551 res = identify_page_state(pfn, p, page_flags); 2552 mutex_unlock(&mf_mutex); 2553 return res; 2554unlock_page: 2555 folio_unlock(folio); 2556unlock_mutex: 2557 mutex_unlock(&mf_mutex); 2558 return res; 2559} 2560EXPORT_SYMBOL_GPL(memory_failure); 2561 2562#define MEMORY_FAILURE_FIFO_ORDER 4 2563#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER) 2564 2565struct memory_failure_entry { 2566 unsigned long pfn; 2567 int flags; 2568}; 2569 2570struct memory_failure_cpu { 2571 DECLARE_KFIFO(fifo, struct memory_failure_entry, 2572 MEMORY_FAILURE_FIFO_SIZE); 2573 raw_spinlock_t lock; 2574 struct work_struct work; 2575}; 2576 2577static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu); 2578 2579/** 2580 * memory_failure_queue - Schedule handling memory failure of a page. 2581 * @pfn: Page Number of the corrupted page 2582 * @flags: Flags for memory failure handling 2583 * 2584 * This function is called by the low level hardware error handler 2585 * when it detects hardware memory corruption of a page. It schedules 2586 * the recovering of error page, including dropping pages, killing 2587 * processes etc. 2588 * 2589 * The function is primarily of use for corruptions that 2590 * happen outside the current execution context (e.g. when 2591 * detected by a background scrubber) 2592 * 2593 * Can run in IRQ context. 2594 */ 2595void memory_failure_queue(unsigned long pfn, int flags) 2596{ 2597 struct memory_failure_cpu *mf_cpu; 2598 unsigned long proc_flags; 2599 bool buffer_overflow; 2600 struct memory_failure_entry entry = { 2601 .pfn = pfn, 2602 .flags = flags, 2603 }; 2604 2605 mf_cpu = &get_cpu_var(memory_failure_cpu); 2606 raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags); 2607 buffer_overflow = !kfifo_put(&mf_cpu->fifo, entry); 2608 if (!buffer_overflow) 2609 schedule_work_on(smp_processor_id(), &mf_cpu->work); 2610 raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 2611 put_cpu_var(memory_failure_cpu); 2612 if (buffer_overflow) 2613 pr_err("buffer overflow when queuing memory failure at %#lx\n", 2614 pfn); 2615} 2616EXPORT_SYMBOL_GPL(memory_failure_queue); 2617 2618static void memory_failure_work_func(struct work_struct *work) 2619{ 2620 struct memory_failure_cpu *mf_cpu; 2621 struct memory_failure_entry entry = { 0, }; 2622 unsigned long proc_flags; 2623 int gotten; 2624 2625 mf_cpu = container_of(work, struct memory_failure_cpu, work); 2626 for (;;) { 2627 raw_spin_lock_irqsave(&mf_cpu->lock, proc_flags); 2628 gotten = kfifo_get(&mf_cpu->fifo, &entry); 2629 raw_spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); 2630 if (!gotten) 2631 break; 2632 if (entry.flags & MF_SOFT_OFFLINE) 2633 soft_offline_page(entry.pfn, entry.flags); 2634 else 2635 memory_failure(entry.pfn, entry.flags); 2636 } 2637} 2638 2639static int __init memory_failure_init(void) 2640{ 2641 struct memory_failure_cpu *mf_cpu; 2642 int cpu; 2643 2644 for_each_possible_cpu(cpu) { 2645 mf_cpu = &per_cpu(memory_failure_cpu, cpu); 2646 raw_spin_lock_init(&mf_cpu->lock); 2647 INIT_KFIFO(mf_cpu->fifo); 2648 INIT_WORK(&mf_cpu->work, memory_failure_work_func); 2649 } 2650 2651 register_sysctl_init("vm", memory_failure_table); 2652 2653 return 0; 2654} 2655core_initcall(memory_failure_init); 2656 2657#undef pr_fmt 2658#define pr_fmt(fmt) "Unpoison: " fmt 2659#define unpoison_pr_info(fmt, pfn, rs) \ 2660({ \ 2661 if (__ratelimit(rs)) \ 2662 pr_info(fmt, pfn); \ 2663}) 2664 2665/** 2666 * unpoison_memory - Unpoison a previously poisoned page 2667 * @pfn: Page number of the to be unpoisoned page 2668 * 2669 * Software-unpoison a page that has been poisoned by 2670 * memory_failure() earlier. 2671 * 2672 * This is only done on the software-level, so it only works 2673 * for linux injected failures, not real hardware failures 2674 * 2675 * Returns 0 for success, otherwise -errno. 2676 */ 2677int unpoison_memory(unsigned long pfn) 2678{ 2679 struct folio *folio; 2680 struct page *p; 2681 int ret = -EBUSY, ghp; 2682 unsigned long count; 2683 bool huge = false; 2684 static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL, 2685 DEFAULT_RATELIMIT_BURST); 2686 2687 p = pfn_to_online_page(pfn); 2688 if (!p) 2689 return -EIO; 2690 folio = page_folio(p); 2691 2692 mutex_lock(&mf_mutex); 2693 2694 if (hw_memory_failure) { 2695 unpoison_pr_info("%#lx: disabled after HW memory failure\n", 2696 pfn, &unpoison_rs); 2697 ret = -EOPNOTSUPP; 2698 goto unlock_mutex; 2699 } 2700 2701 if (is_huge_zero_folio(folio)) { 2702 unpoison_pr_info("%#lx: huge zero page is not supported\n", 2703 pfn, &unpoison_rs); 2704 ret = -EOPNOTSUPP; 2705 goto unlock_mutex; 2706 } 2707 2708 if (!PageHWPoison(p)) { 2709 unpoison_pr_info("%#lx: page was already unpoisoned\n", 2710 pfn, &unpoison_rs); 2711 goto unlock_mutex; 2712 } 2713 2714 if (folio_ref_count(folio) > 1) { 2715 unpoison_pr_info("%#lx: someone grabs the hwpoison page\n", 2716 pfn, &unpoison_rs); 2717 goto unlock_mutex; 2718 } 2719 2720 if (folio_test_slab(folio) || folio_test_pgtable(folio) || 2721 folio_test_reserved(folio) || folio_test_offline(folio)) 2722 goto unlock_mutex; 2723 2724 if (folio_mapped(folio)) { 2725 unpoison_pr_info("%#lx: someone maps the hwpoison page\n", 2726 pfn, &unpoison_rs); 2727 goto unlock_mutex; 2728 } 2729 2730 if (folio_mapping(folio)) { 2731 unpoison_pr_info("%#lx: the hwpoison page has non-NULL mapping\n", 2732 pfn, &unpoison_rs); 2733 goto unlock_mutex; 2734 } 2735 2736 ghp = get_hwpoison_page(p, MF_UNPOISON); 2737 if (!ghp) { 2738 if (folio_test_hugetlb(folio)) { 2739 huge = true; 2740 count = folio_free_raw_hwp(folio, false); 2741 if (count == 0) 2742 goto unlock_mutex; 2743 } 2744 ret = folio_test_clear_hwpoison(folio) ? 0 : -EBUSY; 2745 } else if (ghp < 0) { 2746 if (ghp == -EHWPOISON) { 2747 ret = put_page_back_buddy(p) ? 0 : -EBUSY; 2748 } else { 2749 ret = ghp; 2750 unpoison_pr_info("%#lx: failed to grab page\n", 2751 pfn, &unpoison_rs); 2752 } 2753 } else { 2754 if (folio_test_hugetlb(folio)) { 2755 huge = true; 2756 count = folio_free_raw_hwp(folio, false); 2757 if (count == 0) { 2758 folio_put(folio); 2759 goto unlock_mutex; 2760 } 2761 } 2762 2763 folio_put(folio); 2764 if (TestClearPageHWPoison(p)) { 2765 folio_put(folio); 2766 ret = 0; 2767 } 2768 } 2769 2770unlock_mutex: 2771 mutex_unlock(&mf_mutex); 2772 if (!ret) { 2773 if (!huge) 2774 num_poisoned_pages_sub(pfn, 1); 2775 unpoison_pr_info("%#lx: software-unpoisoned page\n", 2776 page_to_pfn(p), &unpoison_rs); 2777 } 2778 return ret; 2779} 2780EXPORT_SYMBOL(unpoison_memory); 2781 2782#undef pr_fmt 2783#define pr_fmt(fmt) "Soft offline: " fmt 2784 2785/* 2786 * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages. 2787 * If the page is a non-dirty unmapped page-cache page, it simply invalidates. 2788 * If the page is mapped, it migrates the contents over. 2789 */ 2790static int soft_offline_in_use_page(struct page *page) 2791{ 2792 long ret = 0; 2793 unsigned long pfn = page_to_pfn(page); 2794 struct folio *folio = page_folio(page); 2795 char const *msg_page[] = {"page", "hugepage"}; 2796 bool huge = folio_test_hugetlb(folio); 2797 bool isolated; 2798 LIST_HEAD(pagelist); 2799 struct migration_target_control mtc = { 2800 .nid = NUMA_NO_NODE, 2801 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 2802 .reason = MR_MEMORY_FAILURE, 2803 }; 2804 2805 if (!huge && folio_test_large(folio)) { 2806 const int new_order = min_order_for_split(folio); 2807 2808 /* 2809 * If new_order (target split order) is not 0, do not split the 2810 * folio at all to retain the still accessible large folio. 2811 * NOTE: if minimizing the number of soft offline pages is 2812 * preferred, split it to non-zero new_order like it is done in 2813 * memory_failure(). 2814 */ 2815 if (new_order || try_to_split_thp_page(page, /* new_order= */ 0, 2816 /* release= */ true)) { 2817 pr_info("%#lx: thp split failed\n", pfn); 2818 return -EBUSY; 2819 } 2820 folio = page_folio(page); 2821 } 2822 2823 folio_lock(folio); 2824 if (!huge) 2825 folio_wait_writeback(folio); 2826 if (PageHWPoison(page)) { 2827 folio_unlock(folio); 2828 folio_put(folio); 2829 pr_info("%#lx: page already poisoned\n", pfn); 2830 return 0; 2831 } 2832 2833 if (!huge && folio_test_lru(folio) && !folio_test_swapcache(folio)) 2834 /* 2835 * Try to invalidate first. This should work for 2836 * non dirty unmapped page cache pages. 2837 */ 2838 ret = mapping_evict_folio(folio_mapping(folio), folio); 2839 folio_unlock(folio); 2840 2841 if (ret) { 2842 pr_info("%#lx: invalidated\n", pfn); 2843 page_handle_poison(page, false, true); 2844 return 0; 2845 } 2846 2847 isolated = isolate_folio_to_list(folio, &pagelist); 2848 2849 /* 2850 * If we succeed to isolate the folio, we grabbed another refcount on 2851 * the folio, so we can safely drop the one we got from get_any_page(). 2852 * If we failed to isolate the folio, it means that we cannot go further 2853 * and we will return an error, so drop the reference we got from 2854 * get_any_page() as well. 2855 */ 2856 folio_put(folio); 2857 2858 if (isolated) { 2859 ret = migrate_pages(&pagelist, alloc_migration_target, NULL, 2860 (unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_FAILURE, NULL); 2861 if (!ret) { 2862 bool release = !huge; 2863 2864 if (!page_handle_poison(page, huge, release)) 2865 ret = -EBUSY; 2866 } else { 2867 if (!list_empty(&pagelist)) 2868 putback_movable_pages(&pagelist); 2869 2870 pr_info("%#lx: %s migration failed %ld, type %pGp\n", 2871 pfn, msg_page[huge], ret, &page->flags.f); 2872 if (ret > 0) 2873 ret = -EBUSY; 2874 } 2875 } else { 2876 pr_info("%#lx: %s isolation failed, page count %d, type %pGp\n", 2877 pfn, msg_page[huge], page_count(page), &page->flags.f); 2878 ret = -EBUSY; 2879 } 2880 return ret; 2881} 2882 2883/** 2884 * soft_offline_page - Soft offline a page. 2885 * @pfn: pfn to soft-offline 2886 * @flags: flags. Same as memory_failure(). 2887 * 2888 * Returns 0 on success, 2889 * -EOPNOTSUPP for hwpoison_filter() filtered the error event, or 2890 * disabled by /proc/sys/vm/enable_soft_offline, 2891 * < 0 otherwise negated errno. 2892 * 2893 * Soft offline a page, by migration or invalidation, 2894 * without killing anything. This is for the case when 2895 * a page is not corrupted yet (so it's still valid to access), 2896 * but has had a number of corrected errors and is better taken 2897 * out. 2898 * 2899 * The actual policy on when to do that is maintained by 2900 * user space. 2901 * 2902 * This should never impact any application or cause data loss, 2903 * however it might take some time. 2904 * 2905 * This is not a 100% solution for all memory, but tries to be 2906 * ``good enough'' for the majority of memory. 2907 */ 2908int soft_offline_page(unsigned long pfn, int flags) 2909{ 2910 int ret; 2911 bool try_again = true; 2912 struct page *page; 2913 2914 if (!pfn_valid(pfn)) { 2915 WARN_ON_ONCE(flags & MF_COUNT_INCREASED); 2916 return -ENXIO; 2917 } 2918 2919 /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */ 2920 page = pfn_to_online_page(pfn); 2921 if (!page) { 2922 put_ref_page(pfn, flags); 2923 return -EIO; 2924 } 2925 2926 if (!sysctl_enable_soft_offline) { 2927 pr_info_once("disabled by /proc/sys/vm/enable_soft_offline\n"); 2928 put_ref_page(pfn, flags); 2929 return -EOPNOTSUPP; 2930 } 2931 2932 mutex_lock(&mf_mutex); 2933 2934 if (PageHWPoison(page)) { 2935 pr_info("%#lx: page already poisoned\n", pfn); 2936 put_ref_page(pfn, flags); 2937 mutex_unlock(&mf_mutex); 2938 return 0; 2939 } 2940 2941retry: 2942 get_online_mems(); 2943 ret = get_hwpoison_page(page, flags | MF_SOFT_OFFLINE); 2944 put_online_mems(); 2945 2946 if (hwpoison_filter(page)) { 2947 if (ret > 0) 2948 put_page(page); 2949 2950 mutex_unlock(&mf_mutex); 2951 return -EOPNOTSUPP; 2952 } 2953 2954 if (ret > 0) { 2955 ret = soft_offline_in_use_page(page); 2956 } else if (ret == 0) { 2957 if (!page_handle_poison(page, true, false)) { 2958 if (try_again) { 2959 try_again = false; 2960 flags &= ~MF_COUNT_INCREASED; 2961 goto retry; 2962 } 2963 ret = -EBUSY; 2964 } 2965 } 2966 2967 mutex_unlock(&mf_mutex); 2968 2969 return ret; 2970}
~/
Read-only file preview.