FILE
core.c
File preview with syntax highlighting.
linux-6.19.14/arch/x86/kernel/cpu/mce/core.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Machine check handler. 4 * 5 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs. 6 * Rest from unknown author(s). 7 * 2004 Andi Kleen. Rewrote most of it. 8 * Copyright 2008 Intel Corporation 9 * Author: Andi Kleen 10 */ 11 12#include <linux/thread_info.h> 13#include <linux/capability.h> 14#include <linux/miscdevice.h> 15#include <linux/ratelimit.h> 16#include <linux/rcupdate.h> 17#include <linux/kobject.h> 18#include <linux/uaccess.h> 19#include <linux/kdebug.h> 20#include <linux/kernel.h> 21#include <linux/percpu.h> 22#include <linux/string.h> 23#include <linux/device.h> 24#include <linux/syscore_ops.h> 25#include <linux/delay.h> 26#include <linux/ctype.h> 27#include <linux/sched.h> 28#include <linux/sysfs.h> 29#include <linux/types.h> 30#include <linux/slab.h> 31#include <linux/init.h> 32#include <linux/kmod.h> 33#include <linux/poll.h> 34#include <linux/nmi.h> 35#include <linux/cpu.h> 36#include <linux/ras.h> 37#include <linux/smp.h> 38#include <linux/fs.h> 39#include <linux/mm.h> 40#include <linux/debugfs.h> 41#include <linux/irq_work.h> 42#include <linux/export.h> 43#include <linux/set_memory.h> 44#include <linux/sync_core.h> 45#include <linux/task_work.h> 46#include <linux/hardirq.h> 47#include <linux/kexec.h> 48#include <linux/vmcore_info.h> 49 50#include <asm/fred.h> 51#include <asm/cpu_device_id.h> 52#include <asm/processor.h> 53#include <asm/traps.h> 54#include <asm/tlbflush.h> 55#include <asm/mce.h> 56#include <asm/msr.h> 57#include <asm/reboot.h> 58#include <asm/tdx.h> 59 60#include "internal.h" 61 62/* sysfs synchronization */ 63static DEFINE_MUTEX(mce_sysfs_mutex); 64 65#define CREATE_TRACE_POINTS 66#include <trace/events/mce.h> 67 68#define SPINUNIT 100 /* 100ns */ 69 70DEFINE_PER_CPU(unsigned, mce_exception_count); 71 72DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); 73 74DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); 75 76#define ATTR_LEN 16 77/* One object for each MCE bank, shared by all CPUs */ 78struct mce_bank_dev { 79 struct device_attribute attr; /* device attribute */ 80 char attrname[ATTR_LEN]; /* attribute name */ 81 u8 bank; /* bank number */ 82}; 83static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS]; 84 85struct mce_vendor_flags mce_flags __read_mostly; 86 87struct mca_config mca_cfg __read_mostly = { 88 .bootlog = -1, 89 .monarch_timeout = -1 90}; 91 92static DEFINE_PER_CPU(struct mce_hw_err, hw_errs_seen); 93static unsigned long mce_need_notify; 94 95/* 96 * MCA banks polled by the period polling timer for corrected events. 97 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any). 98 */ 99DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = { 100 [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL 101}; 102 103/* 104 * MCA banks controlled through firmware first for corrected errors. 105 * This is a global list of banks for which we won't enable CMCI and we 106 * won't poll. Firmware controls these banks and is responsible for 107 * reporting corrected errors through GHES. Uncorrected/recoverable 108 * errors are still notified through a machine check. 109 */ 110mce_banks_t mce_banks_ce_disabled; 111 112static struct work_struct mce_work; 113static struct irq_work mce_irq_work; 114 115/* 116 * CPU/chipset specific EDAC code can register a notifier call here to print 117 * MCE errors in a human-readable form. 118 */ 119BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); 120 121void mce_prep_record_common(struct mce *m) 122{ 123 m->cpuid = cpuid_eax(1); 124 m->cpuvendor = boot_cpu_data.x86_vendor; 125 m->mcgcap = native_rdmsrq(MSR_IA32_MCG_CAP); 126 /* need the internal __ version to avoid deadlocks */ 127 m->time = __ktime_get_real_seconds(); 128} 129 130void mce_prep_record_per_cpu(unsigned int cpu, struct mce *m) 131{ 132 m->cpu = cpu; 133 m->extcpu = cpu; 134 m->apicid = cpu_data(cpu).topo.initial_apicid; 135 m->microcode = cpu_data(cpu).microcode; 136 m->ppin = topology_ppin(cpu); 137 m->socketid = topology_physical_package_id(cpu); 138} 139 140/* Do initial initialization of struct mce_hw_err */ 141void mce_prep_record(struct mce_hw_err *err) 142{ 143 struct mce *m = &err->m; 144 145 memset(err, 0, sizeof(struct mce_hw_err)); 146 mce_prep_record_common(m); 147 mce_prep_record_per_cpu(smp_processor_id(), m); 148} 149 150DEFINE_PER_CPU(struct mce, injectm); 151EXPORT_PER_CPU_SYMBOL_GPL(injectm); 152 153void mce_log(struct mce_hw_err *err) 154{ 155 if (mce_gen_pool_add(err)) 156 irq_work_queue(&mce_irq_work); 157} 158EXPORT_SYMBOL_GPL(mce_log); 159 160void mce_register_decode_chain(struct notifier_block *nb) 161{ 162 if (WARN_ON(nb->priority < MCE_PRIO_LOWEST || 163 nb->priority > MCE_PRIO_HIGHEST)) 164 return; 165 166 blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); 167} 168EXPORT_SYMBOL_GPL(mce_register_decode_chain); 169 170void mce_unregister_decode_chain(struct notifier_block *nb) 171{ 172 blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb); 173} 174EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); 175 176static void __print_mce(struct mce_hw_err *err) 177{ 178 struct mce *m = &err->m; 179 180 pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n", 181 m->extcpu, 182 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""), 183 m->mcgstatus, m->bank, m->status); 184 185 if (m->ip) { 186 pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ", 187 !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "", 188 m->cs, m->ip); 189 190 if (m->cs == __KERNEL_CS) 191 pr_cont("{%pS}", (void *)(unsigned long)m->ip); 192 pr_cont("\n"); 193 } 194 195 pr_emerg(HW_ERR "TSC %llx ", m->tsc); 196 if (m->addr) 197 pr_cont("ADDR %llx ", m->addr); 198 if (m->misc) 199 pr_cont("MISC %llx ", m->misc); 200 if (m->ppin) 201 pr_cont("PPIN %llx ", m->ppin); 202 203 if (mce_flags.smca) { 204 if (m->synd) 205 pr_cont("SYND %llx ", m->synd); 206 if (err->vendor.amd.synd1) 207 pr_cont("SYND1 %llx ", err->vendor.amd.synd1); 208 if (err->vendor.amd.synd2) 209 pr_cont("SYND2 %llx ", err->vendor.amd.synd2); 210 if (m->ipid) 211 pr_cont("IPID %llx ", m->ipid); 212 } 213 214 pr_cont("\n"); 215 216 /* 217 * Note this output is parsed by external tools and old fields 218 * should not be changed. 219 */ 220 pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n", 221 m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid, 222 m->microcode); 223} 224 225static void print_mce(struct mce_hw_err *err) 226{ 227 struct mce *m = &err->m; 228 229 __print_mce(err); 230 231 if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON) 232 pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n"); 233} 234 235#define PANIC_TIMEOUT 5 /* 5 seconds */ 236 237static atomic_t mce_panicked; 238 239static int fake_panic; 240static atomic_t mce_fake_panicked; 241 242/* Panic in progress. Enable interrupts and wait for final IPI */ 243static void wait_for_panic(void) 244{ 245 long timeout = PANIC_TIMEOUT*USEC_PER_SEC; 246 247 preempt_disable(); 248 local_irq_enable(); 249 while (timeout-- > 0) 250 udelay(1); 251 if (panic_timeout == 0) 252 panic_timeout = mca_cfg.panic_timeout; 253 panic("Panicing machine check CPU died"); 254} 255 256static const char *mce_dump_aux_info(struct mce *m) 257{ 258 if (boot_cpu_has_bug(X86_BUG_TDX_PW_MCE)) 259 return tdx_dump_mce_info(m); 260 261 return NULL; 262} 263 264static noinstr void mce_panic(const char *msg, struct mce_hw_err *final, char *exp) 265{ 266 struct llist_node *pending; 267 struct mce_evt_llist *l; 268 int apei_err = 0; 269 const char *memmsg; 270 271 /* 272 * Allow instrumentation around external facilities usage. Not that it 273 * matters a whole lot since the machine is going to panic anyway. 274 */ 275 instrumentation_begin(); 276 277 if (!fake_panic) { 278 /* 279 * Make sure only one CPU runs in machine check panic 280 */ 281 if (atomic_inc_return(&mce_panicked) > 1) 282 wait_for_panic(); 283 barrier(); 284 285 bust_spinlocks(1); 286 console_verbose(); 287 } else { 288 /* Don't log too much for fake panic */ 289 if (atomic_inc_return(&mce_fake_panicked) > 1) 290 goto out; 291 } 292 pending = mce_gen_pool_prepare_records(); 293 /* First print corrected ones that are still unlogged */ 294 llist_for_each_entry(l, pending, llnode) { 295 struct mce_hw_err *err = &l->err; 296 struct mce *m = &err->m; 297 if (!(m->status & MCI_STATUS_UC)) { 298 print_mce(err); 299 if (!apei_err) 300 apei_err = apei_write_mce(m); 301 } 302 } 303 /* Now print uncorrected but with the final one last */ 304 llist_for_each_entry(l, pending, llnode) { 305 struct mce_hw_err *err = &l->err; 306 struct mce *m = &err->m; 307 if (!(m->status & MCI_STATUS_UC)) 308 continue; 309 if (!final || mce_cmp(m, &final->m)) { 310 print_mce(err); 311 if (!apei_err) 312 apei_err = apei_write_mce(m); 313 } 314 } 315 if (final) { 316 print_mce(final); 317 if (!apei_err) 318 apei_err = apei_write_mce(&final->m); 319 } 320 if (exp) 321 pr_emerg(HW_ERR "Machine check: %s\n", exp); 322 323 memmsg = mce_dump_aux_info(&final->m); 324 if (memmsg) 325 pr_emerg(HW_ERR "Machine check: %s\n", memmsg); 326 327 if (!fake_panic) { 328 if (panic_timeout == 0) 329 panic_timeout = mca_cfg.panic_timeout; 330 331 /* 332 * Kdump skips the poisoned page in order to avoid 333 * touching the error bits again. Poison the page even 334 * if the error is fatal and the machine is about to 335 * panic. 336 */ 337 if (kexec_crash_loaded()) { 338 if (final && (final->m.status & MCI_STATUS_ADDRV)) { 339 struct page *p; 340 p = pfn_to_online_page(final->m.addr >> PAGE_SHIFT); 341 if (p) 342 SetPageHWPoison(p); 343 } 344 } 345 panic(msg); 346 } else 347 pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg); 348 349out: 350 instrumentation_end(); 351} 352 353/* Support code for software error injection */ 354 355static int msr_to_offset(u32 msr) 356{ 357 unsigned bank = __this_cpu_read(injectm.bank); 358 359 if (msr == mca_cfg.rip_msr) 360 return offsetof(struct mce, ip); 361 if (msr == mca_msr_reg(bank, MCA_STATUS)) 362 return offsetof(struct mce, status); 363 if (msr == mca_msr_reg(bank, MCA_ADDR)) 364 return offsetof(struct mce, addr); 365 if (msr == mca_msr_reg(bank, MCA_MISC)) 366 return offsetof(struct mce, misc); 367 if (msr == MSR_IA32_MCG_STATUS) 368 return offsetof(struct mce, mcgstatus); 369 return -1; 370} 371 372void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) 373{ 374 if (wrmsr) { 375 pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 376 (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, 377 regs->ip, (void *)regs->ip); 378 } else { 379 pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 380 (unsigned int)regs->cx, regs->ip, (void *)regs->ip); 381 } 382 383 show_stack_regs(regs); 384 385 panic("MCA architectural violation!\n"); 386 387 while (true) 388 cpu_relax(); 389} 390 391/* MSR access wrappers used for error injection */ 392noinstr u64 mce_rdmsrq(u32 msr) 393{ 394 EAX_EDX_DECLARE_ARGS(val, low, high); 395 396 if (__this_cpu_read(injectm.finished)) { 397 int offset; 398 u64 ret; 399 400 instrumentation_begin(); 401 402 offset = msr_to_offset(msr); 403 if (offset < 0) 404 ret = 0; 405 else 406 ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset); 407 408 instrumentation_end(); 409 410 return ret; 411 } 412 413 /* 414 * RDMSR on MCA MSRs should not fault. If they do, this is very much an 415 * architectural violation and needs to be reported to hw vendor. Panic 416 * the box to not allow any further progress. 417 */ 418 asm volatile("1: rdmsr\n" 419 "2:\n" 420 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE) 421 : EAX_EDX_RET(val, low, high) : "c" (msr)); 422 423 424 return EAX_EDX_VAL(val, low, high); 425} 426 427noinstr void mce_wrmsrq(u32 msr, u64 v) 428{ 429 u32 low, high; 430 431 if (__this_cpu_read(injectm.finished)) { 432 int offset; 433 434 instrumentation_begin(); 435 436 offset = msr_to_offset(msr); 437 if (offset >= 0) 438 *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v; 439 440 instrumentation_end(); 441 442 return; 443 } 444 445 low = (u32)v; 446 high = (u32)(v >> 32); 447 448 /* See comment in mce_rdmsrq() */ 449 asm volatile("1: wrmsr\n" 450 "2:\n" 451 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE) 452 : : "c" (msr), "a"(low), "d" (high) : "memory"); 453} 454 455/* 456 * Collect all global (w.r.t. this processor) status about this machine 457 * check into our "mce" struct so that we can use it later to assess 458 * the severity of the problem as we read per-bank specific details. 459 */ 460static noinstr void mce_gather_info(struct mce_hw_err *err, struct pt_regs *regs) 461{ 462 struct mce *m; 463 /* 464 * Enable instrumentation around mce_prep_record() which calls external 465 * facilities. 466 */ 467 instrumentation_begin(); 468 mce_prep_record(err); 469 instrumentation_end(); 470 471 m = &err->m; 472 m->mcgstatus = mce_rdmsrq(MSR_IA32_MCG_STATUS); 473 if (regs) { 474 /* 475 * Get the address of the instruction at the time of 476 * the machine check error. 477 */ 478 if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) { 479 m->ip = regs->ip; 480 m->cs = regs->cs; 481 482 /* 483 * When in VM86 mode make the cs look like ring 3 484 * always. This is a lie, but it's better than passing 485 * the additional vm86 bit around everywhere. 486 */ 487 if (v8086_mode(regs)) 488 m->cs |= 3; 489 } 490 /* Use accurate RIP reporting if available. */ 491 if (mca_cfg.rip_msr) 492 m->ip = mce_rdmsrq(mca_cfg.rip_msr); 493 } 494} 495 496bool mce_available(struct cpuinfo_x86 *c) 497{ 498 if (mca_cfg.disabled) 499 return false; 500 return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA); 501} 502 503static void mce_schedule_work(void) 504{ 505 if (!mce_gen_pool_empty()) 506 schedule_work(&mce_work); 507} 508 509static void mce_irq_work_cb(struct irq_work *entry) 510{ 511 mce_schedule_work(); 512} 513 514bool mce_usable_address(struct mce *m) 515{ 516 if (!(m->status & MCI_STATUS_ADDRV)) 517 return false; 518 519 switch (m->cpuvendor) { 520 case X86_VENDOR_AMD: 521 return amd_mce_usable_address(m); 522 523 case X86_VENDOR_INTEL: 524 case X86_VENDOR_ZHAOXIN: 525 return intel_mce_usable_address(m); 526 527 default: 528 return true; 529 } 530} 531EXPORT_SYMBOL_GPL(mce_usable_address); 532 533bool mce_is_memory_error(struct mce *m) 534{ 535 switch (m->cpuvendor) { 536 case X86_VENDOR_AMD: 537 case X86_VENDOR_HYGON: 538 return amd_mce_is_memory_error(m); 539 540 case X86_VENDOR_INTEL: 541 case X86_VENDOR_ZHAOXIN: 542 /* 543 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes 544 * 545 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for 546 * indicating a memory error. Bit 8 is used for indicating a 547 * cache hierarchy error. The combination of bit 2 and bit 3 548 * is used for indicating a `generic' cache hierarchy error 549 * But we can't just blindly check the above bits, because if 550 * bit 11 is set, then it is a bus/interconnect error - and 551 * either way the above bits just gives more detail on what 552 * bus/interconnect error happened. Note that bit 12 can be 553 * ignored, as it's the "filter" bit. 554 */ 555 return (m->status & 0xef80) == BIT(7) || 556 (m->status & 0xef00) == BIT(8) || 557 (m->status & 0xeffc) == 0xc; 558 559 default: 560 return false; 561 } 562} 563EXPORT_SYMBOL_GPL(mce_is_memory_error); 564 565static bool whole_page(struct mce *m) 566{ 567 if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV)) 568 return true; 569 570 return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT; 571} 572 573bool mce_is_correctable(struct mce *m) 574{ 575 if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED) 576 return false; 577 578 if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED) 579 return false; 580 581 if (m->status & MCI_STATUS_UC) 582 return false; 583 584 return true; 585} 586EXPORT_SYMBOL_GPL(mce_is_correctable); 587 588/* 589 * Notify the user(s) about new machine check events. 590 * Can be called from interrupt context, but not from machine check/NMI 591 * context. 592 */ 593static bool mce_notify_irq(void) 594{ 595 /* Not more than two messages every minute */ 596 static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2); 597 598 if (test_and_clear_bit(0, &mce_need_notify)) { 599 mce_work_trigger(); 600 601 if (__ratelimit(&ratelimit)) 602 pr_info(HW_ERR "Machine check events logged\n"); 603 604 return true; 605 } 606 607 return false; 608} 609 610static int mce_early_notifier(struct notifier_block *nb, unsigned long val, 611 void *data) 612{ 613 struct mce_hw_err *err = to_mce_hw_err(data); 614 615 if (!err) 616 return NOTIFY_DONE; 617 618 /* Emit the trace record: */ 619 trace_mce_record(err); 620 621 set_bit(0, &mce_need_notify); 622 623 mce_notify_irq(); 624 625 return NOTIFY_DONE; 626} 627 628static struct notifier_block early_nb = { 629 .notifier_call = mce_early_notifier, 630 .priority = MCE_PRIO_EARLY, 631}; 632 633static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, 634 void *data) 635{ 636 struct mce *mce = (struct mce *)data; 637 unsigned long pfn; 638 639 if (!mce || !mce_usable_address(mce)) 640 return NOTIFY_DONE; 641 642 if (mce->severity != MCE_AO_SEVERITY && 643 mce->severity != MCE_DEFERRED_SEVERITY) 644 return NOTIFY_DONE; 645 646 pfn = (mce->addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; 647 if (!memory_failure(pfn, 0)) { 648 set_mce_nospec(pfn); 649 mce->kflags |= MCE_HANDLED_UC; 650 } 651 652 return NOTIFY_OK; 653} 654 655static struct notifier_block mce_uc_nb = { 656 .notifier_call = uc_decode_notifier, 657 .priority = MCE_PRIO_UC, 658}; 659 660static int mce_default_notifier(struct notifier_block *nb, unsigned long val, 661 void *data) 662{ 663 struct mce_hw_err *err = to_mce_hw_err(data); 664 665 if (!err) 666 return NOTIFY_DONE; 667 668 if (mca_cfg.print_all || !(err->m.kflags)) 669 __print_mce(err); 670 671 return NOTIFY_DONE; 672} 673 674static struct notifier_block mce_default_nb = { 675 .notifier_call = mce_default_notifier, 676 /* lowest prio, we want it to run last. */ 677 .priority = MCE_PRIO_LOWEST, 678}; 679 680/* 681 * Read ADDR and MISC registers. 682 */ 683static noinstr void mce_read_aux(struct mce_hw_err *err, int i) 684{ 685 struct mce *m = &err->m; 686 687 if (m->status & MCI_STATUS_MISCV) 688 m->misc = mce_rdmsrq(mca_msr_reg(i, MCA_MISC)); 689 690 if (m->status & MCI_STATUS_ADDRV) { 691 if (m->kflags & MCE_CHECK_DFR_REGS) 692 m->addr = mce_rdmsrq(MSR_AMD64_SMCA_MCx_DEADDR(i)); 693 else 694 m->addr = mce_rdmsrq(mca_msr_reg(i, MCA_ADDR)); 695 696 /* 697 * Mask the reported address by the reported granularity. 698 */ 699 if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) { 700 u8 shift = MCI_MISC_ADDR_LSB(m->misc); 701 m->addr >>= shift; 702 m->addr <<= shift; 703 } 704 705 smca_extract_err_addr(m); 706 } 707 708 if (mce_flags.smca) { 709 m->ipid = mce_rdmsrq(MSR_AMD64_SMCA_MCx_IPID(i)); 710 711 if (m->status & MCI_STATUS_SYNDV) { 712 m->synd = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND(i)); 713 err->vendor.amd.synd1 = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND1(i)); 714 err->vendor.amd.synd2 = mce_rdmsrq(MSR_AMD64_SMCA_MCx_SYND2(i)); 715 } 716 } 717} 718 719DEFINE_PER_CPU(unsigned, mce_poll_count); 720 721/* 722 * We have three scenarios for checking for Deferred errors: 723 * 724 * 1) Non-SMCA systems check MCA_STATUS and log error if found. 725 * 2) SMCA systems check MCA_STATUS. If error is found then log it and also 726 * clear MCA_DESTAT. 727 * 3) SMCA systems check MCA_DESTAT, if error was not found in MCA_STATUS, and 728 * log it. 729 */ 730static bool smca_should_log_poll_error(struct mce *m) 731{ 732 if (m->status & MCI_STATUS_VAL) 733 return true; 734 735 m->status = mce_rdmsrq(MSR_AMD64_SMCA_MCx_DESTAT(m->bank)); 736 if ((m->status & MCI_STATUS_VAL) && (m->status & MCI_STATUS_DEFERRED)) { 737 m->kflags |= MCE_CHECK_DFR_REGS; 738 return true; 739 } 740 741 return false; 742} 743 744/* 745 * Newer Intel systems that support software error 746 * recovery need to make additional checks. Other 747 * CPUs should skip over uncorrected errors, but log 748 * everything else. 749 */ 750static bool ser_should_log_poll_error(struct mce *m) 751{ 752 /* Log "not enabled" (speculative) errors */ 753 if (!(m->status & MCI_STATUS_EN)) 754 return true; 755 756 /* 757 * Log UCNA (SDM: 15.6.3 "UCR Error Classification") 758 * UC == 1 && PCC == 0 && S == 0 759 */ 760 if (!(m->status & MCI_STATUS_PCC) && !(m->status & MCI_STATUS_S)) 761 return true; 762 763 return false; 764} 765 766static bool should_log_poll_error(enum mcp_flags flags, struct mce_hw_err *err) 767{ 768 struct mce *m = &err->m; 769 770 if (mce_flags.smca) 771 return smca_should_log_poll_error(m); 772 773 /* If this entry is not valid, ignore it. */ 774 if (!(m->status & MCI_STATUS_VAL)) 775 return false; 776 777 /* 778 * If we are logging everything (at CPU online) or this 779 * is a corrected error, then we must log it. 780 */ 781 if ((flags & MCP_UC) || !(m->status & MCI_STATUS_UC)) 782 return true; 783 784 if (mca_cfg.ser) 785 return ser_should_log_poll_error(m); 786 787 if (m->status & MCI_STATUS_UC) 788 return false; 789 790 return true; 791} 792 793static void clear_bank(struct mce *m) 794{ 795 if (m->cpuvendor == X86_VENDOR_AMD) 796 return amd_clear_bank(m); 797 798 mce_wrmsrq(mca_msr_reg(m->bank, MCA_STATUS), 0); 799} 800 801/* 802 * Poll for corrected events or events that happened before reset. 803 * Those are just logged through /dev/mcelog. 804 * 805 * This is executed in standard interrupt context. 806 * 807 * Note: spec recommends to panic for fatal unsignalled 808 * errors here. However this would be quite problematic -- 809 * we would need to reimplement the Monarch handling and 810 * it would mess up the exclusion between exception handler 811 * and poll handler -- * so we skip this for now. 812 * These cases should not happen anyways, or only when the CPU 813 * is already totally * confused. In this case it's likely it will 814 * not fully execute the machine check handler either. 815 */ 816void machine_check_poll(enum mcp_flags flags, mce_banks_t *b) 817{ 818 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 819 struct mce_hw_err err; 820 struct mce *m; 821 int i; 822 823 this_cpu_inc(mce_poll_count); 824 825 mce_gather_info(&err, NULL); 826 m = &err.m; 827 828 if (flags & MCP_TIMESTAMP) 829 m->tsc = rdtsc(); 830 831 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 832 if (!mce_banks[i].ctl || !test_bit(i, *b)) 833 continue; 834 835 m->misc = 0; 836 m->addr = 0; 837 m->bank = i; 838 839 barrier(); 840 m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS)); 841 842 /* 843 * Update storm tracking here, before checking for the 844 * MCI_STATUS_VAL bit. Valid corrected errors count 845 * towards declaring, or maintaining, storm status. No 846 * error in a bank counts towards avoiding, or ending, 847 * storm status. 848 */ 849 if (!mca_cfg.cmci_disabled) 850 mce_track_storm(m); 851 852 /* Verify that the error should be logged based on hardware conditions. */ 853 if (!should_log_poll_error(flags, &err)) 854 continue; 855 856 mce_read_aux(&err, i); 857 m->severity = mce_severity(m, NULL, NULL, false); 858 /* 859 * Don't get the IP here because it's unlikely to 860 * have anything to do with the actual error location. 861 */ 862 863 if (mca_cfg.dont_log_ce && !mce_usable_address(m)) 864 goto clear_it; 865 866 if (flags & MCP_QUEUE_LOG) 867 mce_gen_pool_add(&err); 868 else 869 mce_log(&err); 870 871clear_it: 872 clear_bank(m); 873 } 874 875 /* 876 * Don't clear MCG_STATUS here because it's only defined for 877 * exceptions. 878 */ 879 880 sync_core(); 881} 882EXPORT_SYMBOL_GPL(machine_check_poll); 883 884/* 885 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and 886 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM 887 * Vol 3B Table 15-20). But this confuses both the code that determines 888 * whether the machine check occurred in kernel or user mode, and also 889 * the severity assessment code. Pretend that EIPV was set, and take the 890 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. 891 */ 892static __always_inline void 893quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) 894{ 895 if (bank != 0) 896 return; 897 if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) 898 return; 899 if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| 900 MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| 901 MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| 902 MCACOD)) != 903 (MCI_STATUS_UC|MCI_STATUS_EN| 904 MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| 905 MCI_STATUS_AR|MCACOD_INSTR)) 906 return; 907 908 m->mcgstatus |= MCG_STATUS_EIPV; 909 m->ip = regs->ip; 910 m->cs = regs->cs; 911} 912 913/* 914 * Disable fast string copy and return from the MCE handler upon the first SRAR 915 * MCE on bank 1 due to a CPU erratum on Intel Skylake/Cascade Lake/Cooper Lake 916 * CPUs. 917 * The fast string copy instructions ("REP; MOVS*") could consume an 918 * uncorrectable memory error in the cache line _right after_ the desired region 919 * to copy and raise an MCE with RIP pointing to the instruction _after_ the 920 * "REP; MOVS*". 921 * This mitigation addresses the issue completely with the caveat of performance 922 * degradation on the CPU affected. This is still better than the OS crashing on 923 * MCEs raised on an irrelevant process due to "REP; MOVS*" accesses from a 924 * kernel context (e.g., copy_page). 925 * 926 * Returns true when fast string copy on CPU has been disabled. 927 */ 928static noinstr bool quirk_skylake_repmov(void) 929{ 930 u64 mcgstatus = mce_rdmsrq(MSR_IA32_MCG_STATUS); 931 u64 misc_enable = mce_rdmsrq(MSR_IA32_MISC_ENABLE); 932 u64 mc1_status; 933 934 /* 935 * Apply the quirk only to local machine checks, i.e., no broadcast 936 * sync is needed. 937 */ 938 if (!(mcgstatus & MCG_STATUS_LMCES) || 939 !(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) 940 return false; 941 942 mc1_status = mce_rdmsrq(MSR_IA32_MCx_STATUS(1)); 943 944 /* Check for a software-recoverable data fetch error. */ 945 if ((mc1_status & 946 (MCI_STATUS_VAL | MCI_STATUS_OVER | MCI_STATUS_UC | MCI_STATUS_EN | 947 MCI_STATUS_ADDRV | MCI_STATUS_MISCV | MCI_STATUS_PCC | 948 MCI_STATUS_AR | MCI_STATUS_S)) == 949 (MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN | 950 MCI_STATUS_ADDRV | MCI_STATUS_MISCV | 951 MCI_STATUS_AR | MCI_STATUS_S)) { 952 misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING; 953 mce_wrmsrq(MSR_IA32_MISC_ENABLE, misc_enable); 954 mce_wrmsrq(MSR_IA32_MCx_STATUS(1), 0); 955 956 instrumentation_begin(); 957 pr_err_once("Erratum detected, disable fast string copy instructions.\n"); 958 instrumentation_end(); 959 960 return true; 961 } 962 963 return false; 964} 965 966/* 967 * Some Zen-based Instruction Fetch Units set EIPV=RIPV=0 on poison consumption 968 * errors. This means mce_gather_info() will not save the "ip" and "cs" registers. 969 * 970 * However, the context is still valid, so save the "cs" register for later use. 971 * 972 * The "ip" register is truly unknown, so don't save it or fixup EIPV/RIPV. 973 * 974 * The Instruction Fetch Unit is at MCA bank 1 for all affected systems. 975 */ 976static __always_inline void quirk_zen_ifu(int bank, struct mce *m, struct pt_regs *regs) 977{ 978 if (bank != 1) 979 return; 980 if (!(m->status & MCI_STATUS_POISON)) 981 return; 982 983 m->cs = regs->cs; 984} 985 986/* 987 * Do a quick check if any of the events requires a panic. 988 * This decides if we keep the events around or clear them. 989 */ 990static __always_inline int mce_no_way_out(struct mce_hw_err *err, char **msg, unsigned long *validp, 991 struct pt_regs *regs) 992{ 993 struct mce *m = &err->m; 994 char *tmp = *msg; 995 int i; 996 997 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 998 m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS)); 999 if (!(m->status & MCI_STATUS_VAL)) 1000 continue; 1001 1002 arch___set_bit(i, validp); 1003 if (mce_flags.snb_ifu_quirk) 1004 quirk_sandybridge_ifu(i, m, regs); 1005 1006 if (mce_flags.zen_ifu_quirk) 1007 quirk_zen_ifu(i, m, regs); 1008 1009 m->bank = i; 1010 if (mce_severity(m, regs, &tmp, true) >= MCE_PANIC_SEVERITY) { 1011 mce_read_aux(err, i); 1012 *msg = tmp; 1013 return 1; 1014 } 1015 } 1016 return 0; 1017} 1018 1019/* 1020 * Variable to establish order between CPUs while scanning. 1021 * Each CPU spins initially until executing is equal its number. 1022 */ 1023static atomic_t mce_executing; 1024 1025/* 1026 * Defines order of CPUs on entry. First CPU becomes Monarch. 1027 */ 1028static atomic_t mce_callin; 1029 1030/* 1031 * Track which CPUs entered the MCA broadcast synchronization and which not in 1032 * order to print holdouts. 1033 */ 1034static cpumask_t mce_missing_cpus = CPU_MASK_ALL; 1035 1036/* 1037 * Check if a timeout waiting for other CPUs happened. 1038 */ 1039static noinstr int mce_timed_out(u64 *t, const char *msg) 1040{ 1041 int ret = 0; 1042 1043 /* Enable instrumentation around calls to external facilities */ 1044 instrumentation_begin(); 1045 1046 /* 1047 * The others already did panic for some reason. 1048 * Bail out like in a timeout. 1049 * rmb() to tell the compiler that system_state 1050 * might have been modified by someone else. 1051 */ 1052 rmb(); 1053 if (atomic_read(&mce_panicked)) 1054 wait_for_panic(); 1055 if (!mca_cfg.monarch_timeout) 1056 goto out; 1057 if ((s64)*t < SPINUNIT) { 1058 if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus)) 1059 pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n", 1060 cpumask_pr_args(&mce_missing_cpus)); 1061 mce_panic(msg, NULL, NULL); 1062 1063 ret = 1; 1064 goto out; 1065 } 1066 *t -= SPINUNIT; 1067 1068out: 1069 touch_nmi_watchdog(); 1070 1071 instrumentation_end(); 1072 1073 return ret; 1074} 1075 1076/* 1077 * The Monarch's reign. The Monarch is the CPU who entered 1078 * the machine check handler first. It waits for the others to 1079 * raise the exception too and then grades them. When any 1080 * error is fatal panic. Only then let the others continue. 1081 * 1082 * The other CPUs entering the MCE handler will be controlled by the 1083 * Monarch. They are called Subjects. 1084 * 1085 * This way we prevent any potential data corruption in a unrecoverable case 1086 * and also makes sure always all CPU's errors are examined. 1087 * 1088 * Also this detects the case of a machine check event coming from outer 1089 * space (not detected by any CPUs) In this case some external agent wants 1090 * us to shut down, so panic too. 1091 * 1092 * The other CPUs might still decide to panic if the handler happens 1093 * in a unrecoverable place, but in this case the system is in a semi-stable 1094 * state and won't corrupt anything by itself. It's ok to let the others 1095 * continue for a bit first. 1096 * 1097 * All the spin loops have timeouts; when a timeout happens a CPU 1098 * typically elects itself to be Monarch. 1099 */ 1100static void mce_reign(void) 1101{ 1102 struct mce_hw_err *err = NULL; 1103 struct mce *m = NULL; 1104 int global_worst = 0; 1105 char *msg = NULL; 1106 int cpu; 1107 1108 /* 1109 * This CPU is the Monarch and the other CPUs have run 1110 * through their handlers. 1111 * Grade the severity of the errors of all the CPUs. 1112 */ 1113 for_each_possible_cpu(cpu) { 1114 struct mce_hw_err *etmp = &per_cpu(hw_errs_seen, cpu); 1115 struct mce *mtmp = &etmp->m; 1116 1117 if (mtmp->severity > global_worst) { 1118 global_worst = mtmp->severity; 1119 err = &per_cpu(hw_errs_seen, cpu); 1120 m = &err->m; 1121 } 1122 } 1123 1124 /* 1125 * Cannot recover? Panic here then. 1126 * This dumps all the mces in the log buffer and stops the 1127 * other CPUs. 1128 */ 1129 if (m && global_worst >= MCE_PANIC_SEVERITY) { 1130 /* call mce_severity() to get "msg" for panic */ 1131 mce_severity(m, NULL, &msg, true); 1132 mce_panic("Fatal machine check", err, msg); 1133 } 1134 1135 /* 1136 * For UC somewhere we let the CPU who detects it handle it. 1137 * Also must let continue the others, otherwise the handling 1138 * CPU could deadlock on a lock. 1139 */ 1140 1141 /* 1142 * No machine check event found. Must be some external 1143 * source or one CPU is hung. Panic. 1144 */ 1145 if (global_worst <= MCE_KEEP_SEVERITY) 1146 mce_panic("Fatal machine check from unknown source", NULL, NULL); 1147 1148 /* 1149 * Now clear all the hw_errs_seen so that they don't reappear on 1150 * the next mce. 1151 */ 1152 for_each_possible_cpu(cpu) 1153 memset(&per_cpu(hw_errs_seen, cpu), 0, sizeof(struct mce_hw_err)); 1154} 1155 1156static atomic_t global_nwo; 1157 1158/* 1159 * Start of Monarch synchronization. This waits until all CPUs have 1160 * entered the exception handler and then determines if any of them 1161 * saw a fatal event that requires panic. Then it executes them 1162 * in the entry order. 1163 * TBD double check parallel CPU hotunplug 1164 */ 1165static noinstr int mce_start(int *no_way_out) 1166{ 1167 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1168 int order, ret = -1; 1169 1170 if (!timeout) 1171 return ret; 1172 1173 raw_atomic_add(*no_way_out, &global_nwo); 1174 /* 1175 * Rely on the implied barrier below, such that global_nwo 1176 * is updated before mce_callin. 1177 */ 1178 order = raw_atomic_inc_return(&mce_callin); 1179 arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus); 1180 1181 /* Enable instrumentation around calls to external facilities */ 1182 instrumentation_begin(); 1183 1184 /* 1185 * Wait for everyone. 1186 */ 1187 while (raw_atomic_read(&mce_callin) != num_online_cpus()) { 1188 if (mce_timed_out(&timeout, 1189 "Timeout: Not all CPUs entered broadcast exception handler")) { 1190 raw_atomic_set(&global_nwo, 0); 1191 goto out; 1192 } 1193 ndelay(SPINUNIT); 1194 } 1195 1196 /* 1197 * mce_callin should be read before global_nwo 1198 */ 1199 smp_rmb(); 1200 1201 if (order == 1) { 1202 /* 1203 * Monarch: Starts executing now, the others wait. 1204 */ 1205 raw_atomic_set(&mce_executing, 1); 1206 } else { 1207 /* 1208 * Subject: Now start the scanning loop one by one in 1209 * the original callin order. 1210 * This way when there are any shared banks it will be 1211 * only seen by one CPU before cleared, avoiding duplicates. 1212 */ 1213 while (raw_atomic_read(&mce_executing) < order) { 1214 if (mce_timed_out(&timeout, 1215 "Timeout: Subject CPUs unable to finish machine check processing")) { 1216 raw_atomic_set(&global_nwo, 0); 1217 goto out; 1218 } 1219 ndelay(SPINUNIT); 1220 } 1221 } 1222 1223 /* 1224 * Cache the global no_way_out state. 1225 */ 1226 *no_way_out = raw_atomic_read(&global_nwo); 1227 1228 ret = order; 1229 1230out: 1231 instrumentation_end(); 1232 1233 return ret; 1234} 1235 1236/* 1237 * Synchronize between CPUs after main scanning loop. 1238 * This invokes the bulk of the Monarch processing. 1239 */ 1240static noinstr int mce_end(int order) 1241{ 1242 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1243 int ret = -1; 1244 1245 /* Allow instrumentation around external facilities. */ 1246 instrumentation_begin(); 1247 1248 if (!timeout) 1249 goto reset; 1250 if (order < 0) 1251 goto reset; 1252 1253 /* 1254 * Allow others to run. 1255 */ 1256 atomic_inc(&mce_executing); 1257 1258 if (order == 1) { 1259 /* 1260 * Monarch: Wait for everyone to go through their scanning 1261 * loops. 1262 */ 1263 while (atomic_read(&mce_executing) <= num_online_cpus()) { 1264 if (mce_timed_out(&timeout, 1265 "Timeout: Monarch CPU unable to finish machine check processing")) 1266 goto reset; 1267 ndelay(SPINUNIT); 1268 } 1269 1270 mce_reign(); 1271 barrier(); 1272 ret = 0; 1273 } else { 1274 /* 1275 * Subject: Wait for Monarch to finish. 1276 */ 1277 while (atomic_read(&mce_executing) != 0) { 1278 if (mce_timed_out(&timeout, 1279 "Timeout: Monarch CPU did not finish machine check processing")) 1280 goto reset; 1281 ndelay(SPINUNIT); 1282 } 1283 1284 /* 1285 * Don't reset anything. That's done by the Monarch. 1286 */ 1287 ret = 0; 1288 goto out; 1289 } 1290 1291 /* 1292 * Reset all global state. 1293 */ 1294reset: 1295 atomic_set(&global_nwo, 0); 1296 atomic_set(&mce_callin, 0); 1297 cpumask_setall(&mce_missing_cpus); 1298 barrier(); 1299 1300 /* 1301 * Let others run again. 1302 */ 1303 atomic_set(&mce_executing, 0); 1304 1305out: 1306 instrumentation_end(); 1307 1308 return ret; 1309} 1310 1311static __always_inline void mce_clear_state(unsigned long *toclear) 1312{ 1313 int i; 1314 1315 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1316 if (arch_test_bit(i, toclear)) 1317 mce_wrmsrq(mca_msr_reg(i, MCA_STATUS), 0); 1318 } 1319} 1320 1321/* 1322 * Cases where we avoid rendezvous handler timeout: 1323 * 1) If this CPU is offline. 1324 * 1325 * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to 1326 * skip those CPUs which remain looping in the 1st kernel - see 1327 * crash_nmi_callback(). 1328 * 1329 * Note: there still is a small window between kexec-ing and the new, 1330 * kdump kernel establishing a new #MC handler where a broadcasted MCE 1331 * might not get handled properly. 1332 */ 1333static noinstr bool mce_check_crashing_cpu(void) 1334{ 1335 unsigned int cpu = smp_processor_id(); 1336 1337 if (arch_cpu_is_offline(cpu) || 1338 (crashing_cpu != -1 && crashing_cpu != cpu)) { 1339 u64 mcgstatus; 1340 1341 mcgstatus = native_rdmsrq(MSR_IA32_MCG_STATUS); 1342 1343 if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { 1344 if (mcgstatus & MCG_STATUS_LMCES) 1345 return false; 1346 } 1347 1348 if (mcgstatus & MCG_STATUS_RIPV) { 1349 native_wrmsrq(MSR_IA32_MCG_STATUS, 0); 1350 return true; 1351 } 1352 } 1353 return false; 1354} 1355 1356static __always_inline int 1357__mc_scan_banks(struct mce_hw_err *err, struct pt_regs *regs, 1358 struct mce_hw_err *final, unsigned long *toclear, 1359 unsigned long *valid_banks, int no_way_out, int *worst) 1360{ 1361 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1362 struct mca_config *cfg = &mca_cfg; 1363 int severity, i, taint = 0; 1364 struct mce *m = &err->m; 1365 1366 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1367 arch___clear_bit(i, toclear); 1368 if (!arch_test_bit(i, valid_banks)) 1369 continue; 1370 1371 if (!mce_banks[i].ctl) 1372 continue; 1373 1374 m->misc = 0; 1375 m->addr = 0; 1376 m->bank = i; 1377 1378 m->status = mce_rdmsrq(mca_msr_reg(i, MCA_STATUS)); 1379 if (!(m->status & MCI_STATUS_VAL)) 1380 continue; 1381 1382 /* 1383 * Corrected or non-signaled errors are handled by 1384 * machine_check_poll(). Leave them alone, unless this panics. 1385 */ 1386 if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && 1387 !no_way_out) 1388 continue; 1389 1390 /* Set taint even when machine check was not enabled. */ 1391 taint++; 1392 1393 severity = mce_severity(m, regs, NULL, true); 1394 1395 /* 1396 * When machine check was for corrected/deferred handler don't 1397 * touch, unless we're panicking. 1398 */ 1399 if ((severity == MCE_KEEP_SEVERITY || 1400 severity == MCE_UCNA_SEVERITY) && !no_way_out) 1401 continue; 1402 1403 arch___set_bit(i, toclear); 1404 1405 /* Machine check event was not enabled. Clear, but ignore. */ 1406 if (severity == MCE_NO_SEVERITY) 1407 continue; 1408 1409 mce_read_aux(err, i); 1410 1411 /* assuming valid severity level != 0 */ 1412 m->severity = severity; 1413 1414 /* 1415 * Enable instrumentation around the mce_log() call which is 1416 * done in #MC context, where instrumentation is disabled. 1417 */ 1418 instrumentation_begin(); 1419 mce_log(err); 1420 instrumentation_end(); 1421 1422 if (severity > *worst) { 1423 *final = *err; 1424 *worst = severity; 1425 } 1426 } 1427 1428 /* mce_clear_state will clear *final, save locally for use later */ 1429 *err = *final; 1430 1431 return taint; 1432} 1433 1434static void kill_me_now(struct callback_head *ch) 1435{ 1436 struct task_struct *p = container_of(ch, struct task_struct, mce_kill_me); 1437 1438 p->mce_count = 0; 1439 force_sig(SIGBUS); 1440} 1441 1442static void kill_me_maybe(struct callback_head *cb) 1443{ 1444 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 1445 int flags = MF_ACTION_REQUIRED; 1446 unsigned long pfn; 1447 int ret; 1448 1449 p->mce_count = 0; 1450 pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr); 1451 1452 if (!p->mce_ripv) 1453 flags |= MF_MUST_KILL; 1454 1455 pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; 1456 ret = memory_failure(pfn, flags); 1457 if (!ret) { 1458 set_mce_nospec(pfn); 1459 sync_core(); 1460 return; 1461 } 1462 1463 /* 1464 * -EHWPOISON from memory_failure() means that it already sent SIGBUS 1465 * to the current process with the proper error info, 1466 * -EOPNOTSUPP means hwpoison_filter() filtered the error event, 1467 * 1468 * In both cases, no further processing is required. 1469 */ 1470 if (ret == -EHWPOISON || ret == -EOPNOTSUPP) 1471 return; 1472 1473 pr_err("Memory error not recovered"); 1474 kill_me_now(cb); 1475} 1476 1477static void kill_me_never(struct callback_head *cb) 1478{ 1479 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 1480 unsigned long pfn; 1481 1482 p->mce_count = 0; 1483 pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr); 1484 pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; 1485 if (!memory_failure(pfn, 0)) 1486 set_mce_nospec(pfn); 1487} 1488 1489static void queue_task_work(struct mce_hw_err *err, char *msg, void (*func)(struct callback_head *)) 1490{ 1491 int count = ++current->mce_count; 1492 struct mce *m = &err->m; 1493 1494 /* First call, save all the details */ 1495 if (count == 1) { 1496 current->mce_addr = m->addr; 1497 current->mce_kflags = m->kflags; 1498 current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); 1499 current->mce_whole_page = whole_page(m); 1500 current->mce_kill_me.func = func; 1501 } 1502 1503 /* Ten is likely overkill. Don't expect more than two faults before task_work() */ 1504 if (count > 10) 1505 mce_panic("Too many consecutive machine checks while accessing user data", 1506 err, msg); 1507 1508 /* Second or later call, make sure page address matches the one from first call */ 1509 if (count > 1 && (current->mce_addr >> PAGE_SHIFT) != (m->addr >> PAGE_SHIFT)) 1510 mce_panic("Consecutive machine checks to different user pages", err, msg); 1511 1512 /* Do not call task_work_add() more than once */ 1513 if (count > 1) 1514 return; 1515 1516 task_work_add(current, ¤t->mce_kill_me, TWA_RESUME); 1517} 1518 1519/* Handle unconfigured int18 (should never happen) */ 1520static noinstr void unexpected_machine_check(struct pt_regs *regs) 1521{ 1522 instrumentation_begin(); 1523 pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", 1524 smp_processor_id()); 1525 instrumentation_end(); 1526} 1527 1528/* 1529 * The actual machine check handler. This only handles real exceptions when 1530 * something got corrupted coming in through int 18. 1531 * 1532 * This is executed in #MC context not subject to normal locking rules. 1533 * This implies that most kernel services cannot be safely used. Don't even 1534 * think about putting a printk in there! 1535 * 1536 * On Intel systems this is entered on all CPUs in parallel through 1537 * MCE broadcast. However some CPUs might be broken beyond repair, 1538 * so be always careful when synchronizing with others. 1539 * 1540 * Tracing and kprobes are disabled: if we interrupted a kernel context 1541 * with IF=1, we need to minimize stack usage. There are also recursion 1542 * issues: if the machine check was due to a failure of the memory 1543 * backing the user stack, tracing that reads the user stack will cause 1544 * potentially infinite recursion. 1545 * 1546 * Currently, the #MC handler calls out to a number of external facilities 1547 * and, therefore, allows instrumentation around them. The optimal thing to 1548 * have would be to do the absolutely minimal work required in #MC context 1549 * and have instrumentation disabled only around that. Further processing can 1550 * then happen in process context where instrumentation is allowed. Achieving 1551 * that requires careful auditing and modifications. Until then, the code 1552 * allows instrumentation temporarily, where required. * 1553 */ 1554noinstr void do_machine_check(struct pt_regs *regs) 1555{ 1556 int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0; 1557 DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 }; 1558 DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 }; 1559 struct mce_hw_err *final; 1560 struct mce_hw_err err; 1561 char *msg = NULL; 1562 struct mce *m; 1563 1564 if (unlikely(mce_flags.p5)) 1565 return pentium_machine_check(regs); 1566 else if (unlikely(mce_flags.winchip)) 1567 return winchip_machine_check(regs); 1568 else if (unlikely(!mca_cfg.initialized)) 1569 return unexpected_machine_check(regs); 1570 1571 if (mce_flags.skx_repmov_quirk && quirk_skylake_repmov()) 1572 goto clear; 1573 1574 /* 1575 * Establish sequential order between the CPUs entering the machine 1576 * check handler. 1577 */ 1578 order = -1; 1579 1580 /* 1581 * If no_way_out gets set, there is no safe way to recover from this 1582 * MCE. 1583 */ 1584 no_way_out = 0; 1585 1586 /* 1587 * If kill_current_task is not set, there might be a way to recover from this 1588 * error. 1589 */ 1590 kill_current_task = 0; 1591 1592 /* 1593 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES 1594 * on Intel. 1595 */ 1596 lmce = 1; 1597 1598 this_cpu_inc(mce_exception_count); 1599 1600 mce_gather_info(&err, regs); 1601 m = &err.m; 1602 m->tsc = rdtsc(); 1603 1604 final = this_cpu_ptr(&hw_errs_seen); 1605 *final = err; 1606 1607 no_way_out = mce_no_way_out(&err, &msg, valid_banks, regs); 1608 1609 barrier(); 1610 1611 /* 1612 * When no restart IP might need to kill or panic. 1613 * Assume the worst for now, but if we find the 1614 * severity is MCE_AR_SEVERITY we have other options. 1615 */ 1616 if (!(m->mcgstatus & MCG_STATUS_RIPV)) 1617 kill_current_task = 1; 1618 /* 1619 * Check if this MCE is signaled to only this logical processor, 1620 * on Intel, Zhaoxin only. 1621 */ 1622 if (m->cpuvendor == X86_VENDOR_INTEL || 1623 m->cpuvendor == X86_VENDOR_ZHAOXIN) 1624 lmce = m->mcgstatus & MCG_STATUS_LMCES; 1625 1626 /* 1627 * Local machine check may already know that we have to panic. 1628 * Broadcast machine check begins rendezvous in mce_start() 1629 * Go through all banks in exclusion of the other CPUs. This way we 1630 * don't report duplicated events on shared banks because the first one 1631 * to see it will clear it. 1632 */ 1633 if (lmce) { 1634 if (no_way_out) 1635 mce_panic("Fatal local machine check", &err, msg); 1636 } else { 1637 order = mce_start(&no_way_out); 1638 } 1639 1640 taint = __mc_scan_banks(&err, regs, final, toclear, valid_banks, no_way_out, &worst); 1641 1642 if (!no_way_out) 1643 mce_clear_state(toclear); 1644 1645 /* 1646 * Do most of the synchronization with other CPUs. 1647 * When there's any problem use only local no_way_out state. 1648 */ 1649 if (!lmce) { 1650 if (mce_end(order) < 0) { 1651 if (!no_way_out) 1652 no_way_out = worst >= MCE_PANIC_SEVERITY; 1653 1654 if (no_way_out) 1655 mce_panic("Fatal machine check on current CPU", &err, msg); 1656 } 1657 } else { 1658 /* 1659 * If there was a fatal machine check we should have 1660 * already called mce_panic earlier in this function. 1661 * Since we re-read the banks, we might have found 1662 * something new. Check again to see if we found a 1663 * fatal error. We call "mce_severity()" again to 1664 * make sure we have the right "msg". 1665 */ 1666 if (worst >= MCE_PANIC_SEVERITY) { 1667 mce_severity(m, regs, &msg, true); 1668 mce_panic("Local fatal machine check!", &err, msg); 1669 } 1670 } 1671 1672 /* 1673 * Enable instrumentation around the external facilities like task_work_add() 1674 * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this 1675 * properly would need a lot more involved reorganization. 1676 */ 1677 instrumentation_begin(); 1678 1679 if (taint) 1680 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); 1681 1682 if (worst != MCE_AR_SEVERITY && !kill_current_task) 1683 goto out; 1684 1685 /* Fault was in user mode and we need to take some action */ 1686 if ((m->cs & 3) == 3) { 1687 /* If this triggers there is no way to recover. Die hard. */ 1688 BUG_ON(!on_thread_stack() || !user_mode(regs)); 1689 1690 if (!mce_usable_address(m)) 1691 queue_task_work(&err, msg, kill_me_now); 1692 else 1693 queue_task_work(&err, msg, kill_me_maybe); 1694 1695 } else if (m->mcgstatus & MCG_STATUS_SEAM_NR) { 1696 /* 1697 * Saved RIP on stack makes it look like the machine check 1698 * was taken in the kernel on the instruction following 1699 * the entry to SEAM mode. But MCG_STATUS_SEAM_NR indicates 1700 * that the machine check was taken inside SEAM non-root 1701 * mode. CPU core has already marked that guest as dead. 1702 * It is OK for the kernel to resume execution at the 1703 * apparent point of the machine check as the fault did 1704 * not occur there. Mark the page as poisoned so it won't 1705 * be added to free list when the guest is terminated. 1706 */ 1707 if (mce_usable_address(m)) { 1708 struct page *p = pfn_to_online_page(m->addr >> PAGE_SHIFT); 1709 1710 if (p) 1711 SetPageHWPoison(p); 1712 } 1713 } else { 1714 /* 1715 * Handle an MCE which has happened in kernel space but from 1716 * which the kernel can recover: ex_has_fault_handler() has 1717 * already verified that the rIP at which the error happened is 1718 * a rIP from which the kernel can recover (by jumping to 1719 * recovery code specified in _ASM_EXTABLE_FAULT()) and the 1720 * corresponding exception handler which would do that is the 1721 * proper one. 1722 */ 1723 if (m->kflags & MCE_IN_KERNEL_RECOV) { 1724 if (!fixup_exception(regs, X86_TRAP_MC, 0, 0)) 1725 mce_panic("Failed kernel mode recovery", &err, msg); 1726 } 1727 1728 if (m->kflags & MCE_IN_KERNEL_COPYIN) 1729 queue_task_work(&err, msg, kill_me_never); 1730 } 1731 1732out: 1733 /* Given it didn't panic, mark it as recoverable */ 1734 hwerr_log_error_type(HWERR_RECOV_OTHERS); 1735 1736 instrumentation_end(); 1737 1738clear: 1739 mce_wrmsrq(MSR_IA32_MCG_STATUS, 0); 1740} 1741EXPORT_SYMBOL_GPL(do_machine_check); 1742 1743#ifndef CONFIG_MEMORY_FAILURE 1744int memory_failure(unsigned long pfn, int flags) 1745{ 1746 /* mce_severity() should not hand us an ACTION_REQUIRED error */ 1747 BUG_ON(flags & MF_ACTION_REQUIRED); 1748 pr_err("Uncorrected memory error in page 0x%lx ignored\n" 1749 "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n", 1750 pfn); 1751 1752 return 0; 1753} 1754#endif 1755 1756/* 1757 * Periodic polling timer for "silent" machine check errors. If the 1758 * poller finds an MCE, poll 2x faster. When the poller finds no more 1759 * errors, poll 2x slower (up to check_interval seconds). 1760 */ 1761static unsigned long check_interval = INITIAL_CHECK_INTERVAL; 1762 1763static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */ 1764static DEFINE_PER_CPU(struct timer_list, mce_timer); 1765 1766static void __start_timer(struct timer_list *t, unsigned long interval) 1767{ 1768 unsigned long when = jiffies + interval; 1769 unsigned long flags; 1770 1771 local_irq_save(flags); 1772 1773 if (!timer_pending(t) || time_before(when, t->expires)) 1774 mod_timer(t, round_jiffies(when)); 1775 1776 local_irq_restore(flags); 1777} 1778 1779static void mc_poll_banks_default(void) 1780{ 1781 machine_check_poll(0, this_cpu_ptr(&mce_poll_banks)); 1782} 1783 1784void (*mc_poll_banks)(void) = mc_poll_banks_default; 1785 1786static bool should_enable_timer(unsigned long iv) 1787{ 1788 return !mca_cfg.ignore_ce && iv; 1789} 1790 1791static void mce_timer_fn(struct timer_list *t) 1792{ 1793 struct timer_list *cpu_t = this_cpu_ptr(&mce_timer); 1794 unsigned long iv; 1795 1796 WARN_ON(cpu_t != t); 1797 1798 iv = __this_cpu_read(mce_next_interval); 1799 1800 if (mce_available(this_cpu_ptr(&cpu_info))) 1801 mc_poll_banks(); 1802 1803 /* 1804 * Alert userspace if needed. If we logged an MCE, reduce the polling 1805 * interval, otherwise increase the polling interval. 1806 */ 1807 if (mce_notify_irq()) 1808 iv = max(iv / 2, (unsigned long) HZ/100); 1809 else 1810 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ)); 1811 1812 if (mce_get_storm_mode()) { 1813 __start_timer(t, HZ); 1814 } else if (should_enable_timer(iv)) { 1815 __this_cpu_write(mce_next_interval, iv); 1816 __start_timer(t, iv); 1817 } 1818} 1819 1820/* 1821 * When a storm starts on any bank on this CPU, switch to polling 1822 * once per second. When the storm ends, revert to the default 1823 * polling interval. 1824 */ 1825void mce_timer_kick(bool storm) 1826{ 1827 struct timer_list *t = this_cpu_ptr(&mce_timer); 1828 1829 mce_set_storm_mode(storm); 1830 1831 if (storm) 1832 __start_timer(t, HZ); 1833 else 1834 __this_cpu_write(mce_next_interval, check_interval * HZ); 1835} 1836 1837/* Must not be called in IRQ context where timer_delete_sync() can deadlock */ 1838static void mce_timer_delete_all(void) 1839{ 1840 int cpu; 1841 1842 for_each_online_cpu(cpu) 1843 timer_delete_sync(&per_cpu(mce_timer, cpu)); 1844} 1845 1846static void __mcheck_cpu_mce_banks_init(void) 1847{ 1848 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1849 u8 n_banks = this_cpu_read(mce_num_banks); 1850 int i; 1851 1852 for (i = 0; i < n_banks; i++) { 1853 struct mce_bank *b = &mce_banks[i]; 1854 1855 /* 1856 * Init them all by default. 1857 * 1858 * The required vendor quirks will be applied before 1859 * __mcheck_cpu_init_prepare_banks() does the final bank setup. 1860 */ 1861 b->ctl = -1ULL; 1862 b->init = true; 1863 } 1864} 1865 1866/* 1867 * Initialize Machine Checks for a CPU. 1868 */ 1869static void __mcheck_cpu_cap_init(void) 1870{ 1871 u64 cap; 1872 u8 b; 1873 1874 rdmsrq(MSR_IA32_MCG_CAP, cap); 1875 1876 b = cap & MCG_BANKCNT_MASK; 1877 1878 if (b > MAX_NR_BANKS) { 1879 pr_warn("CPU%d: Using only %u machine check banks out of %u\n", 1880 smp_processor_id(), MAX_NR_BANKS, b); 1881 b = MAX_NR_BANKS; 1882 } 1883 1884 this_cpu_write(mce_num_banks, b); 1885 1886 __mcheck_cpu_mce_banks_init(); 1887} 1888 1889static void __mcheck_cpu_init_generic(void) 1890{ 1891 u64 cap; 1892 1893 rdmsrq(MSR_IA32_MCG_CAP, cap); 1894 if (cap & MCG_CTL_P) 1895 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 1896} 1897 1898static void __mcheck_cpu_init_prepare_banks(void) 1899{ 1900 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1901 u64 msrval; 1902 int i; 1903 1904 /* 1905 * Log the machine checks left over from the previous reset. Log them 1906 * only, do not start processing them. That will happen in mcheck_late_init() 1907 * when all consumers have been registered on the notifier chain. 1908 */ 1909 if (mca_cfg.bootlog) { 1910 mce_banks_t all_banks; 1911 1912 bitmap_fill(all_banks, MAX_NR_BANKS); 1913 machine_check_poll(MCP_UC | MCP_QUEUE_LOG, &all_banks); 1914 } 1915 1916 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1917 struct mce_bank *b = &mce_banks[i]; 1918 1919 if (!b->init) 1920 continue; 1921 1922 wrmsrq(mca_msr_reg(i, MCA_CTL), b->ctl); 1923 wrmsrq(mca_msr_reg(i, MCA_STATUS), 0); 1924 1925 rdmsrq(mca_msr_reg(i, MCA_CTL), msrval); 1926 b->init = !!msrval; 1927 } 1928} 1929 1930static void amd_apply_global_quirks(struct cpuinfo_x86 *c) 1931{ 1932 if (c->x86 < 0x11 && mca_cfg.bootlog < 0) { 1933 /* 1934 * Lots of broken BIOS around that don't clear them 1935 * by default and leave crap in there. Don't log: 1936 */ 1937 mca_cfg.bootlog = 0; 1938 } 1939 1940 /* 1941 * overflow_recov is supported for F15h Models 00h-0fh 1942 * even though we don't have a CPUID bit for it. 1943 */ 1944 if (c->x86 == 0x15 && c->x86_model <= 0xf) 1945 mce_flags.overflow_recov = 1; 1946 1947 if (c->x86 >= 0x17 && c->x86 <= 0x1A) 1948 mce_flags.zen_ifu_quirk = 1; 1949} 1950 1951static void intel_apply_global_quirks(struct cpuinfo_x86 *c) 1952{ 1953 /* Older CPUs (prior to family 6) don't need quirks. */ 1954 if (c->x86_vfm < INTEL_PENTIUM_PRO) 1955 return; 1956 1957 /* 1958 * All newer Intel systems support MCE broadcasting. Enable 1959 * synchronization with a one second timeout. 1960 */ 1961 if (c->x86_vfm >= INTEL_CORE_YONAH && mca_cfg.monarch_timeout < 0) 1962 mca_cfg.monarch_timeout = USEC_PER_SEC; 1963 1964 /* 1965 * There are also broken BIOSes on some Pentium M and 1966 * earlier systems: 1967 */ 1968 if (c->x86_vfm < INTEL_CORE_YONAH && mca_cfg.bootlog < 0) 1969 mca_cfg.bootlog = 0; 1970 1971 if (c->x86_vfm == INTEL_SANDYBRIDGE_X) 1972 mce_flags.snb_ifu_quirk = 1; 1973 1974 /* 1975 * Skylake, Cascacde Lake and Cooper Lake require a quirk on 1976 * rep movs. 1977 */ 1978 if (c->x86_vfm == INTEL_SKYLAKE_X) 1979 mce_flags.skx_repmov_quirk = 1; 1980} 1981 1982static void zhaoxin_apply_global_quirks(struct cpuinfo_x86 *c) 1983{ 1984 /* 1985 * All newer Zhaoxin CPUs support MCE broadcasting. Enable 1986 * synchronization with a one second timeout. 1987 */ 1988 if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 1989 if (mca_cfg.monarch_timeout < 0) 1990 mca_cfg.monarch_timeout = USEC_PER_SEC; 1991 } 1992} 1993 1994static bool __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) 1995{ 1996 if (c->x86 != 5) 1997 return false; 1998 1999 switch (c->x86_vendor) { 2000 case X86_VENDOR_INTEL: 2001 intel_p5_mcheck_init(c); 2002 mce_flags.p5 = 1; 2003 return true; 2004 case X86_VENDOR_CENTAUR: 2005 winchip_mcheck_init(c); 2006 mce_flags.winchip = 1; 2007 return true; 2008 default: 2009 return false; 2010 } 2011 2012 return false; 2013} 2014 2015static void mce_centaur_feature_init(struct cpuinfo_x86 *c) 2016{ 2017 struct mca_config *cfg = &mca_cfg; 2018 2019 /* 2020 * All newer Centaur CPUs support MCE broadcasting. Enable 2021 * synchronization with a one second timeout. 2022 */ 2023 if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) || 2024 c->x86 > 6) { 2025 if (cfg->monarch_timeout < 0) 2026 cfg->monarch_timeout = USEC_PER_SEC; 2027 } 2028} 2029 2030static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c) 2031{ 2032 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2033 2034 /* 2035 * These CPUs have MCA bank 8 which reports only one error type called 2036 * SVAD (System View Address Decoder). The reporting of that error is 2037 * controlled by IA32_MC8.CTL.0. 2038 * 2039 * If enabled, prefetching on these CPUs will cause SVAD MCE when 2040 * virtual machines start and result in a system panic. Always disable 2041 * bank 8 SVAD error by default. 2042 */ 2043 if ((c->x86 == 7 && c->x86_model == 0x1b) || 2044 (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 2045 if (this_cpu_read(mce_num_banks) > 8) 2046 mce_banks[8].ctl = 0; 2047 } 2048 2049 intel_init_cmci(); 2050 intel_init_lmce(); 2051} 2052 2053static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c) 2054{ 2055 intel_clear_lmce(); 2056} 2057 2058static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c) 2059{ 2060 switch (c->x86_vendor) { 2061 case X86_VENDOR_INTEL: 2062 mce_intel_feature_init(c); 2063 break; 2064 2065 case X86_VENDOR_AMD: 2066 case X86_VENDOR_HYGON: 2067 mce_amd_feature_init(c); 2068 break; 2069 2070 case X86_VENDOR_CENTAUR: 2071 mce_centaur_feature_init(c); 2072 break; 2073 2074 case X86_VENDOR_ZHAOXIN: 2075 mce_zhaoxin_feature_init(c); 2076 break; 2077 2078 default: 2079 break; 2080 } 2081} 2082 2083static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c) 2084{ 2085 switch (c->x86_vendor) { 2086 case X86_VENDOR_INTEL: 2087 mce_intel_feature_clear(c); 2088 break; 2089 2090 case X86_VENDOR_ZHAOXIN: 2091 mce_zhaoxin_feature_clear(c); 2092 break; 2093 2094 default: 2095 break; 2096 } 2097} 2098 2099static void mce_start_timer(struct timer_list *t) 2100{ 2101 unsigned long iv = check_interval * HZ; 2102 2103 if (should_enable_timer(iv)) { 2104 this_cpu_write(mce_next_interval, iv); 2105 __start_timer(t, iv); 2106 } 2107} 2108 2109static void __mcheck_cpu_setup_timer(void) 2110{ 2111 struct timer_list *t = this_cpu_ptr(&mce_timer); 2112 2113 timer_setup(t, mce_timer_fn, TIMER_PINNED); 2114} 2115 2116static void __mcheck_cpu_init_timer(void) 2117{ 2118 struct timer_list *t = this_cpu_ptr(&mce_timer); 2119 2120 timer_setup(t, mce_timer_fn, TIMER_PINNED); 2121 mce_start_timer(t); 2122} 2123 2124bool filter_mce(struct mce *m) 2125{ 2126 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 2127 return amd_filter_mce(m); 2128 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 2129 return intel_filter_mce(m); 2130 2131 return false; 2132} 2133 2134static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) 2135{ 2136 irqentry_state_t irq_state; 2137 2138 WARN_ON_ONCE(user_mode(regs)); 2139 2140 /* 2141 * Only required when from kernel mode. See 2142 * mce_check_crashing_cpu() for details. 2143 */ 2144 if (mca_cfg.initialized && mce_check_crashing_cpu()) 2145 return; 2146 2147 irq_state = irqentry_nmi_enter(regs); 2148 2149 do_machine_check(regs); 2150 2151 irqentry_nmi_exit(regs, irq_state); 2152} 2153 2154static __always_inline void exc_machine_check_user(struct pt_regs *regs) 2155{ 2156 irqentry_enter_from_user_mode(regs); 2157 2158 do_machine_check(regs); 2159 2160 irqentry_exit_to_user_mode(regs); 2161} 2162 2163#ifdef CONFIG_X86_64 2164/* MCE hit kernel mode */ 2165DEFINE_IDTENTRY_MCE(exc_machine_check) 2166{ 2167 unsigned long dr7; 2168 2169 dr7 = local_db_save(); 2170 exc_machine_check_kernel(regs); 2171 local_db_restore(dr7); 2172} 2173 2174/* The user mode variant. */ 2175DEFINE_IDTENTRY_MCE_USER(exc_machine_check) 2176{ 2177 unsigned long dr7; 2178 2179 dr7 = local_db_save(); 2180 exc_machine_check_user(regs); 2181 local_db_restore(dr7); 2182} 2183 2184#ifdef CONFIG_X86_FRED 2185/* 2186 * When occurred on different ring level, i.e., from user or kernel 2187 * context, #MCE needs to be handled on different stack: User #MCE 2188 * on current task stack, while kernel #MCE on a dedicated stack. 2189 * 2190 * This is exactly how FRED event delivery invokes an exception 2191 * handler: ring 3 event on level 0 stack, i.e., current task stack; 2192 * ring 0 event on the #MCE dedicated stack specified in the 2193 * IA32_FRED_STKLVLS MSR. So unlike IDT, the FRED machine check entry 2194 * stub doesn't do stack switch. 2195 */ 2196DEFINE_FREDENTRY_MCE(exc_machine_check) 2197{ 2198 unsigned long dr7; 2199 2200 dr7 = local_db_save(); 2201 if (user_mode(regs)) 2202 exc_machine_check_user(regs); 2203 else 2204 exc_machine_check_kernel(regs); 2205 local_db_restore(dr7); 2206} 2207#endif 2208#else 2209/* 32bit unified entry point */ 2210DEFINE_IDTENTRY_RAW(exc_machine_check) 2211{ 2212 unsigned long dr7; 2213 2214 dr7 = local_db_save(); 2215 if (user_mode(regs)) 2216 exc_machine_check_user(regs); 2217 else 2218 exc_machine_check_kernel(regs); 2219 local_db_restore(dr7); 2220} 2221#endif 2222 2223void mca_bsp_init(struct cpuinfo_x86 *c) 2224{ 2225 u64 cap; 2226 2227 if (!mce_available(c)) 2228 return; 2229 2230 if (c->x86_vendor == X86_VENDOR_UNKNOWN) { 2231 mca_cfg.disabled = 1; 2232 pr_info("unknown CPU type - not enabling MCE support\n"); 2233 return; 2234 } 2235 2236 mce_flags.overflow_recov = cpu_feature_enabled(X86_FEATURE_OVERFLOW_RECOV); 2237 mce_flags.succor = cpu_feature_enabled(X86_FEATURE_SUCCOR); 2238 mce_flags.smca = cpu_feature_enabled(X86_FEATURE_SMCA); 2239 2240 if (mce_flags.smca) 2241 smca_bsp_init(); 2242 2243 rdmsrq(MSR_IA32_MCG_CAP, cap); 2244 2245 /* Use accurate RIP reporting if available. */ 2246 if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9) 2247 mca_cfg.rip_msr = MSR_IA32_MCG_EIP; 2248 2249 if (cap & MCG_SER_P) 2250 mca_cfg.ser = 1; 2251 2252 switch (c->x86_vendor) { 2253 case X86_VENDOR_AMD: 2254 amd_apply_global_quirks(c); 2255 break; 2256 case X86_VENDOR_INTEL: 2257 intel_apply_global_quirks(c); 2258 break; 2259 case X86_VENDOR_ZHAOXIN: 2260 zhaoxin_apply_global_quirks(c); 2261 break; 2262 } 2263 2264 if (mca_cfg.monarch_timeout < 0) 2265 mca_cfg.monarch_timeout = 0; 2266 if (mca_cfg.bootlog != 0) 2267 mca_cfg.panic_timeout = 30; 2268} 2269 2270/* 2271 * Called for each booted CPU to set up machine checks. 2272 * Must be called with preempt off: 2273 */ 2274void mcheck_cpu_init(struct cpuinfo_x86 *c) 2275{ 2276 if (mca_cfg.disabled) 2277 return; 2278 2279 if (__mcheck_cpu_ancient_init(c)) 2280 return; 2281 2282 if (!mce_available(c)) 2283 return; 2284 2285 __mcheck_cpu_cap_init(); 2286 2287 if (!mce_gen_pool_init()) { 2288 mca_cfg.disabled = 1; 2289 pr_emerg("Couldn't allocate MCE records pool!\n"); 2290 return; 2291 } 2292 2293 mca_cfg.initialized = 1; 2294 2295 __mcheck_cpu_init_generic(); 2296 __mcheck_cpu_init_vendor(c); 2297 __mcheck_cpu_init_prepare_banks(); 2298 __mcheck_cpu_setup_timer(); 2299 cr4_set_bits(X86_CR4_MCE); 2300} 2301 2302/* 2303 * Called for each booted CPU to clear some machine checks opt-ins 2304 */ 2305void mcheck_cpu_clear(struct cpuinfo_x86 *c) 2306{ 2307 if (mca_cfg.disabled) 2308 return; 2309 2310 if (!mce_available(c)) 2311 return; 2312 2313 /* 2314 * Possibly to clear general settings generic to x86 2315 * __mcheck_cpu_clear_generic(c); 2316 */ 2317 __mcheck_cpu_clear_vendor(c); 2318 2319} 2320 2321static void __mce_disable_bank(void *arg) 2322{ 2323 int bank = *((int *)arg); 2324 __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); 2325 cmci_disable_bank(bank); 2326} 2327 2328void mce_disable_bank(int bank) 2329{ 2330 if (bank >= this_cpu_read(mce_num_banks)) { 2331 pr_warn(FW_BUG 2332 "Ignoring request to disable invalid MCA bank %d.\n", 2333 bank); 2334 return; 2335 } 2336 set_bit(bank, mce_banks_ce_disabled); 2337 on_each_cpu(__mce_disable_bank, &bank, 1); 2338} 2339 2340/* 2341 * mce=off Disables machine check 2342 * mce=no_cmci Disables CMCI 2343 * mce=no_lmce Disables LMCE 2344 * mce=dont_log_ce Clears corrected events silently, no log created for CEs. 2345 * mce=print_all Print all machine check logs to console 2346 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared. 2347 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above) 2348 * monarchtimeout is how long to wait for other CPUs on machine 2349 * check, or 0 to not wait 2350 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h 2351 and older. 2352 * mce=nobootlog Don't log MCEs from before booting. 2353 * mce=bios_cmci_threshold Don't program the CMCI threshold 2354 * mce=recovery force enable copy_mc_fragile() 2355 */ 2356static int __init mcheck_enable(char *str) 2357{ 2358 struct mca_config *cfg = &mca_cfg; 2359 2360 if (*str == 0) { 2361 enable_p5_mce(); 2362 return 1; 2363 } 2364 if (*str == '=') 2365 str++; 2366 if (!strcmp(str, "off")) 2367 cfg->disabled = 1; 2368 else if (!strcmp(str, "no_cmci")) 2369 cfg->cmci_disabled = true; 2370 else if (!strcmp(str, "no_lmce")) 2371 cfg->lmce_disabled = 1; 2372 else if (!strcmp(str, "dont_log_ce")) 2373 cfg->dont_log_ce = true; 2374 else if (!strcmp(str, "print_all")) 2375 cfg->print_all = true; 2376 else if (!strcmp(str, "ignore_ce")) 2377 cfg->ignore_ce = true; 2378 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) 2379 cfg->bootlog = (str[0] == 'b'); 2380 else if (!strcmp(str, "bios_cmci_threshold")) 2381 cfg->bios_cmci_threshold = 1; 2382 else if (!strcmp(str, "recovery")) 2383 cfg->recovery = 1; 2384 else if (isdigit(str[0])) 2385 get_option(&str, &(cfg->monarch_timeout)); 2386 else { 2387 pr_info("mce argument %s ignored. Please use /sys\n", str); 2388 return 0; 2389 } 2390 return 1; 2391} 2392__setup("mce", mcheck_enable); 2393 2394int __init mcheck_init(void) 2395{ 2396 mce_register_decode_chain(&early_nb); 2397 mce_register_decode_chain(&mce_uc_nb); 2398 mce_register_decode_chain(&mce_default_nb); 2399 2400 INIT_WORK(&mce_work, mce_gen_pool_process); 2401 init_irq_work(&mce_irq_work, mce_irq_work_cb); 2402 2403 return 0; 2404} 2405 2406/* 2407 * mce_syscore: PM support 2408 */ 2409 2410/* 2411 * Disable machine checks on suspend and shutdown. We can't really handle 2412 * them later. 2413 */ 2414static void mce_disable_error_reporting(void) 2415{ 2416 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2417 int i; 2418 2419 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 2420 struct mce_bank *b = &mce_banks[i]; 2421 2422 if (b->init) 2423 wrmsrq(mca_msr_reg(i, MCA_CTL), 0); 2424 } 2425 return; 2426} 2427 2428static void vendor_disable_error_reporting(void) 2429{ 2430 /* 2431 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these 2432 * MSRs are socket-wide. Disabling them for just a single offlined CPU 2433 * is bad, since it will inhibit reporting for all shared resources on 2434 * the socket like the last level cache (LLC), the integrated memory 2435 * controller (iMC), etc. 2436 */ 2437 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL || 2438 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON || 2439 boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2440 boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) 2441 return; 2442 2443 mce_disable_error_reporting(); 2444} 2445 2446static int mce_syscore_suspend(void *data) 2447{ 2448 vendor_disable_error_reporting(); 2449 return 0; 2450} 2451 2452static void mce_syscore_shutdown(void *data) 2453{ 2454 vendor_disable_error_reporting(); 2455} 2456 2457/* 2458 * On resume clear all MCE state. Don't want to see leftovers from the BIOS. 2459 * Only one CPU is active at this time, the others get re-added later using 2460 * CPU hotplug: 2461 */ 2462static void mce_syscore_resume(void *data) 2463{ 2464 __mcheck_cpu_init_generic(); 2465 __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info)); 2466 __mcheck_cpu_init_prepare_banks(); 2467 cr4_set_bits(X86_CR4_MCE); 2468} 2469 2470static const struct syscore_ops mce_syscore_ops = { 2471 .suspend = mce_syscore_suspend, 2472 .shutdown = mce_syscore_shutdown, 2473 .resume = mce_syscore_resume, 2474}; 2475 2476static struct syscore mce_syscore = { 2477 .ops = &mce_syscore_ops, 2478}; 2479 2480/* 2481 * mce_device: Sysfs support 2482 */ 2483 2484static void mce_cpu_restart(void *data) 2485{ 2486 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2487 return; 2488 __mcheck_cpu_init_generic(); 2489 __mcheck_cpu_init_prepare_banks(); 2490 __mcheck_cpu_init_timer(); 2491 cr4_set_bits(X86_CR4_MCE); 2492} 2493 2494/* Reinit MCEs after user configuration changes */ 2495static void mce_restart(void) 2496{ 2497 mce_timer_delete_all(); 2498 on_each_cpu(mce_cpu_restart, NULL, 1); 2499 mce_schedule_work(); 2500} 2501 2502/* Toggle features for corrected errors */ 2503static void mce_disable_cmci(void *data) 2504{ 2505 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2506 return; 2507 cmci_clear(); 2508} 2509 2510static void mce_enable_ce(void *all) 2511{ 2512 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2513 return; 2514 cmci_reenable(); 2515 cmci_recheck(); 2516 if (all) 2517 __mcheck_cpu_init_timer(); 2518} 2519 2520static const struct bus_type mce_subsys = { 2521 .name = "machinecheck", 2522 .dev_name = "machinecheck", 2523}; 2524 2525DEFINE_PER_CPU(struct device *, mce_device); 2526 2527static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr) 2528{ 2529 return container_of(attr, struct mce_bank_dev, attr); 2530} 2531 2532static ssize_t show_bank(struct device *s, struct device_attribute *attr, 2533 char *buf) 2534{ 2535 u8 bank = attr_to_bank(attr)->bank; 2536 struct mce_bank *b; 2537 2538 if (bank >= per_cpu(mce_num_banks, s->id)) 2539 return -EINVAL; 2540 2541 b = &per_cpu(mce_banks_array, s->id)[bank]; 2542 2543 if (!b->init) 2544 return -ENODEV; 2545 2546 return sprintf(buf, "%llx\n", b->ctl); 2547} 2548 2549static ssize_t set_bank(struct device *s, struct device_attribute *attr, 2550 const char *buf, size_t size) 2551{ 2552 u8 bank = attr_to_bank(attr)->bank; 2553 struct mce_bank *b; 2554 u64 new; 2555 2556 if (kstrtou64(buf, 0, &new) < 0) 2557 return -EINVAL; 2558 2559 if (bank >= per_cpu(mce_num_banks, s->id)) 2560 return -EINVAL; 2561 2562 b = &per_cpu(mce_banks_array, s->id)[bank]; 2563 if (!b->init) 2564 return -ENODEV; 2565 2566 b->ctl = new; 2567 2568 mutex_lock(&mce_sysfs_mutex); 2569 mce_restart(); 2570 mutex_unlock(&mce_sysfs_mutex); 2571 2572 return size; 2573} 2574 2575static ssize_t set_ignore_ce(struct device *s, 2576 struct device_attribute *attr, 2577 const char *buf, size_t size) 2578{ 2579 u64 new; 2580 2581 if (kstrtou64(buf, 0, &new) < 0) 2582 return -EINVAL; 2583 2584 mutex_lock(&mce_sysfs_mutex); 2585 if (mca_cfg.ignore_ce ^ !!new) { 2586 if (new) { 2587 /* disable ce features */ 2588 mce_timer_delete_all(); 2589 on_each_cpu(mce_disable_cmci, NULL, 1); 2590 mca_cfg.ignore_ce = true; 2591 } else { 2592 /* enable ce features */ 2593 mca_cfg.ignore_ce = false; 2594 on_each_cpu(mce_enable_ce, (void *)1, 1); 2595 } 2596 } 2597 mutex_unlock(&mce_sysfs_mutex); 2598 2599 return size; 2600} 2601 2602static ssize_t set_cmci_disabled(struct device *s, 2603 struct device_attribute *attr, 2604 const char *buf, size_t size) 2605{ 2606 u64 new; 2607 2608 if (kstrtou64(buf, 0, &new) < 0) 2609 return -EINVAL; 2610 2611 mutex_lock(&mce_sysfs_mutex); 2612 if (mca_cfg.cmci_disabled ^ !!new) { 2613 if (new) { 2614 /* disable cmci */ 2615 on_each_cpu(mce_disable_cmci, NULL, 1); 2616 mca_cfg.cmci_disabled = true; 2617 } else { 2618 /* enable cmci */ 2619 mca_cfg.cmci_disabled = false; 2620 on_each_cpu(mce_enable_ce, NULL, 1); 2621 } 2622 } 2623 mutex_unlock(&mce_sysfs_mutex); 2624 2625 return size; 2626} 2627 2628static ssize_t store_int_with_restart(struct device *s, 2629 struct device_attribute *attr, 2630 const char *buf, size_t size) 2631{ 2632 unsigned long old_check_interval = check_interval; 2633 ssize_t ret = device_store_ulong(s, attr, buf, size); 2634 2635 if (check_interval == old_check_interval) 2636 return ret; 2637 2638 mutex_lock(&mce_sysfs_mutex); 2639 mce_restart(); 2640 mutex_unlock(&mce_sysfs_mutex); 2641 2642 return ret; 2643} 2644 2645static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout); 2646static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce); 2647static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all); 2648 2649static struct dev_ext_attribute dev_attr_check_interval = { 2650 __ATTR(check_interval, 0644, device_show_int, store_int_with_restart), 2651 &check_interval 2652}; 2653 2654static struct dev_ext_attribute dev_attr_ignore_ce = { 2655 __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce), 2656 &mca_cfg.ignore_ce 2657}; 2658 2659static struct dev_ext_attribute dev_attr_cmci_disabled = { 2660 __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled), 2661 &mca_cfg.cmci_disabled 2662}; 2663 2664static struct device_attribute *mce_device_attrs[] = { 2665 &dev_attr_check_interval.attr, 2666#ifdef CONFIG_X86_MCELOG_LEGACY 2667 &dev_attr_trigger, 2668#endif 2669 &dev_attr_monarch_timeout.attr, 2670 &dev_attr_dont_log_ce.attr, 2671 &dev_attr_print_all.attr, 2672 &dev_attr_ignore_ce.attr, 2673 &dev_attr_cmci_disabled.attr, 2674 NULL 2675}; 2676 2677static cpumask_var_t mce_device_initialized; 2678 2679static void mce_device_release(struct device *dev) 2680{ 2681 kfree(dev); 2682} 2683 2684/* Per CPU device init. All of the CPUs still share the same bank device: */ 2685static int mce_device_create(unsigned int cpu) 2686{ 2687 struct device *dev; 2688 int err; 2689 int i, j; 2690 2691 dev = per_cpu(mce_device, cpu); 2692 if (dev) 2693 return 0; 2694 2695 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2696 if (!dev) 2697 return -ENOMEM; 2698 dev->id = cpu; 2699 dev->bus = &mce_subsys; 2700 dev->release = &mce_device_release; 2701 2702 err = device_register(dev); 2703 if (err) { 2704 put_device(dev); 2705 return err; 2706 } 2707 2708 for (i = 0; mce_device_attrs[i]; i++) { 2709 err = device_create_file(dev, mce_device_attrs[i]); 2710 if (err) 2711 goto error; 2712 } 2713 for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) { 2714 err = device_create_file(dev, &mce_bank_devs[j].attr); 2715 if (err) 2716 goto error2; 2717 } 2718 cpumask_set_cpu(cpu, mce_device_initialized); 2719 per_cpu(mce_device, cpu) = dev; 2720 2721 return 0; 2722error2: 2723 while (--j >= 0) 2724 device_remove_file(dev, &mce_bank_devs[j].attr); 2725error: 2726 while (--i >= 0) 2727 device_remove_file(dev, mce_device_attrs[i]); 2728 2729 device_unregister(dev); 2730 2731 return err; 2732} 2733 2734static void mce_device_remove(unsigned int cpu) 2735{ 2736 struct device *dev = per_cpu(mce_device, cpu); 2737 int i; 2738 2739 if (!cpumask_test_cpu(cpu, mce_device_initialized)) 2740 return; 2741 2742 for (i = 0; mce_device_attrs[i]; i++) 2743 device_remove_file(dev, mce_device_attrs[i]); 2744 2745 for (i = 0; i < per_cpu(mce_num_banks, cpu); i++) 2746 device_remove_file(dev, &mce_bank_devs[i].attr); 2747 2748 device_unregister(dev); 2749 cpumask_clear_cpu(cpu, mce_device_initialized); 2750 per_cpu(mce_device, cpu) = NULL; 2751} 2752 2753/* Make sure there are no machine checks on offlined CPUs. */ 2754static void mce_disable_cpu(void) 2755{ 2756 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2757 return; 2758 2759 if (!cpuhp_tasks_frozen) 2760 cmci_clear(); 2761 2762 vendor_disable_error_reporting(); 2763} 2764 2765static void mce_reenable_cpu(void) 2766{ 2767 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2768 int i; 2769 2770 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2771 return; 2772 2773 if (!cpuhp_tasks_frozen) 2774 cmci_reenable(); 2775 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 2776 struct mce_bank *b = &mce_banks[i]; 2777 2778 if (b->init) 2779 wrmsrq(mca_msr_reg(i, MCA_CTL), b->ctl); 2780 } 2781} 2782 2783static int mce_cpu_dead(unsigned int cpu) 2784{ 2785 /* intentionally ignoring frozen here */ 2786 if (!cpuhp_tasks_frozen) 2787 cmci_rediscover(); 2788 return 0; 2789} 2790 2791static int mce_cpu_online(unsigned int cpu) 2792{ 2793 struct timer_list *t = this_cpu_ptr(&mce_timer); 2794 2795 mce_device_create(cpu); 2796 mce_threshold_create_device(cpu); 2797 mce_reenable_cpu(); 2798 mce_start_timer(t); 2799 return 0; 2800} 2801 2802static int mce_cpu_pre_down(unsigned int cpu) 2803{ 2804 struct timer_list *t = this_cpu_ptr(&mce_timer); 2805 2806 mce_disable_cpu(); 2807 timer_delete_sync(t); 2808 mce_threshold_remove_device(cpu); 2809 mce_device_remove(cpu); 2810 return 0; 2811} 2812 2813static __init void mce_init_banks(void) 2814{ 2815 int i; 2816 2817 for (i = 0; i < MAX_NR_BANKS; i++) { 2818 struct mce_bank_dev *b = &mce_bank_devs[i]; 2819 struct device_attribute *a = &b->attr; 2820 2821 b->bank = i; 2822 2823 sysfs_attr_init(&a->attr); 2824 a->attr.name = b->attrname; 2825 snprintf(b->attrname, ATTR_LEN, "bank%d", i); 2826 2827 a->attr.mode = 0644; 2828 a->show = show_bank; 2829 a->store = set_bank; 2830 } 2831} 2832 2833/* 2834 * When running on XEN, this initcall is ordered against the XEN mcelog 2835 * initcall: 2836 * 2837 * device_initcall(xen_late_init_mcelog); 2838 * device_initcall_sync(mcheck_init_device); 2839 */ 2840static __init int mcheck_init_device(void) 2841{ 2842 int err; 2843 2844 /* 2845 * Check if we have a spare virtual bit. This will only become 2846 * a problem if/when we move beyond 5-level page tables. 2847 */ 2848 MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63); 2849 2850 if (!mce_available(&boot_cpu_data)) { 2851 err = -EIO; 2852 goto err_out; 2853 } 2854 2855 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { 2856 err = -ENOMEM; 2857 goto err_out; 2858 } 2859 2860 mce_init_banks(); 2861 2862 err = subsys_system_register(&mce_subsys, NULL); 2863 if (err) 2864 goto err_out_mem; 2865 2866 err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL, 2867 mce_cpu_dead); 2868 if (err) 2869 goto err_out_mem; 2870 2871 /* 2872 * Invokes mce_cpu_online() on all CPUs which are online when 2873 * the state is installed. 2874 */ 2875 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", 2876 mce_cpu_online, mce_cpu_pre_down); 2877 if (err < 0) 2878 goto err_out_online; 2879 2880 register_syscore(&mce_syscore); 2881 2882 return 0; 2883 2884err_out_online: 2885 cpuhp_remove_state(CPUHP_X86_MCE_DEAD); 2886 2887err_out_mem: 2888 free_cpumask_var(mce_device_initialized); 2889 2890err_out: 2891 pr_err("Unable to init MCE device (rc: %d)\n", err); 2892 2893 return err; 2894} 2895device_initcall_sync(mcheck_init_device); 2896 2897/* 2898 * Old style boot options parsing. Only for compatibility. 2899 */ 2900static int __init mcheck_disable(char *str) 2901{ 2902 mca_cfg.disabled = 1; 2903 return 1; 2904} 2905__setup("nomce", mcheck_disable); 2906 2907#ifdef CONFIG_DEBUG_FS 2908struct dentry *mce_get_debugfs_dir(void) 2909{ 2910 static struct dentry *dmce; 2911 2912 if (!dmce) 2913 dmce = debugfs_create_dir("mce", NULL); 2914 2915 return dmce; 2916} 2917 2918static void mce_reset(void) 2919{ 2920 atomic_set(&mce_fake_panicked, 0); 2921 atomic_set(&mce_executing, 0); 2922 atomic_set(&mce_callin, 0); 2923 atomic_set(&global_nwo, 0); 2924 cpumask_setall(&mce_missing_cpus); 2925} 2926 2927static int fake_panic_get(void *data, u64 *val) 2928{ 2929 *val = fake_panic; 2930 return 0; 2931} 2932 2933static int fake_panic_set(void *data, u64 val) 2934{ 2935 mce_reset(); 2936 fake_panic = val; 2937 return 0; 2938} 2939 2940DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set, 2941 "%llu\n"); 2942 2943static void __init mcheck_debugfs_init(void) 2944{ 2945 struct dentry *dmce; 2946 2947 dmce = mce_get_debugfs_dir(); 2948 debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL, 2949 &fake_panic_fops); 2950} 2951#else 2952static void __init mcheck_debugfs_init(void) { } 2953#endif 2954 2955static int __init mcheck_late_init(void) 2956{ 2957 if (mca_cfg.recovery) 2958 enable_copy_mc_fragile(); 2959 2960 mcheck_debugfs_init(); 2961 2962 /* 2963 * Flush out everything that has been logged during early boot, now that 2964 * everything has been initialized (workqueues, decoders, ...). 2965 */ 2966 mce_schedule_work(); 2967 2968 return 0; 2969} 2970late_initcall(mcheck_late_init);
~/
Read-only file preview.