inject.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Machine check injection support.
  4 * Copyright 2008 Intel Corporation.
  5 *
  6 * Authors:
  7 * Andi Kleen
  8 * Ying Huang
  9 *
 10 * The AMD part (from mce_amd_inj.c): a simple MCE injection facility
 11 * for testing different aspects of the RAS code. This driver should be
 12 * built as module so that it can be loaded on production kernels for
 13 * testing purposes.
 14 *
 15 * Copyright (c) 2010-17:  Borislav Petkov <bp@alien8.de>
 16 *			   Advanced Micro Devices Inc.
 17 */
 18
 19#include <linux/cpu.h>
 20#include <linux/debugfs.h>
 21#include <linux/kernel.h>
 22#include <linux/module.h>
 23#include <linux/notifier.h>
 24#include <linux/pci.h>
 25#include <linux/uaccess.h>
 26
 27#include <asm/amd/nb.h>
 28#include <asm/apic.h>
 29#include <asm/irq_vectors.h>
 30#include <asm/mce.h>
 31#include <asm/msr.h>
 32#include <asm/nmi.h>
 33#include <asm/smp.h>
 34
 35#include "internal.h"
 36
 37static bool hw_injection_possible = true;
 38
 39/*
 40 * Collect all the MCi_XXX settings
 41 */
 42static struct mce i_mce;
 43static struct dentry *dfs_inj;
 44
 45#define MAX_FLAG_OPT_SIZE	4
 46#define NBCFG			0x44
 47
 48enum injection_type {
 49	SW_INJ = 0,	/* SW injection, simply decode the error */
 50	HW_INJ,		/* Trigger a #MC */
 51	DFR_INT_INJ,    /* Trigger Deferred error interrupt */
 52	THR_INT_INJ,    /* Trigger threshold interrupt */
 53	N_INJ_TYPES,
 54};
 55
 56static const char * const flags_options[] = {
 57	[SW_INJ] = "sw",
 58	[HW_INJ] = "hw",
 59	[DFR_INT_INJ] = "df",
 60	[THR_INT_INJ] = "th",
 61	NULL
 62};
 63
 64/* Set default injection to SW_INJ */
 65static enum injection_type inj_type = SW_INJ;
 66
 67#define MCE_INJECT_SET(reg)						\
 68static int inj_##reg##_set(void *data, u64 val)				\
 69{									\
 70	struct mce *m = (struct mce *)data;				\
 71									\
 72	m->reg = val;							\
 73	return 0;							\
 74}
 75
 76MCE_INJECT_SET(status);
 77MCE_INJECT_SET(misc);
 78MCE_INJECT_SET(addr);
 79MCE_INJECT_SET(synd);
 80
 81#define MCE_INJECT_GET(reg)						\
 82static int inj_##reg##_get(void *data, u64 *val)			\
 83{									\
 84	struct mce *m = (struct mce *)data;				\
 85									\
 86	*val = m->reg;							\
 87	return 0;							\
 88}
 89
 90MCE_INJECT_GET(status);
 91MCE_INJECT_GET(misc);
 92MCE_INJECT_GET(addr);
 93MCE_INJECT_GET(synd);
 94MCE_INJECT_GET(ipid);
 95
 96DEFINE_SIMPLE_ATTRIBUTE(status_fops, inj_status_get, inj_status_set, "%llx\n");
 97DEFINE_SIMPLE_ATTRIBUTE(misc_fops, inj_misc_get, inj_misc_set, "%llx\n");
 98DEFINE_SIMPLE_ATTRIBUTE(addr_fops, inj_addr_get, inj_addr_set, "%llx\n");
 99DEFINE_SIMPLE_ATTRIBUTE(synd_fops, inj_synd_get, inj_synd_set, "%llx\n");
100
101/* Use the user provided IPID value on a sw injection. */
102static int inj_ipid_set(void *data, u64 val)
103{
104	struct mce *m = (struct mce *)data;
105
106	if (cpu_feature_enabled(X86_FEATURE_SMCA)) {
107		if (inj_type == SW_INJ)
108			m->ipid = val;
109	}
110
111	return 0;
112}
113
114DEFINE_SIMPLE_ATTRIBUTE(ipid_fops, inj_ipid_get, inj_ipid_set, "%llx\n");
115
116static void setup_inj_struct(struct mce *m)
117{
118	memset(m, 0, sizeof(struct mce));
119
120	m->cpuvendor = boot_cpu_data.x86_vendor;
121	m->time	     = ktime_get_real_seconds();
122	m->cpuid     = cpuid_eax(1);
123	m->microcode = boot_cpu_data.microcode;
124}
125
126/* Update fake mce registers on current CPU. */
127static void inject_mce(struct mce *m)
128{
129	struct mce *i = &per_cpu(injectm, m->extcpu);
130
131	/* Make sure no one reads partially written injectm */
132	i->finished = 0;
133	mb();
134	m->finished = 0;
135	/* First set the fields after finished */
136	i->extcpu = m->extcpu;
137	mb();
138	/* Now write record in order, finished last (except above) */
139	memcpy(i, m, sizeof(struct mce));
140	/* Finally activate it */
141	mb();
142	i->finished = 1;
143}
144
145static void raise_poll(struct mce *m)
146{
147	unsigned long flags;
148	mce_banks_t b;
149
150	memset(&b, 0xff, sizeof(mce_banks_t));
151	local_irq_save(flags);
152	machine_check_poll(0, &b);
153	local_irq_restore(flags);
154	m->finished = 0;
155}
156
157static void raise_exception(struct mce *m, struct pt_regs *pregs)
158{
159	struct pt_regs regs;
160	unsigned long flags;
161
162	if (!pregs) {
163		memset(&regs, 0, sizeof(struct pt_regs));
164		regs.ip = m->ip;
165		regs.cs = m->cs;
166		pregs = &regs;
167	}
168	/* do_machine_check() expects interrupts disabled -- at least */
169	local_irq_save(flags);
170	do_machine_check(pregs);
171	local_irq_restore(flags);
172	m->finished = 0;
173}
174
175static cpumask_var_t mce_inject_cpumask;
176static DEFINE_MUTEX(mce_inject_mutex);
177
178static int mce_raise_notify(unsigned int cmd, struct pt_regs *regs)
179{
180	int cpu = smp_processor_id();
181	struct mce *m = this_cpu_ptr(&injectm);
182	if (!cpumask_test_cpu(cpu, mce_inject_cpumask))
183		return NMI_DONE;
184	cpumask_clear_cpu(cpu, mce_inject_cpumask);
185	if (m->inject_flags & MCJ_EXCEPTION)
186		raise_exception(m, regs);
187	else if (m->status)
188		raise_poll(m);
189	return NMI_HANDLED;
190}
191
192static void mce_irq_ipi(void *info)
193{
194	int cpu = smp_processor_id();
195	struct mce *m = this_cpu_ptr(&injectm);
196
197	if (cpumask_test_cpu(cpu, mce_inject_cpumask) &&
198			m->inject_flags & MCJ_EXCEPTION) {
199		cpumask_clear_cpu(cpu, mce_inject_cpumask);
200		raise_exception(m, NULL);
201	}
202}
203
204/* Inject mce on current CPU */
205static int raise_local(void)
206{
207	struct mce *m = this_cpu_ptr(&injectm);
208	int context = MCJ_CTX(m->inject_flags);
209	int ret = 0;
210	int cpu = m->extcpu;
211
212	if (m->inject_flags & MCJ_EXCEPTION) {
213		pr_info("Triggering MCE exception on CPU %d\n", cpu);
214		switch (context) {
215		case MCJ_CTX_IRQ:
216			/*
217			 * Could do more to fake interrupts like
218			 * calling irq_enter, but the necessary
219			 * machinery isn't exported currently.
220			 */
221			fallthrough;
222		case MCJ_CTX_PROCESS:
223			raise_exception(m, NULL);
224			break;
225		default:
226			pr_info("Invalid MCE context\n");
227			ret = -EINVAL;
228		}
229		pr_info("MCE exception done on CPU %d\n", cpu);
230	} else if (m->status) {
231		pr_info("Starting machine check poll CPU %d\n", cpu);
232		raise_poll(m);
233		pr_info("Machine check poll done on CPU %d\n", cpu);
234	} else
235		m->finished = 0;
236
237	return ret;
238}
239
240static void __maybe_unused raise_mce(struct mce *m)
241{
242	int context = MCJ_CTX(m->inject_flags);
243
244	inject_mce(m);
245
246	if (context == MCJ_CTX_RANDOM)
247		return;
248
249	if (m->inject_flags & (MCJ_IRQ_BROADCAST | MCJ_NMI_BROADCAST)) {
250		unsigned long start;
251		int cpu;
252
253		cpus_read_lock();
254		cpumask_copy(mce_inject_cpumask, cpu_online_mask);
255		cpumask_clear_cpu(get_cpu(), mce_inject_cpumask);
256		for_each_online_cpu(cpu) {
257			struct mce *mcpu = &per_cpu(injectm, cpu);
258			if (!mcpu->finished ||
259			    MCJ_CTX(mcpu->inject_flags) != MCJ_CTX_RANDOM)
260				cpumask_clear_cpu(cpu, mce_inject_cpumask);
261		}
262		if (!cpumask_empty(mce_inject_cpumask)) {
263			if (m->inject_flags & MCJ_IRQ_BROADCAST) {
264				/*
265				 * don't wait because mce_irq_ipi is necessary
266				 * to be sync with following raise_local
267				 */
268				preempt_disable();
269				smp_call_function_many(mce_inject_cpumask,
270					mce_irq_ipi, NULL, 0);
271				preempt_enable();
272			} else if (m->inject_flags & MCJ_NMI_BROADCAST)
273				__apic_send_IPI_mask(mce_inject_cpumask, NMI_VECTOR);
274		}
275		start = jiffies;
276		while (!cpumask_empty(mce_inject_cpumask)) {
277			if (!time_before(jiffies, start + 2*HZ)) {
278				pr_err("Timeout waiting for mce inject %lx\n",
279					*cpumask_bits(mce_inject_cpumask));
280				break;
281			}
282			cpu_relax();
283		}
284		raise_local();
285		put_cpu();
286		cpus_read_unlock();
287	} else {
288		preempt_disable();
289		raise_local();
290		preempt_enable();
291	}
292}
293
294static int mce_inject_raise(struct notifier_block *nb, unsigned long val,
295			    void *data)
296{
297	struct mce *m = (struct mce *)data;
298
299	if (!m)
300		return NOTIFY_DONE;
301
302	mutex_lock(&mce_inject_mutex);
303	raise_mce(m);
304	mutex_unlock(&mce_inject_mutex);
305
306	return NOTIFY_DONE;
307}
308
309static struct notifier_block inject_nb = {
310	.notifier_call  = mce_inject_raise,
311};
312
313/*
314 * Caller needs to be make sure this cpu doesn't disappear
315 * from under us, i.e.: get_cpu/put_cpu.
316 */
317static int toggle_hw_mce_inject(unsigned int cpu, bool enable)
318{
319	u32 l, h;
320	int err;
321
322	err = rdmsr_on_cpu(cpu, MSR_K7_HWCR, &l, &h);
323	if (err) {
324		pr_err("%s: error reading HWCR\n", __func__);
325		return err;
326	}
327
328	enable ? (l |= BIT(18)) : (l &= ~BIT(18));
329
330	err = wrmsr_on_cpu(cpu, MSR_K7_HWCR, l, h);
331	if (err)
332		pr_err("%s: error writing HWCR\n", __func__);
333
334	return err;
335}
336
337static int __set_inj(const char *buf)
338{
339	int i;
340
341	for (i = 0; i < N_INJ_TYPES; i++) {
342		if (!strncmp(flags_options[i], buf, strlen(flags_options[i]))) {
343			if (i > SW_INJ && !hw_injection_possible)
344				continue;
345			inj_type = i;
346			return 0;
347		}
348	}
349	return -EINVAL;
350}
351
352static ssize_t flags_read(struct file *filp, char __user *ubuf,
353			  size_t cnt, loff_t *ppos)
354{
355	char buf[MAX_FLAG_OPT_SIZE];
356	int n;
357
358	n = sprintf(buf, "%s\n", flags_options[inj_type]);
359
360	return simple_read_from_buffer(ubuf, cnt, ppos, buf, n);
361}
362
363static ssize_t flags_write(struct file *filp, const char __user *ubuf,
364			   size_t cnt, loff_t *ppos)
365{
366	char buf[MAX_FLAG_OPT_SIZE], *__buf;
367	int err;
368
369	if (!cnt || cnt > MAX_FLAG_OPT_SIZE)
370		return -EINVAL;
371
372	if (copy_from_user(&buf, ubuf, cnt))
373		return -EFAULT;
374
375	buf[cnt - 1] = 0;
376
377	/* strip whitespace */
378	__buf = strstrip(buf);
379
380	err = __set_inj(__buf);
381	if (err) {
382		pr_err("%s: Invalid flags value: %s\n", __func__, __buf);
383		return err;
384	}
385
386	*ppos += cnt;
387
388	return cnt;
389}
390
391static const struct file_operations flags_fops = {
392	.read           = flags_read,
393	.write          = flags_write,
394	.llseek         = generic_file_llseek,
395};
396
397/*
398 * On which CPU to inject?
399 */
400MCE_INJECT_GET(extcpu);
401
402static int inj_extcpu_set(void *data, u64 val)
403{
404	struct mce *m = (struct mce *)data;
405
406	if (val >= nr_cpu_ids || !cpu_online(val)) {
407		pr_err("%s: Invalid CPU: %llu\n", __func__, val);
408		return -EINVAL;
409	}
410	m->extcpu = val;
411	return 0;
412}
413
414DEFINE_SIMPLE_ATTRIBUTE(extcpu_fops, inj_extcpu_get, inj_extcpu_set, "%llu\n");
415
416static void trigger_mce(void *info)
417{
418	asm volatile("int $18");
419}
420
421static void trigger_dfr_int(void *info)
422{
423	asm volatile("int %0" :: "i" (DEFERRED_ERROR_VECTOR));
424}
425
426static void trigger_thr_int(void *info)
427{
428	asm volatile("int %0" :: "i" (THRESHOLD_APIC_VECTOR));
429}
430
431static u32 get_nbc_for_node(int node_id)
432{
433	u32 cores_per_node;
434
435	cores_per_node = topology_num_threads_per_package() / topology_amd_nodes_per_pkg();
436	return cores_per_node * node_id;
437}
438
439static void toggle_nb_mca_mst_cpu(u16 nid)
440{
441	struct amd_northbridge *nb;
442	struct pci_dev *F3;
443	u32 val;
444	int err;
445
446	nb = node_to_amd_nb(nid);
447	if (!nb)
448		return;
449
450	F3 = nb->misc;
451	if (!F3)
452		return;
453
454	err = pci_read_config_dword(F3, NBCFG, &val);
455	if (err) {
456		pr_err("%s: Error reading F%dx%03x.\n",
457		       __func__, PCI_FUNC(F3->devfn), NBCFG);
458		return;
459	}
460
461	if (val & BIT(27))
462		return;
463
464	pr_err("%s: Set D18F3x44[NbMcaToMstCpuEn] which BIOS hasn't done.\n",
465	       __func__);
466
467	val |= BIT(27);
468	err = pci_write_config_dword(F3, NBCFG, val);
469	if (err)
470		pr_err("%s: Error writing F%dx%03x.\n",
471		       __func__, PCI_FUNC(F3->devfn), NBCFG);
472}
473
474static void prepare_msrs(void *info)
475{
476	struct mce m = *(struct mce *)info;
477	u8 b = m.bank;
478
479	wrmsrq(MSR_IA32_MCG_STATUS, m.mcgstatus);
480
481	if (boot_cpu_has(X86_FEATURE_SMCA)) {
482		if (m.inject_flags == DFR_INT_INJ) {
483			wrmsrq(MSR_AMD64_SMCA_MCx_DESTAT(b), m.status);
484			wrmsrq(MSR_AMD64_SMCA_MCx_DEADDR(b), m.addr);
485		} else {
486			wrmsrq(MSR_AMD64_SMCA_MCx_STATUS(b), m.status);
487			wrmsrq(MSR_AMD64_SMCA_MCx_ADDR(b), m.addr);
488		}
489
490		wrmsrq(MSR_AMD64_SMCA_MCx_SYND(b), m.synd);
491
492		if (m.misc)
493			wrmsrq(MSR_AMD64_SMCA_MCx_MISC(b), m.misc);
494	} else {
495		wrmsrq(MSR_IA32_MCx_STATUS(b), m.status);
496		wrmsrq(MSR_IA32_MCx_ADDR(b), m.addr);
497
498		if (m.misc)
499			wrmsrq(MSR_IA32_MCx_MISC(b), m.misc);
500	}
501}
502
503static void do_inject(void)
504{
505	unsigned int cpu = i_mce.extcpu;
506	struct mce_hw_err err;
507	u64 mcg_status = 0;
508	u8 b = i_mce.bank;
509
510	i_mce.tsc = rdtsc_ordered();
511
512	i_mce.status |= MCI_STATUS_VAL;
513
514	if (i_mce.misc)
515		i_mce.status |= MCI_STATUS_MISCV;
516
517	if (i_mce.synd)
518		i_mce.status |= MCI_STATUS_SYNDV;
519
520	if (inj_type == SW_INJ) {
521		err.m = i_mce;
522		mce_log(&err);
523		return;
524	}
525
526	/* prep MCE global settings for the injection */
527	mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV;
528
529	if (!(i_mce.status & MCI_STATUS_PCC))
530		mcg_status |= MCG_STATUS_RIPV;
531
532	/*
533	 * Ensure necessary status bits for deferred errors:
534	 * - MCx_STATUS[Deferred]: make sure it is a deferred error
535	 * - MCx_STATUS[UC] cleared: deferred errors are _not_ UC
536	 */
537	if (inj_type == DFR_INT_INJ) {
538		i_mce.status |= MCI_STATUS_DEFERRED;
539		i_mce.status &= ~MCI_STATUS_UC;
540	}
541
542	/*
543	 * For multi node CPUs, logging and reporting of bank 4 errors happens
544	 * only on the node base core. Refer to D18F3x44[NbMcaToMstCpuEn] for
545	 * Fam10h and later BKDGs.
546	 */
547	if (boot_cpu_has(X86_FEATURE_AMD_DCM) &&
548	    b == 4 &&
549	    boot_cpu_data.x86 < 0x17) {
550		toggle_nb_mca_mst_cpu(topology_amd_node_id(cpu));
551		cpu = get_nbc_for_node(topology_amd_node_id(cpu));
552	}
553
554	cpus_read_lock();
555	if (!cpu_online(cpu))
556		goto err;
557
558	toggle_hw_mce_inject(cpu, true);
559
560	i_mce.mcgstatus = mcg_status;
561	i_mce.inject_flags = inj_type;
562	smp_call_function_single(cpu, prepare_msrs, &i_mce, 0);
563
564	toggle_hw_mce_inject(cpu, false);
565
566	switch (inj_type) {
567	case DFR_INT_INJ:
568		smp_call_function_single(cpu, trigger_dfr_int, NULL, 0);
569		break;
570	case THR_INT_INJ:
571		smp_call_function_single(cpu, trigger_thr_int, NULL, 0);
572		break;
573	default:
574		smp_call_function_single(cpu, trigger_mce, NULL, 0);
575	}
576
577err:
578	cpus_read_unlock();
579
580}
581
582/*
583 * This denotes into which bank we're injecting and triggers
584 * the injection, at the same time.
585 */
586static int inj_bank_set(void *data, u64 val)
587{
588	struct mce *m = (struct mce *)data;
589	u8 n_banks;
590	u64 cap;
591
592	/* Get bank count on target CPU so we can handle non-uniform values. */
593	rdmsrq_on_cpu(m->extcpu, MSR_IA32_MCG_CAP, &cap);
594	n_banks = cap & MCG_BANKCNT_MASK;
595
596	if (val >= n_banks) {
597		pr_err("MCA bank %llu non-existent on CPU%d\n", val, m->extcpu);
598		return -EINVAL;
599	}
600
601	m->bank = val;
602
603	/*
604	 * sw-only injection allows to write arbitrary values into the MCA
605	 * registers because it tests only the decoding paths.
606	 */
607	if (inj_type == SW_INJ)
608		goto inject;
609
610	/*
611	 * Read IPID value to determine if a bank is populated on the target
612	 * CPU.
613	 */
614	if (cpu_feature_enabled(X86_FEATURE_SMCA)) {
615		u64 ipid;
616
617		if (rdmsrq_on_cpu(m->extcpu, MSR_AMD64_SMCA_MCx_IPID(val), &ipid)) {
618			pr_err("Error reading IPID on CPU%d\n", m->extcpu);
619			return -EINVAL;
620		}
621
622		if (!ipid) {
623			pr_err("Cannot inject into unpopulated bank %llu\n", val);
624			return -ENODEV;
625		}
626	}
627
628inject:
629	do_inject();
630
631	/* Reset injection struct */
632	setup_inj_struct(&i_mce);
633
634	return 0;
635}
636
637MCE_INJECT_GET(bank);
638
639DEFINE_SIMPLE_ATTRIBUTE(bank_fops, inj_bank_get, inj_bank_set, "%llu\n");
640
641static const char readme_msg[] =
642"Description of the files and their usages:\n"
643"\n"
644"Note1: i refers to the bank number below.\n"
645"Note2: See respective BKDGs for the exact bit definitions of the files below\n"
646"as they mirror the hardware registers.\n"
647"\n"
648"status:\t Set MCi_STATUS: the bits in that MSR control the error type and\n"
649"\t attributes of the error which caused the MCE.\n"
650"\n"
651"misc:\t Set MCi_MISC: provide auxiliary info about the error. It is mostly\n"
652"\t used for error thresholding purposes and its validity is indicated by\n"
653"\t MCi_STATUS[MiscV].\n"
654"\n"
655"synd:\t Set MCi_SYND: provide syndrome info about the error. Only valid on\n"
656"\t Scalable MCA systems, and its validity is indicated by MCi_STATUS[SyndV].\n"
657"\n"
658"addr:\t Error address value to be written to MCi_ADDR. Log address information\n"
659"\t associated with the error.\n"
660"\n"
661"cpu:\t The CPU to inject the error on.\n"
662"\n"
663"bank:\t Specify the bank you want to inject the error into: the number of\n"
664"\t banks in a processor varies and is family/model-specific, therefore, the\n"
665"\t supplied value is sanity-checked. Setting the bank value also triggers the\n"
666"\t injection.\n"
667"\n"
668"flags:\t Injection type to be performed. Writing to this file will trigger a\n"
669"\t real machine check, an APIC interrupt or invoke the error decoder routines\n"
670"\t for AMD processors.\n"
671"\n"
672"\t Allowed error injection types:\n"
673"\t  - \"sw\": Software error injection. Decode error to a human-readable \n"
674"\t    format only. Safe to use.\n"
675"\t  - \"hw\": Hardware error injection. Causes the #MC exception handler to \n"
676"\t    handle the error. Be warned: might cause system panic if MCi_STATUS[PCC] \n"
677"\t    is set. Therefore, consider setting (debugfs_mountpoint)/mce/fake_panic \n"
678"\t    before injecting.\n"
679"\t  - \"df\": Trigger APIC interrupt for Deferred error. Causes deferred \n"
680"\t    error APIC interrupt handler to handle the error if the feature is \n"
681"\t    is present in hardware. \n"
682"\t  - \"th\": Trigger APIC interrupt for Threshold errors. Causes threshold \n"
683"\t    APIC interrupt handler to handle the error. \n"
684"\n"
685"ipid:\t IPID (AMD-specific)\n"
686"\n";
687
688static ssize_t
689inj_readme_read(struct file *filp, char __user *ubuf,
690		       size_t cnt, loff_t *ppos)
691{
692	return simple_read_from_buffer(ubuf, cnt, ppos,
693					readme_msg, strlen(readme_msg));
694}
695
696static const struct file_operations readme_fops = {
697	.read		= inj_readme_read,
698};
699
700static struct dfs_node {
701	char *name;
702	const struct file_operations *fops;
703	umode_t perm;
704} dfs_fls[] = {
705	{ .name = "status",	.fops = &status_fops, .perm = S_IRUSR | S_IWUSR },
706	{ .name = "misc",	.fops = &misc_fops,   .perm = S_IRUSR | S_IWUSR },
707	{ .name = "addr",	.fops = &addr_fops,   .perm = S_IRUSR | S_IWUSR },
708	{ .name = "synd",	.fops = &synd_fops,   .perm = S_IRUSR | S_IWUSR },
709	{ .name = "ipid",	.fops = &ipid_fops,   .perm = S_IRUSR | S_IWUSR },
710	{ .name = "bank",	.fops = &bank_fops,   .perm = S_IRUSR | S_IWUSR },
711	{ .name = "flags",	.fops = &flags_fops,  .perm = S_IRUSR | S_IWUSR },
712	{ .name = "cpu",	.fops = &extcpu_fops, .perm = S_IRUSR | S_IWUSR },
713	{ .name = "README",	.fops = &readme_fops, .perm = S_IRUSR | S_IRGRP | S_IROTH },
714};
715
716static void __init debugfs_init(void)
717{
718	unsigned int i;
719
720	dfs_inj = debugfs_create_dir("mce-inject", NULL);
721
722	for (i = 0; i < ARRAY_SIZE(dfs_fls); i++)
723		debugfs_create_file(dfs_fls[i].name, dfs_fls[i].perm, dfs_inj,
724				    &i_mce, dfs_fls[i].fops);
725}
726
727static void check_hw_inj_possible(void)
728{
729	int cpu;
730	u8 bank;
731
732	/*
733	 * This behavior exists only on SMCA systems though its not directly
734	 * related to SMCA.
735	 */
736	if (!cpu_feature_enabled(X86_FEATURE_SMCA))
737		return;
738
739	cpu = get_cpu();
740
741	for (bank = 0; bank < MAX_NR_BANKS; ++bank) {
742		u64 status = MCI_STATUS_VAL, ipid;
743
744		/* Check whether bank is populated */
745		rdmsrq(MSR_AMD64_SMCA_MCx_IPID(bank), ipid);
746		if (!ipid)
747			continue;
748
749		toggle_hw_mce_inject(cpu, true);
750
751		wrmsrq_safe(mca_msr_reg(bank, MCA_STATUS), status);
752		rdmsrq_safe(mca_msr_reg(bank, MCA_STATUS), &status);
753		wrmsrq_safe(mca_msr_reg(bank, MCA_STATUS), 0);
754
755		if (!status) {
756			hw_injection_possible = false;
757			pr_warn("Platform does not allow *hardware* error injection."
758				"Try using APEI EINJ instead.\n");
759		}
760
761		toggle_hw_mce_inject(cpu, false);
762
763		break;
764	}
765
766	put_cpu();
767}
768
769static int __init inject_init(void)
770{
771	if (!alloc_cpumask_var(&mce_inject_cpumask, GFP_KERNEL))
772		return -ENOMEM;
773
774	check_hw_inj_possible();
775
776	debugfs_init();
777
778	register_nmi_handler(NMI_LOCAL, mce_raise_notify, 0, "mce_notify");
779	mce_register_injector_chain(&inject_nb);
780
781	setup_inj_struct(&i_mce);
782
783	pr_info("Machine check injector initialized\n");
784
785	return 0;
786}
787
788static void __exit inject_exit(void)
789{
790
791	mce_unregister_injector_chain(&inject_nb);
792	unregister_nmi_handler(NMI_LOCAL, "mce_notify");
793
794	debugfs_remove_recursive(dfs_inj);
795	dfs_inj = NULL;
796
797	memset(&dfs_fls, 0, sizeof(dfs_fls));
798
799	free_cpumask_var(mce_inject_cpumask);
800}
801
802module_init(inject_init);
803module_exit(inject_exit);
804MODULE_DESCRIPTION("Machine check injection support");
805MODULE_LICENSE("GPL");