einj-core.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * APEI Error INJection support
   4 *
   5 * EINJ provides a hardware error injection mechanism, this is useful
   6 * for debugging and testing of other APEI and RAS features.
   7 *
   8 * For more information about EINJ, please refer to ACPI Specification
   9 * version 4.0, section 17.5.
  10 *
  11 * Copyright 2009-2010 Intel Corp.
  12 *   Author: Huang Ying <ying.huang@intel.com>
  13 */
  14
  15#include <linux/kernel.h>
  16#include <linux/module.h>
  17#include <linux/init.h>
  18#include <linux/io.h>
  19#include <linux/debugfs.h>
  20#include <linux/seq_file.h>
  21#include <linux/nmi.h>
  22#include <linux/delay.h>
  23#include <linux/mm.h>
  24#include <linux/device/faux.h>
  25#include <linux/unaligned.h>
  26
  27#include "apei-internal.h"
  28
  29#undef pr_fmt
  30#define pr_fmt(fmt) "EINJ: " fmt
  31
  32#define SLEEP_UNIT_MIN		1000			/* 1ms */
  33#define SLEEP_UNIT_MAX		5000			/* 5ms */
  34/* Firmware should respond within 1 seconds */
  35#define FIRMWARE_TIMEOUT	(1 * USEC_PER_SEC)
  36#define COMPONENT_LEN		16
  37#define ACPI65_EINJV2_SUPP	BIT(30)
  38#define ACPI5_VENDOR_BIT	BIT(31)
  39#define MEM_ERROR_MASK		(ACPI_EINJ_MEMORY_CORRECTABLE | \
  40				ACPI_EINJ_MEMORY_UNCORRECTABLE | \
  41				ACPI_EINJ_MEMORY_FATAL)
  42#define CXL_ERROR_MASK		(ACPI_EINJ_CXL_CACHE_CORRECTABLE | \
  43				ACPI_EINJ_CXL_CACHE_UNCORRECTABLE | \
  44				ACPI_EINJ_CXL_CACHE_FATAL | \
  45				ACPI_EINJ_CXL_MEM_CORRECTABLE | \
  46				ACPI_EINJ_CXL_MEM_UNCORRECTABLE | \
  47				ACPI_EINJ_CXL_MEM_FATAL)
  48
  49/*
  50 * ACPI version 5 provides a SET_ERROR_TYPE_WITH_ADDRESS action.
  51 */
  52static int acpi5;
  53
  54struct syndrome_array {
  55	union {
  56		u8	acpi_id[COMPONENT_LEN];
  57		u8	device_id[COMPONENT_LEN];
  58		u8	pcie_sbdf[COMPONENT_LEN];
  59		u8	vendor_id[COMPONENT_LEN];
  60	} comp_id;
  61	union {
  62		u8	proc_synd[COMPONENT_LEN];
  63		u8	mem_synd[COMPONENT_LEN];
  64		u8	pcie_synd[COMPONENT_LEN];
  65		u8	vendor_synd[COMPONENT_LEN];
  66	} comp_synd;
  67};
  68
  69struct einjv2_extension_struct {
  70	u32 length;
  71	u16 revision;
  72	u16 component_arr_count;
  73	struct syndrome_array component_arr[] __counted_by(component_arr_count);
  74};
  75
  76struct set_error_type_with_address {
  77	u32	type;
  78	u32	vendor_extension;
  79	u32	flags;
  80	u32	apicid;
  81	u64	memory_address;
  82	u64	memory_address_range;
  83	u32	pcie_sbdf;
  84	struct	einjv2_extension_struct einjv2_struct;
  85};
  86enum {
  87	SETWA_FLAGS_APICID = 1,
  88	SETWA_FLAGS_MEM = 2,
  89	SETWA_FLAGS_PCIE_SBDF = 4,
  90	SETWA_FLAGS_EINJV2 = 8,
  91};
  92
  93/*
  94 * Vendor extensions for platform specific operations
  95 */
  96struct vendor_error_type_extension {
  97	u32	length;
  98	u32	pcie_sbdf;
  99	u16	vendor_id;
 100	u16	device_id;
 101	u8	rev_id;
 102	u8	reserved[3];
 103};
 104
 105static u32 notrigger;
 106
 107static u32 vendor_flags;
 108static struct debugfs_blob_wrapper vendor_blob;
 109static struct debugfs_blob_wrapper vendor_errors;
 110static char vendor_dev[64];
 111
 112static u32 max_nr_components;
 113static u32 available_error_type;
 114static u32 available_error_type_v2;
 115static struct syndrome_array *syndrome_data;
 116
 117/*
 118 * Some BIOSes allow parameters to the SET_ERROR_TYPE entries in the
 119 * EINJ table through an unpublished extension. Use with caution as
 120 * most will ignore the parameter and make their own choice of address
 121 * for error injection.  This extension is used only if
 122 * param_extension module parameter is specified.
 123 */
 124struct einj_parameter {
 125	u64 type;
 126	u64 reserved1;
 127	u64 reserved2;
 128	u64 param1;
 129	u64 param2;
 130};
 131
 132#define EINJ_OP_BUSY			0x1
 133#define EINJ_STATUS_SUCCESS		0x0
 134#define EINJ_STATUS_FAIL		0x1
 135#define EINJ_STATUS_INVAL		0x2
 136
 137#define EINJ_TAB_ENTRY(tab)						\
 138	((struct acpi_whea_header *)((char *)(tab) +			\
 139				    sizeof(struct acpi_table_einj)))
 140
 141static bool param_extension;
 142module_param(param_extension, bool, 0);
 143
 144static struct acpi_table_einj *einj_tab;
 145
 146static struct apei_resources einj_resources;
 147
 148static struct apei_exec_ins_type einj_ins_type[] = {
 149	[ACPI_EINJ_READ_REGISTER] = {
 150		.flags = APEI_EXEC_INS_ACCESS_REGISTER,
 151		.run   = apei_exec_read_register,
 152	},
 153	[ACPI_EINJ_READ_REGISTER_VALUE] = {
 154		.flags = APEI_EXEC_INS_ACCESS_REGISTER,
 155		.run   = apei_exec_read_register_value,
 156	},
 157	[ACPI_EINJ_WRITE_REGISTER] = {
 158		.flags = APEI_EXEC_INS_ACCESS_REGISTER,
 159		.run   = apei_exec_write_register,
 160	},
 161	[ACPI_EINJ_WRITE_REGISTER_VALUE] = {
 162		.flags = APEI_EXEC_INS_ACCESS_REGISTER,
 163		.run   = apei_exec_write_register_value,
 164	},
 165	[ACPI_EINJ_NOOP] = {
 166		.flags = 0,
 167		.run   = apei_exec_noop,
 168	},
 169};
 170
 171/*
 172 * Prevent EINJ interpreter to run simultaneously, because the
 173 * corresponding firmware implementation may not work properly when
 174 * invoked simultaneously.
 175 */
 176static DEFINE_MUTEX(einj_mutex);
 177
 178/*
 179 * Exported APIs use this flag to exit early if einj_probe() failed.
 180 */
 181bool einj_initialized __ro_after_init;
 182
 183static void __iomem *einj_param;
 184static u32 v5param_size;
 185static u32 v66param_size;
 186static bool is_v2;
 187
 188static void einj_exec_ctx_init(struct apei_exec_context *ctx)
 189{
 190	apei_exec_ctx_init(ctx, einj_ins_type, ARRAY_SIZE(einj_ins_type),
 191			   EINJ_TAB_ENTRY(einj_tab), einj_tab->entries);
 192}
 193
 194static int __einj_get_available_error_type(u32 *type, int einj_action)
 195{
 196	struct apei_exec_context ctx;
 197	int rc;
 198
 199	einj_exec_ctx_init(&ctx);
 200	rc = apei_exec_run(&ctx, einj_action);
 201	if (rc)
 202		return rc;
 203	*type = apei_exec_ctx_get_output(&ctx);
 204
 205	return 0;
 206}
 207
 208/* Get error injection capabilities of the platform */
 209int einj_get_available_error_type(u32 *type, int einj_action)
 210{
 211	int rc;
 212
 213	mutex_lock(&einj_mutex);
 214	rc = __einj_get_available_error_type(type, einj_action);
 215	mutex_unlock(&einj_mutex);
 216
 217	return rc;
 218}
 219
 220static int einj_get_available_error_types(u32 *type1, u32 *type2)
 221{
 222	int rc;
 223
 224	rc = einj_get_available_error_type(type1, ACPI_EINJ_GET_ERROR_TYPE);
 225	if (rc)
 226		return rc;
 227	if (*type1 & ACPI65_EINJV2_SUPP) {
 228		rc = einj_get_available_error_type(type2,
 229						   ACPI_EINJV2_GET_ERROR_TYPE);
 230		if (rc)
 231			return rc;
 232	}
 233
 234	return 0;
 235}
 236
 237static int einj_timedout(u64 *t)
 238{
 239	if ((s64)*t < SLEEP_UNIT_MIN) {
 240		pr_warn(FW_WARN "Firmware does not respond in time\n");
 241		return 1;
 242	}
 243	*t -= SLEEP_UNIT_MIN;
 244	usleep_range(SLEEP_UNIT_MIN, SLEEP_UNIT_MAX);
 245
 246	return 0;
 247}
 248
 249static void get_oem_vendor_struct(u64 paddr, int offset,
 250				  struct vendor_error_type_extension *v)
 251{
 252	unsigned long vendor_size;
 253	u64 target_pa = paddr + offset + sizeof(struct vendor_error_type_extension);
 254
 255	vendor_size = v->length - sizeof(struct vendor_error_type_extension);
 256
 257	if (vendor_size)
 258		vendor_errors.data = acpi_os_map_memory(target_pa, vendor_size);
 259
 260	if (vendor_errors.data)
 261		vendor_errors.size = vendor_size;
 262}
 263
 264static void check_vendor_extension(u64 paddr,
 265				   struct set_error_type_with_address *v5param)
 266{
 267	int	offset = v5param->vendor_extension;
 268	struct	vendor_error_type_extension v;
 269	struct vendor_error_type_extension __iomem *p;
 270	u32	sbdf;
 271
 272	if (!offset)
 273		return;
 274	p = acpi_os_map_iomem(paddr + offset, sizeof(*p));
 275	if (!p)
 276		return;
 277	memcpy_fromio(&v, p, sizeof(v));
 278	get_oem_vendor_struct(paddr, offset, &v);
 279	sbdf = v.pcie_sbdf;
 280	sprintf(vendor_dev, "%x:%x:%x.%x vendor_id=%x device_id=%x rev_id=%x\n",
 281		sbdf >> 24, (sbdf >> 16) & 0xff,
 282		(sbdf >> 11) & 0x1f, (sbdf >> 8) & 0x7,
 283		 v.vendor_id, v.device_id, v.rev_id);
 284	acpi_os_unmap_iomem(p, sizeof(v));
 285}
 286
 287static u32 einjv2_init(struct einjv2_extension_struct *e)
 288{
 289	if (e->revision != 1) {
 290		pr_info("Unknown v2 extension revision %u\n", e->revision);
 291		return 0;
 292	}
 293	if (e->length < sizeof(*e) || e->length > PAGE_SIZE) {
 294		pr_info(FW_BUG "Bad1 v2 extension length %u\n", e->length);
 295		return 0;
 296	}
 297	if ((e->length - sizeof(*e)) % sizeof(e->component_arr[0])) {
 298		pr_info(FW_BUG "Bad2 v2 extension length %u\n", e->length);
 299		return 0;
 300	}
 301
 302	return (e->length - sizeof(*e)) / sizeof(e->component_arr[0]);
 303}
 304
 305static void __iomem *einj_get_parameter_address(void)
 306{
 307	int i;
 308	u64 pa_v4 = 0, pa_v5 = 0;
 309	struct acpi_whea_header *entry;
 310
 311	entry = EINJ_TAB_ENTRY(einj_tab);
 312	for (i = 0; i < einj_tab->entries; i++) {
 313		if (entry->action == ACPI_EINJ_SET_ERROR_TYPE &&
 314		    entry->instruction == ACPI_EINJ_WRITE_REGISTER &&
 315		    entry->register_region.space_id ==
 316		    ACPI_ADR_SPACE_SYSTEM_MEMORY)
 317			pa_v4 = get_unaligned(&entry->register_region.address);
 318		if (entry->action == ACPI_EINJ_SET_ERROR_TYPE_WITH_ADDRESS &&
 319		    entry->instruction == ACPI_EINJ_WRITE_REGISTER &&
 320		    entry->register_region.space_id ==
 321		    ACPI_ADR_SPACE_SYSTEM_MEMORY)
 322			pa_v5 = get_unaligned(&entry->register_region.address);
 323		entry++;
 324	}
 325	if (pa_v5) {
 326		struct set_error_type_with_address v5param;
 327		struct set_error_type_with_address __iomem *p;
 328
 329		v5param_size = sizeof(v5param);
 330		p = acpi_os_map_iomem(pa_v5, sizeof(*p));
 331		if (p) {
 332			memcpy_fromio(&v5param, p, v5param_size);
 333			acpi5 = 1;
 334			check_vendor_extension(pa_v5, &v5param);
 335			if (available_error_type & ACPI65_EINJV2_SUPP) {
 336				struct einjv2_extension_struct *e;
 337
 338				e = &v5param.einjv2_struct;
 339				max_nr_components = einjv2_init(e);
 340
 341				/* remap including einjv2_extension_struct */
 342				acpi_os_unmap_iomem(p, v5param_size);
 343				v66param_size = v5param_size - sizeof(*e) + e->length;
 344				p = acpi_os_map_iomem(pa_v5, v66param_size);
 345			}
 346
 347			return p;
 348		}
 349	}
 350	if (param_extension && pa_v4) {
 351		struct einj_parameter v4param;
 352		struct einj_parameter __iomem *p;
 353
 354		p = acpi_os_map_iomem(pa_v4, sizeof(*p));
 355		if (!p)
 356			return NULL;
 357		memcpy_fromio(&v4param, p, sizeof(v4param));
 358		if (v4param.reserved1 || v4param.reserved2) {
 359			acpi_os_unmap_iomem(p, sizeof(v4param));
 360			return NULL;
 361		}
 362		return p;
 363	}
 364
 365	return NULL;
 366}
 367
 368/* do sanity check to trigger table */
 369static int einj_check_trigger_header(struct acpi_einj_trigger *trigger_tab)
 370{
 371	if (trigger_tab->header_size != sizeof(struct acpi_einj_trigger))
 372		return -EINVAL;
 373	if (trigger_tab->table_size > PAGE_SIZE ||
 374	    trigger_tab->table_size < trigger_tab->header_size)
 375		return -EINVAL;
 376	if (trigger_tab->entry_count !=
 377	    (trigger_tab->table_size - trigger_tab->header_size) /
 378	    sizeof(struct acpi_einj_entry))
 379		return -EINVAL;
 380
 381	return 0;
 382}
 383
 384static struct acpi_generic_address *einj_get_trigger_parameter_region(
 385	struct acpi_einj_trigger *trigger_tab, u64 param1, u64 param2)
 386{
 387	int i;
 388	struct acpi_whea_header *entry;
 389
 390	entry = (struct acpi_whea_header *)
 391		((char *)trigger_tab + sizeof(struct acpi_einj_trigger));
 392	for (i = 0; i < trigger_tab->entry_count; i++) {
 393		if (entry->action == ACPI_EINJ_TRIGGER_ERROR &&
 394		entry->instruction <= ACPI_EINJ_WRITE_REGISTER_VALUE &&
 395		entry->register_region.space_id ==
 396			ACPI_ADR_SPACE_SYSTEM_MEMORY &&
 397		(entry->register_region.address & param2) == (param1 & param2))
 398			return &entry->register_region;
 399		entry++;
 400	}
 401
 402	return NULL;
 403}
 404/* Execute instructions in trigger error action table */
 405static int __einj_error_trigger(u64 trigger_paddr, u32 type,
 406				u64 param1, u64 param2)
 407{
 408	struct acpi_einj_trigger trigger_tab;
 409	struct acpi_einj_trigger *full_trigger_tab;
 410	struct apei_exec_context trigger_ctx;
 411	struct apei_resources trigger_resources;
 412	struct acpi_whea_header *trigger_entry;
 413	struct resource *r;
 414	u32 table_size;
 415	int rc = -EIO;
 416	struct acpi_generic_address *trigger_param_region = NULL;
 417	struct acpi_einj_trigger __iomem *p = NULL;
 418
 419	r = request_mem_region(trigger_paddr, sizeof(trigger_tab),
 420			       "APEI EINJ Trigger Table");
 421	if (!r) {
 422		pr_err("Can not request [mem %#010llx-%#010llx] for Trigger table\n",
 423		       (unsigned long long)trigger_paddr,
 424		       (unsigned long long)trigger_paddr +
 425			    sizeof(trigger_tab) - 1);
 426		goto out;
 427	}
 428	p = ioremap_cache(trigger_paddr, sizeof(*p));
 429	if (!p) {
 430		pr_err("Failed to map trigger table!\n");
 431		goto out_rel_header;
 432	}
 433	memcpy_fromio(&trigger_tab, p, sizeof(trigger_tab));
 434	rc = einj_check_trigger_header(&trigger_tab);
 435	if (rc) {
 436		pr_warn(FW_BUG "Invalid trigger error action table.\n");
 437		goto out_rel_header;
 438	}
 439
 440	/* No action structures in the TRIGGER_ERROR table, nothing to do */
 441	if (!trigger_tab.entry_count)
 442		goto out_rel_header;
 443
 444	rc = -EIO;
 445	table_size = trigger_tab.table_size;
 446	full_trigger_tab = kmalloc(table_size, GFP_KERNEL);
 447	if (!full_trigger_tab)
 448		goto out_rel_header;
 449	r = request_mem_region(trigger_paddr + sizeof(trigger_tab),
 450			       table_size - sizeof(trigger_tab),
 451			       "APEI EINJ Trigger Table");
 452	if (!r) {
 453		pr_err("Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n",
 454		       (unsigned long long)trigger_paddr + sizeof(trigger_tab),
 455		       (unsigned long long)trigger_paddr + table_size - 1);
 456		goto out_free_trigger_tab;
 457	}
 458	iounmap(p);
 459	p = ioremap_cache(trigger_paddr, table_size);
 460	if (!p) {
 461		pr_err("Failed to map trigger table!\n");
 462		goto out_rel_entry;
 463	}
 464	memcpy_fromio(full_trigger_tab, p, table_size);
 465	trigger_entry = (struct acpi_whea_header *)
 466		((char *)full_trigger_tab + sizeof(struct acpi_einj_trigger));
 467	apei_resources_init(&trigger_resources);
 468	apei_exec_ctx_init(&trigger_ctx, einj_ins_type,
 469			   ARRAY_SIZE(einj_ins_type),
 470			   trigger_entry, trigger_tab.entry_count);
 471	rc = apei_exec_collect_resources(&trigger_ctx, &trigger_resources);
 472	if (rc)
 473		goto out_fini;
 474	rc = apei_resources_sub(&trigger_resources, &einj_resources);
 475	if (rc)
 476		goto out_fini;
 477	/*
 478	 * Some firmware will access target address specified in
 479	 * param1 to trigger the error when injecting memory error.
 480	 * This will cause resource conflict with regular memory.  So
 481	 * remove it from trigger table resources.
 482	 */
 483	if ((param_extension || acpi5) && (type & MEM_ERROR_MASK) && param2) {
 484		struct apei_resources addr_resources;
 485
 486		apei_resources_init(&addr_resources);
 487		trigger_param_region = einj_get_trigger_parameter_region(
 488			full_trigger_tab, param1, param2);
 489		if (trigger_param_region) {
 490			rc = apei_resources_add(&addr_resources,
 491				trigger_param_region->address,
 492				trigger_param_region->bit_width/8, true);
 493			if (rc)
 494				goto out_fini;
 495			rc = apei_resources_sub(&trigger_resources,
 496					&addr_resources);
 497		}
 498		apei_resources_fini(&addr_resources);
 499		if (rc)
 500			goto out_fini;
 501	}
 502	rc = apei_resources_request(&trigger_resources, "APEI EINJ Trigger");
 503	if (rc)
 504		goto out_fini;
 505	rc = apei_exec_pre_map_gars(&trigger_ctx);
 506	if (rc)
 507		goto out_release;
 508
 509	rc = apei_exec_run(&trigger_ctx, ACPI_EINJ_TRIGGER_ERROR);
 510
 511	apei_exec_post_unmap_gars(&trigger_ctx);
 512out_release:
 513	apei_resources_release(&trigger_resources);
 514out_fini:
 515	apei_resources_fini(&trigger_resources);
 516out_rel_entry:
 517	release_mem_region(trigger_paddr + sizeof(trigger_tab),
 518			   table_size - sizeof(trigger_tab));
 519out_free_trigger_tab:
 520	kfree(full_trigger_tab);
 521out_rel_header:
 522	release_mem_region(trigger_paddr, sizeof(trigger_tab));
 523out:
 524	if (p)
 525		iounmap(p);
 526
 527	return rc;
 528}
 529
 530static bool is_end_of_list(u8 *val)
 531{
 532	for (int i = 0; i < COMPONENT_LEN; ++i) {
 533		if (val[i] != 0xFF)
 534			return false;
 535	}
 536	return true;
 537}
 538static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2,
 539			       u64 param3, u64 param4)
 540{
 541	struct apei_exec_context ctx;
 542	u32 param_size = is_v2 ? v66param_size : v5param_size;
 543	u64 val, trigger_paddr, timeout = FIRMWARE_TIMEOUT;
 544	int i, rc;
 545
 546	einj_exec_ctx_init(&ctx);
 547
 548	rc = apei_exec_run_optional(&ctx, ACPI_EINJ_BEGIN_OPERATION);
 549	if (rc)
 550		return rc;
 551	apei_exec_ctx_set_input(&ctx, type);
 552	if (acpi5) {
 553		struct set_error_type_with_address *v5param;
 554
 555		v5param = kmalloc(param_size, GFP_KERNEL);
 556		if (!v5param)
 557			return -ENOMEM;
 558
 559		memcpy_fromio(v5param, einj_param, param_size);
 560		v5param->type = type;
 561		if (type & ACPI5_VENDOR_BIT) {
 562			switch (vendor_flags) {
 563			case SETWA_FLAGS_APICID:
 564				v5param->apicid = param1;
 565				break;
 566			case SETWA_FLAGS_MEM:
 567				v5param->memory_address = param1;
 568				v5param->memory_address_range = param2;
 569				break;
 570			case SETWA_FLAGS_PCIE_SBDF:
 571				v5param->pcie_sbdf = param1;
 572				break;
 573			}
 574			v5param->flags = vendor_flags;
 575		} else if (flags) {
 576			v5param->flags = flags;
 577			v5param->memory_address = param1;
 578			v5param->memory_address_range = param2;
 579
 580			if (is_v2) {
 581				for (i = 0; i < max_nr_components; i++) {
 582					if (is_end_of_list(syndrome_data[i].comp_id.acpi_id))
 583						break;
 584					v5param->einjv2_struct.component_arr[i].comp_id =
 585						syndrome_data[i].comp_id;
 586					v5param->einjv2_struct.component_arr[i].comp_synd =
 587						syndrome_data[i].comp_synd;
 588				}
 589				v5param->einjv2_struct.component_arr_count = i;
 590			} else {
 591				v5param->apicid = param3;
 592				v5param->pcie_sbdf = param4;
 593			}
 594		} else {
 595			switch (type) {
 596			case ACPI_EINJ_PROCESSOR_CORRECTABLE:
 597			case ACPI_EINJ_PROCESSOR_UNCORRECTABLE:
 598			case ACPI_EINJ_PROCESSOR_FATAL:
 599				v5param->apicid = param1;
 600				v5param->flags = SETWA_FLAGS_APICID;
 601				break;
 602			case ACPI_EINJ_MEMORY_CORRECTABLE:
 603			case ACPI_EINJ_MEMORY_UNCORRECTABLE:
 604			case ACPI_EINJ_MEMORY_FATAL:
 605				v5param->memory_address = param1;
 606				v5param->memory_address_range = param2;
 607				v5param->flags = SETWA_FLAGS_MEM;
 608				break;
 609			case ACPI_EINJ_PCIX_CORRECTABLE:
 610			case ACPI_EINJ_PCIX_UNCORRECTABLE:
 611			case ACPI_EINJ_PCIX_FATAL:
 612				v5param->pcie_sbdf = param1;
 613				v5param->flags = SETWA_FLAGS_PCIE_SBDF;
 614				break;
 615			}
 616		}
 617		memcpy_toio(einj_param, v5param, param_size);
 618		kfree(v5param);
 619	} else {
 620		rc = apei_exec_run(&ctx, ACPI_EINJ_SET_ERROR_TYPE);
 621		if (rc)
 622			return rc;
 623		if (einj_param) {
 624			struct einj_parameter v4param;
 625
 626			memcpy_fromio(&v4param, einj_param, sizeof(v4param));
 627			v4param.param1 = param1;
 628			v4param.param2 = param2;
 629			memcpy_toio(einj_param, &v4param, sizeof(v4param));
 630		}
 631	}
 632	rc = apei_exec_run(&ctx, ACPI_EINJ_EXECUTE_OPERATION);
 633	if (rc)
 634		return rc;
 635	for (;;) {
 636		rc = apei_exec_run(&ctx, ACPI_EINJ_CHECK_BUSY_STATUS);
 637		if (rc)
 638			return rc;
 639		val = apei_exec_ctx_get_output(&ctx);
 640		if (!(val & EINJ_OP_BUSY))
 641			break;
 642		if (einj_timedout(&timeout))
 643			return -EIO;
 644	}
 645	rc = apei_exec_run(&ctx, ACPI_EINJ_GET_COMMAND_STATUS);
 646	if (rc)
 647		return rc;
 648	val = apei_exec_ctx_get_output(&ctx);
 649	if (val == EINJ_STATUS_FAIL)
 650		return -EBUSY;
 651	else if (val == EINJ_STATUS_INVAL)
 652		return -EINVAL;
 653
 654	/*
 655	 * The error is injected into the platform successfully, then it needs
 656	 * to trigger the error.
 657	 */
 658	rc = apei_exec_run(&ctx, ACPI_EINJ_GET_TRIGGER_TABLE);
 659	if (rc)
 660		return rc;
 661	trigger_paddr = apei_exec_ctx_get_output(&ctx);
 662	if (notrigger == 0) {
 663		rc = __einj_error_trigger(trigger_paddr, type, param1, param2);
 664		if (rc)
 665			return rc;
 666	}
 667	rc = apei_exec_run_optional(&ctx, ACPI_EINJ_END_OPERATION);
 668
 669	return rc;
 670}
 671
 672/* Allow almost all types of address except MMIO. */
 673static bool is_allowed_range(u64 base_addr, u64 size)
 674{
 675	int i;
 676	/*
 677	 * MMIO region is usually claimed with IORESOURCE_MEM + IORES_DESC_NONE.
 678	 * However, IORES_DESC_NONE is treated like a wildcard when we check if
 679	 * region intersects with known resource. So do an allow list check for
 680	 * IORES_DESCs that definitely or most likely not MMIO.
 681	 */
 682	int non_mmio_desc[] = {
 683		IORES_DESC_CRASH_KERNEL,
 684		IORES_DESC_ACPI_TABLES,
 685		IORES_DESC_ACPI_NV_STORAGE,
 686		IORES_DESC_PERSISTENT_MEMORY,
 687		IORES_DESC_PERSISTENT_MEMORY_LEGACY,
 688		/* Treat IORES_DESC_DEVICE_PRIVATE_MEMORY as MMIO. */
 689		IORES_DESC_RESERVED,
 690		IORES_DESC_SOFT_RESERVED,
 691	};
 692
 693	if (region_intersects(base_addr, size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)
 694			      == REGION_INTERSECTS)
 695		return true;
 696
 697	for (i = 0; i < ARRAY_SIZE(non_mmio_desc); ++i) {
 698		if (region_intersects(base_addr, size, IORESOURCE_MEM, non_mmio_desc[i])
 699				      == REGION_INTERSECTS)
 700			return true;
 701	}
 702
 703	if (arch_is_platform_page(base_addr))
 704		return true;
 705
 706	return false;
 707}
 708
 709/* Inject the specified hardware error */
 710int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3,
 711		      u64 param4)
 712{
 713	int rc;
 714	u64 base_addr, size;
 715
 716	/* If user manually set "flags", make sure it is legal */
 717	if (flags && (flags & ~(SETWA_FLAGS_APICID | SETWA_FLAGS_MEM |
 718		      SETWA_FLAGS_PCIE_SBDF | SETWA_FLAGS_EINJV2)))
 719		return -EINVAL;
 720
 721	/* check if type is a valid EINJv2 error type */
 722	if (is_v2) {
 723		if (!(type & available_error_type_v2))
 724			return -EINVAL;
 725	}
 726	/*
 727	 * We need extra sanity checks for memory errors.
 728	 * Other types leap directly to injection.
 729	 */
 730
 731	/* ensure param1/param2 existed */
 732	if (!(param_extension || acpi5))
 733		goto inject;
 734
 735	/* ensure injection is memory related */
 736	if (type & ACPI5_VENDOR_BIT) {
 737		if (vendor_flags != SETWA_FLAGS_MEM)
 738			goto inject;
 739	} else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) {
 740		goto inject;
 741	}
 742
 743	/*
 744	 * Injections targeting a CXL 1.0/1.1 port have to be injected
 745	 * via the einj_cxl_rch_error_inject() path as that does the proper
 746	 * validation of the given RCRB base (MMIO) address.
 747	 */
 748	if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM))
 749		return -EINVAL;
 750
 751	/*
 752	 * Disallow crazy address masks that give BIOS leeway to pick
 753	 * injection address almost anywhere. Insist on page or
 754	 * better granularity and that target address is normal RAM or
 755	 * as long as is not MMIO.
 756	 */
 757	base_addr = param1 & param2;
 758	size = ~param2 + 1;
 759
 760	if ((param2 & PAGE_MASK) != PAGE_MASK)
 761		return -EINVAL;
 762
 763	if (!is_allowed_range(base_addr, size))
 764		return -EINVAL;
 765
 766	if (is_zero_pfn(base_addr >> PAGE_SHIFT))
 767		return -EADDRINUSE;
 768
 769inject:
 770	mutex_lock(&einj_mutex);
 771	rc = __einj_error_inject(type, flags, param1, param2, param3, param4);
 772	mutex_unlock(&einj_mutex);
 773
 774	return rc;
 775}
 776
 777int einj_cxl_rch_error_inject(u32 type, u32 flags, u64 param1, u64 param2,
 778			      u64 param3, u64 param4)
 779{
 780	int rc;
 781
 782	if (!(einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM)))
 783		return -EINVAL;
 784
 785	mutex_lock(&einj_mutex);
 786	rc = __einj_error_inject(type, flags, param1, param2, param3, param4);
 787	mutex_unlock(&einj_mutex);
 788
 789	return rc;
 790}
 791
 792static u32 error_type;
 793static u32 error_flags;
 794static u64 error_param1;
 795static u64 error_param2;
 796static u64 error_param3;
 797static u64 error_param4;
 798static struct dentry *einj_debug_dir;
 799static char einj_buf[32];
 800static bool einj_v2_enabled;
 801static struct { u32 mask; const char *str; } const einj_error_type_string[] = {
 802	{ BIT(0), "Processor Correctable" },
 803	{ BIT(1), "Processor Uncorrectable non-fatal" },
 804	{ BIT(2), "Processor Uncorrectable fatal" },
 805	{ BIT(3), "Memory Correctable" },
 806	{ BIT(4), "Memory Uncorrectable non-fatal" },
 807	{ BIT(5), "Memory Uncorrectable fatal" },
 808	{ BIT(6), "PCI Express Correctable" },
 809	{ BIT(7), "PCI Express Uncorrectable non-fatal" },
 810	{ BIT(8), "PCI Express Uncorrectable fatal" },
 811	{ BIT(9), "Platform Correctable" },
 812	{ BIT(10), "Platform Uncorrectable non-fatal" },
 813	{ BIT(11), "Platform Uncorrectable fatal"},
 814	{ BIT(31), "Vendor Defined Error Types" },
 815};
 816
 817static struct { u32 mask; const char *str; } const einjv2_error_type_string[] = {
 818	{ BIT(0), "EINJV2 Processor Error" },
 819	{ BIT(1), "EINJV2 Memory Error" },
 820	{ BIT(2), "EINJV2 PCI Express Error" },
 821};
 822
 823static int available_error_type_show(struct seq_file *m, void *v)
 824{
 825
 826	for (int pos = 0; pos < ARRAY_SIZE(einj_error_type_string); pos++)
 827		if (available_error_type & einj_error_type_string[pos].mask)
 828			seq_printf(m, "0x%08x\t%s\n", einj_error_type_string[pos].mask,
 829				   einj_error_type_string[pos].str);
 830	if ((available_error_type & ACPI65_EINJV2_SUPP) && einj_v2_enabled) {
 831		for (int pos = 0; pos < ARRAY_SIZE(einjv2_error_type_string); pos++) {
 832			if (available_error_type_v2 & einjv2_error_type_string[pos].mask)
 833				seq_printf(m, "V2_0x%08x\t%s\n", einjv2_error_type_string[pos].mask,
 834					   einjv2_error_type_string[pos].str);
 835		}
 836	}
 837	return 0;
 838}
 839
 840DEFINE_SHOW_ATTRIBUTE(available_error_type);
 841
 842static ssize_t error_type_get(struct file *file, char __user *buf,
 843				size_t count, loff_t *ppos)
 844{
 845	return simple_read_from_buffer(buf, count, ppos, einj_buf, strlen(einj_buf));
 846}
 847
 848bool einj_is_cxl_error_type(u64 type)
 849{
 850	return (type & CXL_ERROR_MASK) && (!(type & ACPI5_VENDOR_BIT));
 851}
 852
 853int einj_validate_error_type(u64 type)
 854{
 855	u32 tval, vendor;
 856
 857	/* Only low 32 bits for error type are valid */
 858	if (type & GENMASK_ULL(63, 32))
 859		return -EINVAL;
 860
 861	/*
 862	 * Vendor defined types have 0x80000000 bit set, and
 863	 * are not enumerated by ACPI_EINJ_GET_ERROR_TYPE
 864	 */
 865	vendor = type & ACPI5_VENDOR_BIT;
 866	tval = type & GENMASK(30, 0);
 867
 868	/* Only one error type can be specified */
 869	if (tval & (tval - 1))
 870		return -EINVAL;
 871	if (!vendor)
 872		if (!(type & (available_error_type | available_error_type_v2)))
 873			return -EINVAL;
 874
 875	return 0;
 876}
 877
 878static ssize_t error_type_set(struct file *file, const char __user *buf,
 879				size_t count, loff_t *ppos)
 880{
 881	int rc;
 882	u64 val;
 883
 884	/* Leave the last character for the NUL terminator */
 885	if (count > sizeof(einj_buf) - 1)
 886		return -EINVAL;
 887
 888	memset(einj_buf, 0, sizeof(einj_buf));
 889	if (copy_from_user(einj_buf, buf, count))
 890		return -EFAULT;
 891
 892	if (strncmp(einj_buf, "V2_", 3) == 0) {
 893		if (!sscanf(einj_buf, "V2_%llx", &val))
 894			return -EINVAL;
 895		is_v2 = true;
 896	} else {
 897		if (!sscanf(einj_buf, "%llx", &val))
 898			return -EINVAL;
 899		is_v2 = false;
 900	}
 901
 902	rc = einj_validate_error_type(val);
 903	if (rc)
 904		return rc;
 905
 906	error_type = val;
 907
 908	return count;
 909}
 910
 911static const struct file_operations error_type_fops = {
 912	.read		= error_type_get,
 913	.write		= error_type_set,
 914};
 915
 916static int error_inject_set(void *data, u64 val)
 917{
 918	if (!error_type)
 919		return -EINVAL;
 920
 921	if (is_v2)
 922		error_flags |= SETWA_FLAGS_EINJV2;
 923	else
 924		error_flags &= ~SETWA_FLAGS_EINJV2;
 925
 926	return einj_error_inject(error_type, error_flags, error_param1, error_param2,
 927		error_param3, error_param4);
 928}
 929
 930DEFINE_DEBUGFS_ATTRIBUTE(error_inject_fops, NULL, error_inject_set, "%llu\n");
 931
 932static int einj_check_table(struct acpi_table_einj *einj_tab)
 933{
 934	if ((einj_tab->header_length !=
 935	     (sizeof(struct acpi_table_einj) - sizeof(einj_tab->header)))
 936	    && (einj_tab->header_length != sizeof(struct acpi_table_einj)))
 937		return -EINVAL;
 938	if (einj_tab->header.length < sizeof(struct acpi_table_einj))
 939		return -EINVAL;
 940	if (einj_tab->entries !=
 941	    (einj_tab->header.length - sizeof(struct acpi_table_einj)) /
 942	    sizeof(struct acpi_einj_entry))
 943		return -EINVAL;
 944
 945	return 0;
 946}
 947
 948static ssize_t u128_read(struct file *f, char __user *buf, size_t count, loff_t *off)
 949{
 950	char output[2 * COMPONENT_LEN + 1];
 951	u8 *data = f->f_inode->i_private;
 952	int i;
 953
 954	if (*off >= sizeof(output))
 955		return 0;
 956
 957	for (i = 0; i < COMPONENT_LEN; i++)
 958		sprintf(output + 2 * i, "%.02x", data[COMPONENT_LEN - i - 1]);
 959	output[2 * COMPONENT_LEN] = '\n';
 960
 961	return simple_read_from_buffer(buf, count, off, output, sizeof(output));
 962}
 963
 964static ssize_t u128_write(struct file *f, const char __user *buf, size_t count, loff_t *off)
 965{
 966	char input[2 + 2 * COMPONENT_LEN + 2];
 967	u8 *save = f->f_inode->i_private;
 968	u8 tmp[COMPONENT_LEN];
 969	char byte[3] = {};
 970	char *s, *e;
 971	ssize_t c;
 972	long val;
 973	int i;
 974
 975	/* Require that user supply whole input line in one write(2) syscall */
 976	if (*off)
 977		return -EINVAL;
 978
 979	c = simple_write_to_buffer(input, sizeof(input), off, buf, count);
 980	if (c < 0)
 981		return c;
 982
 983	if (c < 1 || input[c - 1] != '\n')
 984		return -EINVAL;
 985
 986	/* Empty line means invalidate this entry */
 987	if (c == 1) {
 988		memset(save, 0xff, COMPONENT_LEN);
 989		return c;
 990	}
 991
 992	if (input[0] == '0' && (input[1] == 'x' || input[1] == 'X'))
 993		s = input + 2;
 994	else
 995		s = input;
 996	e = input + c - 1;
 997
 998	for (i = 0; i < COMPONENT_LEN; i++) {
 999		byte[1] = *--e;
1000		byte[0] = e > s ? *--e : '0';
1001		if (kstrtol(byte, 16, &val))
1002			return -EINVAL;
1003		tmp[i] = val;
1004		if (e <= s)
1005			break;
1006	}
1007	while (++i < COMPONENT_LEN)
1008		tmp[i] = 0;
1009
1010	memcpy(save, tmp, COMPONENT_LEN);
1011
1012	return c;
1013}
1014
1015static const struct file_operations u128_fops = {
1016	.read	= u128_read,
1017	.write	= u128_write,
1018};
1019
1020static bool setup_einjv2_component_files(void)
1021{
1022	char name[32];
1023
1024	syndrome_data = kcalloc(max_nr_components, sizeof(syndrome_data[0]), GFP_KERNEL);
1025	if (!syndrome_data)
1026		return false;
1027
1028	for (int i = 0; i < max_nr_components; i++) {
1029		sprintf(name, "component_id%d", i);
1030		debugfs_create_file(name, 0600, einj_debug_dir,
1031				    &syndrome_data[i].comp_id, &u128_fops);
1032		sprintf(name, "component_syndrome%d", i);
1033		debugfs_create_file(name, 0600, einj_debug_dir,
1034				    &syndrome_data[i].comp_synd, &u128_fops);
1035	}
1036
1037	return true;
1038}
1039
1040static int __init einj_probe(struct faux_device *fdev)
1041{
1042	int rc;
1043	acpi_status status;
1044	struct apei_exec_context ctx;
1045
1046	status = acpi_get_table(ACPI_SIG_EINJ, 0,
1047				(struct acpi_table_header **)&einj_tab);
1048	if (status == AE_NOT_FOUND) {
1049		pr_debug("EINJ table not found.\n");
1050		return -ENODEV;
1051	} else if (ACPI_FAILURE(status)) {
1052		pr_err("Failed to get EINJ table: %s\n",
1053				acpi_format_exception(status));
1054		return -EINVAL;
1055	}
1056
1057	rc = einj_check_table(einj_tab);
1058	if (rc) {
1059		pr_warn(FW_BUG "Invalid EINJ table.\n");
1060		goto err_put_table;
1061	}
1062
1063	rc = einj_get_available_error_types(&available_error_type, &available_error_type_v2);
1064	if (rc)
1065		goto err_put_table;
1066
1067	rc = -ENOMEM;
1068	einj_debug_dir = debugfs_create_dir("einj", apei_get_debugfs_dir());
1069
1070	debugfs_create_file("available_error_type", S_IRUSR, einj_debug_dir,
1071			    NULL, &available_error_type_fops);
1072	debugfs_create_file_unsafe("error_type", 0600, einj_debug_dir,
1073				   NULL, &error_type_fops);
1074	debugfs_create_file_unsafe("error_inject", 0200, einj_debug_dir,
1075				   NULL, &error_inject_fops);
1076
1077	apei_resources_init(&einj_resources);
1078	einj_exec_ctx_init(&ctx);
1079	rc = apei_exec_collect_resources(&ctx, &einj_resources);
1080	if (rc) {
1081		pr_err("Error collecting EINJ resources.\n");
1082		goto err_fini;
1083	}
1084
1085	rc = apei_resources_request(&einj_resources, "APEI EINJ");
1086	if (rc) {
1087		pr_err("Error requesting memory/port resources.\n");
1088		goto err_fini;
1089	}
1090
1091	rc = apei_exec_pre_map_gars(&ctx);
1092	if (rc) {
1093		pr_err("Error pre-mapping GARs.\n");
1094		goto err_release;
1095	}
1096
1097	einj_param = einj_get_parameter_address();
1098	if ((param_extension || acpi5) && einj_param) {
1099		debugfs_create_x32("flags", S_IRUSR | S_IWUSR, einj_debug_dir,
1100				   &error_flags);
1101		debugfs_create_x64("param1", S_IRUSR | S_IWUSR, einj_debug_dir,
1102				   &error_param1);
1103		debugfs_create_x64("param2", S_IRUSR | S_IWUSR, einj_debug_dir,
1104				   &error_param2);
1105		debugfs_create_x64("param3", S_IRUSR | S_IWUSR, einj_debug_dir,
1106				   &error_param3);
1107		debugfs_create_x64("param4", S_IRUSR | S_IWUSR, einj_debug_dir,
1108				   &error_param4);
1109		debugfs_create_x32("notrigger", S_IRUSR | S_IWUSR,
1110				   einj_debug_dir, &notrigger);
1111		if (available_error_type & ACPI65_EINJV2_SUPP)
1112			einj_v2_enabled = setup_einjv2_component_files();
1113	}
1114
1115	if (vendor_dev[0]) {
1116		vendor_blob.data = vendor_dev;
1117		vendor_blob.size = strlen(vendor_dev);
1118		debugfs_create_blob("vendor", S_IRUSR, einj_debug_dir,
1119				    &vendor_blob);
1120		debugfs_create_x32("vendor_flags", S_IRUSR | S_IWUSR,
1121				   einj_debug_dir, &vendor_flags);
1122	}
1123
1124	if (vendor_errors.size)
1125		debugfs_create_blob("oem_error", 0600, einj_debug_dir,
1126				    &vendor_errors);
1127
1128	pr_info("Error INJection is initialized.\n");
1129
1130	return 0;
1131
1132err_release:
1133	apei_resources_release(&einj_resources);
1134err_fini:
1135	apei_resources_fini(&einj_resources);
1136	debugfs_remove_recursive(einj_debug_dir);
1137err_put_table:
1138	acpi_put_table((struct acpi_table_header *)einj_tab);
1139
1140	return rc;
1141}
1142
1143static void einj_remove(struct faux_device *fdev)
1144{
1145	struct apei_exec_context ctx;
1146
1147	if (einj_param) {
1148		acpi_size size;
1149
1150		if (v66param_size)
1151			size = v66param_size;
1152		else if (acpi5)
1153			size = v5param_size;
1154		else
1155			size = sizeof(struct einj_parameter);
1156
1157		acpi_os_unmap_iomem(einj_param, size);
1158		if (vendor_errors.size)
1159			acpi_os_unmap_memory(vendor_errors.data, vendor_errors.size);
1160	}
1161	einj_exec_ctx_init(&ctx);
1162	apei_exec_post_unmap_gars(&ctx);
1163	apei_resources_release(&einj_resources);
1164	apei_resources_fini(&einj_resources);
1165	debugfs_remove_recursive(einj_debug_dir);
1166	kfree(syndrome_data);
1167	acpi_put_table((struct acpi_table_header *)einj_tab);
1168}
1169
1170static struct faux_device *einj_dev;
1171static struct faux_device_ops einj_device_ops = {
1172	.probe = einj_probe,
1173	.remove = einj_remove,
1174};
1175
1176static int __init einj_init(void)
1177{
1178	if (acpi_disabled) {
1179		pr_debug("ACPI disabled.\n");
1180		return -ENODEV;
1181	}
1182
1183	einj_dev = faux_device_create("acpi-einj", NULL, &einj_device_ops);
1184
1185	if (einj_dev)
1186		einj_initialized = true;
1187
1188	return 0;
1189}
1190
1191static void __exit einj_exit(void)
1192{
1193	faux_device_destroy(einj_dev);
1194}
1195
1196module_init(einj_init);
1197module_exit(einj_exit);
1198
1199MODULE_AUTHOR("Huang Ying");
1200MODULE_DESCRIPTION("APEI Error INJection support");
1201MODULE_LICENSE("GPL");