aer.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Implement the AER root port service driver. The driver registers an IRQ
   4 * handler. When a root port triggers an AER interrupt, the IRQ handler
   5 * collects Root Port status and schedules work.
   6 *
   7 * Copyright (C) 2006 Intel Corp.
   8 *	Tom Long Nguyen (tom.l.nguyen@intel.com)
   9 *	Zhang Yanmin (yanmin.zhang@intel.com)
  10 *
  11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
  12 *    Andrew Patterson <andrew.patterson@hp.com>
  13 */
  14
  15#define pr_fmt(fmt) "AER: " fmt
  16#define dev_fmt pr_fmt
  17
  18#include <linux/bitops.h>
  19#include <linux/cper.h>
  20#include <linux/dev_printk.h>
  21#include <linux/pci.h>
  22#include <linux/pci-acpi.h>
  23#include <linux/sched.h>
  24#include <linux/kernel.h>
  25#include <linux/errno.h>
  26#include <linux/pm.h>
  27#include <linux/init.h>
  28#include <linux/interrupt.h>
  29#include <linux/delay.h>
  30#include <linux/kfifo.h>
  31#include <linux/ratelimit.h>
  32#include <linux/slab.h>
  33#include <linux/vmcore_info.h>
  34#include <acpi/apei.h>
  35#include <acpi/ghes.h>
  36#include <ras/ras_event.h>
  37
  38#include "../pci.h"
  39#include "portdrv.h"
  40
  41#define aer_printk(level, pdev, fmt, arg...) \
  42	dev_printk(level, &(pdev)->dev, fmt, ##arg)
  43
  44#define AER_ERROR_SOURCES_MAX		128
  45
  46#define AER_MAX_TYPEOF_COR_ERRS		16	/* as per PCI_ERR_COR_STATUS */
  47#define AER_MAX_TYPEOF_UNCOR_ERRS	32	/* as per PCI_ERR_UNCOR_STATUS*/
  48
  49struct aer_err_source {
  50	u32 status;			/* PCI_ERR_ROOT_STATUS */
  51	u32 id;				/* PCI_ERR_ROOT_ERR_SRC */
  52};
  53
  54struct aer_rpc {
  55	struct pci_dev *rpd;		/* Root Port device */
  56	DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX);
  57};
  58
  59/* AER info for the device */
  60struct aer_info {
  61
  62	/*
  63	 * Fields for all AER capable devices. They indicate the errors
  64	 * "as seen by this device". Note that this may mean that if an
  65	 * Endpoint is causing problems, the AER counters may increment
  66	 * at its link partner (e.g. Root Port) because the errors will be
  67	 * "seen" by the link partner and not the problematic Endpoint
  68	 * itself (which may report all counters as 0 as it never saw any
  69	 * problems).
  70	 */
  71	/* Counters for different type of correctable errors */
  72	u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS];
  73	/* Counters for different type of fatal uncorrectable errors */
  74	u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];
  75	/* Counters for different type of nonfatal uncorrectable errors */
  76	u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS];
  77	/* Total number of ERR_COR sent by this device */
  78	u64 dev_total_cor_errs;
  79	/* Total number of ERR_FATAL sent by this device */
  80	u64 dev_total_fatal_errs;
  81	/* Total number of ERR_NONFATAL sent by this device */
  82	u64 dev_total_nonfatal_errs;
  83
  84	/*
  85	 * Fields for Root Ports & Root Complex Event Collectors only; these
  86	 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL
  87	 * messages received by the Root Port / Event Collector, INCLUDING the
  88	 * ones that are generated internally (by the Root Port itself)
  89	 */
  90	u64 rootport_total_cor_errs;
  91	u64 rootport_total_fatal_errs;
  92	u64 rootport_total_nonfatal_errs;
  93
  94	/* Ratelimits for errors */
  95	struct ratelimit_state correctable_ratelimit;
  96	struct ratelimit_state nonfatal_ratelimit;
  97};
  98
  99#define AER_LOG_TLP_MASKS		(PCI_ERR_UNC_POISON_TLP|	\
 100					PCI_ERR_UNC_POISON_BLK |	\
 101					PCI_ERR_UNC_ECRC|		\
 102					PCI_ERR_UNC_UNSUP|		\
 103					PCI_ERR_UNC_COMP_ABORT|		\
 104					PCI_ERR_UNC_UNX_COMP|		\
 105					PCI_ERR_UNC_ACSV |		\
 106					PCI_ERR_UNC_MCBTLP |		\
 107					PCI_ERR_UNC_ATOMEG |		\
 108					PCI_ERR_UNC_DMWR_BLK |		\
 109					PCI_ERR_UNC_XLAT_BLK |		\
 110					PCI_ERR_UNC_TLPPRE |		\
 111					PCI_ERR_UNC_MALF_TLP |		\
 112					PCI_ERR_UNC_IDE_CHECK |		\
 113					PCI_ERR_UNC_MISR_IDE |		\
 114					PCI_ERR_UNC_PCRC_CHECK)
 115
 116#define SYSTEM_ERROR_INTR_ON_MESG_MASK	(PCI_EXP_RTCTL_SECEE|	\
 117					PCI_EXP_RTCTL_SENFEE|	\
 118					PCI_EXP_RTCTL_SEFEE)
 119#define ROOT_PORT_INTR_ON_MESG_MASK	(PCI_ERR_ROOT_CMD_COR_EN|	\
 120					PCI_ERR_ROOT_CMD_NONFATAL_EN|	\
 121					PCI_ERR_ROOT_CMD_FATAL_EN)
 122#define ERR_COR_ID(d)			(d & 0xffff)
 123#define ERR_UNCOR_ID(d)			(d >> 16)
 124
 125#define AER_ERR_STATUS_MASK		(PCI_ERR_ROOT_UNCOR_RCV |	\
 126					PCI_ERR_ROOT_COR_RCV |		\
 127					PCI_ERR_ROOT_MULTI_COR_RCV |	\
 128					PCI_ERR_ROOT_MULTI_UNCOR_RCV)
 129
 130static bool pcie_aer_disable;
 131static pci_ers_result_t aer_root_reset(struct pci_dev *dev);
 132
 133void pci_no_aer(void)
 134{
 135	pcie_aer_disable = true;
 136}
 137
 138bool pci_aer_available(void)
 139{
 140	return !pcie_aer_disable && pci_msi_enabled();
 141}
 142
 143#ifdef CONFIG_PCIE_ECRC
 144
 145#define ECRC_POLICY_DEFAULT 0		/* ECRC set by BIOS */
 146#define ECRC_POLICY_OFF     1		/* ECRC off for performance */
 147#define ECRC_POLICY_ON      2		/* ECRC on for data integrity */
 148
 149static int ecrc_policy = ECRC_POLICY_DEFAULT;
 150
 151static const char * const ecrc_policy_str[] = {
 152	[ECRC_POLICY_DEFAULT] = "bios",
 153	[ECRC_POLICY_OFF] = "off",
 154	[ECRC_POLICY_ON] = "on"
 155};
 156
 157/**
 158 * enable_ecrc_checking - enable PCIe ECRC checking for a device
 159 * @dev: the PCI device
 160 *
 161 * Return: 0 on success, or negative on failure.
 162 */
 163static int enable_ecrc_checking(struct pci_dev *dev)
 164{
 165	int aer = dev->aer_cap;
 166	u32 reg32;
 167
 168	if (!aer)
 169		return -ENODEV;
 170
 171	pci_read_config_dword(dev, aer + PCI_ERR_CAP, &reg32);
 172	if (reg32 & PCI_ERR_CAP_ECRC_GENC)
 173		reg32 |= PCI_ERR_CAP_ECRC_GENE;
 174	if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
 175		reg32 |= PCI_ERR_CAP_ECRC_CHKE;
 176	pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
 177
 178	return 0;
 179}
 180
 181/**
 182 * disable_ecrc_checking - disable PCIe ECRC checking for a device
 183 * @dev: the PCI device
 184 *
 185 * Return: 0 on success, or negative on failure.
 186 */
 187static int disable_ecrc_checking(struct pci_dev *dev)
 188{
 189	int aer = dev->aer_cap;
 190	u32 reg32;
 191
 192	if (!aer)
 193		return -ENODEV;
 194
 195	pci_read_config_dword(dev, aer + PCI_ERR_CAP, &reg32);
 196	reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
 197	pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
 198
 199	return 0;
 200}
 201
 202/**
 203 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based
 204 * on global policy
 205 * @dev: the PCI device
 206 */
 207void pcie_set_ecrc_checking(struct pci_dev *dev)
 208{
 209	if (!pcie_aer_is_native(dev))
 210		return;
 211
 212	switch (ecrc_policy) {
 213	case ECRC_POLICY_DEFAULT:
 214		return;
 215	case ECRC_POLICY_OFF:
 216		disable_ecrc_checking(dev);
 217		break;
 218	case ECRC_POLICY_ON:
 219		enable_ecrc_checking(dev);
 220		break;
 221	default:
 222		return;
 223	}
 224}
 225
 226/**
 227 * pcie_ecrc_get_policy - parse kernel command-line ecrc option
 228 * @str: ECRC policy from kernel command line to use
 229 */
 230void pcie_ecrc_get_policy(char *str)
 231{
 232	int i;
 233
 234	i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str);
 235	if (i < 0)
 236		return;
 237
 238	ecrc_policy = i;
 239}
 240#endif	/* CONFIG_PCIE_ECRC */
 241
 242int pcie_aer_is_native(struct pci_dev *dev)
 243{
 244	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
 245
 246	if (!dev->aer_cap)
 247		return 0;
 248
 249	return pcie_ports_native || host->native_aer;
 250}
 251EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL");
 252
 253static int pci_enable_pcie_error_reporting(struct pci_dev *dev)
 254{
 255	int rc;
 256
 257	if (!pcie_aer_is_native(dev))
 258		return -EIO;
 259
 260	rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
 261	return pcibios_err_to_errno(rc);
 262}
 263
 264int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
 265{
 266	int aer = dev->aer_cap;
 267	u32 status, sev;
 268
 269	if (!pcie_aer_is_native(dev))
 270		return -EIO;
 271
 272	/* Clear status bits for ERR_NONFATAL errors only */
 273	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
 274	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
 275	status &= ~sev;
 276	if (status)
 277		pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
 278
 279	return 0;
 280}
 281EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status);
 282
 283void pci_aer_clear_fatal_status(struct pci_dev *dev)
 284{
 285	int aer = dev->aer_cap;
 286	u32 status, sev;
 287
 288	if (!pcie_aer_is_native(dev))
 289		return;
 290
 291	/* Clear status bits for ERR_FATAL errors only */
 292	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
 293	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
 294	status &= sev;
 295	if (status)
 296		pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
 297}
 298
 299/**
 300 * pci_aer_raw_clear_status - Clear AER error registers.
 301 * @dev: the PCI device
 302 *
 303 * Clear AER error status registers unconditionally, regardless of
 304 * whether they're owned by firmware or the OS.
 305 *
 306 * Return: 0 on success, or negative on failure.
 307 */
 308int pci_aer_raw_clear_status(struct pci_dev *dev)
 309{
 310	int aer = dev->aer_cap;
 311	u32 status;
 312	int port_type;
 313
 314	if (!aer)
 315		return -EIO;
 316
 317	port_type = pci_pcie_type(dev);
 318	if (port_type == PCI_EXP_TYPE_ROOT_PORT ||
 319	    port_type == PCI_EXP_TYPE_RC_EC) {
 320		pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status);
 321		pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status);
 322	}
 323
 324	pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
 325	pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status);
 326
 327	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
 328	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
 329
 330	return 0;
 331}
 332
 333int pci_aer_clear_status(struct pci_dev *dev)
 334{
 335	if (!pcie_aer_is_native(dev))
 336		return -EIO;
 337
 338	return pci_aer_raw_clear_status(dev);
 339}
 340
 341void pci_save_aer_state(struct pci_dev *dev)
 342{
 343	int aer = dev->aer_cap;
 344	struct pci_cap_saved_state *save_state;
 345	u32 *cap;
 346
 347	if (!aer)
 348		return;
 349
 350	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
 351	if (!save_state)
 352		return;
 353
 354	cap = &save_state->cap.data[0];
 355	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++);
 356	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++);
 357	pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++);
 358	pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++);
 359	if (pcie_cap_has_rtctl(dev))
 360		pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++);
 361}
 362
 363void pci_restore_aer_state(struct pci_dev *dev)
 364{
 365	int aer = dev->aer_cap;
 366	struct pci_cap_saved_state *save_state;
 367	u32 *cap;
 368
 369	if (!aer)
 370		return;
 371
 372	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
 373	if (!save_state)
 374		return;
 375
 376	cap = &save_state->cap.data[0];
 377	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++);
 378	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++);
 379	pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++);
 380	pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++);
 381	if (pcie_cap_has_rtctl(dev))
 382		pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++);
 383}
 384
 385void pci_aer_init(struct pci_dev *dev)
 386{
 387	int n;
 388
 389	dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 390	if (!dev->aer_cap)
 391		return;
 392
 393	dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL);
 394	if (!dev->aer_info) {
 395		dev->aer_cap = 0;
 396		return;
 397	}
 398
 399	ratelimit_state_init(&dev->aer_info->correctable_ratelimit,
 400			     DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
 401	ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit,
 402			     DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
 403
 404	/*
 405	 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER,
 406	 * PCI_ERR_COR_MASK, and PCI_ERR_CAP.  Root and Root Complex Event
 407	 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec
 408	 * 7.8.4.9).
 409	 */
 410	n = pcie_cap_has_rtctl(dev) ? 5 : 4;
 411	pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n);
 412
 413	pci_aer_clear_status(dev);
 414
 415	if (pci_aer_available())
 416		pci_enable_pcie_error_reporting(dev);
 417
 418	pcie_set_ecrc_checking(dev);
 419}
 420
 421void pci_aer_exit(struct pci_dev *dev)
 422{
 423	kfree(dev->aer_info);
 424	dev->aer_info = NULL;
 425}
 426
 427#define AER_AGENT_RECEIVER		0
 428#define AER_AGENT_REQUESTER		1
 429#define AER_AGENT_COMPLETER		2
 430#define AER_AGENT_TRANSMITTER		3
 431
 432#define AER_AGENT_REQUESTER_MASK(t)	((t == AER_CORRECTABLE) ?	\
 433	0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP))
 434#define AER_AGENT_COMPLETER_MASK(t)	((t == AER_CORRECTABLE) ?	\
 435	0 : PCI_ERR_UNC_COMP_ABORT)
 436#define AER_AGENT_TRANSMITTER_MASK(t)	((t == AER_CORRECTABLE) ?	\
 437	(PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0)
 438
 439#define AER_GET_AGENT(t, e)						\
 440	((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER :	\
 441	(e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER :	\
 442	(e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER :	\
 443	AER_AGENT_RECEIVER)
 444
 445#define AER_PHYSICAL_LAYER_ERROR	0
 446#define AER_DATA_LINK_LAYER_ERROR	1
 447#define AER_TRANSACTION_LAYER_ERROR	2
 448
 449#define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ?	\
 450	PCI_ERR_COR_RCVR : 0)
 451#define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ?	\
 452	(PCI_ERR_COR_BAD_TLP|						\
 453	PCI_ERR_COR_BAD_DLLP|						\
 454	PCI_ERR_COR_REP_ROLL|						\
 455	PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP)
 456
 457#define AER_GET_LAYER_ERROR(t, e)					\
 458	((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \
 459	(e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \
 460	AER_TRANSACTION_LAYER_ERROR)
 461
 462/*
 463 * AER error strings
 464 */
 465static const char * const aer_error_severity_string[] = {
 466	"Uncorrectable (Non-Fatal)",
 467	"Uncorrectable (Fatal)",
 468	"Correctable"
 469};
 470
 471static const char *aer_error_layer[] = {
 472	"Physical Layer",
 473	"Data Link Layer",
 474	"Transaction Layer"
 475};
 476
 477static const char *aer_correctable_error_string[] = {
 478	"RxErr",			/* Bit Position 0	*/
 479	NULL,
 480	NULL,
 481	NULL,
 482	NULL,
 483	NULL,
 484	"BadTLP",			/* Bit Position 6	*/
 485	"BadDLLP",			/* Bit Position 7	*/
 486	"Rollover",			/* Bit Position 8	*/
 487	NULL,
 488	NULL,
 489	NULL,
 490	"Timeout",			/* Bit Position 12	*/
 491	"NonFatalErr",			/* Bit Position 13	*/
 492	"CorrIntErr",			/* Bit Position 14	*/
 493	"HeaderOF",			/* Bit Position 15	*/
 494	NULL,				/* Bit Position 16	*/
 495	NULL,				/* Bit Position 17	*/
 496	NULL,				/* Bit Position 18	*/
 497	NULL,				/* Bit Position 19	*/
 498	NULL,				/* Bit Position 20	*/
 499	NULL,				/* Bit Position 21	*/
 500	NULL,				/* Bit Position 22	*/
 501	NULL,				/* Bit Position 23	*/
 502	NULL,				/* Bit Position 24	*/
 503	NULL,				/* Bit Position 25	*/
 504	NULL,				/* Bit Position 26	*/
 505	NULL,				/* Bit Position 27	*/
 506	NULL,				/* Bit Position 28	*/
 507	NULL,				/* Bit Position 29	*/
 508	NULL,				/* Bit Position 30	*/
 509	NULL,				/* Bit Position 31	*/
 510};
 511
 512static const char *aer_uncorrectable_error_string[] = {
 513	"Undefined",			/* Bit Position 0	*/
 514	NULL,
 515	NULL,
 516	NULL,
 517	"DLP",				/* Bit Position 4	*/
 518	"SDES",				/* Bit Position 5	*/
 519	NULL,
 520	NULL,
 521	NULL,
 522	NULL,
 523	NULL,
 524	NULL,
 525	"TLP",				/* Bit Position 12	*/
 526	"FCP",				/* Bit Position 13	*/
 527	"CmpltTO",			/* Bit Position 14	*/
 528	"CmpltAbrt",			/* Bit Position 15	*/
 529	"UnxCmplt",			/* Bit Position 16	*/
 530	"RxOF",				/* Bit Position 17	*/
 531	"MalfTLP",			/* Bit Position 18	*/
 532	"ECRC",				/* Bit Position 19	*/
 533	"UnsupReq",			/* Bit Position 20	*/
 534	"ACSViol",			/* Bit Position 21	*/
 535	"UncorrIntErr",			/* Bit Position 22	*/
 536	"BlockedTLP",			/* Bit Position 23	*/
 537	"AtomicOpBlocked",		/* Bit Position 24	*/
 538	"TLPBlockedErr",		/* Bit Position 25	*/
 539	"PoisonTLPBlocked",		/* Bit Position 26	*/
 540	"DMWrReqBlocked",		/* Bit Position 27	*/
 541	"IDECheck",			/* Bit Position 28	*/
 542	"MisIDETLP",			/* Bit Position 29	*/
 543	"PCRC_CHECK",			/* Bit Position 30	*/
 544	"TLPXlatBlocked",		/* Bit Position 31	*/
 545};
 546
 547static const char *aer_agent_string[] = {
 548	"Receiver ID",
 549	"Requester ID",
 550	"Completer ID",
 551	"Transmitter ID"
 552};
 553
 554#define aer_stats_dev_attr(name, stats_array, strings_array,		\
 555			   total_string, total_field)			\
 556	static ssize_t							\
 557	name##_show(struct device *dev, struct device_attribute *attr,	\
 558		     char *buf)						\
 559{									\
 560	unsigned int i;							\
 561	struct pci_dev *pdev = to_pci_dev(dev);				\
 562	u64 *stats = pdev->aer_info->stats_array;			\
 563	size_t len = 0;							\
 564									\
 565	for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) {	\
 566		if (strings_array[i])					\
 567			len += sysfs_emit_at(buf, len, "%s %llu\n",	\
 568					     strings_array[i],		\
 569					     stats[i]);			\
 570		else if (stats[i])					\
 571			len += sysfs_emit_at(buf, len,			\
 572					     #stats_array "_bit[%d] %llu\n",\
 573					     i, stats[i]);		\
 574	}								\
 575	len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string,	\
 576			     pdev->aer_info->total_field);		\
 577	return len;							\
 578}									\
 579static DEVICE_ATTR_RO(name)
 580
 581aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs,
 582		   aer_correctable_error_string, "ERR_COR",
 583		   dev_total_cor_errs);
 584aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs,
 585		   aer_uncorrectable_error_string, "ERR_FATAL",
 586		   dev_total_fatal_errs);
 587aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs,
 588		   aer_uncorrectable_error_string, "ERR_NONFATAL",
 589		   dev_total_nonfatal_errs);
 590
 591#define aer_stats_rootport_attr(name, field)				\
 592	static ssize_t							\
 593	name##_show(struct device *dev, struct device_attribute *attr,	\
 594		     char *buf)						\
 595{									\
 596	struct pci_dev *pdev = to_pci_dev(dev);				\
 597	return sysfs_emit(buf, "%llu\n", pdev->aer_info->field);	\
 598}									\
 599static DEVICE_ATTR_RO(name)
 600
 601aer_stats_rootport_attr(aer_rootport_total_err_cor,
 602			 rootport_total_cor_errs);
 603aer_stats_rootport_attr(aer_rootport_total_err_fatal,
 604			 rootport_total_fatal_errs);
 605aer_stats_rootport_attr(aer_rootport_total_err_nonfatal,
 606			 rootport_total_nonfatal_errs);
 607
 608static struct attribute *aer_stats_attrs[] __ro_after_init = {
 609	&dev_attr_aer_dev_correctable.attr,
 610	&dev_attr_aer_dev_fatal.attr,
 611	&dev_attr_aer_dev_nonfatal.attr,
 612	&dev_attr_aer_rootport_total_err_cor.attr,
 613	&dev_attr_aer_rootport_total_err_fatal.attr,
 614	&dev_attr_aer_rootport_total_err_nonfatal.attr,
 615	NULL
 616};
 617
 618static umode_t aer_stats_attrs_are_visible(struct kobject *kobj,
 619					   struct attribute *a, int n)
 620{
 621	struct device *dev = kobj_to_dev(kobj);
 622	struct pci_dev *pdev = to_pci_dev(dev);
 623
 624	if (!pdev->aer_info)
 625		return 0;
 626
 627	if ((a == &dev_attr_aer_rootport_total_err_cor.attr ||
 628	     a == &dev_attr_aer_rootport_total_err_fatal.attr ||
 629	     a == &dev_attr_aer_rootport_total_err_nonfatal.attr) &&
 630	    ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
 631	     (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC)))
 632		return 0;
 633
 634	return a->mode;
 635}
 636
 637const struct attribute_group aer_stats_attr_group = {
 638	.attrs  = aer_stats_attrs,
 639	.is_visible = aer_stats_attrs_are_visible,
 640};
 641
 642/*
 643 * Ratelimit interval
 644 * <=0: disabled with ratelimit.interval = 0
 645 * >0: enabled with ratelimit.interval in ms
 646 */
 647#define aer_ratelimit_interval_attr(name, ratelimit)			\
 648	static ssize_t							\
 649	name##_show(struct device *dev, struct device_attribute *attr,	\
 650					 char *buf)			\
 651	{								\
 652		struct pci_dev *pdev = to_pci_dev(dev);			\
 653									\
 654		return sysfs_emit(buf, "%d\n",				\
 655				  pdev->aer_info->ratelimit.interval);	\
 656	}								\
 657									\
 658	static ssize_t							\
 659	name##_store(struct device *dev, struct device_attribute *attr, \
 660		     const char *buf, size_t count) 			\
 661	{								\
 662		struct pci_dev *pdev = to_pci_dev(dev);			\
 663		int interval;						\
 664									\
 665		if (!capable(CAP_SYS_ADMIN))				\
 666			return -EPERM;					\
 667									\
 668		if (kstrtoint(buf, 0, &interval) < 0)			\
 669			return -EINVAL;					\
 670									\
 671		if (interval <= 0)					\
 672			interval = 0;					\
 673		else							\
 674			interval = msecs_to_jiffies(interval); 		\
 675									\
 676		pdev->aer_info->ratelimit.interval = interval;		\
 677									\
 678		return count;						\
 679	}								\
 680	static DEVICE_ATTR_RW(name);
 681
 682#define aer_ratelimit_burst_attr(name, ratelimit)			\
 683	static ssize_t							\
 684	name##_show(struct device *dev, struct device_attribute *attr,	\
 685		    char *buf)						\
 686	{								\
 687		struct pci_dev *pdev = to_pci_dev(dev);			\
 688									\
 689		return sysfs_emit(buf, "%d\n",				\
 690				  pdev->aer_info->ratelimit.burst);	\
 691	}								\
 692									\
 693	static ssize_t							\
 694	name##_store(struct device *dev, struct device_attribute *attr,	\
 695		     const char *buf, size_t count)			\
 696	{								\
 697		struct pci_dev *pdev = to_pci_dev(dev);			\
 698		int burst;						\
 699									\
 700		if (!capable(CAP_SYS_ADMIN))				\
 701			return -EPERM;					\
 702									\
 703		if (kstrtoint(buf, 0, &burst) < 0)			\
 704			return -EINVAL;					\
 705									\
 706		pdev->aer_info->ratelimit.burst = burst;		\
 707									\
 708		return count;						\
 709	}								\
 710	static DEVICE_ATTR_RW(name);
 711
 712#define aer_ratelimit_attrs(name)					\
 713	aer_ratelimit_interval_attr(name##_ratelimit_interval_ms,	\
 714				    name##_ratelimit)			\
 715	aer_ratelimit_burst_attr(name##_ratelimit_burst,		\
 716				 name##_ratelimit)
 717
 718aer_ratelimit_attrs(correctable)
 719aer_ratelimit_attrs(nonfatal)
 720
 721static struct attribute *aer_attrs[] = {
 722	&dev_attr_correctable_ratelimit_interval_ms.attr,
 723	&dev_attr_correctable_ratelimit_burst.attr,
 724	&dev_attr_nonfatal_ratelimit_interval_ms.attr,
 725	&dev_attr_nonfatal_ratelimit_burst.attr,
 726	NULL
 727};
 728
 729static umode_t aer_attrs_are_visible(struct kobject *kobj,
 730				     struct attribute *a, int n)
 731{
 732	struct device *dev = kobj_to_dev(kobj);
 733	struct pci_dev *pdev = to_pci_dev(dev);
 734
 735	if (!pdev->aer_info)
 736		return 0;
 737
 738	return a->mode;
 739}
 740
 741const struct attribute_group aer_attr_group = {
 742	.name = "aer",
 743	.attrs = aer_attrs,
 744	.is_visible = aer_attrs_are_visible,
 745};
 746
 747static void pci_dev_aer_stats_incr(struct pci_dev *pdev,
 748				   struct aer_err_info *info)
 749{
 750	unsigned long status = info->status & ~info->mask;
 751	int i, max = -1;
 752	u64 *counter = NULL;
 753	struct aer_info *aer_info = pdev->aer_info;
 754
 755	if (!aer_info)
 756		return;
 757
 758	switch (info->severity) {
 759	case AER_CORRECTABLE:
 760		aer_info->dev_total_cor_errs++;
 761		counter = &aer_info->dev_cor_errs[0];
 762		max = AER_MAX_TYPEOF_COR_ERRS;
 763		break;
 764	case AER_NONFATAL:
 765		aer_info->dev_total_nonfatal_errs++;
 766		hwerr_log_error_type(HWERR_RECOV_PCI);
 767		counter = &aer_info->dev_nonfatal_errs[0];
 768		max = AER_MAX_TYPEOF_UNCOR_ERRS;
 769		break;
 770	case AER_FATAL:
 771		aer_info->dev_total_fatal_errs++;
 772		counter = &aer_info->dev_fatal_errs[0];
 773		max = AER_MAX_TYPEOF_UNCOR_ERRS;
 774		break;
 775	}
 776
 777	for_each_set_bit(i, &status, max)
 778		counter[i]++;
 779}
 780
 781static void pci_rootport_aer_stats_incr(struct pci_dev *pdev,
 782				 struct aer_err_source *e_src)
 783{
 784	struct aer_info *aer_info = pdev->aer_info;
 785
 786	if (!aer_info)
 787		return;
 788
 789	if (e_src->status & PCI_ERR_ROOT_COR_RCV)
 790		aer_info->rootport_total_cor_errs++;
 791
 792	if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {
 793		if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)
 794			aer_info->rootport_total_fatal_errs++;
 795		else
 796			aer_info->rootport_total_nonfatal_errs++;
 797	}
 798}
 799
 800static int aer_ratelimit(struct pci_dev *dev, unsigned int severity)
 801{
 802	if (!dev->aer_info)
 803		return 1;
 804
 805	switch (severity) {
 806	case AER_NONFATAL:
 807		return __ratelimit(&dev->aer_info->nonfatal_ratelimit);
 808	case AER_CORRECTABLE:
 809		return __ratelimit(&dev->aer_info->correctable_ratelimit);
 810	default:
 811		return 1;	/* Don't ratelimit fatal errors */
 812	}
 813}
 814
 815static bool tlp_header_logged(u32 status, u32 capctl)
 816{
 817	/* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */
 818	if (status & AER_LOG_TLP_MASKS)
 819		return true;
 820
 821	/* Completion Timeout header is only logged on capable devices */
 822	if (status & PCI_ERR_UNC_COMP_TIME &&
 823	    capctl & PCI_ERR_CAP_COMP_TIME_LOG)
 824		return true;
 825
 826	return false;
 827}
 828
 829static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info)
 830{
 831	const char **strings;
 832	unsigned long status = info->status & ~info->mask;
 833	const char *level = info->level;
 834	const char *errmsg;
 835	int i;
 836
 837	if (info->severity == AER_CORRECTABLE)
 838		strings = aer_correctable_error_string;
 839	else
 840		strings = aer_uncorrectable_error_string;
 841
 842	for_each_set_bit(i, &status, 32) {
 843		errmsg = strings[i];
 844		if (!errmsg)
 845			errmsg = "Unknown Error Bit";
 846
 847		aer_printk(level, dev, "   [%2d] %-22s%s\n", i, errmsg,
 848				info->first_error == i ? " (First)" : "");
 849	}
 850}
 851
 852static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info,
 853			     bool found)
 854{
 855	u16 source = info->id;
 856
 857	pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n",
 858		 info->multi_error_valid ? "Multiple " : "",
 859		 aer_error_severity_string[info->severity],
 860		 pci_domain_nr(dev->bus), PCI_BUS_NUM(source),
 861		 PCI_SLOT(source), PCI_FUNC(source),
 862		 found ? "" : " (no details found");
 863}
 864
 865void aer_print_error(struct aer_err_info *info, int i)
 866{
 867	struct pci_dev *dev;
 868	int layer, agent, id;
 869	const char *level = info->level;
 870
 871	if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES))
 872		return;
 873
 874	dev = info->dev[i];
 875	id = pci_dev_id(dev);
 876
 877	pci_dev_aer_stats_incr(dev, info);
 878	trace_aer_event(pci_name(dev), (info->status & ~info->mask),
 879			info->severity, info->tlp_header_valid, &info->tlp);
 880
 881	if (!info->ratelimit_print[i])
 882		return;
 883
 884	if (!info->status) {
 885		pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
 886			aer_error_severity_string[info->severity]);
 887		goto out;
 888	}
 889
 890	layer = AER_GET_LAYER_ERROR(info->severity, info->status);
 891	agent = AER_GET_AGENT(info->severity, info->status);
 892
 893	aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
 894		   aer_error_severity_string[info->severity],
 895		   aer_error_layer[layer], aer_agent_string[agent]);
 896
 897	aer_printk(level, dev, "  device [%04x:%04x] error status/mask=%08x/%08x\n",
 898		   dev->vendor, dev->device, info->status, info->mask);
 899
 900	__aer_print_error(dev, info);
 901
 902	if (info->tlp_header_valid)
 903		pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt("  "));
 904
 905out:
 906	if (info->id && info->error_dev_num > 1 && info->id == id)
 907		pci_err(dev, "  Error of this Agent is reported first\n");
 908}
 909
 910#ifdef CONFIG_ACPI_APEI_PCIEAER
 911int cper_severity_to_aer(int cper_severity)
 912{
 913	switch (cper_severity) {
 914	case CPER_SEV_RECOVERABLE:
 915		return AER_NONFATAL;
 916	case CPER_SEV_FATAL:
 917		return AER_FATAL;
 918	default:
 919		return AER_CORRECTABLE;
 920	}
 921}
 922EXPORT_SYMBOL_GPL(cper_severity_to_aer);
 923#endif
 924
 925void pci_print_aer(struct pci_dev *dev, int aer_severity,
 926		   struct aer_capability_regs *aer)
 927{
 928	int layer, agent, tlp_header_valid = 0;
 929	u32 status, mask;
 930	struct aer_err_info info = {
 931		.severity = aer_severity,
 932		.first_error = PCI_ERR_CAP_FEP(aer->cap_control),
 933	};
 934
 935	if (aer_severity == AER_CORRECTABLE) {
 936		status = aer->cor_status;
 937		mask = aer->cor_mask;
 938		info.level = KERN_WARNING;
 939	} else {
 940		status = aer->uncor_status;
 941		mask = aer->uncor_mask;
 942		info.level = KERN_ERR;
 943		tlp_header_valid = tlp_header_logged(status, aer->cap_control);
 944	}
 945
 946	info.status = status;
 947	info.mask = mask;
 948
 949	pci_dev_aer_stats_incr(dev, &info);
 950	trace_aer_event(pci_name(dev), (status & ~mask),
 951			aer_severity, tlp_header_valid, &aer->header_log);
 952
 953	if (!aer_ratelimit(dev, info.severity))
 954		return;
 955
 956	layer = AER_GET_LAYER_ERROR(aer_severity, status);
 957	agent = AER_GET_AGENT(aer_severity, status);
 958
 959	aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n",
 960		   status, mask);
 961	__aer_print_error(dev, &info);
 962	aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n",
 963		   aer_error_layer[layer], aer_agent_string[agent]);
 964
 965	if (aer_severity != AER_CORRECTABLE)
 966		aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n",
 967			   aer->uncor_severity);
 968
 969	if (tlp_header_valid)
 970		pcie_print_tlp_log(dev, &aer->header_log, info.level,
 971				   dev_fmt("  "));
 972}
 973EXPORT_SYMBOL_NS_GPL(pci_print_aer, "CXL");
 974
 975/**
 976 * add_error_device - list device to be handled
 977 * @e_info: pointer to error info
 978 * @dev: pointer to pci_dev to be added
 979 */
 980static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
 981{
 982	int i = e_info->error_dev_num;
 983
 984	if (i >= AER_MAX_MULTI_ERR_DEVICES)
 985		return -ENOSPC;
 986
 987	e_info->dev[i] = pci_dev_get(dev);
 988	e_info->error_dev_num++;
 989
 990	/*
 991	 * Ratelimit AER log messages.  "dev" is either the source
 992	 * identified by the root's Error Source ID or it has an unmasked
 993	 * error logged in its own AER Capability.  Messages are emitted
 994	 * when "ratelimit_print[i]" is non-zero.  If we will print detail
 995	 * for a downstream device, make sure we print the Error Source ID
 996	 * from the root as well.
 997	 */
 998	if (aer_ratelimit(dev, e_info->severity)) {
 999		e_info->ratelimit_print[i] = 1;
1000		e_info->root_ratelimit_print = 1;
1001	}
1002	return 0;
1003}
1004
1005/**
1006 * is_error_source - check whether the device is source of reported error
1007 * @dev: pointer to pci_dev to be checked
1008 * @e_info: pointer to reported error info
1009 */
1010static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
1011{
1012	int aer = dev->aer_cap;
1013	u32 status, mask;
1014	u16 reg16;
1015
1016	/*
1017	 * When bus ID is equal to 0, it might be a bad ID
1018	 * reported by Root Port.
1019	 */
1020	if ((PCI_BUS_NUM(e_info->id) != 0) &&
1021	    !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {
1022		/* Device ID match? */
1023		if (e_info->id == pci_dev_id(dev))
1024			return true;
1025
1026		/* Continue ID comparing if there is no multiple error */
1027		if (!e_info->multi_error_valid)
1028			return false;
1029	}
1030
1031	/*
1032	 * When either
1033	 *      1) bus ID is equal to 0. Some ports might lose the bus
1034	 *              ID of error source id;
1035	 *      2) bus flag PCI_BUS_FLAGS_NO_AERSID is set
1036	 *      3) There are multiple errors and prior ID comparing fails;
1037	 * We check AER status registers to find possible reporter.
1038	 */
1039	if (atomic_read(&dev->enable_cnt) == 0)
1040		return false;
1041
1042	/* Check if AER is enabled */
1043	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &reg16);
1044	if (!(reg16 & PCI_EXP_AER_FLAGS))
1045		return false;
1046
1047	if (!aer)
1048		return false;
1049
1050	/* Check if error is recorded */
1051	if (e_info->severity == AER_CORRECTABLE) {
1052		pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
1053		pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);
1054	} else {
1055		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
1056		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);
1057	}
1058	if (status & ~mask)
1059		return true;
1060
1061	return false;
1062}
1063
1064static int find_device_iter(struct pci_dev *dev, void *data)
1065{
1066	struct aer_err_info *e_info = (struct aer_err_info *)data;
1067
1068	if (is_error_source(dev, e_info)) {
1069		/* List this device */
1070		if (add_error_device(e_info, dev)) {
1071			/* We cannot handle more... Stop iteration */
1072			pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n",
1073				AER_MAX_MULTI_ERR_DEVICES);
1074			return 1;
1075		}
1076
1077		/* If there is only a single error, stop iteration */
1078		if (!e_info->multi_error_valid)
1079			return 1;
1080	}
1081	return 0;
1082}
1083
1084/**
1085 * find_source_device - search through device hierarchy for source device
1086 * @parent: pointer to Root Port pci_dev data structure
1087 * @e_info: including detailed error information such as ID
1088 *
1089 * Return: true if found.
1090 *
1091 * Invoked by DPC when error is detected at the Root Port.
1092 * Caller of this function must set id, severity, and multi_error_valid of
1093 * struct aer_err_info pointed by @e_info properly.  This function must fill
1094 * e_info->error_dev_num and e_info->dev[], based on the given information.
1095 */
1096static bool find_source_device(struct pci_dev *parent,
1097			       struct aer_err_info *e_info)
1098{
1099	struct pci_dev *dev = parent;
1100	int result;
1101
1102	/* Must reset in this function */
1103	e_info->error_dev_num = 0;
1104
1105	/* Is Root Port an agent that sends error message? */
1106	result = find_device_iter(dev, e_info);
1107	if (result)
1108		return true;
1109
1110	if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC)
1111		pcie_walk_rcec(parent, find_device_iter, e_info);
1112	else
1113		pci_walk_bus(parent->subordinate, find_device_iter, e_info);
1114
1115	if (!e_info->error_dev_num)
1116		return false;
1117	return true;
1118}
1119
1120#ifdef CONFIG_PCIEAER_CXL
1121
1122/**
1123 * pci_aer_unmask_internal_errors - unmask internal errors
1124 * @dev: pointer to the pci_dev data structure
1125 *
1126 * Unmask internal errors in the Uncorrectable and Correctable Error
1127 * Mask registers.
1128 *
1129 * Note: AER must be enabled and supported by the device which must be
1130 * checked in advance, e.g. with pcie_aer_is_native().
1131 */
1132static void pci_aer_unmask_internal_errors(struct pci_dev *dev)
1133{
1134	int aer = dev->aer_cap;
1135	u32 mask;
1136
1137	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);
1138	mask &= ~PCI_ERR_UNC_INTN;
1139	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask);
1140
1141	pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);
1142	mask &= ~PCI_ERR_COR_INTERNAL;
1143	pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask);
1144}
1145
1146static bool is_cxl_mem_dev(struct pci_dev *dev)
1147{
1148	/*
1149	 * The capability, status, and control fields in Device 0,
1150	 * Function 0 DVSEC control the CXL functionality of the
1151	 * entire device (CXL 3.0, 8.1.3).
1152	 */
1153	if (dev->devfn != PCI_DEVFN(0, 0))
1154		return false;
1155
1156	/*
1157	 * CXL Memory Devices must have the 502h class code set (CXL
1158	 * 3.0, 8.1.12.1).
1159	 */
1160	if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)
1161		return false;
1162
1163	return true;
1164}
1165
1166static bool cxl_error_is_native(struct pci_dev *dev)
1167{
1168	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
1169
1170	return (pcie_ports_native || host->native_aer);
1171}
1172
1173static bool is_internal_error(struct aer_err_info *info)
1174{
1175	if (info->severity == AER_CORRECTABLE)
1176		return info->status & PCI_ERR_COR_INTERNAL;
1177
1178	return info->status & PCI_ERR_UNC_INTN;
1179}
1180
1181static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data)
1182{
1183	struct aer_err_info *info = (struct aer_err_info *)data;
1184	const struct pci_error_handlers *err_handler;
1185
1186	if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev))
1187		return 0;
1188
1189	/* Protect dev->driver */
1190	device_lock(&dev->dev);
1191
1192	err_handler = dev->driver ? dev->driver->err_handler : NULL;
1193	if (!err_handler)
1194		goto out;
1195
1196	if (info->severity == AER_CORRECTABLE) {
1197		if (err_handler->cor_error_detected)
1198			err_handler->cor_error_detected(dev);
1199	} else if (err_handler->error_detected) {
1200		if (info->severity == AER_NONFATAL)
1201			err_handler->error_detected(dev, pci_channel_io_normal);
1202		else if (info->severity == AER_FATAL)
1203			err_handler->error_detected(dev, pci_channel_io_frozen);
1204	}
1205out:
1206	device_unlock(&dev->dev);
1207	return 0;
1208}
1209
1210static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info)
1211{
1212	/*
1213	 * Internal errors of an RCEC indicate an AER error in an
1214	 * RCH's downstream port. Check and handle them in the CXL.mem
1215	 * device driver.
1216	 */
1217	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC &&
1218	    is_internal_error(info))
1219		pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info);
1220}
1221
1222static int handles_cxl_error_iter(struct pci_dev *dev, void *data)
1223{
1224	bool *handles_cxl = data;
1225
1226	if (!*handles_cxl)
1227		*handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev);
1228
1229	/* Non-zero terminates iteration */
1230	return *handles_cxl;
1231}
1232
1233static bool handles_cxl_errors(struct pci_dev *rcec)
1234{
1235	bool handles_cxl = false;
1236
1237	if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC &&
1238	    pcie_aer_is_native(rcec))
1239		pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl);
1240
1241	return handles_cxl;
1242}
1243
1244static void cxl_rch_enable_rcec(struct pci_dev *rcec)
1245{
1246	if (!handles_cxl_errors(rcec))
1247		return;
1248
1249	pci_aer_unmask_internal_errors(rcec);
1250	pci_info(rcec, "CXL: Internal errors unmasked");
1251}
1252
1253#else
1254static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { }
1255static inline void cxl_rch_handle_error(struct pci_dev *dev,
1256					struct aer_err_info *info) { }
1257#endif
1258
1259/**
1260 * pci_aer_handle_error - handle logging error into an event log
1261 * @dev: pointer to pci_dev data structure of error source device
1262 * @info: comprehensive error information
1263 *
1264 * Invoked when an error being detected by Root Port.
1265 */
1266static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info)
1267{
1268	int aer = dev->aer_cap;
1269
1270	if (info->severity == AER_CORRECTABLE) {
1271		/*
1272		 * Correctable error does not need software intervention.
1273		 * No need to go through error recovery process.
1274		 */
1275		if (aer)
1276			pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS,
1277					info->status);
1278		if (pcie_aer_is_native(dev)) {
1279			struct pci_driver *pdrv = dev->driver;
1280
1281			if (pdrv && pdrv->err_handler &&
1282			    pdrv->err_handler->cor_error_detected)
1283				pdrv->err_handler->cor_error_detected(dev);
1284			pcie_clear_device_status(dev);
1285		}
1286	} else if (info->severity == AER_NONFATAL)
1287		pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset);
1288	else if (info->severity == AER_FATAL)
1289		pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset);
1290}
1291
1292static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info)
1293{
1294	cxl_rch_handle_error(dev, info);
1295	pci_aer_handle_error(dev, info);
1296	pci_dev_put(dev);
1297}
1298
1299#ifdef CONFIG_ACPI_APEI_PCIEAER
1300
1301#define AER_RECOVER_RING_SIZE		16
1302
1303struct aer_recover_entry {
1304	u8	bus;
1305	u8	devfn;
1306	u16	domain;
1307	int	severity;
1308	struct aer_capability_regs *regs;
1309};
1310
1311static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry,
1312		    AER_RECOVER_RING_SIZE);
1313
1314static void aer_recover_work_func(struct work_struct *work)
1315{
1316	struct aer_recover_entry entry;
1317	struct pci_dev *pdev;
1318
1319	while (kfifo_get(&aer_recover_ring, &entry)) {
1320		pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus,
1321						   entry.devfn);
1322		if (!pdev) {
1323			pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n",
1324					   entry.domain, entry.bus,
1325					   PCI_SLOT(entry.devfn),
1326					   PCI_FUNC(entry.devfn));
1327			continue;
1328		}
1329		pci_print_aer(pdev, entry.severity, entry.regs);
1330
1331		/*
1332		 * Memory for aer_capability_regs(entry.regs) is being
1333		 * allocated from the ghes_estatus_pool to protect it from
1334		 * overwriting when multiple sections are present in the
1335		 * error status. Thus free the same after processing the
1336		 * data.
1337		 */
1338		ghes_estatus_pool_region_free((unsigned long)entry.regs,
1339					    sizeof(struct aer_capability_regs));
1340
1341		if (entry.severity == AER_NONFATAL)
1342			pcie_do_recovery(pdev, pci_channel_io_normal,
1343					 aer_root_reset);
1344		else if (entry.severity == AER_FATAL)
1345			pcie_do_recovery(pdev, pci_channel_io_frozen,
1346					 aer_root_reset);
1347		pci_dev_put(pdev);
1348	}
1349}
1350
1351/*
1352 * Mutual exclusion for writers of aer_recover_ring, reader side don't
1353 * need lock, because there is only one reader and lock is not needed
1354 * between reader and writer.
1355 */
1356static DEFINE_SPINLOCK(aer_recover_ring_lock);
1357static DECLARE_WORK(aer_recover_work, aer_recover_work_func);
1358
1359void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn,
1360		       int severity, struct aer_capability_regs *aer_regs)
1361{
1362	struct aer_recover_entry entry = {
1363		.bus		= bus,
1364		.devfn		= devfn,
1365		.domain		= domain,
1366		.severity	= severity,
1367		.regs		= aer_regs,
1368	};
1369
1370	if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1,
1371				 &aer_recover_ring_lock))
1372		schedule_work(&aer_recover_work);
1373	else
1374		pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n",
1375		       domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1376}
1377EXPORT_SYMBOL_GPL(aer_recover_queue);
1378#endif
1379
1380/**
1381 * aer_get_device_error_info - read error status from dev and store it to info
1382 * @info: pointer to structure to store the error record
1383 * @i: index into info->dev[]
1384 *
1385 * Return: 1 on success, 0 on error.
1386 *
1387 * Note that @info is reused among all error devices. Clear fields properly.
1388 */
1389int aer_get_device_error_info(struct aer_err_info *info, int i)
1390{
1391	struct pci_dev *dev;
1392	int type, aer;
1393	u32 aercc;
1394
1395	if (i >= AER_MAX_MULTI_ERR_DEVICES)
1396		return 0;
1397
1398	dev = info->dev[i];
1399	aer = dev->aer_cap;
1400	type = pci_pcie_type(dev);
1401
1402	/* Must reset in this function */
1403	info->status = 0;
1404	info->tlp_header_valid = 0;
1405
1406	/* The device might not support AER */
1407	if (!aer)
1408		return 0;
1409
1410	if (info->severity == AER_CORRECTABLE) {
1411		pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS,
1412			&info->status);
1413		pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK,
1414			&info->mask);
1415		if (!(info->status & ~info->mask))
1416			return 0;
1417	} else if (type == PCI_EXP_TYPE_ROOT_PORT ||
1418		   type == PCI_EXP_TYPE_RC_EC ||
1419		   type == PCI_EXP_TYPE_DOWNSTREAM ||
1420		   info->severity == AER_NONFATAL) {
1421
1422		/* Link is still healthy for IO reads */
1423		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS,
1424			&info->status);
1425		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK,
1426			&info->mask);
1427		if (!(info->status & ~info->mask))
1428			return 0;
1429
1430		/* Get First Error Pointer */
1431		pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc);
1432		info->first_error = PCI_ERR_CAP_FEP(aercc);
1433
1434		if (tlp_header_logged(info->status, aercc)) {
1435			info->tlp_header_valid = 1;
1436			pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG,
1437					  aer + PCI_ERR_PREFIX_LOG,
1438					  aer_tlp_log_len(dev, aercc),
1439					  aercc & PCI_ERR_CAP_TLP_LOG_FLIT,
1440					  &info->tlp);
1441		}
1442	}
1443
1444	return 1;
1445}
1446
1447static inline void aer_process_err_devices(struct aer_err_info *e_info)
1448{
1449	int i;
1450
1451	/* Report all before handling them, to not lose records by reset etc. */
1452	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
1453		if (aer_get_device_error_info(e_info, i))
1454			aer_print_error(e_info, i);
1455	}
1456	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
1457		if (aer_get_device_error_info(e_info, i))
1458			handle_error_source(e_info->dev[i], e_info);
1459	}
1460}
1461
1462/**
1463 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error
1464 *			    detected by Root Port or RCEC
1465 * @root: pointer to Root Port or RCEC that signaled AER interrupt
1466 * @info: pointer to AER error info
1467 */
1468static void aer_isr_one_error_type(struct pci_dev *root,
1469				   struct aer_err_info *info)
1470{
1471	bool found;
1472
1473	found = find_source_device(root, info);
1474
1475	/*
1476	 * If we're going to log error messages, we've already set
1477	 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to
1478	 * non-zero (which enables printing) because this is either an
1479	 * ERR_FATAL or we found a device with an error logged in its AER
1480	 * Capability.
1481	 *
1482	 * If we didn't find the Error Source device, at least log the
1483	 * Requester ID from the ERR_* Message received by the Root Port or
1484	 * RCEC, ratelimited by the RP or RCEC.
1485	 */
1486	if (info->root_ratelimit_print ||
1487	    (!found && aer_ratelimit(root, info->severity)))
1488		aer_print_source(root, info, found);
1489
1490	if (found)
1491		aer_process_err_devices(info);
1492}
1493
1494/**
1495 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from
1496 *		       Root Port or RCEC
1497 * @root: pointer to Root Port or RCEC that signaled AER interrupt
1498 * @e_src: pointer to an error source
1499 */
1500static void aer_isr_one_error(struct pci_dev *root,
1501			      struct aer_err_source *e_src)
1502{
1503	u32 status = e_src->status;
1504
1505	pci_rootport_aer_stats_incr(root, e_src);
1506
1507	/*
1508	 * There is a possibility that both correctable error and
1509	 * uncorrectable error being logged. Report correctable error first.
1510	 */
1511	if (status & PCI_ERR_ROOT_COR_RCV) {
1512		int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV;
1513		struct aer_err_info e_info = {
1514			.id = ERR_COR_ID(e_src->id),
1515			.severity = AER_CORRECTABLE,
1516			.level = KERN_WARNING,
1517			.multi_error_valid = multi ? 1 : 0,
1518		};
1519
1520		aer_isr_one_error_type(root, &e_info);
1521	}
1522
1523	if (status & PCI_ERR_ROOT_UNCOR_RCV) {
1524		int fatal = status & PCI_ERR_ROOT_FATAL_RCV;
1525		int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV;
1526		struct aer_err_info e_info = {
1527			.id = ERR_UNCOR_ID(e_src->id),
1528			.severity = fatal ? AER_FATAL : AER_NONFATAL,
1529			.level = KERN_ERR,
1530			.multi_error_valid = multi ? 1 : 0,
1531		};
1532
1533		aer_isr_one_error_type(root, &e_info);
1534	}
1535}
1536
1537/**
1538 * aer_isr - consume errors detected by Root Port
1539 * @irq: IRQ assigned to Root Port
1540 * @context: pointer to Root Port data structure
1541 *
1542 * Invoked, as DPC, when Root Port records new detected error
1543 */
1544static irqreturn_t aer_isr(int irq, void *context)
1545{
1546	struct pcie_device *dev = (struct pcie_device *)context;
1547	struct aer_rpc *rpc = get_service_data(dev);
1548	struct aer_err_source e_src;
1549
1550	if (kfifo_is_empty(&rpc->aer_fifo))
1551		return IRQ_NONE;
1552
1553	while (kfifo_get(&rpc->aer_fifo, &e_src))
1554		aer_isr_one_error(rpc->rpd, &e_src);
1555	return IRQ_HANDLED;
1556}
1557
1558/**
1559 * aer_irq - Root Port's ISR
1560 * @irq: IRQ assigned to Root Port
1561 * @context: pointer to Root Port data structure
1562 *
1563 * Invoked when Root Port detects AER messages.
1564 */
1565static irqreturn_t aer_irq(int irq, void *context)
1566{
1567	struct pcie_device *pdev = (struct pcie_device *)context;
1568	struct aer_rpc *rpc = get_service_data(pdev);
1569	struct pci_dev *rp = rpc->rpd;
1570	int aer = rp->aer_cap;
1571	struct aer_err_source e_src = {};
1572
1573	pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status);
1574	if (!(e_src.status & AER_ERR_STATUS_MASK))
1575		return IRQ_NONE;
1576
1577	pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id);
1578	pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status);
1579
1580	if (!kfifo_put(&rpc->aer_fifo, e_src))
1581		return IRQ_HANDLED;
1582
1583	return IRQ_WAKE_THREAD;
1584}
1585
1586static void aer_enable_irq(struct pci_dev *pdev)
1587{
1588	int aer = pdev->aer_cap;
1589	u32 reg32;
1590
1591	/* Enable Root Port's interrupt in response to error messages */
1592	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
1593	reg32 |= ROOT_PORT_INTR_ON_MESG_MASK;
1594	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
1595}
1596
1597static void aer_disable_irq(struct pci_dev *pdev)
1598{
1599	int aer = pdev->aer_cap;
1600	u32 reg32;
1601
1602	/* Disable Root Port's interrupt in response to error messages */
1603	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
1604	reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
1605	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
1606}
1607
1608static int clear_status_iter(struct pci_dev *dev, void *data)
1609{
1610	u16 devctl;
1611
1612	/* Skip if pci_enable_pcie_error_reporting() hasn't been called yet */
1613	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &devctl);
1614	if (!(devctl & PCI_EXP_AER_FLAGS))
1615		return 0;
1616
1617	pci_aer_clear_status(dev);
1618	pcie_clear_device_status(dev);
1619	return 0;
1620}
1621
1622/**
1623 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
1624 * @rpc: pointer to a Root Port data structure
1625 *
1626 * Invoked when PCIe bus loads AER service driver.
1627 */
1628static void aer_enable_rootport(struct aer_rpc *rpc)
1629{
1630	struct pci_dev *pdev = rpc->rpd;
1631	int aer = pdev->aer_cap;
1632	u16 reg16;
1633	u32 reg32;
1634
1635	/* Clear PCIe Capability's Device Status */
1636	pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, &reg16);
1637	pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16);
1638
1639	/* Disable system error generation in response to error messages */
1640	pcie_capability_clear_word(pdev, PCI_EXP_RTCTL,
1641				   SYSTEM_ERROR_INTR_ON_MESG_MASK);
1642
1643	/* Clear error status of this Root Port or RCEC */
1644	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, &reg32);
1645	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
1646
1647	/* Clear error status of agents reporting to this Root Port or RCEC */
1648	if (reg32 & AER_ERR_STATUS_MASK) {
1649		if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_EC)
1650			pcie_walk_rcec(pdev, clear_status_iter, NULL);
1651		else if (pdev->subordinate)
1652			pci_walk_bus(pdev->subordinate, clear_status_iter,
1653				     NULL);
1654	}
1655
1656	pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, &reg32);
1657	pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32);
1658	pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, &reg32);
1659	pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32);
1660
1661	aer_enable_irq(pdev);
1662}
1663
1664/**
1665 * aer_disable_rootport - disable Root Port's interrupts when receiving messages
1666 * @rpc: pointer to a Root Port data structure
1667 *
1668 * Invoked when PCIe bus unloads AER service driver.
1669 */
1670static void aer_disable_rootport(struct aer_rpc *rpc)
1671{
1672	struct pci_dev *pdev = rpc->rpd;
1673	int aer = pdev->aer_cap;
1674	u32 reg32;
1675
1676	aer_disable_irq(pdev);
1677
1678	/* Clear Root's error status reg */
1679	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, &reg32);
1680	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
1681}
1682
1683/**
1684 * aer_remove - clean up resources
1685 * @dev: pointer to the pcie_dev data structure
1686 *
1687 * Invoked when PCI Express bus unloads or AER probe fails.
1688 */
1689static void aer_remove(struct pcie_device *dev)
1690{
1691	struct aer_rpc *rpc = get_service_data(dev);
1692
1693	aer_disable_rootport(rpc);
1694}
1695
1696/**
1697 * aer_probe - initialize resources
1698 * @dev: pointer to the pcie_dev data structure
1699 *
1700 * Invoked when PCI Express bus loads AER service driver.
1701 */
1702static int aer_probe(struct pcie_device *dev)
1703{
1704	int status;
1705	struct aer_rpc *rpc;
1706	struct device *device = &dev->device;
1707	struct pci_dev *port = dev->port;
1708
1709	BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) <
1710		     AER_MAX_TYPEOF_COR_ERRS);
1711	BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) <
1712		     AER_MAX_TYPEOF_UNCOR_ERRS);
1713
1714	/* Limit to Root Ports or Root Complex Event Collectors */
1715	if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&
1716	    (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))
1717		return -ENODEV;
1718
1719	rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL);
1720	if (!rpc)
1721		return -ENOMEM;
1722
1723	rpc->rpd = port;
1724	INIT_KFIFO(rpc->aer_fifo);
1725	set_service_data(dev, rpc);
1726
1727	status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,
1728					   IRQF_SHARED, "aerdrv", dev);
1729	if (status) {
1730		pci_err(port, "request AER IRQ %d failed\n", dev->irq);
1731		return status;
1732	}
1733
1734	cxl_rch_enable_rcec(port);
1735	aer_enable_rootport(rpc);
1736	pci_info(port, "enabled with IRQ %d\n", dev->irq);
1737	return 0;
1738}
1739
1740static int aer_suspend(struct pcie_device *dev)
1741{
1742	struct aer_rpc *rpc = get_service_data(dev);
1743
1744	aer_disable_rootport(rpc);
1745	return 0;
1746}
1747
1748static int aer_resume(struct pcie_device *dev)
1749{
1750	struct aer_rpc *rpc = get_service_data(dev);
1751
1752	aer_enable_rootport(rpc);
1753	return 0;
1754}
1755
1756/**
1757 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP
1758 * @dev: pointer to Root Port, RCEC, or RCiEP
1759 *
1760 * Invoked by Port Bus driver when performing reset.
1761 */
1762static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
1763{
1764	int type = pci_pcie_type(dev);
1765	struct pci_dev *root;
1766	int aer;
1767	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
1768	u32 reg32;
1769	int rc;
1770
1771	/*
1772	 * Only Root Ports and RCECs have AER Root Command and Root Status
1773	 * registers.  If "dev" is an RCiEP, the relevant registers are in
1774	 * the RCEC.
1775	 */
1776	if (type == PCI_EXP_TYPE_RC_END)
1777		root = dev->rcec;
1778	else
1779		root = pcie_find_root_port(dev);
1780
1781	/*
1782	 * If the platform retained control of AER, an RCiEP may not have
1783	 * an RCEC visible to us, so dev->rcec ("root") may be NULL.  In
1784	 * that case, firmware is responsible for these registers.
1785	 */
1786	aer = root ? root->aer_cap : 0;
1787
1788	if ((host->native_aer || pcie_ports_native) && aer)
1789		aer_disable_irq(root);
1790
1791	if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) {
1792		rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET);
1793		if (!rc)
1794			pci_info(dev, "has been reset\n");
1795		else
1796			pci_info(dev, "not reset (no FLR support: %d)\n", rc);
1797	} else {
1798		rc = pci_bus_error_reset(dev);
1799		pci_info(dev, "%s Port link has been reset (%d)\n",
1800			pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);
1801	}
1802
1803	if ((host->native_aer || pcie_ports_native) && aer) {
1804		/* Clear Root Error Status */
1805		pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, &reg32);
1806		pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32);
1807
1808		aer_enable_irq(root);
1809	}
1810
1811	return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1812}
1813
1814static struct pcie_port_service_driver aerdriver = {
1815	.name		= "aer",
1816	.port_type	= PCIE_ANY_PORT,
1817	.service	= PCIE_PORT_SERVICE_AER,
1818
1819	.probe		= aer_probe,
1820	.suspend	= aer_suspend,
1821	.resume		= aer_resume,
1822	.remove		= aer_remove,
1823};
1824
1825/**
1826 * pcie_aer_init - register AER service driver
1827 *
1828 * Invoked when AER service driver is loaded.
1829 */
1830int __init pcie_aer_init(void)
1831{
1832	if (!pci_aer_available())
1833		return -ENXIO;
1834	return pcie_port_service_register(&aerdriver);
1835}