igen6_edac.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Driver for Intel client SoC with integrated memory controller using IBECC
   4 *
   5 * Copyright (C) 2020 Intel Corporation
   6 *
   7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific
   8 * regions of the physical memory space. It's used for memory controllers
   9 * that don't support the out-of-band ECC which often needs an additional
  10 * storage device to each channel for storing ECC data.
  11 */
  12
  13#include <linux/module.h>
  14#include <linux/init.h>
  15#include <linux/pci.h>
  16#include <linux/slab.h>
  17#include <linux/irq_work.h>
  18#include <linux/llist.h>
  19#include <linux/genalloc.h>
  20#include <linux/edac.h>
  21#include <linux/bits.h>
  22#include <linux/io.h>
  23#include <asm/mach_traps.h>
  24#include <asm/nmi.h>
  25#include <asm/mce.h>
  26
  27#include "edac_mc.h"
  28#include "edac_module.h"
  29
  30#define IGEN6_REVISION	"v2.5.1"
  31
  32#define EDAC_MOD_STR	"igen6_edac"
  33#define IGEN6_NMI_NAME	"igen6_ibecc"
  34
  35/* Debug macros */
  36#define igen6_printk(level, fmt, arg...)		\
  37	edac_printk(level, "igen6", fmt, ##arg)
  38
  39#define igen6_mc_printk(mci, level, fmt, arg...)	\
  40	edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg)
  41
  42#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
  43
  44#define NUM_IMC				2 /* Max memory controllers */
  45#define NUM_CHANNELS			2 /* Max channels */
  46#define NUM_DIMMS			2 /* Max DIMMs per channel */
  47
  48#define _4GB				BIT_ULL(32)
  49
  50/* Size of physical memory */
  51#define TOM_OFFSET			0xa0
  52/* Top of low usable DRAM */
  53#define TOLUD_OFFSET			0xbc
  54/* Capability register C */
  55#define CAPID_C_OFFSET			0xec
  56#define CAPID_C_IBECC			BIT(15)
  57
  58/* Capability register E */
  59#define CAPID_E_OFFSET			0xf0
  60#define CAPID_E_IBECC			BIT(12)
  61#define CAPID_E_IBECC_BIT18		BIT(18)
  62
  63/* Error Status */
  64#define ERRSTS_OFFSET			0xc8
  65#define ERRSTS_CE			BIT_ULL(6)
  66#define ERRSTS_UE			BIT_ULL(7)
  67
  68/* Error Command */
  69#define ERRCMD_OFFSET			0xca
  70#define ERRCMD_CE			BIT_ULL(6)
  71#define ERRCMD_UE			BIT_ULL(7)
  72
  73/* IBECC MMIO base address */
  74#define IBECC_BASE			(res_cfg->ibecc_base)
  75#define IBECC_ACTIVATE_OFFSET		IBECC_BASE
  76#define IBECC_ACTIVATE_EN		BIT(0)
  77
  78/* IBECC error log */
  79#define ECC_ERROR_LOG_OFFSET		(IBECC_BASE + res_cfg->ibecc_error_log_offset)
  80#define ECC_ERROR_LOG_CE		BIT_ULL(62)
  81#define ECC_ERROR_LOG_UE		BIT_ULL(63)
  82#define ECC_ERROR_LOG_ADDR_SHIFT	5
  83#define ECC_ERROR_LOG_ADDR(v)		GET_BITFIELD(v, 5, 38)
  84#define ECC_ERROR_LOG_ADDR45(v)		GET_BITFIELD(v, 5, 45)
  85#define ECC_ERROR_LOG_SYND(v)		GET_BITFIELD(v, 46, 61)
  86
  87/* Host MMIO base address */
  88#define MCHBAR_OFFSET			0x48
  89#define MCHBAR_EN			BIT_ULL(0)
  90#define MCHBAR_BASE(v)			(GET_BITFIELD(v, 16, 38) << 16)
  91#define MCHBAR_SIZE			0x10000
  92
  93/* Parameters for the channel decode stage */
  94#define IMC_BASE			(res_cfg->imc_base)
  95#define MAD_INTER_CHANNEL_OFFSET	IMC_BASE
  96#define MAD_INTER_CHANNEL_DDR_TYPE(v)	GET_BITFIELD(v, 0, 2)
  97#define MAD_INTER_CHANNEL_ECHM(v)	GET_BITFIELD(v, 3, 3)
  98#define MAD_INTER_CHANNEL_CH_L_MAP(v)	GET_BITFIELD(v, 4, 4)
  99#define MAD_INTER_CHANNEL_CH_S_SIZE(v)	((u64)GET_BITFIELD(v, 12, 19) << 29)
 100
 101/* Parameters for DRAM decode stage */
 102#define MAD_INTRA_CH0_OFFSET		(IMC_BASE + 4)
 103#define MAD_INTRA_CH_DIMM_L_MAP(v)	GET_BITFIELD(v, 0, 0)
 104
 105/* DIMM characteristics */
 106#define MAD_DIMM_CH0_OFFSET		(IMC_BASE + 0xc)
 107#define MAD_DIMM_CH_DIMM_L_SIZE(v)	((u64)GET_BITFIELD(v, 0, 6) << 29)
 108#define MAD_DIMM_CH_DLW(v)		GET_BITFIELD(v, 7, 8)
 109#define MAD_DIMM_CH_DIMM_S_SIZE(v)	((u64)GET_BITFIELD(v, 16, 22) << 29)
 110#define MAD_DIMM_CH_DSW(v)		GET_BITFIELD(v, 24, 25)
 111
 112/* Hash for memory controller selection */
 113#define MAD_MC_HASH_OFFSET		(IMC_BASE + 0x1b8)
 114#define MAC_MC_HASH_LSB(v)		GET_BITFIELD(v, 1, 3)
 115
 116/* Hash for channel selection */
 117#define CHANNEL_HASH_OFFSET		(IMC_BASE + 0x24)
 118/* Hash for enhanced channel selection */
 119#define CHANNEL_EHASH_OFFSET		(IMC_BASE + 0x28)
 120#define CHANNEL_HASH_MASK(v)		(GET_BITFIELD(v, 6, 19) << 6)
 121#define CHANNEL_HASH_LSB_MASK_BIT(v)	GET_BITFIELD(v, 24, 26)
 122#define CHANNEL_HASH_MODE(v)		GET_BITFIELD(v, 28, 28)
 123
 124/* Parameters for memory slice decode stage */
 125#define MEM_SLICE_HASH_MASK(v)		(GET_BITFIELD(v, 6, 19) << 6)
 126#define MEM_SLICE_HASH_LSB_MASK_BIT(v)	GET_BITFIELD(v, 24, 26)
 127
 128static struct res_config {
 129	bool machine_check;
 130	/* The number of present memory controllers. */
 131	int num_imc;
 132	u32 imc_base;
 133	u32 cmf_base;
 134	u32 cmf_size;
 135	u32 ms_hash_offset;
 136	u32 ibecc_base;
 137	u32 ibecc_error_log_offset;
 138	bool (*ibecc_available)(struct pci_dev *pdev);
 139	/* Extract error address logged in IBECC */
 140	u64 (*err_addr)(u64 ecclog);
 141	/* Convert error address logged in IBECC to system physical address */
 142	u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc);
 143	/* Convert error address logged in IBECC to integrated memory controller address */
 144	u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc);
 145} *res_cfg;
 146
 147struct igen6_imc {
 148	int mc;
 149	struct mem_ctl_info *mci;
 150	struct pci_dev *pdev;
 151	struct device dev;
 152	void __iomem *window;
 153	u64 size;
 154	u64 ch_s_size;
 155	int ch_l_map;
 156	u64 dimm_s_size[NUM_CHANNELS];
 157	u64 dimm_l_size[NUM_CHANNELS];
 158	int dimm_l_map[NUM_CHANNELS];
 159};
 160
 161static struct igen6_pvt {
 162	struct igen6_imc imc[NUM_IMC];
 163	u64 ms_hash;
 164	u64 ms_s_size;
 165	int ms_l_map;
 166} *igen6_pvt;
 167
 168/* The top of low usable DRAM */
 169static u32 igen6_tolud;
 170/* The size of physical memory */
 171static u64 igen6_tom;
 172
 173struct decoded_addr {
 174	int mc;
 175	u64 imc_addr;
 176	u64 sys_addr;
 177	int channel_idx;
 178	u64 channel_addr;
 179	int sub_channel_idx;
 180	u64 sub_channel_addr;
 181};
 182
 183struct ecclog_node {
 184	struct llist_node llnode;
 185	int mc;
 186	u64 ecclog;
 187};
 188
 189/*
 190 * In the NMI handler, the driver uses the lock-less memory allocator
 191 * to allocate memory to store the IBECC error logs and links the logs
 192 * to the lock-less list. Delay printk() and the work of error reporting
 193 * to EDAC core in a worker.
 194 */
 195#define ECCLOG_POOL_SIZE	PAGE_SIZE
 196static LLIST_HEAD(ecclog_llist);
 197static struct gen_pool *ecclog_pool;
 198static char ecclog_buf[ECCLOG_POOL_SIZE];
 199static struct irq_work ecclog_irq_work;
 200static struct work_struct ecclog_work;
 201
 202/* Compute die IDs for Elkhart Lake with IBECC */
 203#define DID_EHL_SKU5	0x4514
 204#define DID_EHL_SKU6	0x4528
 205#define DID_EHL_SKU7	0x452a
 206#define DID_EHL_SKU8	0x4516
 207#define DID_EHL_SKU9	0x452c
 208#define DID_EHL_SKU10	0x452e
 209#define DID_EHL_SKU11	0x4532
 210#define DID_EHL_SKU12	0x4518
 211#define DID_EHL_SKU13	0x451a
 212#define DID_EHL_SKU14	0x4534
 213#define DID_EHL_SKU15	0x4536
 214
 215/* Compute die IDs for ICL-NNPI with IBECC */
 216#define DID_ICL_SKU8	0x4581
 217#define DID_ICL_SKU10	0x4585
 218#define DID_ICL_SKU11	0x4589
 219#define DID_ICL_SKU12	0x458d
 220
 221/* Compute die IDs for Tiger Lake with IBECC */
 222#define DID_TGL_SKU	0x9a14
 223
 224/* Compute die IDs for Alder Lake with IBECC */
 225#define DID_ADL_SKU1	0x4601
 226#define DID_ADL_SKU2	0x4602
 227#define DID_ADL_SKU3	0x4621
 228#define DID_ADL_SKU4	0x4641
 229
 230/* Compute die IDs for Alder Lake-N with IBECC */
 231#define DID_ADL_N_SKU1	0x4614
 232#define DID_ADL_N_SKU2	0x4617
 233#define DID_ADL_N_SKU3	0x461b
 234#define DID_ADL_N_SKU4	0x461c
 235#define DID_ADL_N_SKU5	0x4673
 236#define DID_ADL_N_SKU6	0x4674
 237#define DID_ADL_N_SKU7	0x4675
 238#define DID_ADL_N_SKU8	0x4677
 239#define DID_ADL_N_SKU9	0x4678
 240#define DID_ADL_N_SKU10	0x4679
 241#define DID_ADL_N_SKU11	0x467c
 242#define DID_ADL_N_SKU12	0x4632
 243
 244/* Compute die IDs for Arizona Beach with IBECC */
 245#define DID_AZB_SKU1	0x4676
 246
 247/* Compute did IDs for Amston Lake with IBECC */
 248#define DID_ASL_SKU1	0x464a
 249#define DID_ASL_SKU2	0x4646
 250#define DID_ASL_SKU3	0x4652
 251
 252/* Compute die IDs for Raptor Lake-P with IBECC */
 253#define DID_RPL_P_SKU1	0xa706
 254#define DID_RPL_P_SKU2	0xa707
 255#define DID_RPL_P_SKU3	0xa708
 256#define DID_RPL_P_SKU4	0xa716
 257#define DID_RPL_P_SKU5	0xa718
 258
 259/* Compute die IDs for Meteor Lake-PS with IBECC */
 260#define DID_MTL_PS_SKU1	0x7d21
 261#define DID_MTL_PS_SKU2	0x7d22
 262#define DID_MTL_PS_SKU3	0x7d23
 263#define DID_MTL_PS_SKU4	0x7d24
 264
 265/* Compute die IDs for Meteor Lake-P with IBECC */
 266#define DID_MTL_P_SKU1	0x7d01
 267#define DID_MTL_P_SKU2	0x7d02
 268#define DID_MTL_P_SKU3	0x7d14
 269
 270/* Compute die IDs for Arrow Lake-UH with IBECC */
 271#define DID_ARL_UH_SKU1	0x7d06
 272#define DID_ARL_UH_SKU2	0x7d20
 273#define DID_ARL_UH_SKU3	0x7d30
 274
 275/* Compute die IDs for Panther Lake-H with IBECC */
 276#define DID_PTL_H_SKU1	0xb000
 277#define DID_PTL_H_SKU2	0xb001
 278#define DID_PTL_H_SKU3	0xb002
 279#define DID_PTL_H_SKU4	0xb003
 280#define DID_PTL_H_SKU5	0xb004
 281#define DID_PTL_H_SKU6	0xb005
 282#define DID_PTL_H_SKU7	0xb008
 283#define DID_PTL_H_SKU8	0xb011
 284#define DID_PTL_H_SKU9	0xb014
 285#define DID_PTL_H_SKU10	0xb015
 286#define DID_PTL_H_SKU11	0xb028
 287#define DID_PTL_H_SKU12	0xb029
 288#define DID_PTL_H_SKU13	0xb02a
 289
 290/* Compute die IDs for Wildcat Lake with IBECC */
 291#define DID_WCL_SKU1	0xfd00
 292
 293static int get_mchbar(struct pci_dev *pdev, u64 *mchbar)
 294{
 295	union  {
 296		u64 v;
 297		struct {
 298			u32 v_lo;
 299			u32 v_hi;
 300		};
 301	} u;
 302
 303	if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) {
 304		igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n");
 305		return -ENODEV;
 306	}
 307
 308	if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) {
 309		igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n");
 310		return -ENODEV;
 311	}
 312
 313	if (!(u.v & MCHBAR_EN)) {
 314		igen6_printk(KERN_ERR, "MCHBAR is disabled\n");
 315		return -ENODEV;
 316	}
 317
 318	*mchbar = MCHBAR_BASE(u.v);
 319
 320	return 0;
 321}
 322
 323static bool ehl_ibecc_available(struct pci_dev *pdev)
 324{
 325	u32 v;
 326
 327	if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
 328		return false;
 329
 330	return !!(CAPID_C_IBECC & v);
 331}
 332
 333static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc)
 334{
 335	return eaddr;
 336}
 337
 338static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc)
 339{
 340	if (eaddr < igen6_tolud)
 341		return eaddr;
 342
 343	if (igen6_tom <= _4GB)
 344		return eaddr + igen6_tolud - _4GB;
 345
 346	if (eaddr >= igen6_tom)
 347		return eaddr + igen6_tolud - igen6_tom;
 348
 349	return eaddr;
 350}
 351
 352static bool icl_ibecc_available(struct pci_dev *pdev)
 353{
 354	u32 v;
 355
 356	if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
 357		return false;
 358
 359	return !(CAPID_C_IBECC & v) &&
 360		(boot_cpu_data.x86_stepping >= 1);
 361}
 362
 363static bool tgl_ibecc_available(struct pci_dev *pdev)
 364{
 365	u32 v;
 366
 367	if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
 368		return false;
 369
 370	return !(CAPID_E_IBECC & v);
 371}
 372
 373static bool mtl_p_ibecc_available(struct pci_dev *pdev)
 374{
 375	u32 v;
 376
 377	if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
 378		return false;
 379
 380	return !(CAPID_E_IBECC_BIT18 & v);
 381}
 382
 383static bool mtl_ps_ibecc_available(struct pci_dev *pdev)
 384{
 385#define MCHBAR_MEMSS_IBECCDIS	0x13c00
 386	void __iomem *window;
 387	u64 mchbar;
 388	u32 val;
 389
 390	if (get_mchbar(pdev, &mchbar))
 391		return false;
 392
 393	window = ioremap(mchbar, MCHBAR_SIZE * 2);
 394	if (!window) {
 395		igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar);
 396		return false;
 397	}
 398
 399	val = readl(window + MCHBAR_MEMSS_IBECCDIS);
 400	iounmap(window);
 401
 402	/* Bit6: 1 - IBECC is disabled, 0 - IBECC isn't disabled */
 403	return !GET_BITFIELD(val, 6, 6);
 404}
 405
 406static u64 mem_addr_to_sys_addr(u64 maddr)
 407{
 408	if (maddr < igen6_tolud)
 409		return maddr;
 410
 411	if (igen6_tom <= _4GB)
 412		return maddr - igen6_tolud + _4GB;
 413
 414	if (maddr < _4GB)
 415		return maddr - igen6_tolud + igen6_tom;
 416
 417	return maddr;
 418}
 419
 420static u64 mem_slice_hash(u64 addr, u64 mask, u64 hash_init, int intlv_bit)
 421{
 422	u64 hash_addr = addr & mask, hash = hash_init;
 423	u64 intlv = (addr >> intlv_bit) & 1;
 424	int i;
 425
 426	for (i = 6; i < 20; i++)
 427		hash ^= (hash_addr >> i) & 1;
 428
 429	return hash ^ intlv;
 430}
 431
 432static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc)
 433{
 434	u64 maddr, hash, mask, ms_s_size;
 435	int intlv_bit;
 436	u32 ms_hash;
 437
 438	ms_s_size = igen6_pvt->ms_s_size;
 439	if (eaddr >= ms_s_size)
 440		return eaddr + ms_s_size;
 441
 442	ms_hash = igen6_pvt->ms_hash;
 443
 444	mask = MEM_SLICE_HASH_MASK(ms_hash);
 445	intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6;
 446
 447	maddr = GET_BITFIELD(eaddr, intlv_bit, 63) << (intlv_bit + 1) |
 448		GET_BITFIELD(eaddr, 0, intlv_bit - 1);
 449
 450	hash = mem_slice_hash(maddr, mask, mc, intlv_bit);
 451
 452	return maddr | (hash << intlv_bit);
 453}
 454
 455static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc)
 456{
 457	u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc);
 458
 459	return mem_addr_to_sys_addr(maddr);
 460}
 461
 462static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc)
 463{
 464	return eaddr;
 465}
 466
 467static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc)
 468{
 469	return mem_addr_to_sys_addr(eaddr);
 470}
 471
 472static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc)
 473{
 474	u64 imc_addr, ms_s_size = igen6_pvt->ms_s_size;
 475	struct igen6_imc *imc = &igen6_pvt->imc[mc];
 476	int intlv_bit;
 477	u32 mc_hash;
 478
 479	if (eaddr >= 2 * ms_s_size)
 480		return eaddr - ms_s_size;
 481
 482	mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET);
 483
 484	intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6;
 485
 486	imc_addr = GET_BITFIELD(eaddr, intlv_bit + 1, 63) << intlv_bit |
 487		   GET_BITFIELD(eaddr, 0, intlv_bit - 1);
 488
 489	return imc_addr;
 490}
 491
 492static u64 rpl_p_err_addr(u64 ecclog)
 493{
 494	return ECC_ERROR_LOG_ADDR45(ecclog);
 495}
 496
 497static struct res_config ehl_cfg = {
 498	.num_imc		= 1,
 499	.imc_base		= 0x5000,
 500	.ibecc_base		= 0xdc00,
 501	.ibecc_available	= ehl_ibecc_available,
 502	.ibecc_error_log_offset	= 0x170,
 503	.err_addr_to_sys_addr	= ehl_err_addr_to_sys_addr,
 504	.err_addr_to_imc_addr	= ehl_err_addr_to_imc_addr,
 505};
 506
 507static struct res_config icl_cfg = {
 508	.num_imc		= 1,
 509	.imc_base		= 0x5000,
 510	.ibecc_base		= 0xd800,
 511	.ibecc_error_log_offset	= 0x170,
 512	.ibecc_available	= icl_ibecc_available,
 513	.err_addr_to_sys_addr	= ehl_err_addr_to_sys_addr,
 514	.err_addr_to_imc_addr	= ehl_err_addr_to_imc_addr,
 515};
 516
 517static struct res_config tgl_cfg = {
 518	.machine_check		= true,
 519	.num_imc		= 2,
 520	.imc_base		= 0x5000,
 521	.cmf_base		= 0x11000,
 522	.cmf_size		= 0x800,
 523	.ms_hash_offset		= 0xac,
 524	.ibecc_base		= 0xd400,
 525	.ibecc_error_log_offset	= 0x170,
 526	.ibecc_available	= tgl_ibecc_available,
 527	.err_addr_to_sys_addr	= tgl_err_addr_to_sys_addr,
 528	.err_addr_to_imc_addr	= tgl_err_addr_to_imc_addr,
 529};
 530
 531static struct res_config adl_cfg = {
 532	.machine_check		= true,
 533	.num_imc		= 2,
 534	.imc_base		= 0xd800,
 535	.ibecc_base		= 0xd400,
 536	.ibecc_error_log_offset	= 0x68,
 537	.ibecc_available	= tgl_ibecc_available,
 538	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
 539	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 540};
 541
 542static struct res_config adl_n_cfg = {
 543	.machine_check		= true,
 544	.num_imc		= 1,
 545	.imc_base		= 0xd800,
 546	.ibecc_base		= 0xd400,
 547	.ibecc_error_log_offset	= 0x68,
 548	.ibecc_available	= tgl_ibecc_available,
 549	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
 550	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 551};
 552
 553static struct res_config rpl_p_cfg = {
 554	.machine_check		= true,
 555	.num_imc		= 2,
 556	.imc_base		= 0xd800,
 557	.ibecc_base		= 0xd400,
 558	.ibecc_error_log_offset	= 0x68,
 559	.ibecc_available	= tgl_ibecc_available,
 560	.err_addr		= rpl_p_err_addr,
 561	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
 562	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 563};
 564
 565static struct res_config mtl_ps_cfg = {
 566	.machine_check		= true,
 567	.num_imc		= 2,
 568	.imc_base		= 0xd800,
 569	.ibecc_base		= 0xd400,
 570	.ibecc_error_log_offset	= 0x170,
 571	.ibecc_available	= mtl_ps_ibecc_available,
 572	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
 573	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 574};
 575
 576static struct res_config mtl_p_cfg = {
 577	.machine_check		= true,
 578	.num_imc		= 2,
 579	.imc_base		= 0xd800,
 580	.ibecc_base		= 0xd400,
 581	.ibecc_error_log_offset	= 0x170,
 582	.ibecc_available	= mtl_p_ibecc_available,
 583	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
 584	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 585};
 586
 587static struct res_config wcl_cfg = {
 588	.machine_check		= true,
 589	.num_imc		= 1,
 590	.imc_base		= 0xd800,
 591	.ibecc_base		= 0xd400,
 592	.ibecc_error_log_offset	= 0x170,
 593	.ibecc_available	= mtl_p_ibecc_available,
 594	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
 595	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 596};
 597
 598static struct pci_device_id igen6_pci_tbl[] = {
 599	{ PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg },
 600	{ PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg },
 601	{ PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg },
 602	{ PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg },
 603	{ PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg },
 604	{ PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg },
 605	{ PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg },
 606	{ PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg },
 607	{ PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg },
 608	{ PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg },
 609	{ PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg },
 610	{ PCI_VDEVICE(INTEL, DID_ICL_SKU8), (kernel_ulong_t)&icl_cfg },
 611	{ PCI_VDEVICE(INTEL, DID_ICL_SKU10), (kernel_ulong_t)&icl_cfg },
 612	{ PCI_VDEVICE(INTEL, DID_ICL_SKU11), (kernel_ulong_t)&icl_cfg },
 613	{ PCI_VDEVICE(INTEL, DID_ICL_SKU12), (kernel_ulong_t)&icl_cfg },
 614	{ PCI_VDEVICE(INTEL, DID_TGL_SKU), (kernel_ulong_t)&tgl_cfg },
 615	{ PCI_VDEVICE(INTEL, DID_ADL_SKU1), (kernel_ulong_t)&adl_cfg },
 616	{ PCI_VDEVICE(INTEL, DID_ADL_SKU2), (kernel_ulong_t)&adl_cfg },
 617	{ PCI_VDEVICE(INTEL, DID_ADL_SKU3), (kernel_ulong_t)&adl_cfg },
 618	{ PCI_VDEVICE(INTEL, DID_ADL_SKU4), (kernel_ulong_t)&adl_cfg },
 619	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), (kernel_ulong_t)&adl_n_cfg },
 620	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), (kernel_ulong_t)&adl_n_cfg },
 621	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), (kernel_ulong_t)&adl_n_cfg },
 622	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), (kernel_ulong_t)&adl_n_cfg },
 623	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), (kernel_ulong_t)&adl_n_cfg },
 624	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), (kernel_ulong_t)&adl_n_cfg },
 625	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), (kernel_ulong_t)&adl_n_cfg },
 626	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), (kernel_ulong_t)&adl_n_cfg },
 627	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), (kernel_ulong_t)&adl_n_cfg },
 628	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), (kernel_ulong_t)&adl_n_cfg },
 629	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), (kernel_ulong_t)&adl_n_cfg },
 630	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), (kernel_ulong_t)&adl_n_cfg },
 631	{ PCI_VDEVICE(INTEL, DID_AZB_SKU1), (kernel_ulong_t)&adl_n_cfg },
 632	{ PCI_VDEVICE(INTEL, DID_ASL_SKU1), (kernel_ulong_t)&adl_n_cfg },
 633	{ PCI_VDEVICE(INTEL, DID_ASL_SKU2), (kernel_ulong_t)&adl_n_cfg },
 634	{ PCI_VDEVICE(INTEL, DID_ASL_SKU3), (kernel_ulong_t)&adl_n_cfg },
 635	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), (kernel_ulong_t)&rpl_p_cfg },
 636	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), (kernel_ulong_t)&rpl_p_cfg },
 637	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), (kernel_ulong_t)&rpl_p_cfg },
 638	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), (kernel_ulong_t)&rpl_p_cfg },
 639	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), (kernel_ulong_t)&rpl_p_cfg },
 640	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), (kernel_ulong_t)&mtl_ps_cfg },
 641	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), (kernel_ulong_t)&mtl_ps_cfg },
 642	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), (kernel_ulong_t)&mtl_ps_cfg },
 643	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), (kernel_ulong_t)&mtl_ps_cfg },
 644	{ PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), (kernel_ulong_t)&mtl_p_cfg },
 645	{ PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), (kernel_ulong_t)&mtl_p_cfg },
 646	{ PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), (kernel_ulong_t)&mtl_p_cfg },
 647	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), (kernel_ulong_t)&mtl_p_cfg },
 648	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), (kernel_ulong_t)&mtl_p_cfg },
 649	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), (kernel_ulong_t)&mtl_p_cfg },
 650	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg },
 651	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg },
 652	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg },
 653	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg },
 654	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg },
 655	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg },
 656	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg },
 657	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg },
 658	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg },
 659	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg },
 660	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg },
 661	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg },
 662	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg },
 663	{ PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg },
 664	{ },
 665};
 666MODULE_DEVICE_TABLE(pci, igen6_pci_tbl);
 667
 668static enum dev_type get_width(int dimm_l, u32 mad_dimm)
 669{
 670	u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) :
 671			 MAD_DIMM_CH_DSW(mad_dimm);
 672
 673	switch (w) {
 674	case 0:
 675		return DEV_X8;
 676	case 1:
 677		return DEV_X16;
 678	case 2:
 679		return DEV_X32;
 680	default:
 681		return DEV_UNKNOWN;
 682	}
 683}
 684
 685static enum mem_type get_memory_type(u32 mad_inter)
 686{
 687	u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter);
 688
 689	switch (t) {
 690	case 0:
 691		return MEM_DDR4;
 692	case 1:
 693		return MEM_DDR3;
 694	case 2:
 695		return MEM_LPDDR3;
 696	case 3:
 697		return MEM_LPDDR4;
 698	case 4:
 699		return MEM_WIO2;
 700	default:
 701		return MEM_UNKNOWN;
 702	}
 703}
 704
 705static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit)
 706{
 707	u64 hash_addr = addr & mask, hash = 0;
 708	u64 intlv = (addr >> intlv_bit) & 1;
 709	int i;
 710
 711	for (i = 6; i < 20; i++)
 712		hash ^= (hash_addr >> i) & 1;
 713
 714	return (int)hash ^ intlv;
 715}
 716
 717static u64 decode_channel_addr(u64 addr, int intlv_bit)
 718{
 719	u64 channel_addr;
 720
 721	/* Remove the interleave bit and shift upper part down to fill gap */
 722	channel_addr  = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit;
 723	channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
 724
 725	return channel_addr;
 726}
 727
 728static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map,
 729			int *idx, u64 *sub_addr)
 730{
 731	int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
 732
 733	if (addr > 2 * s_size) {
 734		*sub_addr = addr - s_size;
 735		*idx = l_map;
 736		return;
 737	}
 738
 739	if (CHANNEL_HASH_MODE(hash)) {
 740		*sub_addr = decode_channel_addr(addr, intlv_bit);
 741		*idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit);
 742	} else {
 743		*sub_addr = decode_channel_addr(addr, 6);
 744		*idx = GET_BITFIELD(addr, 6, 6);
 745	}
 746}
 747
 748static int igen6_decode(struct decoded_addr *res)
 749{
 750	struct igen6_imc *imc = &igen6_pvt->imc[res->mc];
 751	u64 addr = res->imc_addr, sub_addr, s_size;
 752	int idx, l_map;
 753	u32 hash;
 754
 755	if (addr >= igen6_tom) {
 756		edac_dbg(0, "Address 0x%llx out of range\n", addr);
 757		return -EINVAL;
 758	}
 759
 760	/* Decode channel */
 761	hash   = readl(imc->window + CHANNEL_HASH_OFFSET);
 762	s_size = imc->ch_s_size;
 763	l_map  = imc->ch_l_map;
 764	decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr);
 765	res->channel_idx  = idx;
 766	res->channel_addr = sub_addr;
 767
 768	/* Decode sub-channel/DIMM */
 769	hash   = readl(imc->window + CHANNEL_EHASH_OFFSET);
 770	s_size = imc->dimm_s_size[idx];
 771	l_map  = imc->dimm_l_map[idx];
 772	decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr);
 773	res->sub_channel_idx  = idx;
 774	res->sub_channel_addr = sub_addr;
 775
 776	return 0;
 777}
 778
 779static void igen6_output_error(struct decoded_addr *res,
 780			       struct mem_ctl_info *mci, u64 ecclog)
 781{
 782	enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ?
 783					 HW_EVENT_ERR_UNCORRECTED :
 784					 HW_EVENT_ERR_CORRECTED;
 785
 786	edac_mc_handle_error(type, mci, 1,
 787			     res->sys_addr >> PAGE_SHIFT,
 788			     res->sys_addr & ~PAGE_MASK,
 789			     ECC_ERROR_LOG_SYND(ecclog),
 790			     res->channel_idx, res->sub_channel_idx,
 791			     -1, "", "");
 792}
 793
 794static struct gen_pool *ecclog_gen_pool_create(void)
 795{
 796	struct gen_pool *pool;
 797
 798	pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1);
 799	if (!pool)
 800		return NULL;
 801
 802	if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) {
 803		gen_pool_destroy(pool);
 804		return NULL;
 805	}
 806
 807	return pool;
 808}
 809
 810static int ecclog_gen_pool_add(int mc, u64 ecclog)
 811{
 812	struct ecclog_node *node;
 813
 814	node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node));
 815	if (!node)
 816		return -ENOMEM;
 817
 818	node->mc = mc;
 819	node->ecclog = ecclog;
 820	llist_add(&node->llnode, &ecclog_llist);
 821
 822	return 0;
 823}
 824
 825/*
 826 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI
 827 * configuration space status register ERRSTS can indicate whether a
 828 * correctable error or an uncorrectable error occurred. We only use the
 829 * ECC_ERROR_LOG register to check error type, but need to clear both
 830 * registers to enable future error events.
 831 */
 832static u64 ecclog_read_and_clear(struct igen6_imc *imc)
 833{
 834	u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET);
 835
 836	/*
 837	 * Quirk: The ECC_ERROR_LOG register of certain SoCs may contain
 838	 *        the invalid value ~0. This will result in a flood of invalid
 839	 *        error reports in polling mode. Skip it.
 840	 */
 841	if (ecclog == ~0)
 842		return 0;
 843
 844	/* Neither a CE nor a UE. Skip it.*/
 845	if (!(ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)))
 846		return 0;
 847
 848	/* Clear CE/UE bits by writing 1s */
 849	writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET);
 850
 851	return ecclog;
 852}
 853
 854static void errsts_clear(struct igen6_imc *imc)
 855{
 856	u16 errsts;
 857
 858	if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) {
 859		igen6_printk(KERN_ERR, "Failed to read ERRSTS\n");
 860		return;
 861	}
 862
 863	/* Clear CE/UE bits by writing 1s */
 864	if (errsts & (ERRSTS_CE | ERRSTS_UE))
 865		pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts);
 866}
 867
 868static int errcmd_enable_error_reporting(bool enable)
 869{
 870	struct igen6_imc *imc = &igen6_pvt->imc[0];
 871	u16 errcmd;
 872	int rc;
 873
 874	rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd);
 875	if (rc)
 876		return pcibios_err_to_errno(rc);
 877
 878	if (enable)
 879		errcmd |= ERRCMD_CE | ERRSTS_UE;
 880	else
 881		errcmd &= ~(ERRCMD_CE | ERRSTS_UE);
 882
 883	rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd);
 884	if (rc)
 885		return pcibios_err_to_errno(rc);
 886
 887	return 0;
 888}
 889
 890static int ecclog_handler(void)
 891{
 892	struct igen6_imc *imc;
 893	int i, n = 0;
 894	u64 ecclog;
 895
 896	for (i = 0; i < res_cfg->num_imc; i++) {
 897		imc = &igen6_pvt->imc[i];
 898
 899		/* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
 900
 901		ecclog = ecclog_read_and_clear(imc);
 902		if (!ecclog)
 903			continue;
 904
 905		if (!ecclog_gen_pool_add(i, ecclog))
 906			irq_work_queue(&ecclog_irq_work);
 907
 908		n++;
 909	}
 910
 911	return n;
 912}
 913
 914static void ecclog_work_cb(struct work_struct *work)
 915{
 916	struct ecclog_node *node, *tmp;
 917	struct mem_ctl_info *mci;
 918	struct llist_node *head;
 919	struct decoded_addr res;
 920	u64 eaddr;
 921
 922	head = llist_del_all(&ecclog_llist);
 923	if (!head)
 924		return;
 925
 926	llist_for_each_entry_safe(node, tmp, head, llnode) {
 927		memset(&res, 0, sizeof(res));
 928		if (res_cfg->err_addr)
 929			eaddr = res_cfg->err_addr(node->ecclog);
 930		else
 931			eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) <<
 932				ECC_ERROR_LOG_ADDR_SHIFT;
 933		res.mc	     = node->mc;
 934		res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc);
 935		res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc);
 936
 937		mci = igen6_pvt->imc[res.mc].mci;
 938
 939		edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog);
 940		igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n");
 941		igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr);
 942
 943		if (!igen6_decode(&res))
 944			igen6_output_error(&res, mci, node->ecclog);
 945
 946		gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node));
 947	}
 948}
 949
 950static void ecclog_irq_work_cb(struct irq_work *irq_work)
 951{
 952	int i;
 953
 954	for (i = 0; i < res_cfg->num_imc; i++)
 955		errsts_clear(&igen6_pvt->imc[i]);
 956
 957	if (!llist_empty(&ecclog_llist))
 958		schedule_work(&ecclog_work);
 959}
 960
 961static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs)
 962{
 963	unsigned char reason;
 964
 965	if (!ecclog_handler())
 966		return NMI_DONE;
 967
 968	/*
 969	 * Both In-Band ECC correctable error and uncorrectable error are
 970	 * reported by SERR# NMI. The NMI generic code (see pci_serr_error())
 971	 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to
 972	 * re-enable the SERR# NMI after NMI handling. So clear this bit here
 973	 * to re-enable SERR# NMI for receiving future In-Band ECC errors.
 974	 */
 975	reason  = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK;
 976	reason |= NMI_REASON_CLEAR_SERR;
 977	outb(reason, NMI_REASON_PORT);
 978	reason &= ~NMI_REASON_CLEAR_SERR;
 979	outb(reason, NMI_REASON_PORT);
 980
 981	return NMI_HANDLED;
 982}
 983
 984static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val,
 985			      void *data)
 986{
 987	struct mce *mce = (struct mce *)data;
 988	char *type;
 989
 990	if (mce->kflags & MCE_HANDLED_CEC)
 991		return NOTIFY_DONE;
 992
 993	/*
 994	 * Ignore unless this is a memory related error.
 995	 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here,
 996	 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3).
 997	 */
 998	if ((mce->status & 0xefff) >> 7 != 1)
 999		return NOTIFY_DONE;
1000
1001	if (mce->mcgstatus & MCG_STATUS_MCIP)
1002		type = "Exception";
1003	else
1004		type = "Event";
1005
1006	edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n",
1007		 mce->extcpu, type, mce->mcgstatus,
1008		 mce->bank, mce->status);
1009	edac_dbg(0, "TSC 0x%llx\n", mce->tsc);
1010	edac_dbg(0, "ADDR 0x%llx\n", mce->addr);
1011	edac_dbg(0, "MISC 0x%llx\n", mce->misc);
1012	edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n",
1013		 mce->cpuvendor, mce->cpuid, mce->time,
1014		 mce->socketid, mce->apicid);
1015	/*
1016	 * We just use the Machine Check for the memory error notification.
1017	 * Each memory controller is associated with an IBECC instance.
1018	 * Directly read and clear the error information(error address and
1019	 * error type) on all the IBECC instances so that we know on which
1020	 * memory controller the memory error(s) occurred.
1021	 */
1022	if (!ecclog_handler())
1023		return NOTIFY_DONE;
1024
1025	mce->kflags |= MCE_HANDLED_EDAC;
1026
1027	return NOTIFY_DONE;
1028}
1029
1030static struct notifier_block ecclog_mce_dec = {
1031	.notifier_call	= ecclog_mce_handler,
1032	.priority	= MCE_PRIO_EDAC,
1033};
1034
1035static bool igen6_check_ecc(struct igen6_imc *imc)
1036{
1037	u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET);
1038
1039	return !!(activate & IBECC_ACTIVATE_EN);
1040}
1041
1042static int igen6_get_dimm_config(struct mem_ctl_info *mci)
1043{
1044	struct igen6_imc *imc = mci->pvt_info;
1045	u32 mad_inter, mad_intra, mad_dimm;
1046	int i, j, ndimms, mc = imc->mc;
1047	struct dimm_info *dimm;
1048	enum mem_type mtype;
1049	enum dev_type dtype;
1050	u64 dsize;
1051	bool ecc;
1052
1053	edac_dbg(2, "\n");
1054
1055	mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
1056	mtype = get_memory_type(mad_inter);
1057	ecc = igen6_check_ecc(imc);
1058	imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter);
1059	imc->ch_l_map  = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter);
1060
1061	for (i = 0; i < NUM_CHANNELS; i++) {
1062		mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4);
1063		mad_dimm  = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4);
1064
1065		imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm);
1066		imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm);
1067		imc->dimm_l_map[i]  = MAD_INTRA_CH_DIMM_L_MAP(mad_intra);
1068		imc->size += imc->dimm_s_size[i];
1069		imc->size += imc->dimm_l_size[i];
1070		ndimms = 0;
1071
1072		for (j = 0; j < NUM_DIMMS; j++) {
1073			dimm = edac_get_dimm(mci, i, j, 0);
1074
1075			if (j ^ imc->dimm_l_map[i]) {
1076				dtype = get_width(0, mad_dimm);
1077				dsize = imc->dimm_s_size[i];
1078			} else {
1079				dtype = get_width(1, mad_dimm);
1080				dsize = imc->dimm_l_size[i];
1081			}
1082
1083			if (!dsize)
1084				continue;
1085
1086			dimm->grain = 64;
1087			dimm->mtype = mtype;
1088			dimm->dtype = dtype;
1089			dimm->nr_pages  = MiB_TO_PAGES(dsize >> 20);
1090			dimm->edac_mode = EDAC_SECDED;
1091			snprintf(dimm->label, sizeof(dimm->label),
1092				 "MC#%d_Chan#%d_DIMM#%d", mc, i, j);
1093			edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n",
1094				 mc, i, j, dsize >> 20, dimm->nr_pages);
1095
1096			ndimms++;
1097		}
1098
1099		if (ndimms && !ecc) {
1100			igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc);
1101			return -ENODEV;
1102		}
1103	}
1104
1105	edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20);
1106
1107	return 0;
1108}
1109
1110#ifdef CONFIG_EDAC_DEBUG
1111/* Top of upper usable DRAM */
1112static u64 igen6_touud;
1113#define TOUUD_OFFSET	0xa8
1114
1115static void igen6_reg_dump(struct igen6_imc *imc)
1116{
1117	int i;
1118
1119	edac_dbg(2, "CHANNEL_HASH     : 0x%x\n",
1120		 readl(imc->window + CHANNEL_HASH_OFFSET));
1121	edac_dbg(2, "CHANNEL_EHASH    : 0x%x\n",
1122		 readl(imc->window + CHANNEL_EHASH_OFFSET));
1123	edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n",
1124		 readl(imc->window + MAD_INTER_CHANNEL_OFFSET));
1125	edac_dbg(2, "ECC_ERROR_LOG    : 0x%llx\n",
1126		 readq(imc->window + ECC_ERROR_LOG_OFFSET));
1127
1128	for (i = 0; i < NUM_CHANNELS; i++) {
1129		edac_dbg(2, "MAD_INTRA_CH%d    : 0x%x\n", i,
1130			 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4));
1131		edac_dbg(2, "MAD_DIMM_CH%d     : 0x%x\n", i,
1132			 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4));
1133	}
1134	edac_dbg(2, "TOLUD            : 0x%x", igen6_tolud);
1135	edac_dbg(2, "TOUUD            : 0x%llx", igen6_touud);
1136	edac_dbg(2, "TOM              : 0x%llx", igen6_tom);
1137}
1138
1139static struct dentry *igen6_test;
1140
1141static int debugfs_u64_set(void *data, u64 val)
1142{
1143	u64 ecclog;
1144
1145	if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) {
1146		edac_dbg(0, "Address 0x%llx out of range\n", val);
1147		return 0;
1148	}
1149
1150	pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
1151
1152	val  >>= ECC_ERROR_LOG_ADDR_SHIFT;
1153	ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE;
1154
1155	if (!ecclog_gen_pool_add(0, ecclog))
1156		irq_work_queue(&ecclog_irq_work);
1157
1158	return 0;
1159}
1160DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
1161
1162static void igen6_debug_setup(void)
1163{
1164	igen6_test = edac_debugfs_create_dir("igen6_test");
1165	if (!igen6_test)
1166		return;
1167
1168	if (!edac_debugfs_create_file("addr", 0200, igen6_test,
1169				      NULL, &fops_u64_wo)) {
1170		debugfs_remove(igen6_test);
1171		igen6_test = NULL;
1172	}
1173}
1174
1175static void igen6_debug_teardown(void)
1176{
1177	debugfs_remove_recursive(igen6_test);
1178}
1179#else
1180static void igen6_reg_dump(struct igen6_imc *imc) {}
1181static void igen6_debug_setup(void) {}
1182static void igen6_debug_teardown(void) {}
1183#endif
1184
1185static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar)
1186{
1187	union  {
1188		u64 v;
1189		struct {
1190			u32 v_lo;
1191			u32 v_hi;
1192		};
1193	} u;
1194
1195	edac_dbg(2, "\n");
1196
1197	if (!res_cfg->ibecc_available(pdev)) {
1198		edac_dbg(2, "No In-Band ECC IP\n");
1199		goto fail;
1200	}
1201
1202	if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) {
1203		igen6_printk(KERN_ERR, "Failed to read TOLUD\n");
1204		goto fail;
1205	}
1206
1207	igen6_tolud &= GENMASK(31, 20);
1208
1209	if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) {
1210		igen6_printk(KERN_ERR, "Failed to read lower TOM\n");
1211		goto fail;
1212	}
1213
1214	if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) {
1215		igen6_printk(KERN_ERR, "Failed to read upper TOM\n");
1216		goto fail;
1217	}
1218
1219	igen6_tom = u.v & GENMASK_ULL(38, 20);
1220
1221	if (get_mchbar(pdev, mchbar))
1222		goto fail;
1223
1224#ifdef CONFIG_EDAC_DEBUG
1225	if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo))
1226		edac_dbg(2, "Failed to read lower TOUUD\n");
1227	else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi))
1228		edac_dbg(2, "Failed to read upper TOUUD\n");
1229	else
1230		igen6_touud = u.v & GENMASK_ULL(38, 20);
1231#endif
1232
1233	return 0;
1234fail:
1235	return -ENODEV;
1236}
1237
1238static void igen6_check(struct mem_ctl_info *mci)
1239{
1240	struct igen6_imc *imc = mci->pvt_info;
1241	u64 ecclog;
1242
1243	/* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
1244	ecclog = ecclog_read_and_clear(imc);
1245	if (!ecclog)
1246		return;
1247
1248	if (!ecclog_gen_pool_add(imc->mc, ecclog))
1249		irq_work_queue(&ecclog_irq_work);
1250}
1251
1252/* Check whether the memory controller is absent. */
1253static bool igen6_imc_absent(void __iomem *window)
1254{
1255	return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0;
1256}
1257
1258static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev)
1259{
1260	struct edac_mc_layer layers[2];
1261	struct mem_ctl_info *mci;
1262	struct igen6_imc *imc;
1263	int rc;
1264
1265	edac_dbg(2, "\n");
1266
1267	layers[0].type = EDAC_MC_LAYER_CHANNEL;
1268	layers[0].size = NUM_CHANNELS;
1269	layers[0].is_virt_csrow = false;
1270	layers[1].type = EDAC_MC_LAYER_SLOT;
1271	layers[1].size = NUM_DIMMS;
1272	layers[1].is_virt_csrow = true;
1273
1274	mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0);
1275	if (!mci) {
1276		rc = -ENOMEM;
1277		goto fail;
1278	}
1279
1280	mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc);
1281	if (!mci->ctl_name) {
1282		rc = -ENOMEM;
1283		goto fail2;
1284	}
1285
1286	mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4;
1287	mci->edac_ctl_cap = EDAC_FLAG_SECDED;
1288	mci->edac_cap = EDAC_FLAG_SECDED;
1289	mci->mod_name = EDAC_MOD_STR;
1290	mci->dev_name = pci_name(pdev);
1291	if (edac_op_state == EDAC_OPSTATE_POLL)
1292		mci->edac_check = igen6_check;
1293	mci->pvt_info = &igen6_pvt->imc[mc];
1294
1295	imc = mci->pvt_info;
1296	device_initialize(&imc->dev);
1297	/*
1298	 * EDAC core uses mci->pdev(pointer of structure device) as
1299	 * memory controller ID. The client SoCs attach one or more
1300	 * memory controllers to single pci_dev (single pci_dev->dev
1301	 * can be for multiple memory controllers).
1302	 *
1303	 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev
1304	 * for the first memory controller and assign a unique imc->dev
1305	 * to mci->pdev for each non-first memory controller.
1306	 */
1307	mci->pdev = mc ? &imc->dev : &pdev->dev;
1308	imc->mc	= mc;
1309	imc->pdev = pdev;
1310	imc->window = window;
1311
1312	igen6_reg_dump(imc);
1313
1314	rc = igen6_get_dimm_config(mci);
1315	if (rc)
1316		goto fail3;
1317
1318	rc = edac_mc_add_mc(mci);
1319	if (rc) {
1320		igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc);
1321		goto fail3;
1322	}
1323
1324	imc->mci = mci;
1325	return 0;
1326fail3:
1327	put_device(&imc->dev);
1328	mci->pvt_info = NULL;
1329	kfree(mci->ctl_name);
1330fail2:
1331	edac_mc_free(mci);
1332fail:
1333	return rc;
1334}
1335
1336static void igen6_unregister_mcis(void)
1337{
1338	struct mem_ctl_info *mci;
1339	struct igen6_imc *imc;
1340	int i;
1341
1342	edac_dbg(2, "\n");
1343
1344	for (i = 0; i < res_cfg->num_imc; i++) {
1345		imc = &igen6_pvt->imc[i];
1346		mci = imc->mci;
1347		if (!mci)
1348			continue;
1349
1350		edac_mc_del_mc(mci->pdev);
1351		kfree(mci->ctl_name);
1352		mci->pvt_info = NULL;
1353		edac_mc_free(mci);
1354		put_device(&imc->dev);
1355		iounmap(imc->window);
1356	}
1357}
1358
1359static int igen6_register_mcis(struct pci_dev *pdev, u64 mchbar)
1360{
1361	void __iomem *window;
1362	int lmc, pmc, rc;
1363	u64 base;
1364
1365	for (lmc = 0, pmc = 0; pmc < NUM_IMC; pmc++) {
1366		base   = mchbar + pmc * MCHBAR_SIZE;
1367		window = ioremap(base, MCHBAR_SIZE);
1368		if (!window) {
1369			igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx for mc%d\n", base, pmc);
1370			rc = -ENOMEM;
1371			goto out_unregister_mcis;
1372		}
1373
1374		if (igen6_imc_absent(window)) {
1375			iounmap(window);
1376			edac_dbg(2, "Skip absent mc%d\n", pmc);
1377			continue;
1378		}
1379
1380		rc = igen6_register_mci(lmc, window, pdev);
1381		if (rc)
1382			goto out_iounmap;
1383
1384		/* Done, if all present MCs are detected and registered. */
1385		if (++lmc >= res_cfg->num_imc)
1386			break;
1387	}
1388
1389	if (!lmc) {
1390		igen6_printk(KERN_ERR, "No mc found.\n");
1391		return -ENODEV;
1392	}
1393
1394	if (lmc < res_cfg->num_imc) {
1395		igen6_printk(KERN_DEBUG, "Expected %d mcs, but only %d detected.",
1396			     res_cfg->num_imc, lmc);
1397		res_cfg->num_imc = lmc;
1398	}
1399
1400	return 0;
1401
1402out_iounmap:
1403	iounmap(window);
1404
1405out_unregister_mcis:
1406	igen6_unregister_mcis();
1407
1408	return rc;
1409}
1410
1411static int igen6_mem_slice_setup(u64 mchbar)
1412{
1413	struct igen6_imc *imc = &igen6_pvt->imc[0];
1414	u64 base = mchbar + res_cfg->cmf_base;
1415	u32 offset = res_cfg->ms_hash_offset;
1416	u32 size = res_cfg->cmf_size;
1417	u64 ms_s_size, ms_hash;
1418	void __iomem *cmf;
1419	int ms_l_map;
1420
1421	edac_dbg(2, "\n");
1422
1423	if (imc[0].size < imc[1].size) {
1424		ms_s_size = imc[0].size;
1425		ms_l_map  = 1;
1426	} else {
1427		ms_s_size = imc[1].size;
1428		ms_l_map  = 0;
1429	}
1430
1431	igen6_pvt->ms_s_size = ms_s_size;
1432	igen6_pvt->ms_l_map  = ms_l_map;
1433
1434	edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n",
1435		 ms_s_size >> 20, ms_l_map);
1436
1437	if (!size)
1438		return 0;
1439
1440	cmf = ioremap(base, size);
1441	if (!cmf) {
1442		igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base);
1443		return -ENODEV;
1444	}
1445
1446	ms_hash = readq(cmf + offset);
1447	igen6_pvt->ms_hash = ms_hash;
1448
1449	edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash);
1450
1451	iounmap(cmf);
1452
1453	return 0;
1454}
1455
1456static int register_err_handler(void)
1457{
1458	int rc;
1459
1460	if (res_cfg->machine_check) {
1461		mce_register_decode_chain(&ecclog_mce_dec);
1462		return 0;
1463	}
1464
1465	rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler,
1466				  0, IGEN6_NMI_NAME);
1467	if (rc) {
1468		igen6_printk(KERN_ERR, "Failed to register NMI handler\n");
1469		return rc;
1470	}
1471
1472	return 0;
1473}
1474
1475static void unregister_err_handler(void)
1476{
1477	if (res_cfg->machine_check) {
1478		mce_unregister_decode_chain(&ecclog_mce_dec);
1479		return;
1480	}
1481
1482	unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1483}
1484
1485static void opstate_set(const struct res_config *cfg, const struct pci_device_id *ent)
1486{
1487	/*
1488	 * Quirk: Certain SoCs' error reporting interrupts don't work.
1489	 *        Force polling mode for them to ensure that memory error
1490	 *        events can be handled.
1491	 */
1492	if (ent->device == DID_ADL_N_SKU4) {
1493		edac_op_state = EDAC_OPSTATE_POLL;
1494		return;
1495	}
1496
1497	/* Set the mode according to the configuration data. */
1498	if (cfg->machine_check)
1499		edac_op_state = EDAC_OPSTATE_INT;
1500	else
1501		edac_op_state = EDAC_OPSTATE_NMI;
1502}
1503
1504static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1505{
1506	u64 mchbar;
1507	int rc;
1508
1509	edac_dbg(2, "\n");
1510
1511	igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL);
1512	if (!igen6_pvt)
1513		return -ENOMEM;
1514
1515	res_cfg = (struct res_config *)ent->driver_data;
1516
1517	rc = igen6_pci_setup(pdev, &mchbar);
1518	if (rc)
1519		goto fail;
1520
1521	opstate_set(res_cfg, ent);
1522
1523	rc = igen6_register_mcis(pdev, mchbar);
1524	if (rc)
1525		goto fail;
1526
1527	if (res_cfg->num_imc > 1) {
1528		rc = igen6_mem_slice_setup(mchbar);
1529		if (rc)
1530			goto fail2;
1531	}
1532
1533	ecclog_pool = ecclog_gen_pool_create();
1534	if (!ecclog_pool) {
1535		rc = -ENOMEM;
1536		goto fail2;
1537	}
1538
1539	INIT_WORK(&ecclog_work, ecclog_work_cb);
1540	init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb);
1541
1542	rc = register_err_handler();
1543	if (rc)
1544		goto fail3;
1545
1546	/* Enable error reporting */
1547	rc = errcmd_enable_error_reporting(true);
1548	if (rc) {
1549		igen6_printk(KERN_ERR, "Failed to enable error reporting\n");
1550		goto fail4;
1551	}
1552
1553	/* Check if any pending errors before/during the registration of the error handler */
1554	ecclog_handler();
1555
1556	igen6_debug_setup();
1557	return 0;
1558fail4:
1559	unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
1560fail3:
1561	gen_pool_destroy(ecclog_pool);
1562fail2:
1563	igen6_unregister_mcis();
1564fail:
1565	kfree(igen6_pvt);
1566	return rc;
1567}
1568
1569static void igen6_remove(struct pci_dev *pdev)
1570{
1571	edac_dbg(2, "\n");
1572
1573	igen6_debug_teardown();
1574	errcmd_enable_error_reporting(false);
1575	unregister_err_handler();
1576	irq_work_sync(&ecclog_irq_work);
1577	flush_work(&ecclog_work);
1578	gen_pool_destroy(ecclog_pool);
1579	igen6_unregister_mcis();
1580	kfree(igen6_pvt);
1581}
1582
1583static struct pci_driver igen6_driver = {
1584	.name     = EDAC_MOD_STR,
1585	.probe    = igen6_probe,
1586	.remove   = igen6_remove,
1587	.id_table = igen6_pci_tbl,
1588};
1589
1590static int __init igen6_init(void)
1591{
1592	const char *owner;
1593	int rc;
1594
1595	edac_dbg(2, "\n");
1596
1597	if (ghes_get_devices())
1598		return -EBUSY;
1599
1600	owner = edac_get_owner();
1601	if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
1602		return -EBUSY;
1603
1604	rc = pci_register_driver(&igen6_driver);
1605	if (rc)
1606		return rc;
1607
1608	igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION);
1609
1610	return 0;
1611}
1612
1613static void __exit igen6_exit(void)
1614{
1615	edac_dbg(2, "\n");
1616
1617	pci_unregister_driver(&igen6_driver);
1618}
1619
1620module_init(igen6_init);
1621module_exit(igen6_exit);
1622
1623MODULE_LICENSE("GPL v2");
1624MODULE_AUTHOR("Qiuxu Zhuo");
1625MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC");