FILE
amd64_edac.c
File preview with syntax highlighting.
linux-6.19.14/drivers/edac/amd64_edac.c
1// SPDX-License-Identifier: GPL-2.0-only 2#include <linux/ras.h> 3#include <linux/string_choices.h> 4#include "amd64_edac.h" 5#include <asm/amd/nb.h> 6#include <asm/amd/node.h> 7 8static struct edac_pci_ctl_info *pci_ctl; 9 10/* 11 * Set by command line parameter. If BIOS has enabled the ECC, this override is 12 * cleared to prevent re-enabling the hardware by this driver. 13 */ 14static int ecc_enable_override; 15module_param(ecc_enable_override, int, 0644); 16 17static struct msr __percpu *msrs; 18 19static inline u32 get_umc_reg(struct amd64_pvt *pvt, u32 reg) 20{ 21 if (!pvt->flags.zn_regs_v2) 22 return reg; 23 24 switch (reg) { 25 case UMCCH_ADDR_MASK_SEC: return UMCCH_ADDR_MASK_SEC_DDR5; 26 case UMCCH_DIMM_CFG: return UMCCH_DIMM_CFG_DDR5; 27 } 28 29 WARN_ONCE(1, "%s: unknown register 0x%x", __func__, reg); 30 return 0; 31} 32 33/* Per-node stuff */ 34static struct ecc_settings **ecc_stngs; 35 36/* Device for the PCI component */ 37static struct device *pci_ctl_dev; 38 39/* 40 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 41 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 42 * or higher value'. 43 * 44 *FIXME: Produce a better mapping/linearisation. 45 */ 46static const struct scrubrate { 47 u32 scrubval; /* bit pattern for scrub rate */ 48 u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 49} scrubrates[] = { 50 { 0x01, 1600000000UL}, 51 { 0x02, 800000000UL}, 52 { 0x03, 400000000UL}, 53 { 0x04, 200000000UL}, 54 { 0x05, 100000000UL}, 55 { 0x06, 50000000UL}, 56 { 0x07, 25000000UL}, 57 { 0x08, 12284069UL}, 58 { 0x09, 6274509UL}, 59 { 0x0A, 3121951UL}, 60 { 0x0B, 1560975UL}, 61 { 0x0C, 781440UL}, 62 { 0x0D, 390720UL}, 63 { 0x0E, 195300UL}, 64 { 0x0F, 97650UL}, 65 { 0x10, 48854UL}, 66 { 0x11, 24427UL}, 67 { 0x12, 12213UL}, 68 { 0x13, 6101UL}, 69 { 0x14, 3051UL}, 70 { 0x15, 1523UL}, 71 { 0x16, 761UL}, 72 { 0x00, 0UL}, /* scrubbing off */ 73}; 74 75int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 76 u32 *val, const char *func) 77{ 78 int err = 0; 79 80 err = pci_read_config_dword(pdev, offset, val); 81 if (err) 82 amd64_warn("%s: error reading F%dx%03x.\n", 83 func, PCI_FUNC(pdev->devfn), offset); 84 85 return pcibios_err_to_errno(err); 86} 87 88int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 89 u32 val, const char *func) 90{ 91 int err = 0; 92 93 err = pci_write_config_dword(pdev, offset, val); 94 if (err) 95 amd64_warn("%s: error writing to F%dx%03x.\n", 96 func, PCI_FUNC(pdev->devfn), offset); 97 98 return pcibios_err_to_errno(err); 99} 100 101/* 102 * Select DCT to which PCI cfg accesses are routed 103 */ 104static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) 105{ 106 u32 reg = 0; 107 108 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 109 reg &= (pvt->model == 0x30) ? ~3 : ~1; 110 reg |= dct; 111 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 112} 113 114/* 115 * 116 * Depending on the family, F2 DCT reads need special handling: 117 * 118 * K8: has a single DCT only and no address offsets >= 0x100 119 * 120 * F10h: each DCT has its own set of regs 121 * DCT0 -> F2x040.. 122 * DCT1 -> F2x140.. 123 * 124 * F16h: has only 1 DCT 125 * 126 * F15h: we select which DCT we access using F1x10C[DctCfgSel] 127 */ 128static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct, 129 int offset, u32 *val) 130{ 131 switch (pvt->fam) { 132 case 0xf: 133 if (dct || offset >= 0x100) 134 return -EINVAL; 135 break; 136 137 case 0x10: 138 if (dct) { 139 /* 140 * Note: If ganging is enabled, barring the regs 141 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx 142 * return 0. (cf. Section 2.8.1 F10h BKDG) 143 */ 144 if (dct_ganging_enabled(pvt)) 145 return 0; 146 147 offset += 0x100; 148 } 149 break; 150 151 case 0x15: 152 /* 153 * F15h: F2x1xx addresses do not map explicitly to DCT1. 154 * We should select which DCT we access using F1x10C[DctCfgSel] 155 */ 156 dct = (dct && pvt->model == 0x30) ? 3 : dct; 157 f15h_select_dct(pvt, dct); 158 break; 159 160 case 0x16: 161 if (dct) 162 return -EINVAL; 163 break; 164 165 default: 166 break; 167 } 168 return amd64_read_pci_cfg(pvt->F2, offset, val); 169} 170 171/* 172 * Memory scrubber control interface. For K8, memory scrubbing is handled by 173 * hardware and can involve L2 cache, dcache as well as the main memory. With 174 * F10, this is extended to L3 cache scrubbing on CPU models sporting that 175 * functionality. 176 * 177 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 178 * (dram) over to cache lines. This is nasty, so we will use bandwidth in 179 * bytes/sec for the setting. 180 * 181 * Currently, we only do dram scrubbing. If the scrubbing is done in software on 182 * other archs, we might not have access to the caches directly. 183 */ 184 185/* 186 * Scan the scrub rate mapping table for a close or matching bandwidth value to 187 * issue. If requested is too big, then use last maximum value found. 188 */ 189static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate) 190{ 191 u32 scrubval; 192 int i; 193 194 /* 195 * map the configured rate (new_bw) to a value specific to the AMD64 196 * memory controller and apply to register. Search for the first 197 * bandwidth entry that is greater or equal than the setting requested 198 * and program that. If at last entry, turn off DRAM scrubbing. 199 * 200 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely 201 * by falling back to the last element in scrubrates[]. 202 */ 203 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) { 204 /* 205 * skip scrub rates which aren't recommended 206 * (see F10 BKDG, F3x58) 207 */ 208 if (scrubrates[i].scrubval < min_rate) 209 continue; 210 211 if (scrubrates[i].bandwidth <= new_bw) 212 break; 213 } 214 215 scrubval = scrubrates[i].scrubval; 216 217 if (pvt->fam == 0x15 && pvt->model == 0x60) { 218 f15h_select_dct(pvt, 0); 219 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); 220 f15h_select_dct(pvt, 1); 221 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F); 222 } else { 223 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F); 224 } 225 226 if (scrubval) 227 return scrubrates[i].bandwidth; 228 229 return 0; 230} 231 232static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 233{ 234 struct amd64_pvt *pvt = mci->pvt_info; 235 u32 min_scrubrate = 0x5; 236 237 if (pvt->fam == 0xf) 238 min_scrubrate = 0x0; 239 240 if (pvt->fam == 0x15) { 241 /* Erratum #505 */ 242 if (pvt->model < 0x10) 243 f15h_select_dct(pvt, 0); 244 245 if (pvt->model == 0x60) 246 min_scrubrate = 0x6; 247 } 248 return __set_scrub_rate(pvt, bw, min_scrubrate); 249} 250 251static int get_scrub_rate(struct mem_ctl_info *mci) 252{ 253 struct amd64_pvt *pvt = mci->pvt_info; 254 int i, retval = -EINVAL; 255 u32 scrubval = 0; 256 257 if (pvt->fam == 0x15) { 258 /* Erratum #505 */ 259 if (pvt->model < 0x10) 260 f15h_select_dct(pvt, 0); 261 262 if (pvt->model == 0x60) 263 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval); 264 else 265 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 266 } else { 267 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 268 } 269 270 scrubval = scrubval & 0x001F; 271 272 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 273 if (scrubrates[i].scrubval == scrubval) { 274 retval = scrubrates[i].bandwidth; 275 break; 276 } 277 } 278 return retval; 279} 280 281/* 282 * returns true if the SysAddr given by sys_addr matches the 283 * DRAM base/limit associated with node_id 284 */ 285static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid) 286{ 287 u64 addr; 288 289 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 290 * all ones if the most significant implemented address bit is 1. 291 * Here we discard bits 63-40. See section 3.4.2 of AMD publication 292 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 293 * Application Programming. 294 */ 295 addr = sys_addr & 0x000000ffffffffffull; 296 297 return ((addr >= get_dram_base(pvt, nid)) && 298 (addr <= get_dram_limit(pvt, nid))); 299} 300 301/* 302 * Attempt to map a SysAddr to a node. On success, return a pointer to the 303 * mem_ctl_info structure for the node that the SysAddr maps to. 304 * 305 * On failure, return NULL. 306 */ 307static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 308 u64 sys_addr) 309{ 310 struct amd64_pvt *pvt; 311 u8 node_id; 312 u32 intlv_en, bits; 313 314 /* 315 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 316 * 3.4.4.2) registers to map the SysAddr to a node ID. 317 */ 318 pvt = mci->pvt_info; 319 320 /* 321 * The value of this field should be the same for all DRAM Base 322 * registers. Therefore we arbitrarily choose to read it from the 323 * register for node 0. 324 */ 325 intlv_en = dram_intlv_en(pvt, 0); 326 327 if (intlv_en == 0) { 328 for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 329 if (base_limit_match(pvt, sys_addr, node_id)) 330 goto found; 331 } 332 goto err_no_match; 333 } 334 335 if (unlikely((intlv_en != 0x01) && 336 (intlv_en != 0x03) && 337 (intlv_en != 0x07))) { 338 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 339 return NULL; 340 } 341 342 bits = (((u32) sys_addr) >> 12) & intlv_en; 343 344 for (node_id = 0; ; ) { 345 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 346 break; /* intlv_sel field matches */ 347 348 if (++node_id >= DRAM_RANGES) 349 goto err_no_match; 350 } 351 352 /* sanity test for sys_addr */ 353 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) { 354 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 355 "range for node %d with node interleaving enabled.\n", 356 __func__, sys_addr, node_id); 357 return NULL; 358 } 359 360found: 361 return edac_mc_find((int)node_id); 362 363err_no_match: 364 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n", 365 (unsigned long)sys_addr); 366 367 return NULL; 368} 369 370/* 371 * compute the CS base address of the @csrow on the DRAM controller @dct. 372 * For details see F2x[5C:40] in the processor's BKDG 373 */ 374static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 375 u64 *base, u64 *mask) 376{ 377 u64 csbase, csmask, base_bits, mask_bits; 378 u8 addr_shift; 379 380 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 381 csbase = pvt->csels[dct].csbases[csrow]; 382 csmask = pvt->csels[dct].csmasks[csrow]; 383 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9); 384 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9); 385 addr_shift = 4; 386 387 /* 388 * F16h and F15h, models 30h and later need two addr_shift values: 389 * 8 for high and 6 for low (cf. F16h BKDG). 390 */ 391 } else if (pvt->fam == 0x16 || 392 (pvt->fam == 0x15 && pvt->model >= 0x30)) { 393 csbase = pvt->csels[dct].csbases[csrow]; 394 csmask = pvt->csels[dct].csmasks[csrow >> 1]; 395 396 *base = (csbase & GENMASK_ULL(15, 5)) << 6; 397 *base |= (csbase & GENMASK_ULL(30, 19)) << 8; 398 399 *mask = ~0ULL; 400 /* poke holes for the csmask */ 401 *mask &= ~((GENMASK_ULL(15, 5) << 6) | 402 (GENMASK_ULL(30, 19) << 8)); 403 404 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6; 405 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8; 406 407 return; 408 } else { 409 csbase = pvt->csels[dct].csbases[csrow]; 410 csmask = pvt->csels[dct].csmasks[csrow >> 1]; 411 addr_shift = 8; 412 413 if (pvt->fam == 0x15) 414 base_bits = mask_bits = 415 GENMASK_ULL(30,19) | GENMASK_ULL(13,5); 416 else 417 base_bits = mask_bits = 418 GENMASK_ULL(28,19) | GENMASK_ULL(13,5); 419 } 420 421 *base = (csbase & base_bits) << addr_shift; 422 423 *mask = ~0ULL; 424 /* poke holes for the csmask */ 425 *mask &= ~(mask_bits << addr_shift); 426 /* OR them in */ 427 *mask |= (csmask & mask_bits) << addr_shift; 428} 429 430#define for_each_chip_select(i, dct, pvt) \ 431 for (i = 0; i < pvt->csels[dct].b_cnt; i++) 432 433#define chip_select_base(i, dct, pvt) \ 434 pvt->csels[dct].csbases[i] 435 436#define for_each_chip_select_mask(i, dct, pvt) \ 437 for (i = 0; i < pvt->csels[dct].m_cnt; i++) 438 439#define for_each_umc(i) \ 440 for (i = 0; i < pvt->max_mcs; i++) 441 442/* 443 * @input_addr is an InputAddr associated with the node given by mci. Return the 444 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 445 */ 446static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 447{ 448 struct amd64_pvt *pvt; 449 int csrow; 450 u64 base, mask; 451 452 pvt = mci->pvt_info; 453 454 for_each_chip_select(csrow, 0, pvt) { 455 if (!csrow_enabled(csrow, 0, pvt)) 456 continue; 457 458 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 459 460 mask = ~mask; 461 462 if ((input_addr & mask) == (base & mask)) { 463 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n", 464 (unsigned long)input_addr, csrow, 465 pvt->mc_node_id); 466 467 return csrow; 468 } 469 } 470 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n", 471 (unsigned long)input_addr, pvt->mc_node_id); 472 473 return -1; 474} 475 476/* 477 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 478 * for the node represented by mci. Info is passed back in *hole_base, 479 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 480 * info is invalid. Info may be invalid for either of the following reasons: 481 * 482 * - The revision of the node is not E or greater. In this case, the DRAM Hole 483 * Address Register does not exist. 484 * 485 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 486 * indicating that its contents are not valid. 487 * 488 * The values passed back in *hole_base, *hole_offset, and *hole_size are 489 * complete 32-bit values despite the fact that the bitfields in the DHAR 490 * only represent bits 31-24 of the base and offset values. 491 */ 492static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 493 u64 *hole_offset, u64 *hole_size) 494{ 495 struct amd64_pvt *pvt = mci->pvt_info; 496 497 /* only revE and later have the DRAM Hole Address Register */ 498 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) { 499 edac_dbg(1, " revision %d for node %d does not support DHAR\n", 500 pvt->ext_model, pvt->mc_node_id); 501 return 1; 502 } 503 504 /* valid for Fam10h and above */ 505 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 506 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n"); 507 return 1; 508 } 509 510 if (!dhar_valid(pvt)) { 511 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n", 512 pvt->mc_node_id); 513 return 1; 514 } 515 516 /* This node has Memory Hoisting */ 517 518 /* +------------------+--------------------+--------------------+----- 519 * | memory | DRAM hole | relocated | 520 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 521 * | | | DRAM hole | 522 * | | | [0x100000000, | 523 * | | | (0x100000000+ | 524 * | | | (0xffffffff-x))] | 525 * +------------------+--------------------+--------------------+----- 526 * 527 * Above is a diagram of physical memory showing the DRAM hole and the 528 * relocated addresses from the DRAM hole. As shown, the DRAM hole 529 * starts at address x (the base address) and extends through address 530 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 531 * addresses in the hole so that they start at 0x100000000. 532 */ 533 534 *hole_base = dhar_base(pvt); 535 *hole_size = (1ULL << 32) - *hole_base; 536 537 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt) 538 : k8_dhar_offset(pvt); 539 540 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 541 pvt->mc_node_id, (unsigned long)*hole_base, 542 (unsigned long)*hole_offset, (unsigned long)*hole_size); 543 544 return 0; 545} 546 547#ifdef CONFIG_EDAC_DEBUG 548#define EDAC_DCT_ATTR_SHOW(reg) \ 549static ssize_t reg##_show(struct device *dev, \ 550 struct device_attribute *mattr, char *data) \ 551{ \ 552 struct mem_ctl_info *mci = to_mci(dev); \ 553 struct amd64_pvt *pvt = mci->pvt_info; \ 554 \ 555 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \ 556} 557 558EDAC_DCT_ATTR_SHOW(dhar); 559EDAC_DCT_ATTR_SHOW(dbam0); 560EDAC_DCT_ATTR_SHOW(top_mem); 561EDAC_DCT_ATTR_SHOW(top_mem2); 562 563static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr, 564 char *data) 565{ 566 struct mem_ctl_info *mci = to_mci(dev); 567 568 u64 hole_base = 0; 569 u64 hole_offset = 0; 570 u64 hole_size = 0; 571 572 get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size); 573 574 return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset, 575 hole_size); 576} 577 578/* 579 * update NUM_DBG_ATTRS in case you add new members 580 */ 581static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL); 582static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL); 583static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL); 584static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL); 585static DEVICE_ATTR_RO(dram_hole); 586 587static struct attribute *dbg_attrs[] = { 588 &dev_attr_dhar.attr, 589 &dev_attr_dbam.attr, 590 &dev_attr_topmem.attr, 591 &dev_attr_topmem2.attr, 592 &dev_attr_dram_hole.attr, 593 NULL 594}; 595 596static const struct attribute_group dbg_group = { 597 .attrs = dbg_attrs, 598}; 599 600static ssize_t inject_section_show(struct device *dev, 601 struct device_attribute *mattr, char *buf) 602{ 603 struct mem_ctl_info *mci = to_mci(dev); 604 struct amd64_pvt *pvt = mci->pvt_info; 605 return sprintf(buf, "0x%x\n", pvt->injection.section); 606} 607 608/* 609 * store error injection section value which refers to one of 4 16-byte sections 610 * within a 64-byte cacheline 611 * 612 * range: 0..3 613 */ 614static ssize_t inject_section_store(struct device *dev, 615 struct device_attribute *mattr, 616 const char *data, size_t count) 617{ 618 struct mem_ctl_info *mci = to_mci(dev); 619 struct amd64_pvt *pvt = mci->pvt_info; 620 unsigned long value; 621 int ret; 622 623 ret = kstrtoul(data, 10, &value); 624 if (ret < 0) 625 return ret; 626 627 if (value > 3) { 628 amd64_warn("%s: invalid section 0x%lx\n", __func__, value); 629 return -EINVAL; 630 } 631 632 pvt->injection.section = (u32) value; 633 return count; 634} 635 636static ssize_t inject_word_show(struct device *dev, 637 struct device_attribute *mattr, char *buf) 638{ 639 struct mem_ctl_info *mci = to_mci(dev); 640 struct amd64_pvt *pvt = mci->pvt_info; 641 return sprintf(buf, "0x%x\n", pvt->injection.word); 642} 643 644/* 645 * store error injection word value which refers to one of 9 16-bit word of the 646 * 16-byte (128-bit + ECC bits) section 647 * 648 * range: 0..8 649 */ 650static ssize_t inject_word_store(struct device *dev, 651 struct device_attribute *mattr, 652 const char *data, size_t count) 653{ 654 struct mem_ctl_info *mci = to_mci(dev); 655 struct amd64_pvt *pvt = mci->pvt_info; 656 unsigned long value; 657 int ret; 658 659 ret = kstrtoul(data, 10, &value); 660 if (ret < 0) 661 return ret; 662 663 if (value > 8) { 664 amd64_warn("%s: invalid word 0x%lx\n", __func__, value); 665 return -EINVAL; 666 } 667 668 pvt->injection.word = (u32) value; 669 return count; 670} 671 672static ssize_t inject_ecc_vector_show(struct device *dev, 673 struct device_attribute *mattr, 674 char *buf) 675{ 676 struct mem_ctl_info *mci = to_mci(dev); 677 struct amd64_pvt *pvt = mci->pvt_info; 678 return sprintf(buf, "0x%x\n", pvt->injection.bit_map); 679} 680 681/* 682 * store 16 bit error injection vector which enables injecting errors to the 683 * corresponding bit within the error injection word above. When used during a 684 * DRAM ECC read, it holds the contents of the of the DRAM ECC bits. 685 */ 686static ssize_t inject_ecc_vector_store(struct device *dev, 687 struct device_attribute *mattr, 688 const char *data, size_t count) 689{ 690 struct mem_ctl_info *mci = to_mci(dev); 691 struct amd64_pvt *pvt = mci->pvt_info; 692 unsigned long value; 693 int ret; 694 695 ret = kstrtoul(data, 16, &value); 696 if (ret < 0) 697 return ret; 698 699 if (value & 0xFFFF0000) { 700 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value); 701 return -EINVAL; 702 } 703 704 pvt->injection.bit_map = (u32) value; 705 return count; 706} 707 708/* 709 * Do a DRAM ECC read. Assemble staged values in the pvt area, format into 710 * fields needed by the injection registers and read the NB Array Data Port. 711 */ 712static ssize_t inject_read_store(struct device *dev, 713 struct device_attribute *mattr, 714 const char *data, size_t count) 715{ 716 struct mem_ctl_info *mci = to_mci(dev); 717 struct amd64_pvt *pvt = mci->pvt_info; 718 unsigned long value; 719 u32 section, word_bits; 720 int ret; 721 722 ret = kstrtoul(data, 10, &value); 723 if (ret < 0) 724 return ret; 725 726 /* Form value to choose 16-byte section of cacheline */ 727 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section); 728 729 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section); 730 731 word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection); 732 733 /* Issue 'word' and 'bit' along with the READ request */ 734 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits); 735 736 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits); 737 738 return count; 739} 740 741/* 742 * Do a DRAM ECC write. Assemble staged values in the pvt area and format into 743 * fields needed by the injection registers. 744 */ 745static ssize_t inject_write_store(struct device *dev, 746 struct device_attribute *mattr, 747 const char *data, size_t count) 748{ 749 struct mem_ctl_info *mci = to_mci(dev); 750 struct amd64_pvt *pvt = mci->pvt_info; 751 u32 section, word_bits, tmp; 752 unsigned long value; 753 int ret; 754 755 ret = kstrtoul(data, 10, &value); 756 if (ret < 0) 757 return ret; 758 759 /* Form value to choose 16-byte section of cacheline */ 760 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section); 761 762 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section); 763 764 word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection); 765 766 pr_notice_once("Don't forget to decrease MCE polling interval in\n" 767 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n" 768 "so that you can get the error report faster.\n"); 769 770 on_each_cpu(disable_caches, NULL, 1); 771 772 /* Issue 'word' and 'bit' along with the READ request */ 773 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits); 774 775 retry: 776 /* wait until injection happens */ 777 amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp); 778 if (tmp & F10_NB_ARR_ECC_WR_REQ) { 779 cpu_relax(); 780 goto retry; 781 } 782 783 on_each_cpu(enable_caches, NULL, 1); 784 785 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits); 786 787 return count; 788} 789 790/* 791 * update NUM_INJ_ATTRS in case you add new members 792 */ 793 794static DEVICE_ATTR_RW(inject_section); 795static DEVICE_ATTR_RW(inject_word); 796static DEVICE_ATTR_RW(inject_ecc_vector); 797static DEVICE_ATTR_WO(inject_write); 798static DEVICE_ATTR_WO(inject_read); 799 800static struct attribute *inj_attrs[] = { 801 &dev_attr_inject_section.attr, 802 &dev_attr_inject_word.attr, 803 &dev_attr_inject_ecc_vector.attr, 804 &dev_attr_inject_write.attr, 805 &dev_attr_inject_read.attr, 806 NULL 807}; 808 809static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx) 810{ 811 struct device *dev = kobj_to_dev(kobj); 812 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev); 813 struct amd64_pvt *pvt = mci->pvt_info; 814 815 /* Families which have that injection hw */ 816 if (pvt->fam >= 0x10 && pvt->fam <= 0x16) 817 return attr->mode; 818 819 return 0; 820} 821 822static const struct attribute_group inj_group = { 823 .attrs = inj_attrs, 824 .is_visible = inj_is_visible, 825}; 826#endif /* CONFIG_EDAC_DEBUG */ 827 828/* 829 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 830 * assumed that sys_addr maps to the node given by mci. 831 * 832 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 833 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 834 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 835 * then it is also involved in translating a SysAddr to a DramAddr. Sections 836 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 837 * These parts of the documentation are unclear. I interpret them as follows: 838 * 839 * When node n receives a SysAddr, it processes the SysAddr as follows: 840 * 841 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 842 * Limit registers for node n. If the SysAddr is not within the range 843 * specified by the base and limit values, then node n ignores the Sysaddr 844 * (since it does not map to node n). Otherwise continue to step 2 below. 845 * 846 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 847 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 848 * the range of relocated addresses (starting at 0x100000000) from the DRAM 849 * hole. If not, skip to step 3 below. Else get the value of the 850 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 851 * offset defined by this value from the SysAddr. 852 * 853 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 854 * Base register for node n. To obtain the DramAddr, subtract the base 855 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 856 */ 857static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 858{ 859 struct amd64_pvt *pvt = mci->pvt_info; 860 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 861 int ret; 862 863 dram_base = get_dram_base(pvt, pvt->mc_node_id); 864 865 ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size); 866 if (!ret) { 867 if ((sys_addr >= (1ULL << 32)) && 868 (sys_addr < ((1ULL << 32) + hole_size))) { 869 /* use DHAR to translate SysAddr to DramAddr */ 870 dram_addr = sys_addr - hole_offset; 871 872 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 873 (unsigned long)sys_addr, 874 (unsigned long)dram_addr); 875 876 return dram_addr; 877 } 878 } 879 880 /* 881 * Translate the SysAddr to a DramAddr as shown near the start of 882 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 883 * only deals with 40-bit values. Therefore we discard bits 63-40 of 884 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 885 * discard are all 1s. Otherwise the bits we discard are all 0s. See 886 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 887 * Programmer's Manual Volume 1 Application Programming. 888 */ 889 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base; 890 891 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n", 892 (unsigned long)sys_addr, (unsigned long)dram_addr); 893 return dram_addr; 894} 895 896/* 897 * @intlv_en is the value of the IntlvEn field from a DRAM Base register 898 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 899 * for node interleaving. 900 */ 901static int num_node_interleave_bits(unsigned intlv_en) 902{ 903 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 904 int n; 905 906 BUG_ON(intlv_en > 7); 907 n = intlv_shift_table[intlv_en]; 908 return n; 909} 910 911/* Translate the DramAddr given by @dram_addr to an InputAddr. */ 912static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 913{ 914 struct amd64_pvt *pvt; 915 int intlv_shift; 916 u64 input_addr; 917 918 pvt = mci->pvt_info; 919 920 /* 921 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 922 * concerning translating a DramAddr to an InputAddr. 923 */ 924 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 925 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) + 926 (dram_addr & 0xfff); 927 928 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 929 intlv_shift, (unsigned long)dram_addr, 930 (unsigned long)input_addr); 931 932 return input_addr; 933} 934 935/* 936 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 937 * assumed that @sys_addr maps to the node given by mci. 938 */ 939static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 940{ 941 u64 input_addr; 942 943 input_addr = 944 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 945 946 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n", 947 (unsigned long)sys_addr, (unsigned long)input_addr); 948 949 return input_addr; 950} 951 952/* Map the Error address to a PAGE and PAGE OFFSET. */ 953static inline void error_address_to_page_and_offset(u64 error_address, 954 struct err_info *err) 955{ 956 err->page = (u32) (error_address >> PAGE_SHIFT); 957 err->offset = ((u32) error_address) & ~PAGE_MASK; 958} 959 960/* 961 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 962 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 963 * of a node that detected an ECC memory error. mci represents the node that 964 * the error address maps to (possibly different from the node that detected 965 * the error). Return the number of the csrow that sys_addr maps to, or -1 on 966 * error. 967 */ 968static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 969{ 970 int csrow; 971 972 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 973 974 if (csrow == -1) 975 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 976 "address 0x%lx\n", (unsigned long)sys_addr); 977 return csrow; 978} 979 980/* 981 * See AMD PPR DF::LclNodeTypeMap 982 * 983 * This register gives information for nodes of the same type within a system. 984 * 985 * Reading this register from a GPU node will tell how many GPU nodes are in the 986 * system and what the lowest AMD Node ID value is for the GPU nodes. Use this 987 * info to fixup the Linux logical "Node ID" value set in the AMD NB code and EDAC. 988 */ 989static struct local_node_map { 990 u16 node_count; 991 u16 base_node_id; 992} gpu_node_map; 993 994#define PCI_DEVICE_ID_AMD_MI200_DF_F1 0x14d1 995#define REG_LOCAL_NODE_TYPE_MAP 0x144 996 997/* Local Node Type Map (LNTM) fields */ 998#define LNTM_NODE_COUNT GENMASK(27, 16) 999#define LNTM_BASE_NODE_ID GENMASK(11, 0) 1000 1001static int gpu_get_node_map(struct amd64_pvt *pvt) 1002{ 1003 struct pci_dev *pdev; 1004 int ret; 1005 u32 tmp; 1006 1007 /* 1008 * Mapping of nodes from hardware-provided AMD Node ID to a 1009 * Linux logical one is applicable for MI200 models. Therefore, 1010 * return early for other heterogeneous systems. 1011 */ 1012 if (pvt->F3->device != PCI_DEVICE_ID_AMD_MI200_DF_F3) 1013 return 0; 1014 1015 /* 1016 * Node ID 0 is reserved for CPUs. Therefore, a non-zero Node ID 1017 * means the values have been already cached. 1018 */ 1019 if (gpu_node_map.base_node_id) 1020 return 0; 1021 1022 pdev = pci_get_device(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI200_DF_F1, NULL); 1023 if (!pdev) { 1024 ret = -ENODEV; 1025 goto out; 1026 } 1027 1028 ret = pci_read_config_dword(pdev, REG_LOCAL_NODE_TYPE_MAP, &tmp); 1029 if (ret) { 1030 ret = pcibios_err_to_errno(ret); 1031 goto out; 1032 } 1033 1034 gpu_node_map.node_count = FIELD_GET(LNTM_NODE_COUNT, tmp); 1035 gpu_node_map.base_node_id = FIELD_GET(LNTM_BASE_NODE_ID, tmp); 1036 1037out: 1038 pci_dev_put(pdev); 1039 return ret; 1040} 1041 1042static int fixup_node_id(int node_id, struct mce *m) 1043{ 1044 /* MCA_IPID[InstanceIdHi] give the AMD Node ID for the bank. */ 1045 u8 nid = (m->ipid >> 44) & 0xF; 1046 1047 if (smca_get_bank_type(m->extcpu, m->bank) != SMCA_UMC_V2) 1048 return node_id; 1049 1050 /* Nodes below the GPU base node are CPU nodes and don't need a fixup. */ 1051 if (nid < gpu_node_map.base_node_id) 1052 return node_id; 1053 1054 /* Convert the hardware-provided AMD Node ID to a Linux logical one. */ 1055 return nid - gpu_node_map.base_node_id + 1; 1056} 1057 1058static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 1059 1060/* 1061 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 1062 * are ECC capable. 1063 */ 1064static unsigned long dct_determine_edac_cap(struct amd64_pvt *pvt) 1065{ 1066 unsigned long edac_cap = EDAC_FLAG_NONE; 1067 u8 bit; 1068 1069 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F) 1070 ? 19 1071 : 17; 1072 1073 if (pvt->dclr0 & BIT(bit)) 1074 edac_cap = EDAC_FLAG_SECDED; 1075 1076 return edac_cap; 1077} 1078 1079static unsigned long umc_determine_edac_cap(struct amd64_pvt *pvt) 1080{ 1081 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0; 1082 unsigned long edac_cap = EDAC_FLAG_NONE; 1083 1084 for_each_umc(i) { 1085 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT)) 1086 continue; 1087 1088 umc_en_mask |= BIT(i); 1089 1090 /* UMC Configuration bit 12 (DimmEccEn) */ 1091 if (pvt->umc[i].umc_cfg & BIT(12)) 1092 dimm_ecc_en_mask |= BIT(i); 1093 } 1094 1095 if (umc_en_mask == dimm_ecc_en_mask) 1096 edac_cap = EDAC_FLAG_SECDED; 1097 1098 return edac_cap; 1099} 1100 1101/* 1102 * debug routine to display the memory sizes of all logical DIMMs and its 1103 * CSROWs 1104 */ 1105static void dct_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 1106{ 1107 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 1108 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1109 int dimm, size0, size1; 1110 1111 if (pvt->fam == 0xf) { 1112 /* K8 families < revF not supported yet */ 1113 if (pvt->ext_model < K8_REV_F) 1114 return; 1115 1116 WARN_ON(ctrl != 0); 1117 } 1118 1119 if (pvt->fam == 0x10) { 1120 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 1121 : pvt->dbam0; 1122 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? 1123 pvt->csels[1].csbases : 1124 pvt->csels[0].csbases; 1125 } else if (ctrl) { 1126 dbam = pvt->dbam0; 1127 dcsb = pvt->csels[1].csbases; 1128 } 1129 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", 1130 ctrl, dbam); 1131 1132 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 1133 1134 /* Dump memory sizes for DIMM and its CSROWs */ 1135 for (dimm = 0; dimm < 4; dimm++) { 1136 size0 = 0; 1137 if (dcsb[dimm * 2] & DCSB_CS_ENABLE) 1138 /* 1139 * For F15m60h, we need multiplier for LRDIMM cs_size 1140 * calculation. We pass dimm value to the dbam_to_cs 1141 * mapper so we can find the multiplier from the 1142 * corresponding DCSM. 1143 */ 1144 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 1145 DBAM_DIMM(dimm, dbam), 1146 dimm); 1147 1148 size1 = 0; 1149 if (dcsb[dimm * 2 + 1] & DCSB_CS_ENABLE) 1150 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 1151 DBAM_DIMM(dimm, dbam), 1152 dimm); 1153 1154 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1155 dimm * 2, size0, 1156 dimm * 2 + 1, size1); 1157 } 1158} 1159 1160 1161static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan) 1162{ 1163 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 1164 1165 if (pvt->dram_type == MEM_LRDDR3) { 1166 u32 dcsm = pvt->csels[chan].csmasks[0]; 1167 /* 1168 * It's assumed all LRDIMMs in a DCT are going to be of 1169 * same 'type' until proven otherwise. So, use a cs 1170 * value of '0' here to get dcsm value. 1171 */ 1172 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3)); 1173 } 1174 1175 edac_dbg(1, "All DIMMs support ECC: %s\n", str_yes_no(dclr & BIT(19))); 1176 1177 1178 edac_dbg(1, " PAR/ERR parity: %s\n", 1179 str_enabled_disabled(dclr & BIT(8))); 1180 1181 if (pvt->fam == 0x10) 1182 edac_dbg(1, " DCT 128bit mode width: %s\n", 1183 (dclr & BIT(11)) ? "128b" : "64b"); 1184 1185 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 1186 str_yes_no(dclr & BIT(12)), 1187 str_yes_no(dclr & BIT(13)), 1188 str_yes_no(dclr & BIT(14)), 1189 str_yes_no(dclr & BIT(15))); 1190} 1191 1192#define CS_EVEN_PRIMARY BIT(0) 1193#define CS_ODD_PRIMARY BIT(1) 1194#define CS_EVEN_SECONDARY BIT(2) 1195#define CS_ODD_SECONDARY BIT(3) 1196#define CS_3R_INTERLEAVE BIT(4) 1197 1198#define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY) 1199#define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY) 1200 1201static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt) 1202{ 1203 u8 base, count = 0; 1204 int cs_mode = 0; 1205 1206 if (csrow_enabled(2 * dimm, ctrl, pvt)) 1207 cs_mode |= CS_EVEN_PRIMARY; 1208 1209 if (csrow_enabled(2 * dimm + 1, ctrl, pvt)) 1210 cs_mode |= CS_ODD_PRIMARY; 1211 1212 if (csrow_sec_enabled(2 * dimm, ctrl, pvt)) 1213 cs_mode |= CS_EVEN_SECONDARY; 1214 1215 if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt)) 1216 cs_mode |= CS_ODD_SECONDARY; 1217 1218 /* 1219 * 3 Rank inteleaving support. 1220 * There should be only three bases enabled and their two masks should 1221 * be equal. 1222 */ 1223 for_each_chip_select(base, ctrl, pvt) 1224 count += csrow_enabled(base, ctrl, pvt); 1225 1226 if (count == 3 && 1227 pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) { 1228 edac_dbg(1, "3R interleaving in use.\n"); 1229 cs_mode |= CS_3R_INTERLEAVE; 1230 } 1231 1232 return cs_mode; 1233} 1234 1235static int calculate_cs_size(u32 mask, unsigned int cs_mode) 1236{ 1237 int msb, weight, num_zero_bits; 1238 u32 deinterleaved_mask; 1239 1240 if (!mask) 1241 return 0; 1242 1243 /* 1244 * The number of zero bits in the mask is equal to the number of bits 1245 * in a full mask minus the number of bits in the current mask. 1246 * 1247 * The MSB is the number of bits in the full mask because BIT[0] is 1248 * always 0. 1249 * 1250 * In the special 3 Rank interleaving case, a single bit is flipped 1251 * without swapping with the most significant bit. This can be handled 1252 * by keeping the MSB where it is and ignoring the single zero bit. 1253 */ 1254 msb = fls(mask) - 1; 1255 weight = hweight_long(mask); 1256 num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE); 1257 1258 /* Take the number of zero bits off from the top of the mask. */ 1259 deinterleaved_mask = GENMASK(msb - num_zero_bits, 1); 1260 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", deinterleaved_mask); 1261 1262 return (deinterleaved_mask >> 2) + 1; 1263} 1264 1265static int __addr_mask_to_cs_size(u32 addr_mask, u32 addr_mask_sec, 1266 unsigned int cs_mode, int csrow_nr, int dimm) 1267{ 1268 int size; 1269 1270 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm); 1271 edac_dbg(1, " Primary AddrMask: 0x%x\n", addr_mask); 1272 1273 /* Register [31:1] = Address [39:9]. Size is in kBs here. */ 1274 size = calculate_cs_size(addr_mask, cs_mode); 1275 1276 edac_dbg(1, " Secondary AddrMask: 0x%x\n", addr_mask_sec); 1277 size += calculate_cs_size(addr_mask_sec, cs_mode); 1278 1279 /* Return size in MBs. */ 1280 return size >> 10; 1281} 1282 1283static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, 1284 unsigned int cs_mode, int csrow_nr) 1285{ 1286 u32 addr_mask = 0, addr_mask_sec = 0; 1287 int cs_mask_nr = csrow_nr; 1288 int dimm, size = 0; 1289 1290 /* No Chip Selects are enabled. */ 1291 if (!cs_mode) 1292 return size; 1293 1294 /* Requested size of an even CS but none are enabled. */ 1295 if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1)) 1296 return size; 1297 1298 /* Requested size of an odd CS but none are enabled. */ 1299 if (!(cs_mode & CS_ODD) && (csrow_nr & 1)) 1300 return size; 1301 1302 /* 1303 * Family 17h introduced systems with one mask per DIMM, 1304 * and two Chip Selects per DIMM. 1305 * 1306 * CS0 and CS1 -> MASK0 / DIMM0 1307 * CS2 and CS3 -> MASK1 / DIMM1 1308 * 1309 * Family 19h Model 10h introduced systems with one mask per Chip Select, 1310 * and two Chip Selects per DIMM. 1311 * 1312 * CS0 -> MASK0 -> DIMM0 1313 * CS1 -> MASK1 -> DIMM0 1314 * CS2 -> MASK2 -> DIMM1 1315 * CS3 -> MASK3 -> DIMM1 1316 * 1317 * Keep the mask number equal to the Chip Select number for newer systems, 1318 * and shift the mask number for older systems. 1319 */ 1320 dimm = csrow_nr >> 1; 1321 1322 if (!pvt->flags.zn_regs_v2) 1323 cs_mask_nr >>= 1; 1324 1325 if (cs_mode & (CS_EVEN_PRIMARY | CS_ODD_PRIMARY)) 1326 addr_mask = pvt->csels[umc].csmasks[cs_mask_nr]; 1327 1328 if (cs_mode & (CS_EVEN_SECONDARY | CS_ODD_SECONDARY)) 1329 addr_mask_sec = pvt->csels[umc].csmasks_sec[cs_mask_nr]; 1330 1331 return __addr_mask_to_cs_size(addr_mask, addr_mask_sec, cs_mode, csrow_nr, dimm); 1332} 1333 1334static void umc_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 1335{ 1336 int dimm, size0, size1, cs0, cs1, cs_mode; 1337 1338 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl); 1339 1340 for (dimm = 0; dimm < 2; dimm++) { 1341 cs0 = dimm * 2; 1342 cs1 = dimm * 2 + 1; 1343 1344 cs_mode = umc_get_cs_mode(dimm, ctrl, pvt); 1345 1346 size0 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs0); 1347 size1 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs1); 1348 1349 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1350 cs0, size0, 1351 cs1, size1); 1352 } 1353} 1354 1355static void umc_dump_misc_regs(struct amd64_pvt *pvt) 1356{ 1357 struct amd64_umc *umc; 1358 u32 i; 1359 1360 for_each_umc(i) { 1361 umc = &pvt->umc[i]; 1362 1363 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg); 1364 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg); 1365 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl); 1366 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl); 1367 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi); 1368 1369 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n", 1370 i, str_yes_no(umc->umc_cap_hi & BIT(30)), 1371 str_yes_no(umc->umc_cap_hi & BIT(31))); 1372 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n", 1373 i, str_yes_no(umc->umc_cfg & BIT(12))); 1374 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n", 1375 i, str_yes_no(umc->dimm_cfg & BIT(6))); 1376 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n", 1377 i, str_yes_no(umc->dimm_cfg & BIT(7))); 1378 1379 umc_debug_display_dimm_sizes(pvt, i); 1380 } 1381} 1382 1383static void dct_dump_misc_regs(struct amd64_pvt *pvt) 1384{ 1385 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 1386 1387 edac_dbg(1, " NB two channel DRAM capable: %s\n", 1388 str_yes_no(pvt->nbcap & NBCAP_DCT_DUAL)); 1389 1390 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n", 1391 str_yes_no(pvt->nbcap & NBCAP_SECDED), 1392 str_yes_no(pvt->nbcap & NBCAP_CHIPKILL)); 1393 1394 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0); 1395 1396 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 1397 1398 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n", 1399 pvt->dhar, dhar_base(pvt), 1400 (pvt->fam == 0xf) ? k8_dhar_offset(pvt) 1401 : f10_dhar_offset(pvt)); 1402 1403 dct_debug_display_dimm_sizes(pvt, 0); 1404 1405 /* everything below this point is Fam10h and above */ 1406 if (pvt->fam == 0xf) 1407 return; 1408 1409 dct_debug_display_dimm_sizes(pvt, 1); 1410 1411 /* Only if NOT ganged does dclr1 have valid info */ 1412 if (!dct_ganging_enabled(pvt)) 1413 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); 1414 1415 edac_dbg(1, " DramHoleValid: %s\n", str_yes_no(dhar_valid(pvt))); 1416 1417 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz); 1418} 1419 1420/* 1421 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 1422 */ 1423static void dct_prep_chip_selects(struct amd64_pvt *pvt) 1424{ 1425 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { 1426 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 1427 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 1428 } else if (pvt->fam == 0x15 && pvt->model == 0x30) { 1429 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4; 1430 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2; 1431 } else { 1432 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 1433 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 1434 } 1435} 1436 1437static void umc_prep_chip_selects(struct amd64_pvt *pvt) 1438{ 1439 int umc; 1440 1441 for_each_umc(umc) { 1442 pvt->csels[umc].b_cnt = 4; 1443 pvt->csels[umc].m_cnt = pvt->flags.zn_regs_v2 ? 4 : 2; 1444 } 1445} 1446 1447static void umc_read_base_mask(struct amd64_pvt *pvt) 1448{ 1449 u32 umc_base_reg, umc_base_reg_sec; 1450 u32 umc_mask_reg, umc_mask_reg_sec; 1451 u32 base_reg, base_reg_sec; 1452 u32 mask_reg, mask_reg_sec; 1453 u32 *base, *base_sec; 1454 u32 *mask, *mask_sec; 1455 int cs, umc; 1456 u32 tmp; 1457 1458 for_each_umc(umc) { 1459 umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR; 1460 umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC; 1461 1462 for_each_chip_select(cs, umc, pvt) { 1463 base = &pvt->csels[umc].csbases[cs]; 1464 base_sec = &pvt->csels[umc].csbases_sec[cs]; 1465 1466 base_reg = umc_base_reg + (cs * 4); 1467 base_reg_sec = umc_base_reg_sec + (cs * 4); 1468 1469 if (!amd_smn_read(pvt->mc_node_id, base_reg, &tmp)) { 1470 *base = tmp; 1471 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n", 1472 umc, cs, *base, base_reg); 1473 } 1474 1475 if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, &tmp)) { 1476 *base_sec = tmp; 1477 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n", 1478 umc, cs, *base_sec, base_reg_sec); 1479 } 1480 } 1481 1482 umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK; 1483 umc_mask_reg_sec = get_umc_base(umc) + get_umc_reg(pvt, UMCCH_ADDR_MASK_SEC); 1484 1485 for_each_chip_select_mask(cs, umc, pvt) { 1486 mask = &pvt->csels[umc].csmasks[cs]; 1487 mask_sec = &pvt->csels[umc].csmasks_sec[cs]; 1488 1489 mask_reg = umc_mask_reg + (cs * 4); 1490 mask_reg_sec = umc_mask_reg_sec + (cs * 4); 1491 1492 if (!amd_smn_read(pvt->mc_node_id, mask_reg, &tmp)) { 1493 *mask = tmp; 1494 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n", 1495 umc, cs, *mask, mask_reg); 1496 } 1497 1498 if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, &tmp)) { 1499 *mask_sec = tmp; 1500 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n", 1501 umc, cs, *mask_sec, mask_reg_sec); 1502 } 1503 } 1504 } 1505} 1506 1507/* 1508 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 1509 */ 1510static void dct_read_base_mask(struct amd64_pvt *pvt) 1511{ 1512 int cs; 1513 1514 for_each_chip_select(cs, 0, pvt) { 1515 int reg0 = DCSB0 + (cs * 4); 1516 int reg1 = DCSB1 + (cs * 4); 1517 u32 *base0 = &pvt->csels[0].csbases[cs]; 1518 u32 *base1 = &pvt->csels[1].csbases[cs]; 1519 1520 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0)) 1521 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n", 1522 cs, *base0, reg0); 1523 1524 if (pvt->fam == 0xf) 1525 continue; 1526 1527 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1)) 1528 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n", 1529 cs, *base1, (pvt->fam == 0x10) ? reg1 1530 : reg0); 1531 } 1532 1533 for_each_chip_select_mask(cs, 0, pvt) { 1534 int reg0 = DCSM0 + (cs * 4); 1535 int reg1 = DCSM1 + (cs * 4); 1536 u32 *mask0 = &pvt->csels[0].csmasks[cs]; 1537 u32 *mask1 = &pvt->csels[1].csmasks[cs]; 1538 1539 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0)) 1540 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n", 1541 cs, *mask0, reg0); 1542 1543 if (pvt->fam == 0xf) 1544 continue; 1545 1546 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1)) 1547 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n", 1548 cs, *mask1, (pvt->fam == 0x10) ? reg1 1549 : reg0); 1550 } 1551} 1552 1553static void umc_determine_memory_type(struct amd64_pvt *pvt) 1554{ 1555 struct amd64_umc *umc; 1556 u32 i; 1557 1558 for_each_umc(i) { 1559 umc = &pvt->umc[i]; 1560 1561 if (!(umc->sdp_ctrl & UMC_SDP_INIT)) { 1562 umc->dram_type = MEM_EMPTY; 1563 continue; 1564 } 1565 1566 /* 1567 * Check if the system supports the "DDR Type" field in UMC Config 1568 * and has DDR5 DIMMs in use. 1569 */ 1570 if (pvt->flags.zn_regs_v2 && ((umc->umc_cfg & GENMASK(2, 0)) == 0x1)) { 1571 if (umc->dimm_cfg & BIT(5)) 1572 umc->dram_type = MEM_LRDDR5; 1573 else if (umc->dimm_cfg & BIT(4)) 1574 umc->dram_type = MEM_RDDR5; 1575 else 1576 umc->dram_type = MEM_DDR5; 1577 } else { 1578 if (umc->dimm_cfg & BIT(5)) 1579 umc->dram_type = MEM_LRDDR4; 1580 else if (umc->dimm_cfg & BIT(4)) 1581 umc->dram_type = MEM_RDDR4; 1582 else 1583 umc->dram_type = MEM_DDR4; 1584 } 1585 1586 edac_dbg(1, " UMC%d DIMM type: %s\n", i, edac_mem_types[umc->dram_type]); 1587 } 1588} 1589 1590static void dct_determine_memory_type(struct amd64_pvt *pvt) 1591{ 1592 u32 dram_ctrl, dcsm; 1593 1594 switch (pvt->fam) { 1595 case 0xf: 1596 if (pvt->ext_model >= K8_REV_F) 1597 goto ddr3; 1598 1599 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 1600 return; 1601 1602 case 0x10: 1603 if (pvt->dchr0 & DDR3_MODE) 1604 goto ddr3; 1605 1606 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 1607 return; 1608 1609 case 0x15: 1610 if (pvt->model < 0x60) 1611 goto ddr3; 1612 1613 /* 1614 * Model 0x60h needs special handling: 1615 * 1616 * We use a Chip Select value of '0' to obtain dcsm. 1617 * Theoretically, it is possible to populate LRDIMMs of different 1618 * 'Rank' value on a DCT. But this is not the common case. So, 1619 * it's reasonable to assume all DIMMs are going to be of same 1620 * 'type' until proven otherwise. 1621 */ 1622 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl); 1623 dcsm = pvt->csels[0].csmasks[0]; 1624 1625 if (((dram_ctrl >> 8) & 0x7) == 0x2) 1626 pvt->dram_type = MEM_DDR4; 1627 else if (pvt->dclr0 & BIT(16)) 1628 pvt->dram_type = MEM_DDR3; 1629 else if (dcsm & 0x3) 1630 pvt->dram_type = MEM_LRDDR3; 1631 else 1632 pvt->dram_type = MEM_RDDR3; 1633 1634 return; 1635 1636 case 0x16: 1637 goto ddr3; 1638 1639 default: 1640 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam); 1641 pvt->dram_type = MEM_EMPTY; 1642 } 1643 1644 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]); 1645 return; 1646 1647ddr3: 1648 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 1649} 1650 1651/* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 1652static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) 1653{ 1654 u16 mce_nid = topology_amd_node_id(m->extcpu); 1655 struct mem_ctl_info *mci; 1656 u8 start_bit = 1; 1657 u8 end_bit = 47; 1658 u64 addr; 1659 1660 mci = edac_mc_find(mce_nid); 1661 if (!mci) 1662 return 0; 1663 1664 pvt = mci->pvt_info; 1665 1666 if (pvt->fam == 0xf) { 1667 start_bit = 3; 1668 end_bit = 39; 1669 } 1670 1671 addr = m->addr & GENMASK_ULL(end_bit, start_bit); 1672 1673 /* 1674 * Erratum 637 workaround 1675 */ 1676 if (pvt->fam == 0x15) { 1677 u64 cc6_base, tmp_addr; 1678 u32 tmp; 1679 u8 intlv_en; 1680 1681 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7) 1682 return addr; 1683 1684 1685 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); 1686 intlv_en = tmp >> 21 & 0x7; 1687 1688 /* add [47:27] + 3 trailing bits */ 1689 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3; 1690 1691 /* reverse and add DramIntlvEn */ 1692 cc6_base |= intlv_en ^ 0x7; 1693 1694 /* pin at [47:24] */ 1695 cc6_base <<= 24; 1696 1697 if (!intlv_en) 1698 return cc6_base | (addr & GENMASK_ULL(23, 0)); 1699 1700 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); 1701 1702 /* faster log2 */ 1703 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1); 1704 1705 /* OR DramIntlvSel into bits [14:12] */ 1706 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9; 1707 1708 /* add remaining [11:0] bits from original MC4_ADDR */ 1709 tmp_addr |= addr & GENMASK_ULL(11, 0); 1710 1711 return cc6_base | tmp_addr; 1712 } 1713 1714 return addr; 1715} 1716 1717static struct pci_dev *pci_get_related_function(unsigned int vendor, 1718 unsigned int device, 1719 struct pci_dev *related) 1720{ 1721 struct pci_dev *dev = NULL; 1722 1723 while ((dev = pci_get_device(vendor, device, dev))) { 1724 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) && 1725 (dev->bus->number == related->bus->number) && 1726 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 1727 break; 1728 } 1729 1730 return dev; 1731} 1732 1733static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 1734{ 1735 struct amd_northbridge *nb; 1736 struct pci_dev *f1 = NULL; 1737 unsigned int pci_func; 1738 int off = range << 3; 1739 u32 llim; 1740 1741 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 1742 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 1743 1744 if (pvt->fam == 0xf) 1745 return; 1746 1747 if (!dram_rw(pvt, range)) 1748 return; 1749 1750 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 1751 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 1752 1753 /* F15h: factor in CC6 save area by reading dst node's limit reg */ 1754 if (pvt->fam != 0x15) 1755 return; 1756 1757 nb = node_to_amd_nb(dram_dst_node(pvt, range)); 1758 if (WARN_ON(!nb)) 1759 return; 1760 1761 if (pvt->model == 0x60) 1762 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; 1763 else if (pvt->model == 0x30) 1764 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; 1765 else 1766 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1; 1767 1768 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc); 1769 if (WARN_ON(!f1)) 1770 return; 1771 1772 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); 1773 1774 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0); 1775 1776 /* {[39:27],111b} */ 1777 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; 1778 1779 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0); 1780 1781 /* [47:40] */ 1782 pvt->ranges[range].lim.hi |= llim >> 13; 1783 1784 pci_dev_put(f1); 1785} 1786 1787static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 1788 struct err_info *err) 1789{ 1790 struct amd64_pvt *pvt = mci->pvt_info; 1791 1792 error_address_to_page_and_offset(sys_addr, err); 1793 1794 /* 1795 * Find out which node the error address belongs to. This may be 1796 * different from the node that detected the error. 1797 */ 1798 err->src_mci = find_mc_by_sys_addr(mci, sys_addr); 1799 if (!err->src_mci) { 1800 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 1801 (unsigned long)sys_addr); 1802 err->err_code = ERR_NODE; 1803 return; 1804 } 1805 1806 /* Now map the sys_addr to a CSROW */ 1807 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr); 1808 if (err->csrow < 0) { 1809 err->err_code = ERR_CSROW; 1810 return; 1811 } 1812 1813 /* CHIPKILL enabled */ 1814 if (pvt->nbcfg & NBCFG_CHIPKILL) { 1815 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 1816 if (err->channel < 0) { 1817 /* 1818 * Syndrome didn't map, so we don't know which of the 1819 * 2 DIMMs is in error. So we need to ID 'both' of them 1820 * as suspect. 1821 */ 1822 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - " 1823 "possible error reporting race\n", 1824 err->syndrome); 1825 err->err_code = ERR_CHANNEL; 1826 return; 1827 } 1828 } else { 1829 /* 1830 * non-chipkill ecc mode 1831 * 1832 * The k8 documentation is unclear about how to determine the 1833 * channel number when using non-chipkill memory. This method 1834 * was obtained from email communication with someone at AMD. 1835 * (Wish the email was placed in this comment - norsk) 1836 */ 1837 err->channel = ((sys_addr & BIT(3)) != 0); 1838 } 1839} 1840 1841static int ddr2_cs_size(unsigned i, bool dct_width) 1842{ 1843 unsigned shift = 0; 1844 1845 if (i <= 2) 1846 shift = i; 1847 else if (!(i & 0x1)) 1848 shift = i >> 1; 1849 else 1850 shift = (i + 1) >> 1; 1851 1852 return 128 << (shift + !!dct_width); 1853} 1854 1855static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1856 unsigned cs_mode, int cs_mask_nr) 1857{ 1858 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 1859 1860 if (pvt->ext_model >= K8_REV_F) { 1861 WARN_ON(cs_mode > 11); 1862 return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 1863 } 1864 else if (pvt->ext_model >= K8_REV_D) { 1865 unsigned diff; 1866 WARN_ON(cs_mode > 10); 1867 1868 /* 1869 * the below calculation, besides trying to win an obfuscated C 1870 * contest, maps cs_mode values to DIMM chip select sizes. The 1871 * mappings are: 1872 * 1873 * cs_mode CS size (mb) 1874 * ======= ============ 1875 * 0 32 1876 * 1 64 1877 * 2 128 1878 * 3 128 1879 * 4 256 1880 * 5 512 1881 * 6 256 1882 * 7 512 1883 * 8 1024 1884 * 9 1024 1885 * 10 2048 1886 * 1887 * Basically, it calculates a value with which to shift the 1888 * smallest CS size of 32MB. 1889 * 1890 * ddr[23]_cs_size have a similar purpose. 1891 */ 1892 diff = cs_mode/3 + (unsigned)(cs_mode > 5); 1893 1894 return 32 << (cs_mode - diff); 1895 } 1896 else { 1897 WARN_ON(cs_mode > 6); 1898 return 32 << cs_mode; 1899 } 1900} 1901 1902static int ddr3_cs_size(unsigned i, bool dct_width) 1903{ 1904 unsigned shift = 0; 1905 int cs_size = 0; 1906 1907 if (i == 0 || i == 3 || i == 4) 1908 cs_size = -1; 1909 else if (i <= 2) 1910 shift = i; 1911 else if (i == 12) 1912 shift = 7; 1913 else if (!(i & 0x1)) 1914 shift = i >> 1; 1915 else 1916 shift = (i + 1) >> 1; 1917 1918 if (cs_size != -1) 1919 cs_size = (128 * (1 << !!dct_width)) << shift; 1920 1921 return cs_size; 1922} 1923 1924static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply) 1925{ 1926 unsigned shift = 0; 1927 int cs_size = 0; 1928 1929 if (i < 4 || i == 6) 1930 cs_size = -1; 1931 else if (i == 12) 1932 shift = 7; 1933 else if (!(i & 0x1)) 1934 shift = i >> 1; 1935 else 1936 shift = (i + 1) >> 1; 1937 1938 if (cs_size != -1) 1939 cs_size = rank_multiply * (128 << shift); 1940 1941 return cs_size; 1942} 1943 1944static int ddr4_cs_size(unsigned i) 1945{ 1946 int cs_size = 0; 1947 1948 if (i == 0) 1949 cs_size = -1; 1950 else if (i == 1) 1951 cs_size = 1024; 1952 else 1953 /* Min cs_size = 1G */ 1954 cs_size = 1024 * (1 << (i >> 1)); 1955 1956 return cs_size; 1957} 1958 1959static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1960 unsigned cs_mode, int cs_mask_nr) 1961{ 1962 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 1963 1964 WARN_ON(cs_mode > 11); 1965 1966 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 1967 return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 1968 else 1969 return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 1970} 1971 1972/* 1973 * F15h supports only 64bit DCT interfaces 1974 */ 1975static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1976 unsigned cs_mode, int cs_mask_nr) 1977{ 1978 WARN_ON(cs_mode > 12); 1979 1980 return ddr3_cs_size(cs_mode, false); 1981} 1982 1983/* F15h M60h supports DDR4 mapping as well.. */ 1984static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 1985 unsigned cs_mode, int cs_mask_nr) 1986{ 1987 int cs_size; 1988 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr]; 1989 1990 WARN_ON(cs_mode > 12); 1991 1992 if (pvt->dram_type == MEM_DDR4) { 1993 if (cs_mode > 9) 1994 return -1; 1995 1996 cs_size = ddr4_cs_size(cs_mode); 1997 } else if (pvt->dram_type == MEM_LRDDR3) { 1998 unsigned rank_multiply = dcsm & 0xf; 1999 2000 if (rank_multiply == 3) 2001 rank_multiply = 4; 2002 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply); 2003 } else { 2004 /* Minimum cs size is 512mb for F15hM60h*/ 2005 if (cs_mode == 0x1) 2006 return -1; 2007 2008 cs_size = ddr3_cs_size(cs_mode, false); 2009 } 2010 2011 return cs_size; 2012} 2013 2014/* 2015 * F16h and F15h model 30h have only limited cs_modes. 2016 */ 2017static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 2018 unsigned cs_mode, int cs_mask_nr) 2019{ 2020 WARN_ON(cs_mode > 12); 2021 2022 if (cs_mode == 6 || cs_mode == 8 || 2023 cs_mode == 9 || cs_mode == 12) 2024 return -1; 2025 else 2026 return ddr3_cs_size(cs_mode, false); 2027} 2028 2029static void read_dram_ctl_register(struct amd64_pvt *pvt) 2030{ 2031 2032 if (pvt->fam == 0xf) 2033 return; 2034 2035 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) { 2036 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 2037 pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 2038 2039 edac_dbg(0, " DCTs operate in %s mode\n", 2040 (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 2041 2042 if (!dct_ganging_enabled(pvt)) 2043 edac_dbg(0, " Address range split per DCT: %s\n", 2044 str_yes_no(dct_high_range_enabled(pvt))); 2045 2046 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n", 2047 str_enabled_disabled(dct_data_intlv_enabled(pvt)), 2048 str_yes_no(dct_memory_cleared(pvt))); 2049 2050 edac_dbg(0, " channel interleave: %s, " 2051 "interleave bits selector: 0x%x\n", 2052 str_enabled_disabled(dct_interleave_enabled(pvt)), 2053 dct_sel_interleave_addr(pvt)); 2054 } 2055 2056 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi); 2057} 2058 2059/* 2060 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG, 2061 * 2.10.12 Memory Interleaving Modes). 2062 */ 2063static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 2064 u8 intlv_en, int num_dcts_intlv, 2065 u32 dct_sel) 2066{ 2067 u8 channel = 0; 2068 u8 select; 2069 2070 if (!(intlv_en)) 2071 return (u8)(dct_sel); 2072 2073 if (num_dcts_intlv == 2) { 2074 select = (sys_addr >> 8) & 0x3; 2075 channel = select ? 0x3 : 0; 2076 } else if (num_dcts_intlv == 4) { 2077 u8 intlv_addr = dct_sel_interleave_addr(pvt); 2078 switch (intlv_addr) { 2079 case 0x4: 2080 channel = (sys_addr >> 8) & 0x3; 2081 break; 2082 case 0x5: 2083 channel = (sys_addr >> 9) & 0x3; 2084 break; 2085 } 2086 } 2087 return channel; 2088} 2089 2090/* 2091 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 2092 * Interleaving Modes. 2093 */ 2094static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 2095 bool hi_range_sel, u8 intlv_en) 2096{ 2097 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 2098 2099 if (dct_ganging_enabled(pvt)) 2100 return 0; 2101 2102 if (hi_range_sel) 2103 return dct_sel_high; 2104 2105 /* 2106 * see F2x110[DctSelIntLvAddr] - channel interleave mode 2107 */ 2108 if (dct_interleave_enabled(pvt)) { 2109 u8 intlv_addr = dct_sel_interleave_addr(pvt); 2110 2111 /* return DCT select function: 0=DCT0, 1=DCT1 */ 2112 if (!intlv_addr) 2113 return sys_addr >> 6 & 1; 2114 2115 if (intlv_addr & 0x2) { 2116 u8 shift = intlv_addr & 0x1 ? 9 : 6; 2117 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1; 2118 2119 return ((sys_addr >> shift) & 1) ^ temp; 2120 } 2121 2122 if (intlv_addr & 0x4) { 2123 u8 shift = intlv_addr & 0x1 ? 9 : 8; 2124 2125 return (sys_addr >> shift) & 1; 2126 } 2127 2128 return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 2129 } 2130 2131 if (dct_high_range_enabled(pvt)) 2132 return ~dct_sel_high & 1; 2133 2134 return 0; 2135} 2136 2137/* Convert the sys_addr to the normalized DCT address */ 2138static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, 2139 u64 sys_addr, bool hi_rng, 2140 u32 dct_sel_base_addr) 2141{ 2142 u64 chan_off; 2143 u64 dram_base = get_dram_base(pvt, range); 2144 u64 hole_off = f10_dhar_offset(pvt); 2145 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16; 2146 2147 if (hi_rng) { 2148 /* 2149 * if 2150 * base address of high range is below 4Gb 2151 * (bits [47:27] at [31:11]) 2152 * DRAM address space on this DCT is hoisted above 4Gb && 2153 * sys_addr > 4Gb 2154 * 2155 * remove hole offset from sys_addr 2156 * else 2157 * remove high range offset from sys_addr 2158 */ 2159 if ((!(dct_sel_base_addr >> 16) || 2160 dct_sel_base_addr < dhar_base(pvt)) && 2161 dhar_valid(pvt) && 2162 (sys_addr >= BIT_64(32))) 2163 chan_off = hole_off; 2164 else 2165 chan_off = dct_sel_base_off; 2166 } else { 2167 /* 2168 * if 2169 * we have a valid hole && 2170 * sys_addr > 4Gb 2171 * 2172 * remove hole 2173 * else 2174 * remove dram base to normalize to DCT address 2175 */ 2176 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 2177 chan_off = hole_off; 2178 else 2179 chan_off = dram_base; 2180 } 2181 2182 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23)); 2183} 2184 2185/* 2186 * checks if the csrow passed in is marked as SPARED, if so returns the new 2187 * spare row 2188 */ 2189static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 2190{ 2191 int tmp_cs; 2192 2193 if (online_spare_swap_done(pvt, dct) && 2194 csrow == online_spare_bad_dramcs(pvt, dct)) { 2195 2196 for_each_chip_select(tmp_cs, dct, pvt) { 2197 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 2198 csrow = tmp_cs; 2199 break; 2200 } 2201 } 2202 } 2203 return csrow; 2204} 2205 2206/* 2207 * Iterate over the DRAM DCT "base" and "mask" registers looking for a 2208 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 2209 * 2210 * Return: 2211 * -EINVAL: NOT FOUND 2212 * 0..csrow = Chip-Select Row 2213 */ 2214static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct) 2215{ 2216 struct mem_ctl_info *mci; 2217 struct amd64_pvt *pvt; 2218 u64 cs_base, cs_mask; 2219 int cs_found = -EINVAL; 2220 int csrow; 2221 2222 mci = edac_mc_find(nid); 2223 if (!mci) 2224 return cs_found; 2225 2226 pvt = mci->pvt_info; 2227 2228 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct); 2229 2230 for_each_chip_select(csrow, dct, pvt) { 2231 if (!csrow_enabled(csrow, dct, pvt)) 2232 continue; 2233 2234 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 2235 2236 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 2237 csrow, cs_base, cs_mask); 2238 2239 cs_mask = ~cs_mask; 2240 2241 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n", 2242 (in_addr & cs_mask), (cs_base & cs_mask)); 2243 2244 if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 2245 if (pvt->fam == 0x15 && pvt->model >= 0x30) { 2246 cs_found = csrow; 2247 break; 2248 } 2249 cs_found = f10_process_possible_spare(pvt, dct, csrow); 2250 2251 edac_dbg(1, " MATCH csrow=%d\n", cs_found); 2252 break; 2253 } 2254 } 2255 return cs_found; 2256} 2257 2258/* 2259 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 2260 * swapped with a region located at the bottom of memory so that the GPU can use 2261 * the interleaved region and thus two channels. 2262 */ 2263static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 2264{ 2265 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 2266 2267 if (pvt->fam == 0x10) { 2268 /* only revC3 and revE have that feature */ 2269 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3)) 2270 return sys_addr; 2271 } 2272 2273 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg); 2274 2275 if (!(swap_reg & 0x1)) 2276 return sys_addr; 2277 2278 swap_base = (swap_reg >> 3) & 0x7f; 2279 swap_limit = (swap_reg >> 11) & 0x7f; 2280 rgn_size = (swap_reg >> 20) & 0x7f; 2281 tmp_addr = sys_addr >> 27; 2282 2283 if (!(sys_addr >> 34) && 2284 (((tmp_addr >= swap_base) && 2285 (tmp_addr <= swap_limit)) || 2286 (tmp_addr < rgn_size))) 2287 return sys_addr ^ (u64)swap_base << 27; 2288 2289 return sys_addr; 2290} 2291 2292/* For a given @dram_range, check if @sys_addr falls within it. */ 2293static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 2294 u64 sys_addr, int *chan_sel) 2295{ 2296 int cs_found = -EINVAL; 2297 u64 chan_addr; 2298 u32 dct_sel_base; 2299 u8 channel; 2300 bool high_range = false; 2301 2302 u8 node_id = dram_dst_node(pvt, range); 2303 u8 intlv_en = dram_intlv_en(pvt, range); 2304 u32 intlv_sel = dram_intlv_sel(pvt, range); 2305 2306 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 2307 range, sys_addr, get_dram_limit(pvt, range)); 2308 2309 if (dhar_valid(pvt) && 2310 dhar_base(pvt) <= sys_addr && 2311 sys_addr < BIT_64(32)) { 2312 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 2313 sys_addr); 2314 return -EINVAL; 2315 } 2316 2317 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) 2318 return -EINVAL; 2319 2320 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 2321 2322 dct_sel_base = dct_sel_baseaddr(pvt); 2323 2324 /* 2325 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 2326 * select between DCT0 and DCT1. 2327 */ 2328 if (dct_high_range_enabled(pvt) && 2329 !dct_ganging_enabled(pvt) && 2330 ((sys_addr >> 27) >= (dct_sel_base >> 11))) 2331 high_range = true; 2332 2333 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 2334 2335 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 2336 high_range, dct_sel_base); 2337 2338 /* Remove node interleaving, see F1x120 */ 2339 if (intlv_en) 2340 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 2341 (chan_addr & 0xfff); 2342 2343 /* remove channel interleave */ 2344 if (dct_interleave_enabled(pvt) && 2345 !dct_high_range_enabled(pvt) && 2346 !dct_ganging_enabled(pvt)) { 2347 2348 if (dct_sel_interleave_addr(pvt) != 1) { 2349 if (dct_sel_interleave_addr(pvt) == 0x3) 2350 /* hash 9 */ 2351 chan_addr = ((chan_addr >> 10) << 9) | 2352 (chan_addr & 0x1ff); 2353 else 2354 /* A[6] or hash 6 */ 2355 chan_addr = ((chan_addr >> 7) << 6) | 2356 (chan_addr & 0x3f); 2357 } else 2358 /* A[12] */ 2359 chan_addr = ((chan_addr >> 13) << 12) | 2360 (chan_addr & 0xfff); 2361 } 2362 2363 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 2364 2365 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 2366 2367 if (cs_found >= 0) 2368 *chan_sel = channel; 2369 2370 return cs_found; 2371} 2372 2373static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 2374 u64 sys_addr, int *chan_sel) 2375{ 2376 int cs_found = -EINVAL; 2377 int num_dcts_intlv = 0; 2378 u64 chan_addr, chan_offset; 2379 u64 dct_base, dct_limit; 2380 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp; 2381 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en; 2382 2383 u64 dhar_offset = f10_dhar_offset(pvt); 2384 u8 intlv_addr = dct_sel_interleave_addr(pvt); 2385 u8 node_id = dram_dst_node(pvt, range); 2386 u8 intlv_en = dram_intlv_en(pvt, range); 2387 2388 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg); 2389 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg); 2390 2391 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0)); 2392 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7); 2393 2394 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 2395 range, sys_addr, get_dram_limit(pvt, range)); 2396 2397 if (!(get_dram_base(pvt, range) <= sys_addr) && 2398 !(get_dram_limit(pvt, range) >= sys_addr)) 2399 return -EINVAL; 2400 2401 if (dhar_valid(pvt) && 2402 dhar_base(pvt) <= sys_addr && 2403 sys_addr < BIT_64(32)) { 2404 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 2405 sys_addr); 2406 return -EINVAL; 2407 } 2408 2409 /* Verify sys_addr is within DCT Range. */ 2410 dct_base = (u64) dct_sel_baseaddr(pvt); 2411 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF; 2412 2413 if (!(dct_cont_base_reg & BIT(0)) && 2414 !(dct_base <= (sys_addr >> 27) && 2415 dct_limit >= (sys_addr >> 27))) 2416 return -EINVAL; 2417 2418 /* Verify number of dct's that participate in channel interleaving. */ 2419 num_dcts_intlv = (int) hweight8(intlv_en); 2420 2421 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4)) 2422 return -EINVAL; 2423 2424 if (pvt->model >= 0x60) 2425 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en); 2426 else 2427 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en, 2428 num_dcts_intlv, dct_sel); 2429 2430 /* Verify we stay within the MAX number of channels allowed */ 2431 if (channel > 3) 2432 return -EINVAL; 2433 2434 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0)); 2435 2436 /* Get normalized DCT addr */ 2437 if (leg_mmio_hole && (sys_addr >= BIT_64(32))) 2438 chan_offset = dhar_offset; 2439 else 2440 chan_offset = dct_base << 27; 2441 2442 chan_addr = sys_addr - chan_offset; 2443 2444 /* remove channel interleave */ 2445 if (num_dcts_intlv == 2) { 2446 if (intlv_addr == 0x4) 2447 chan_addr = ((chan_addr >> 9) << 8) | 2448 (chan_addr & 0xff); 2449 else if (intlv_addr == 0x5) 2450 chan_addr = ((chan_addr >> 10) << 9) | 2451 (chan_addr & 0x1ff); 2452 else 2453 return -EINVAL; 2454 2455 } else if (num_dcts_intlv == 4) { 2456 if (intlv_addr == 0x4) 2457 chan_addr = ((chan_addr >> 10) << 8) | 2458 (chan_addr & 0xff); 2459 else if (intlv_addr == 0x5) 2460 chan_addr = ((chan_addr >> 11) << 9) | 2461 (chan_addr & 0x1ff); 2462 else 2463 return -EINVAL; 2464 } 2465 2466 if (dct_offset_en) { 2467 amd64_read_pci_cfg(pvt->F1, 2468 DRAM_CONT_HIGH_OFF + (int) channel * 4, 2469 &tmp); 2470 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27; 2471 } 2472 2473 f15h_select_dct(pvt, channel); 2474 2475 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); 2476 2477 /* 2478 * Find Chip select: 2479 * if channel = 3, then alias it to 1. This is because, in F15 M30h, 2480 * there is support for 4 DCT's, but only 2 are currently functional. 2481 * They are DCT0 and DCT3. But we have read all registers of DCT3 into 2482 * pvt->csels[1]. So we need to use '1' here to get correct info. 2483 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications. 2484 */ 2485 alias_channel = (channel == 3) ? 1 : channel; 2486 2487 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel); 2488 2489 if (cs_found >= 0) 2490 *chan_sel = alias_channel; 2491 2492 return cs_found; 2493} 2494 2495static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, 2496 u64 sys_addr, 2497 int *chan_sel) 2498{ 2499 int cs_found = -EINVAL; 2500 unsigned range; 2501 2502 for (range = 0; range < DRAM_RANGES; range++) { 2503 if (!dram_rw(pvt, range)) 2504 continue; 2505 2506 if (pvt->fam == 0x15 && pvt->model >= 0x30) 2507 cs_found = f15_m30h_match_to_this_node(pvt, range, 2508 sys_addr, 2509 chan_sel); 2510 2511 else if ((get_dram_base(pvt, range) <= sys_addr) && 2512 (get_dram_limit(pvt, range) >= sys_addr)) { 2513 cs_found = f1x_match_to_this_node(pvt, range, 2514 sys_addr, chan_sel); 2515 if (cs_found >= 0) 2516 break; 2517 } 2518 } 2519 return cs_found; 2520} 2521 2522/* 2523 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 2524 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 2525 * 2526 * The @sys_addr is usually an error address received from the hardware 2527 * (MCX_ADDR). 2528 */ 2529static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 2530 struct err_info *err) 2531{ 2532 struct amd64_pvt *pvt = mci->pvt_info; 2533 2534 error_address_to_page_and_offset(sys_addr, err); 2535 2536 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel); 2537 if (err->csrow < 0) { 2538 err->err_code = ERR_CSROW; 2539 return; 2540 } 2541 2542 /* 2543 * We need the syndromes for channel detection only when we're 2544 * ganged. Otherwise @chan should already contain the channel at 2545 * this point. 2546 */ 2547 if (dct_ganging_enabled(pvt)) 2548 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); 2549} 2550 2551/* 2552 * These are tables of eigenvectors (one per line) which can be used for the 2553 * construction of the syndrome tables. The modified syndrome search algorithm 2554 * uses those to find the symbol in error and thus the DIMM. 2555 * 2556 * Algorithm courtesy of Ross LaFetra from AMD. 2557 */ 2558static const u16 x4_vectors[] = { 2559 0x2f57, 0x1afe, 0x66cc, 0xdd88, 2560 0x11eb, 0x3396, 0x7f4c, 0xeac8, 2561 0x0001, 0x0002, 0x0004, 0x0008, 2562 0x1013, 0x3032, 0x4044, 0x8088, 2563 0x106b, 0x30d6, 0x70fc, 0xe0a8, 2564 0x4857, 0xc4fe, 0x13cc, 0x3288, 2565 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 2566 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 2567 0x15c1, 0x2a42, 0x89ac, 0x4758, 2568 0x2b03, 0x1602, 0x4f0c, 0xca08, 2569 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 2570 0x8ba7, 0x465e, 0x244c, 0x1cc8, 2571 0x2b87, 0x164e, 0x642c, 0xdc18, 2572 0x40b9, 0x80de, 0x1094, 0x20e8, 2573 0x27db, 0x1eb6, 0x9dac, 0x7b58, 2574 0x11c1, 0x2242, 0x84ac, 0x4c58, 2575 0x1be5, 0x2d7a, 0x5e34, 0xa718, 2576 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 2577 0x4c97, 0xc87e, 0x11fc, 0x33a8, 2578 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 2579 0x16b3, 0x3d62, 0x4f34, 0x8518, 2580 0x1e2f, 0x391a, 0x5cac, 0xf858, 2581 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 2582 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 2583 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 2584 0x4397, 0xc27e, 0x17fc, 0x3ea8, 2585 0x1617, 0x3d3e, 0x6464, 0xb8b8, 2586 0x23ff, 0x12aa, 0xab6c, 0x56d8, 2587 0x2dfb, 0x1ba6, 0x913c, 0x7328, 2588 0x185d, 0x2ca6, 0x7914, 0x9e28, 2589 0x171b, 0x3e36, 0x7d7c, 0xebe8, 2590 0x4199, 0x82ee, 0x19f4, 0x2e58, 2591 0x4807, 0xc40e, 0x130c, 0x3208, 2592 0x1905, 0x2e0a, 0x5804, 0xac08, 2593 0x213f, 0x132a, 0xadfc, 0x5ba8, 2594 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 2595}; 2596 2597static const u16 x8_vectors[] = { 2598 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 2599 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 2600 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 2601 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 2602 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 2603 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 2604 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 2605 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 2606 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 2607 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 2608 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 2609 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 2610 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 2611 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 2612 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 2613 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 2614 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 2615 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 2616 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 2617}; 2618 2619static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs, 2620 unsigned v_dim) 2621{ 2622 unsigned int i, err_sym; 2623 2624 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 2625 u16 s = syndrome; 2626 unsigned v_idx = err_sym * v_dim; 2627 unsigned v_end = (err_sym + 1) * v_dim; 2628 2629 /* walk over all 16 bits of the syndrome */ 2630 for (i = 1; i < (1U << 16); i <<= 1) { 2631 2632 /* if bit is set in that eigenvector... */ 2633 if (v_idx < v_end && vectors[v_idx] & i) { 2634 u16 ev_comp = vectors[v_idx++]; 2635 2636 /* ... and bit set in the modified syndrome, */ 2637 if (s & i) { 2638 /* remove it. */ 2639 s ^= ev_comp; 2640 2641 if (!s) 2642 return err_sym; 2643 } 2644 2645 } else if (s & i) 2646 /* can't get to zero, move to next symbol */ 2647 break; 2648 } 2649 } 2650 2651 edac_dbg(0, "syndrome(%x) not found\n", syndrome); 2652 return -1; 2653} 2654 2655static int map_err_sym_to_channel(int err_sym, int sym_size) 2656{ 2657 if (sym_size == 4) 2658 switch (err_sym) { 2659 case 0x20: 2660 case 0x21: 2661 return 0; 2662 case 0x22: 2663 case 0x23: 2664 return 1; 2665 default: 2666 return err_sym >> 4; 2667 } 2668 /* x8 symbols */ 2669 else 2670 switch (err_sym) { 2671 /* imaginary bits not in a DIMM */ 2672 case 0x10: 2673 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 2674 err_sym); 2675 return -1; 2676 case 0x11: 2677 return 0; 2678 case 0x12: 2679 return 1; 2680 default: 2681 return err_sym >> 3; 2682 } 2683 return -1; 2684} 2685 2686static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 2687{ 2688 struct amd64_pvt *pvt = mci->pvt_info; 2689 int err_sym = -1; 2690 2691 if (pvt->ecc_sym_sz == 8) 2692 err_sym = decode_syndrome(syndrome, x8_vectors, 2693 ARRAY_SIZE(x8_vectors), 2694 pvt->ecc_sym_sz); 2695 else if (pvt->ecc_sym_sz == 4) 2696 err_sym = decode_syndrome(syndrome, x4_vectors, 2697 ARRAY_SIZE(x4_vectors), 2698 pvt->ecc_sym_sz); 2699 else { 2700 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 2701 return err_sym; 2702 } 2703 2704 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 2705} 2706 2707static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err, 2708 u8 ecc_type) 2709{ 2710 enum hw_event_mc_err_type err_type; 2711 const char *string; 2712 2713 if (ecc_type == 2) 2714 err_type = HW_EVENT_ERR_CORRECTED; 2715 else if (ecc_type == 1) 2716 err_type = HW_EVENT_ERR_UNCORRECTED; 2717 else if (ecc_type == 3) 2718 err_type = HW_EVENT_ERR_DEFERRED; 2719 else { 2720 WARN(1, "Something is rotten in the state of Denmark.\n"); 2721 return; 2722 } 2723 2724 switch (err->err_code) { 2725 case DECODE_OK: 2726 string = ""; 2727 break; 2728 case ERR_NODE: 2729 string = "Failed to map error addr to a node"; 2730 break; 2731 case ERR_CSROW: 2732 string = "Failed to map error addr to a csrow"; 2733 break; 2734 case ERR_CHANNEL: 2735 string = "Unknown syndrome - possible error reporting race"; 2736 break; 2737 case ERR_SYND: 2738 string = "MCA_SYND not valid - unknown syndrome and csrow"; 2739 break; 2740 case ERR_NORM_ADDR: 2741 string = "Cannot decode normalized address"; 2742 break; 2743 default: 2744 string = "WTF error"; 2745 break; 2746 } 2747 2748 edac_mc_handle_error(err_type, mci, 1, 2749 err->page, err->offset, err->syndrome, 2750 err->csrow, err->channel, -1, 2751 string, ""); 2752} 2753 2754static inline void decode_bus_error(int node_id, struct mce *m) 2755{ 2756 struct mem_ctl_info *mci; 2757 struct amd64_pvt *pvt; 2758 u8 ecc_type = (m->status >> 45) & 0x3; 2759 u8 xec = XEC(m->status, 0x1f); 2760 u16 ec = EC(m->status); 2761 u64 sys_addr; 2762 struct err_info err; 2763 2764 mci = edac_mc_find(node_id); 2765 if (!mci) 2766 return; 2767 2768 pvt = mci->pvt_info; 2769 2770 /* Bail out early if this was an 'observed' error */ 2771 if (PP(ec) == NBSL_PP_OBS) 2772 return; 2773 2774 /* Do only ECC errors */ 2775 if (xec && xec != F10_NBSL_EXT_ERR_ECC) 2776 return; 2777 2778 memset(&err, 0, sizeof(err)); 2779 2780 sys_addr = get_error_address(pvt, m); 2781 2782 if (ecc_type == 2) 2783 err.syndrome = extract_syndrome(m->status); 2784 2785 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err); 2786 2787 __log_ecc_error(mci, &err, ecc_type); 2788} 2789 2790/* 2791 * To find the UMC channel represented by this bank we need to match on its 2792 * instance_id. The instance_id of a bank is held in the lower 32 bits of its 2793 * IPID. 2794 * 2795 * Currently, we can derive the channel number by looking at the 6th nibble in 2796 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel 2797 * number. 2798 * 2799 * For DRAM ECC errors, the Chip Select number is given in bits [2:0] of 2800 * the MCA_SYND[ErrorInformation] field. 2801 */ 2802static void umc_get_err_info(struct mce *m, struct err_info *err) 2803{ 2804 err->channel = (m->ipid & GENMASK(31, 0)) >> 20; 2805 err->csrow = m->synd & 0x7; 2806} 2807 2808static void decode_umc_error(int node_id, struct mce *m) 2809{ 2810 u8 ecc_type = (m->status >> 45) & 0x3; 2811 struct mem_ctl_info *mci; 2812 unsigned long sys_addr; 2813 struct amd64_pvt *pvt; 2814 struct atl_err a_err; 2815 struct err_info err; 2816 2817 node_id = fixup_node_id(node_id, m); 2818 2819 mci = edac_mc_find(node_id); 2820 if (!mci) 2821 return; 2822 2823 pvt = mci->pvt_info; 2824 2825 memset(&err, 0, sizeof(err)); 2826 2827 if (m->status & MCI_STATUS_DEFERRED) 2828 ecc_type = 3; 2829 2830 if (!(m->status & MCI_STATUS_SYNDV)) { 2831 err.err_code = ERR_SYND; 2832 goto log_error; 2833 } 2834 2835 if (ecc_type == 2) { 2836 u8 length = (m->synd >> 18) & 0x3f; 2837 2838 if (length) 2839 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0); 2840 else 2841 err.err_code = ERR_CHANNEL; 2842 } 2843 2844 pvt->ops->get_err_info(m, &err); 2845 2846 a_err.addr = m->addr; 2847 a_err.ipid = m->ipid; 2848 a_err.cpu = m->extcpu; 2849 2850 sys_addr = amd_convert_umc_mca_addr_to_sys_addr(&a_err); 2851 if (IS_ERR_VALUE(sys_addr)) { 2852 err.err_code = ERR_NORM_ADDR; 2853 goto log_error; 2854 } 2855 2856 error_address_to_page_and_offset(sys_addr, &err); 2857 2858log_error: 2859 __log_ecc_error(mci, &err, ecc_type); 2860} 2861 2862/* 2863 * Use pvt->F3 which contains the F3 CPU PCI device to get the related 2864 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error. 2865 */ 2866static int 2867reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) 2868{ 2869 /* Reserve the ADDRESS MAP Device */ 2870 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3); 2871 if (!pvt->F1) { 2872 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1); 2873 return -ENODEV; 2874 } 2875 2876 /* Reserve the DCT Device */ 2877 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3); 2878 if (!pvt->F2) { 2879 pci_dev_put(pvt->F1); 2880 pvt->F1 = NULL; 2881 2882 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2); 2883 return -ENODEV; 2884 } 2885 2886 if (!pci_ctl_dev) 2887 pci_ctl_dev = &pvt->F2->dev; 2888 2889 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); 2890 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); 2891 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); 2892 2893 return 0; 2894} 2895 2896static void determine_ecc_sym_sz(struct amd64_pvt *pvt) 2897{ 2898 pvt->ecc_sym_sz = 4; 2899 2900 if (pvt->fam >= 0x10) { 2901 u32 tmp; 2902 2903 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2904 /* F16h has only DCT0, so no need to read dbam1. */ 2905 if (pvt->fam != 0x16) 2906 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1); 2907 2908 /* F10h, revD and later can do x8 ECC too. */ 2909 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25)) 2910 pvt->ecc_sym_sz = 8; 2911 } 2912} 2913 2914/* 2915 * Retrieve the hardware registers of the memory controller. 2916 */ 2917static void umc_read_mc_regs(struct amd64_pvt *pvt) 2918{ 2919 u8 nid = pvt->mc_node_id; 2920 struct amd64_umc *umc; 2921 u32 i, tmp, umc_base; 2922 2923 /* Read registers from each UMC */ 2924 for_each_umc(i) { 2925 2926 umc_base = get_umc_base(i); 2927 umc = &pvt->umc[i]; 2928 2929 if (!amd_smn_read(nid, umc_base + get_umc_reg(pvt, UMCCH_DIMM_CFG), &tmp)) 2930 umc->dimm_cfg = tmp; 2931 2932 if (!amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &tmp)) 2933 umc->umc_cfg = tmp; 2934 2935 if (!amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &tmp)) 2936 umc->sdp_ctrl = tmp; 2937 2938 if (!amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &tmp)) 2939 umc->ecc_ctrl = tmp; 2940 2941 if (!amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &tmp)) 2942 umc->umc_cap_hi = tmp; 2943 } 2944} 2945 2946/* 2947 * Retrieve the hardware registers of the memory controller (this includes the 2948 * 'Address Map' and 'Misc' device regs) 2949 */ 2950static void dct_read_mc_regs(struct amd64_pvt *pvt) 2951{ 2952 unsigned int range; 2953 u64 msr_val; 2954 2955 /* 2956 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 2957 * those are Read-As-Zero. 2958 */ 2959 rdmsrq(MSR_K8_TOP_MEM1, pvt->top_mem); 2960 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); 2961 2962 /* Check first whether TOP_MEM2 is enabled: */ 2963 rdmsrq(MSR_AMD64_SYSCFG, msr_val); 2964 if (msr_val & BIT(21)) { 2965 rdmsrq(MSR_K8_TOP_MEM2, pvt->top_mem2); 2966 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 2967 } else { 2968 edac_dbg(0, " TOP_MEM2 disabled\n"); 2969 } 2970 2971 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 2972 2973 read_dram_ctl_register(pvt); 2974 2975 for (range = 0; range < DRAM_RANGES; range++) { 2976 u8 rw; 2977 2978 /* read settings for this DRAM range */ 2979 read_dram_base_limit_regs(pvt, range); 2980 2981 rw = dram_rw(pvt, range); 2982 if (!rw) 2983 continue; 2984 2985 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 2986 range, 2987 get_dram_base(pvt, range), 2988 get_dram_limit(pvt, range)); 2989 2990 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 2991 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 2992 (rw & 0x1) ? "R" : "-", 2993 (rw & 0x2) ? "W" : "-", 2994 dram_intlv_sel(pvt, range), 2995 dram_dst_node(pvt, range)); 2996 } 2997 2998 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 2999 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0); 3000 3001 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 3002 3003 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0); 3004 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0); 3005 3006 if (!dct_ganging_enabled(pvt)) { 3007 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1); 3008 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1); 3009 } 3010 3011 determine_ecc_sym_sz(pvt); 3012} 3013 3014/* 3015 * NOTE: CPU Revision Dependent code 3016 * 3017 * Input: 3018 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 3019 * k8 private pointer to --> 3020 * DRAM Bank Address mapping register 3021 * node_id 3022 * DCL register where dual_channel_active is 3023 * 3024 * The DBAM register consists of 4 sets of 4 bits each definitions: 3025 * 3026 * Bits: CSROWs 3027 * 0-3 CSROWs 0 and 1 3028 * 4-7 CSROWs 2 and 3 3029 * 8-11 CSROWs 4 and 5 3030 * 12-15 CSROWs 6 and 7 3031 * 3032 * Values range from: 0 to 15 3033 * The meaning of the values depends on CPU revision and dual-channel state, 3034 * see relevant BKDG more info. 3035 * 3036 * The memory controller provides for total of only 8 CSROWs in its current 3037 * architecture. Each "pair" of CSROWs normally represents just one DIMM in 3038 * single channel or two (2) DIMMs in dual channel mode. 3039 * 3040 * The following code logic collapses the various tables for CSROW based on CPU 3041 * revision. 3042 * 3043 * Returns: 3044 * The number of PAGE_SIZE pages on the specified CSROW number it 3045 * encompasses 3046 * 3047 */ 3048static u32 dct_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 3049{ 3050 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0; 3051 u32 cs_mode, nr_pages; 3052 3053 csrow_nr >>= 1; 3054 cs_mode = DBAM_DIMM(csrow_nr, dbam); 3055 3056 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr); 3057 nr_pages <<= 20 - PAGE_SHIFT; 3058 3059 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n", 3060 csrow_nr, dct, cs_mode); 3061 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 3062 3063 return nr_pages; 3064} 3065 3066static u32 umc_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig) 3067{ 3068 int csrow_nr = csrow_nr_orig; 3069 u32 cs_mode, nr_pages; 3070 3071 cs_mode = umc_get_cs_mode(csrow_nr >> 1, dct, pvt); 3072 3073 nr_pages = umc_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr); 3074 nr_pages <<= 20 - PAGE_SHIFT; 3075 3076 edac_dbg(0, "csrow: %d, channel: %d, cs_mode %d\n", 3077 csrow_nr_orig, dct, cs_mode); 3078 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 3079 3080 return nr_pages; 3081} 3082 3083static void umc_init_csrows(struct mem_ctl_info *mci) 3084{ 3085 struct amd64_pvt *pvt = mci->pvt_info; 3086 enum edac_type edac_mode = EDAC_NONE; 3087 enum dev_type dev_type = DEV_UNKNOWN; 3088 struct dimm_info *dimm; 3089 u8 umc, cs; 3090 3091 if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) { 3092 edac_mode = EDAC_S16ECD16ED; 3093 dev_type = DEV_X16; 3094 } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) { 3095 edac_mode = EDAC_S8ECD8ED; 3096 dev_type = DEV_X8; 3097 } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) { 3098 edac_mode = EDAC_S4ECD4ED; 3099 dev_type = DEV_X4; 3100 } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) { 3101 edac_mode = EDAC_SECDED; 3102 } 3103 3104 for_each_umc(umc) { 3105 for_each_chip_select(cs, umc, pvt) { 3106 if (!csrow_enabled(cs, umc, pvt)) 3107 continue; 3108 3109 dimm = mci->csrows[cs]->channels[umc]->dimm; 3110 3111 edac_dbg(1, "MC node: %d, csrow: %d\n", 3112 pvt->mc_node_id, cs); 3113 3114 dimm->nr_pages = umc_get_csrow_nr_pages(pvt, umc, cs); 3115 dimm->mtype = pvt->umc[umc].dram_type; 3116 dimm->edac_mode = edac_mode; 3117 dimm->dtype = dev_type; 3118 dimm->grain = 64; 3119 } 3120 } 3121} 3122 3123/* 3124 * Initialize the array of csrow attribute instances, based on the values 3125 * from pci config hardware registers. 3126 */ 3127static void dct_init_csrows(struct mem_ctl_info *mci) 3128{ 3129 struct amd64_pvt *pvt = mci->pvt_info; 3130 enum edac_type edac_mode = EDAC_NONE; 3131 struct csrow_info *csrow; 3132 struct dimm_info *dimm; 3133 int nr_pages = 0; 3134 int i, j; 3135 u32 val; 3136 3137 amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 3138 3139 pvt->nbcfg = val; 3140 3141 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 3142 pvt->mc_node_id, val, 3143 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 3144 3145 /* 3146 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed. 3147 */ 3148 for_each_chip_select(i, 0, pvt) { 3149 bool row_dct0 = !!csrow_enabled(i, 0, pvt); 3150 bool row_dct1 = false; 3151 3152 if (pvt->fam != 0xf) 3153 row_dct1 = !!csrow_enabled(i, 1, pvt); 3154 3155 if (!row_dct0 && !row_dct1) 3156 continue; 3157 3158 csrow = mci->csrows[i]; 3159 3160 edac_dbg(1, "MC node: %d, csrow: %d\n", 3161 pvt->mc_node_id, i); 3162 3163 if (row_dct0) { 3164 nr_pages = dct_get_csrow_nr_pages(pvt, 0, i); 3165 csrow->channels[0]->dimm->nr_pages = nr_pages; 3166 } 3167 3168 /* K8 has only one DCT */ 3169 if (pvt->fam != 0xf && row_dct1) { 3170 int row_dct1_pages = dct_get_csrow_nr_pages(pvt, 1, i); 3171 3172 csrow->channels[1]->dimm->nr_pages = row_dct1_pages; 3173 nr_pages += row_dct1_pages; 3174 } 3175 3176 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages); 3177 3178 /* Determine DIMM ECC mode: */ 3179 if (pvt->nbcfg & NBCFG_ECC_ENABLE) { 3180 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) 3181 ? EDAC_S4ECD4ED 3182 : EDAC_SECDED; 3183 } 3184 3185 for (j = 0; j < pvt->max_mcs; j++) { 3186 dimm = csrow->channels[j]->dimm; 3187 dimm->mtype = pvt->dram_type; 3188 dimm->edac_mode = edac_mode; 3189 dimm->grain = 64; 3190 } 3191 } 3192} 3193 3194/* get all cores on this DCT */ 3195static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) 3196{ 3197 int cpu; 3198 3199 for_each_online_cpu(cpu) 3200 if (topology_amd_node_id(cpu) == nid) 3201 cpumask_set_cpu(cpu, mask); 3202} 3203 3204/* check MCG_CTL on all the cpus on this node */ 3205static bool nb_mce_bank_enabled_on_node(u16 nid) 3206{ 3207 cpumask_var_t mask; 3208 int cpu, nbe; 3209 bool ret = false; 3210 3211 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 3212 amd64_warn("%s: Error allocating mask\n", __func__); 3213 return false; 3214 } 3215 3216 get_cpus_on_this_dct_cpumask(mask, nid); 3217 3218 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 3219 3220 for_each_cpu(cpu, mask) { 3221 struct msr *reg = per_cpu_ptr(msrs, cpu); 3222 nbe = reg->l & MSR_MCGCTL_NBE; 3223 3224 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 3225 cpu, reg->q, str_enabled_disabled(nbe)); 3226 3227 if (!nbe) 3228 goto out; 3229 } 3230 ret = true; 3231 3232out: 3233 free_cpumask_var(mask); 3234 return ret; 3235} 3236 3237static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on) 3238{ 3239 cpumask_var_t cmask; 3240 int cpu; 3241 3242 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 3243 amd64_warn("%s: error allocating mask\n", __func__); 3244 return -ENOMEM; 3245 } 3246 3247 get_cpus_on_this_dct_cpumask(cmask, nid); 3248 3249 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 3250 3251 for_each_cpu(cpu, cmask) { 3252 3253 struct msr *reg = per_cpu_ptr(msrs, cpu); 3254 3255 if (on) { 3256 if (reg->l & MSR_MCGCTL_NBE) 3257 s->flags.nb_mce_enable = 1; 3258 3259 reg->l |= MSR_MCGCTL_NBE; 3260 } else { 3261 /* 3262 * Turn off NB MCE reporting only when it was off before 3263 */ 3264 if (!s->flags.nb_mce_enable) 3265 reg->l &= ~MSR_MCGCTL_NBE; 3266 } 3267 } 3268 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 3269 3270 free_cpumask_var(cmask); 3271 3272 return 0; 3273} 3274 3275static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid, 3276 struct pci_dev *F3) 3277{ 3278 bool ret = true; 3279 u32 value, mask = 0x3; /* UECC/CECC enable */ 3280 3281 if (toggle_ecc_err_reporting(s, nid, ON)) { 3282 amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 3283 return false; 3284 } 3285 3286 amd64_read_pci_cfg(F3, NBCTL, &value); 3287 3288 s->old_nbctl = value & mask; 3289 s->nbctl_valid = true; 3290 3291 value |= mask; 3292 amd64_write_pci_cfg(F3, NBCTL, value); 3293 3294 amd64_read_pci_cfg(F3, NBCFG, &value); 3295 3296 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 3297 nid, value, !!(value & NBCFG_ECC_ENABLE)); 3298 3299 if (!(value & NBCFG_ECC_ENABLE)) { 3300 amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 3301 3302 s->flags.nb_ecc_prev = 0; 3303 3304 /* Attempt to turn on DRAM ECC Enable */ 3305 value |= NBCFG_ECC_ENABLE; 3306 amd64_write_pci_cfg(F3, NBCFG, value); 3307 3308 amd64_read_pci_cfg(F3, NBCFG, &value); 3309 3310 if (!(value & NBCFG_ECC_ENABLE)) { 3311 amd64_warn("Hardware rejected DRAM ECC enable," 3312 "check memory DIMM configuration.\n"); 3313 ret = false; 3314 } else { 3315 amd64_info("Hardware accepted DRAM ECC Enable\n"); 3316 } 3317 } else { 3318 s->flags.nb_ecc_prev = 1; 3319 } 3320 3321 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 3322 nid, value, !!(value & NBCFG_ECC_ENABLE)); 3323 3324 return ret; 3325} 3326 3327static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid, 3328 struct pci_dev *F3) 3329{ 3330 u32 value, mask = 0x3; /* UECC/CECC enable */ 3331 3332 if (!s->nbctl_valid) 3333 return; 3334 3335 amd64_read_pci_cfg(F3, NBCTL, &value); 3336 value &= ~mask; 3337 value |= s->old_nbctl; 3338 3339 amd64_write_pci_cfg(F3, NBCTL, value); 3340 3341 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 3342 if (!s->flags.nb_ecc_prev) { 3343 amd64_read_pci_cfg(F3, NBCFG, &value); 3344 value &= ~NBCFG_ECC_ENABLE; 3345 amd64_write_pci_cfg(F3, NBCFG, value); 3346 } 3347 3348 /* restore the NB Enable MCGCTL bit */ 3349 if (toggle_ecc_err_reporting(s, nid, OFF)) 3350 amd64_warn("Error restoring NB MCGCTL settings!\n"); 3351} 3352 3353static bool dct_ecc_enabled(struct amd64_pvt *pvt) 3354{ 3355 u16 nid = pvt->mc_node_id; 3356 bool nb_mce_en = false; 3357 u8 ecc_en = 0; 3358 u32 value; 3359 3360 amd64_read_pci_cfg(pvt->F3, NBCFG, &value); 3361 3362 ecc_en = !!(value & NBCFG_ECC_ENABLE); 3363 3364 nb_mce_en = nb_mce_bank_enabled_on_node(nid); 3365 if (!nb_mce_en) 3366 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n", 3367 MSR_IA32_MCG_CTL, nid); 3368 3369 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, str_enabled_disabled(ecc_en)); 3370 3371 return ecc_en && nb_mce_en; 3372} 3373 3374static bool umc_ecc_enabled(struct amd64_pvt *pvt) 3375{ 3376 struct amd64_umc *umc; 3377 bool ecc_en = false; 3378 int i; 3379 3380 /* Check whether at least one UMC is enabled: */ 3381 for_each_umc(i) { 3382 umc = &pvt->umc[i]; 3383 3384 if (umc->sdp_ctrl & UMC_SDP_INIT && 3385 umc->umc_cap_hi & UMC_ECC_ENABLED) { 3386 ecc_en = true; 3387 break; 3388 } 3389 } 3390 3391 edac_dbg(3, "Node %d: DRAM ECC %s.\n", pvt->mc_node_id, str_enabled_disabled(ecc_en)); 3392 3393 return ecc_en; 3394} 3395 3396static inline void 3397umc_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt) 3398{ 3399 u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1; 3400 3401 for_each_umc(i) { 3402 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) { 3403 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED); 3404 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP); 3405 3406 dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6)); 3407 dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7)); 3408 } 3409 } 3410 3411 /* Set chipkill only if ECC is enabled: */ 3412 if (ecc_en) { 3413 mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 3414 3415 if (!cpk_en) 3416 return; 3417 3418 if (dev_x4) 3419 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 3420 else if (dev_x16) 3421 mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED; 3422 else 3423 mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED; 3424 } 3425} 3426 3427static void dct_setup_mci_misc_attrs(struct mem_ctl_info *mci) 3428{ 3429 struct amd64_pvt *pvt = mci->pvt_info; 3430 3431 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 3432 mci->edac_ctl_cap = EDAC_FLAG_NONE; 3433 3434 if (pvt->nbcap & NBCAP_SECDED) 3435 mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 3436 3437 if (pvt->nbcap & NBCAP_CHIPKILL) 3438 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 3439 3440 mci->edac_cap = dct_determine_edac_cap(pvt); 3441 mci->mod_name = EDAC_MOD_STR; 3442 mci->ctl_name = pvt->ctl_name; 3443 mci->dev_name = pci_name(pvt->F3); 3444 mci->ctl_page_to_phys = NULL; 3445 3446 /* memory scrubber interface */ 3447 mci->set_sdram_scrub_rate = set_scrub_rate; 3448 mci->get_sdram_scrub_rate = get_scrub_rate; 3449 3450 dct_init_csrows(mci); 3451} 3452 3453static void umc_setup_mci_misc_attrs(struct mem_ctl_info *mci) 3454{ 3455 struct amd64_pvt *pvt = mci->pvt_info; 3456 3457 mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_RDDR4; 3458 mci->edac_ctl_cap = EDAC_FLAG_NONE; 3459 3460 umc_determine_edac_ctl_cap(mci, pvt); 3461 3462 mci->edac_cap = umc_determine_edac_cap(pvt); 3463 mci->mod_name = EDAC_MOD_STR; 3464 mci->ctl_name = pvt->ctl_name; 3465 mci->dev_name = pci_name(pvt->F3); 3466 mci->ctl_page_to_phys = NULL; 3467 3468 umc_init_csrows(mci); 3469} 3470 3471static int dct_hw_info_get(struct amd64_pvt *pvt) 3472{ 3473 int ret = reserve_mc_sibling_devs(pvt, pvt->f1_id, pvt->f2_id); 3474 3475 if (ret) 3476 return ret; 3477 3478 dct_prep_chip_selects(pvt); 3479 dct_read_base_mask(pvt); 3480 dct_read_mc_regs(pvt); 3481 dct_determine_memory_type(pvt); 3482 3483 return 0; 3484} 3485 3486static int umc_hw_info_get(struct amd64_pvt *pvt) 3487{ 3488 pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL); 3489 if (!pvt->umc) 3490 return -ENOMEM; 3491 3492 umc_prep_chip_selects(pvt); 3493 umc_read_base_mask(pvt); 3494 umc_read_mc_regs(pvt); 3495 umc_determine_memory_type(pvt); 3496 3497 return 0; 3498} 3499 3500/* 3501 * The CPUs have one channel per UMC, so UMC number is equivalent to a 3502 * channel number. The GPUs have 8 channels per UMC, so the UMC number no 3503 * longer works as a channel number. 3504 * 3505 * The channel number within a GPU UMC is given in MCA_IPID[15:12]. 3506 * However, the IDs are split such that two UMC values go to one UMC, and 3507 * the channel numbers are split in two groups of four. 3508 * 3509 * Refer to comment on gpu_get_umc_base(). 3510 * 3511 * For example, 3512 * UMC0 CH[3:0] = 0x0005[3:0]000 3513 * UMC0 CH[7:4] = 0x0015[3:0]000 3514 * UMC1 CH[3:0] = 0x0025[3:0]000 3515 * UMC1 CH[7:4] = 0x0035[3:0]000 3516 */ 3517static void gpu_get_err_info(struct mce *m, struct err_info *err) 3518{ 3519 u8 ch = (m->ipid & GENMASK(31, 0)) >> 20; 3520 u8 phy = ((m->ipid >> 12) & 0xf); 3521 3522 err->channel = ch % 2 ? phy + 4 : phy; 3523 err->csrow = phy; 3524} 3525 3526static int gpu_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc, 3527 unsigned int cs_mode, int csrow_nr) 3528{ 3529 u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr]; 3530 u32 addr_mask_sec = pvt->csels[umc].csmasks_sec[csrow_nr]; 3531 3532 return __addr_mask_to_cs_size(addr_mask, addr_mask_sec, cs_mode, csrow_nr, csrow_nr >> 1); 3533} 3534 3535static void gpu_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 3536{ 3537 int size, cs_mode, cs = 0; 3538 3539 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl); 3540 3541 cs_mode = CS_EVEN_PRIMARY | CS_ODD_PRIMARY; 3542 3543 for_each_chip_select(cs, ctrl, pvt) { 3544 size = gpu_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs); 3545 amd64_info(EDAC_MC ": %d: %5dMB\n", cs, size); 3546 } 3547} 3548 3549static void gpu_dump_misc_regs(struct amd64_pvt *pvt) 3550{ 3551 struct amd64_umc *umc; 3552 u32 i; 3553 3554 for_each_umc(i) { 3555 umc = &pvt->umc[i]; 3556 3557 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg); 3558 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl); 3559 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl); 3560 edac_dbg(1, "UMC%d All HBMs support ECC: yes\n", i); 3561 3562 gpu_debug_display_dimm_sizes(pvt, i); 3563 } 3564} 3565 3566static u32 gpu_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 3567{ 3568 u32 nr_pages; 3569 int cs_mode = CS_EVEN_PRIMARY | CS_ODD_PRIMARY; 3570 3571 nr_pages = gpu_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr); 3572 nr_pages <<= 20 - PAGE_SHIFT; 3573 3574 edac_dbg(0, "csrow: %d, channel: %d\n", csrow_nr, dct); 3575 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); 3576 3577 return nr_pages; 3578} 3579 3580static void gpu_init_csrows(struct mem_ctl_info *mci) 3581{ 3582 struct amd64_pvt *pvt = mci->pvt_info; 3583 struct dimm_info *dimm; 3584 u8 umc, cs; 3585 3586 for_each_umc(umc) { 3587 for_each_chip_select(cs, umc, pvt) { 3588 if (!csrow_enabled(cs, umc, pvt)) 3589 continue; 3590 3591 dimm = mci->csrows[umc]->channels[cs]->dimm; 3592 3593 edac_dbg(1, "MC node: %d, csrow: %d\n", 3594 pvt->mc_node_id, cs); 3595 3596 dimm->nr_pages = gpu_get_csrow_nr_pages(pvt, umc, cs); 3597 dimm->edac_mode = EDAC_SECDED; 3598 dimm->mtype = pvt->dram_type; 3599 dimm->dtype = DEV_X16; 3600 dimm->grain = 64; 3601 } 3602 } 3603} 3604 3605static void gpu_setup_mci_misc_attrs(struct mem_ctl_info *mci) 3606{ 3607 struct amd64_pvt *pvt = mci->pvt_info; 3608 3609 mci->mtype_cap = MEM_FLAG_HBM2; 3610 mci->edac_ctl_cap = EDAC_FLAG_SECDED; 3611 3612 mci->edac_cap = EDAC_FLAG_EC; 3613 mci->mod_name = EDAC_MOD_STR; 3614 mci->ctl_name = pvt->ctl_name; 3615 mci->dev_name = pci_name(pvt->F3); 3616 mci->ctl_page_to_phys = NULL; 3617 3618 gpu_init_csrows(mci); 3619} 3620 3621/* ECC is enabled by default on GPU nodes */ 3622static bool gpu_ecc_enabled(struct amd64_pvt *pvt) 3623{ 3624 return true; 3625} 3626 3627static inline u32 gpu_get_umc_base(struct amd64_pvt *pvt, u8 umc, u8 channel) 3628{ 3629 /* 3630 * On CPUs, there is one channel per UMC, so UMC numbering equals 3631 * channel numbering. On GPUs, there are eight channels per UMC, 3632 * so the channel numbering is different from UMC numbering. 3633 * 3634 * On CPU nodes channels are selected in 6th nibble 3635 * UMC chY[3:0]= [(chY*2 + 1) : (chY*2)]50000; 3636 * 3637 * On GPU nodes channels are selected in 3rd nibble 3638 * HBM chX[3:0]= [Y ]5X[3:0]000; 3639 * HBM chX[7:4]= [Y+1]5X[3:0]000 3640 * 3641 * On MI300 APU nodes, same as GPU nodes but channels are selected 3642 * in the base address of 0x90000 3643 */ 3644 umc *= 2; 3645 3646 if (channel >= 4) 3647 umc++; 3648 3649 return pvt->gpu_umc_base + (umc << 20) + ((channel % 4) << 12); 3650} 3651 3652static void gpu_read_mc_regs(struct amd64_pvt *pvt) 3653{ 3654 u8 nid = pvt->mc_node_id; 3655 struct amd64_umc *umc; 3656 u32 i, tmp, umc_base; 3657 3658 /* Read registers from each UMC */ 3659 for_each_umc(i) { 3660 umc_base = gpu_get_umc_base(pvt, i, 0); 3661 umc = &pvt->umc[i]; 3662 3663 if (!amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &tmp)) 3664 umc->umc_cfg = tmp; 3665 3666 if (!amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &tmp)) 3667 umc->sdp_ctrl = tmp; 3668 3669 if (!amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &tmp)) 3670 umc->ecc_ctrl = tmp; 3671 } 3672} 3673 3674static void gpu_read_base_mask(struct amd64_pvt *pvt) 3675{ 3676 u32 base_reg, mask_reg; 3677 u32 *base, *mask; 3678 int umc, cs; 3679 3680 for_each_umc(umc) { 3681 for_each_chip_select(cs, umc, pvt) { 3682 base_reg = gpu_get_umc_base(pvt, umc, cs) + UMCCH_BASE_ADDR; 3683 base = &pvt->csels[umc].csbases[cs]; 3684 3685 if (!amd_smn_read(pvt->mc_node_id, base_reg, base)) { 3686 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n", 3687 umc, cs, *base, base_reg); 3688 } 3689 3690 mask_reg = gpu_get_umc_base(pvt, umc, cs) + UMCCH_ADDR_MASK; 3691 mask = &pvt->csels[umc].csmasks[cs]; 3692 3693 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask)) { 3694 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n", 3695 umc, cs, *mask, mask_reg); 3696 } 3697 } 3698 } 3699} 3700 3701static void gpu_prep_chip_selects(struct amd64_pvt *pvt) 3702{ 3703 int umc; 3704 3705 for_each_umc(umc) { 3706 pvt->csels[umc].b_cnt = 8; 3707 pvt->csels[umc].m_cnt = 8; 3708 } 3709} 3710 3711static int gpu_hw_info_get(struct amd64_pvt *pvt) 3712{ 3713 int ret; 3714 3715 ret = gpu_get_node_map(pvt); 3716 if (ret) 3717 return ret; 3718 3719 pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL); 3720 if (!pvt->umc) 3721 return -ENOMEM; 3722 3723 gpu_prep_chip_selects(pvt); 3724 gpu_read_base_mask(pvt); 3725 gpu_read_mc_regs(pvt); 3726 3727 return 0; 3728} 3729 3730static void hw_info_put(struct amd64_pvt *pvt) 3731{ 3732 pci_dev_put(pvt->F1); 3733 pci_dev_put(pvt->F2); 3734 kfree(pvt->umc); 3735 kfree(pvt->csels); 3736} 3737 3738static struct low_ops umc_ops = { 3739 .hw_info_get = umc_hw_info_get, 3740 .ecc_enabled = umc_ecc_enabled, 3741 .setup_mci_misc_attrs = umc_setup_mci_misc_attrs, 3742 .dump_misc_regs = umc_dump_misc_regs, 3743 .get_err_info = umc_get_err_info, 3744}; 3745 3746static struct low_ops gpu_ops = { 3747 .hw_info_get = gpu_hw_info_get, 3748 .ecc_enabled = gpu_ecc_enabled, 3749 .setup_mci_misc_attrs = gpu_setup_mci_misc_attrs, 3750 .dump_misc_regs = gpu_dump_misc_regs, 3751 .get_err_info = gpu_get_err_info, 3752}; 3753 3754/* Use Family 16h versions for defaults and adjust as needed below. */ 3755static struct low_ops dct_ops = { 3756 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 3757 .dbam_to_cs = f16_dbam_to_chip_select, 3758 .hw_info_get = dct_hw_info_get, 3759 .ecc_enabled = dct_ecc_enabled, 3760 .setup_mci_misc_attrs = dct_setup_mci_misc_attrs, 3761 .dump_misc_regs = dct_dump_misc_regs, 3762}; 3763 3764static int per_family_init(struct amd64_pvt *pvt) 3765{ 3766 pvt->ext_model = boot_cpu_data.x86_model >> 4; 3767 pvt->stepping = boot_cpu_data.x86_stepping; 3768 pvt->model = boot_cpu_data.x86_model; 3769 pvt->fam = boot_cpu_data.x86; 3770 char *tmp_name = NULL; 3771 pvt->max_mcs = 2; 3772 3773 /* 3774 * Decide on which ops group to use here and do any family/model 3775 * overrides below. 3776 */ 3777 if (pvt->fam >= 0x17) 3778 pvt->ops = &umc_ops; 3779 else 3780 pvt->ops = &dct_ops; 3781 3782 switch (pvt->fam) { 3783 case 0xf: 3784 tmp_name = (pvt->ext_model >= K8_REV_F) ? 3785 "K8 revF or later" : "K8 revE or earlier"; 3786 pvt->f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP; 3787 pvt->f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL; 3788 pvt->ops->map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow; 3789 pvt->ops->dbam_to_cs = k8_dbam_to_chip_select; 3790 break; 3791 3792 case 0x10: 3793 pvt->f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP; 3794 pvt->f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM; 3795 pvt->ops->dbam_to_cs = f10_dbam_to_chip_select; 3796 break; 3797 3798 case 0x15: 3799 switch (pvt->model) { 3800 case 0x30: 3801 pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; 3802 pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2; 3803 break; 3804 case 0x60: 3805 pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; 3806 pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2; 3807 pvt->ops->dbam_to_cs = f15_m60h_dbam_to_chip_select; 3808 break; 3809 case 0x13: 3810 /* Richland is only client */ 3811 return -ENODEV; 3812 default: 3813 pvt->f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1; 3814 pvt->f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2; 3815 pvt->ops->dbam_to_cs = f15_dbam_to_chip_select; 3816 break; 3817 } 3818 break; 3819 3820 case 0x16: 3821 switch (pvt->model) { 3822 case 0x30: 3823 pvt->f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1; 3824 pvt->f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2; 3825 break; 3826 default: 3827 pvt->f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1; 3828 pvt->f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2; 3829 break; 3830 } 3831 break; 3832 3833 case 0x17: 3834 switch (pvt->model) { 3835 case 0x30 ... 0x3f: 3836 pvt->max_mcs = 8; 3837 break; 3838 default: 3839 break; 3840 } 3841 break; 3842 3843 case 0x18: 3844 break; 3845 3846 case 0x19: 3847 switch (pvt->model) { 3848 case 0x00 ... 0x0f: 3849 pvt->max_mcs = 8; 3850 break; 3851 case 0x10 ... 0x1f: 3852 pvt->max_mcs = 12; 3853 pvt->flags.zn_regs_v2 = 1; 3854 break; 3855 case 0x30 ... 0x3f: 3856 if (pvt->F3->device == PCI_DEVICE_ID_AMD_MI200_DF_F3) { 3857 tmp_name = "MI200"; 3858 pvt->max_mcs = 4; 3859 pvt->dram_type = MEM_HBM2; 3860 pvt->gpu_umc_base = 0x50000; 3861 pvt->ops = &gpu_ops; 3862 } else { 3863 pvt->max_mcs = 8; 3864 } 3865 break; 3866 case 0x60 ... 0x6f: 3867 pvt->flags.zn_regs_v2 = 1; 3868 break; 3869 case 0x70 ... 0x7f: 3870 pvt->max_mcs = 4; 3871 pvt->flags.zn_regs_v2 = 1; 3872 break; 3873 case 0x90 ... 0x9f: 3874 pvt->max_mcs = 4; 3875 pvt->dram_type = MEM_HBM3; 3876 pvt->gpu_umc_base = 0x90000; 3877 pvt->ops = &gpu_ops; 3878 break; 3879 case 0xa0 ... 0xaf: 3880 pvt->max_mcs = 12; 3881 pvt->flags.zn_regs_v2 = 1; 3882 break; 3883 } 3884 break; 3885 3886 case 0x1A: 3887 switch (pvt->model) { 3888 case 0x00 ... 0x1f: 3889 pvt->max_mcs = 12; 3890 pvt->flags.zn_regs_v2 = 1; 3891 break; 3892 case 0x40 ... 0x4f: 3893 pvt->flags.zn_regs_v2 = 1; 3894 break; 3895 case 0x50 ... 0x57: 3896 case 0xc0 ... 0xc7: 3897 pvt->max_mcs = 16; 3898 pvt->flags.zn_regs_v2 = 1; 3899 break; 3900 case 0x90 ... 0x9f: 3901 case 0xa0 ... 0xaf: 3902 pvt->max_mcs = 8; 3903 pvt->flags.zn_regs_v2 = 1; 3904 break; 3905 } 3906 break; 3907 3908 default: 3909 amd64_err("Unsupported family!\n"); 3910 return -ENODEV; 3911 } 3912 3913 if (tmp_name) 3914 scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "%s", tmp_name); 3915 else 3916 scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "F%02Xh_M%02Xh", 3917 pvt->fam, pvt->model); 3918 3919 pvt->csels = kcalloc(pvt->max_mcs, sizeof(*pvt->csels), GFP_KERNEL); 3920 if (!pvt->csels) 3921 return -ENOMEM; 3922 3923 return 0; 3924} 3925 3926static const struct attribute_group *amd64_edac_attr_groups[] = { 3927#ifdef CONFIG_EDAC_DEBUG 3928 &dbg_group, 3929 &inj_group, 3930#endif 3931 NULL 3932}; 3933 3934/* 3935 * For heterogeneous and APU models EDAC CHIP_SELECT and CHANNEL layers 3936 * should be swapped to fit into the layers. 3937 */ 3938static unsigned int get_layer_size(struct amd64_pvt *pvt, u8 layer) 3939{ 3940 bool is_gpu = (pvt->ops == &gpu_ops); 3941 3942 if (!layer) 3943 return is_gpu ? pvt->max_mcs 3944 : pvt->csels[0].b_cnt; 3945 else 3946 return is_gpu ? pvt->csels[0].b_cnt 3947 : pvt->max_mcs; 3948} 3949 3950static int init_one_instance(struct amd64_pvt *pvt) 3951{ 3952 struct mem_ctl_info *mci = NULL; 3953 struct edac_mc_layer layers[2]; 3954 int ret = -ENOMEM; 3955 3956 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; 3957 layers[0].size = get_layer_size(pvt, 0); 3958 layers[0].is_virt_csrow = true; 3959 layers[1].type = EDAC_MC_LAYER_CHANNEL; 3960 layers[1].size = get_layer_size(pvt, 1); 3961 layers[1].is_virt_csrow = false; 3962 3963 mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0); 3964 if (!mci) 3965 return ret; 3966 3967 mci->pvt_info = pvt; 3968 mci->pdev = &pvt->F3->dev; 3969 3970 pvt->ops->setup_mci_misc_attrs(mci); 3971 3972 ret = -ENODEV; 3973 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) { 3974 edac_dbg(1, "failed edac_mc_add_mc()\n"); 3975 edac_mc_free(mci); 3976 return ret; 3977 } 3978 3979 return 0; 3980} 3981 3982static bool instance_has_memory(struct amd64_pvt *pvt) 3983{ 3984 bool cs_enabled = false; 3985 int cs = 0, dct = 0; 3986 3987 for (dct = 0; dct < pvt->max_mcs; dct++) { 3988 for_each_chip_select(cs, dct, pvt) 3989 cs_enabled |= csrow_enabled(cs, dct, pvt); 3990 } 3991 3992 return cs_enabled; 3993} 3994 3995static int probe_one_instance(unsigned int nid) 3996{ 3997 struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 3998 struct amd64_pvt *pvt = NULL; 3999 struct ecc_settings *s; 4000 int ret; 4001 4002 ret = -ENOMEM; 4003 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 4004 if (!s) 4005 goto err_out; 4006 4007 ecc_stngs[nid] = s; 4008 4009 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 4010 if (!pvt) 4011 goto err_settings; 4012 4013 pvt->mc_node_id = nid; 4014 pvt->F3 = F3; 4015 4016 ret = per_family_init(pvt); 4017 if (ret < 0) 4018 goto err_enable; 4019 4020 ret = pvt->ops->hw_info_get(pvt); 4021 if (ret < 0) 4022 goto err_enable; 4023 4024 ret = 0; 4025 if (!instance_has_memory(pvt)) { 4026 amd64_info("Node %d: No DIMMs detected.\n", nid); 4027 goto err_enable; 4028 } 4029 4030 if (!pvt->ops->ecc_enabled(pvt)) { 4031 ret = -ENODEV; 4032 4033 if (!ecc_enable_override) 4034 goto err_enable; 4035 4036 if (boot_cpu_data.x86 >= 0x17) { 4037 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS."); 4038 goto err_enable; 4039 } else 4040 amd64_warn("Forcing ECC on!\n"); 4041 4042 if (!enable_ecc_error_reporting(s, nid, F3)) 4043 goto err_enable; 4044 } 4045 4046 ret = init_one_instance(pvt); 4047 if (ret < 0) { 4048 amd64_err("Error probing instance: %d\n", nid); 4049 4050 if (boot_cpu_data.x86 < 0x17) 4051 restore_ecc_error_reporting(s, nid, F3); 4052 4053 goto err_enable; 4054 } 4055 4056 amd64_info("%s detected (node %d).\n", pvt->ctl_name, pvt->mc_node_id); 4057 4058 /* Display and decode various registers for debug purposes. */ 4059 pvt->ops->dump_misc_regs(pvt); 4060 4061 return ret; 4062 4063err_enable: 4064 hw_info_put(pvt); 4065 kfree(pvt); 4066 4067err_settings: 4068 kfree(s); 4069 ecc_stngs[nid] = NULL; 4070 4071err_out: 4072 return ret; 4073} 4074 4075static void remove_one_instance(unsigned int nid) 4076{ 4077 struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 4078 struct ecc_settings *s = ecc_stngs[nid]; 4079 struct mem_ctl_info *mci; 4080 struct amd64_pvt *pvt; 4081 4082 /* Remove from EDAC CORE tracking list */ 4083 mci = edac_mc_del_mc(&F3->dev); 4084 if (!mci) 4085 return; 4086 4087 pvt = mci->pvt_info; 4088 4089 restore_ecc_error_reporting(s, nid, F3); 4090 4091 kfree(ecc_stngs[nid]); 4092 ecc_stngs[nid] = NULL; 4093 4094 /* Free the EDAC CORE resources */ 4095 mci->pvt_info = NULL; 4096 4097 hw_info_put(pvt); 4098 kfree(pvt); 4099 edac_mc_free(mci); 4100} 4101 4102static void setup_pci_device(void) 4103{ 4104 if (pci_ctl) 4105 return; 4106 4107 pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR); 4108 if (!pci_ctl) { 4109 pr_warn("%s(): Unable to create PCI control\n", __func__); 4110 pr_warn("%s(): PCI error report via EDAC not set\n", __func__); 4111 } 4112} 4113 4114static const struct x86_cpu_id amd64_cpuids[] = { 4115 X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL), 4116 X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL), 4117 X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL), 4118 X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL), 4119 X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL), 4120 X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL), 4121 X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL), 4122 X86_MATCH_VENDOR_FAM(AMD, 0x1A, NULL), 4123 { } 4124}; 4125MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids); 4126 4127static int __init amd64_edac_init(void) 4128{ 4129 const char *owner; 4130 int err = -ENODEV; 4131 int i; 4132 4133 if (ghes_get_devices()) 4134 return -EBUSY; 4135 4136 owner = edac_get_owner(); 4137 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 4138 return -EBUSY; 4139 4140 if (!x86_match_cpu(amd64_cpuids)) 4141 return -ENODEV; 4142 4143 if (!amd_nb_num()) 4144 return -ENODEV; 4145 4146 opstate_init(); 4147 4148 err = -ENOMEM; 4149 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL); 4150 if (!ecc_stngs) 4151 goto err_free; 4152 4153 msrs = msrs_alloc(); 4154 if (!msrs) 4155 goto err_free; 4156 4157 for (i = 0; i < amd_nb_num(); i++) { 4158 err = probe_one_instance(i); 4159 if (err) { 4160 /* unwind properly */ 4161 while (--i >= 0) 4162 remove_one_instance(i); 4163 4164 goto err_pci; 4165 } 4166 } 4167 4168 if (!edac_has_mcs()) { 4169 err = -ENODEV; 4170 goto err_pci; 4171 } 4172 4173 /* register stuff with EDAC MCE */ 4174 if (boot_cpu_data.x86 >= 0x17) { 4175 amd_register_ecc_decoder(decode_umc_error); 4176 } else { 4177 amd_register_ecc_decoder(decode_bus_error); 4178 setup_pci_device(); 4179 } 4180 4181#ifdef CONFIG_X86_32 4182 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR); 4183#endif 4184 4185 return 0; 4186 4187err_pci: 4188 pci_ctl_dev = NULL; 4189 4190 msrs_free(msrs); 4191 msrs = NULL; 4192 4193err_free: 4194 kfree(ecc_stngs); 4195 ecc_stngs = NULL; 4196 4197 return err; 4198} 4199 4200static void __exit amd64_edac_exit(void) 4201{ 4202 int i; 4203 4204 if (pci_ctl) 4205 edac_pci_release_generic_ctl(pci_ctl); 4206 4207 /* unregister from EDAC MCE */ 4208 if (boot_cpu_data.x86 >= 0x17) 4209 amd_unregister_ecc_decoder(decode_umc_error); 4210 else 4211 amd_unregister_ecc_decoder(decode_bus_error); 4212 4213 for (i = 0; i < amd_nb_num(); i++) 4214 remove_one_instance(i); 4215 4216 kfree(ecc_stngs); 4217 ecc_stngs = NULL; 4218 4219 pci_ctl_dev = NULL; 4220 4221 msrs_free(msrs); 4222 msrs = NULL; 4223} 4224 4225module_init(amd64_edac_init); 4226module_exit(amd64_edac_exit); 4227 4228MODULE_LICENSE("GPL"); 4229MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, Dave Peterson, Thayne Harbaugh; AMD"); 4230MODULE_DESCRIPTION("MC support for AMD64 memory controllers"); 4231 4232module_param(edac_op_state, int, 0444); 4233MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
~/
Read-only file preview.