FILE
amdgpu_ras.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
1/* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24#include <linux/debugfs.h> 25#include <linux/list.h> 26#include <linux/module.h> 27#include <linux/uaccess.h> 28#include <linux/reboot.h> 29#include <linux/syscalls.h> 30#include <linux/pm_runtime.h> 31#include <linux/list_sort.h> 32 33#include "amdgpu.h" 34#include "amdgpu_ras.h" 35#include "amdgpu_atomfirmware.h" 36#include "amdgpu_xgmi.h" 37#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" 38#include "nbio_v4_3.h" 39#include "nbif_v6_3_1.h" 40#include "nbio_v7_9.h" 41#include "atom.h" 42#include "amdgpu_reset.h" 43#include "amdgpu_psp.h" 44#include "amdgpu_ras_mgr.h" 45 46#ifdef CONFIG_X86_MCE_AMD 47#include <asm/mce.h> 48 49static bool notifier_registered; 50#endif 51static const char *RAS_FS_NAME = "ras"; 52 53const char *ras_error_string[] = { 54 "none", 55 "parity", 56 "single_correctable", 57 "multi_uncorrectable", 58 "poison", 59}; 60 61const char *ras_block_string[] = { 62 "umc", 63 "sdma", 64 "gfx", 65 "mmhub", 66 "athub", 67 "pcie_bif", 68 "hdp", 69 "xgmi_wafl", 70 "df", 71 "smn", 72 "sem", 73 "mp0", 74 "mp1", 75 "fuse", 76 "mca", 77 "vcn", 78 "jpeg", 79 "ih", 80 "mpio", 81 "mmsch", 82}; 83 84const char *ras_mca_block_string[] = { 85 "mca_mp0", 86 "mca_mp1", 87 "mca_mpio", 88 "mca_iohc", 89}; 90 91struct amdgpu_ras_block_list { 92 /* ras block link */ 93 struct list_head node; 94 95 struct amdgpu_ras_block_object *ras_obj; 96}; 97 98const char *get_ras_block_str(struct ras_common_if *ras_block) 99{ 100 if (!ras_block) 101 return "NULL"; 102 103 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT || 104 ras_block->block >= ARRAY_SIZE(ras_block_string)) 105 return "OUT OF RANGE"; 106 107 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA) 108 return ras_mca_block_string[ras_block->sub_block_index]; 109 110 return ras_block_string[ras_block->block]; 111} 112 113#define ras_block_str(_BLOCK_) \ 114 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range") 115 116#define ras_err_str(i) (ras_error_string[ffs(i)]) 117 118#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS) 119 120/* inject address is 52 bits */ 121#define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52) 122 123/* typical ECC bad page rate is 1 bad page per 100MB VRAM */ 124#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) 125 126#define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 127 128#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms 129 130#define MAX_FLUSH_RETIRE_DWORK_TIMES 100 131 132#define BYPASS_ALLOCATED_ADDRESS 0x0 133#define BYPASS_INITIALIZATION_ADDRESS 0x1 134 135enum amdgpu_ras_retire_page_reservation { 136 AMDGPU_RAS_RETIRE_PAGE_RESERVED, 137 AMDGPU_RAS_RETIRE_PAGE_PENDING, 138 AMDGPU_RAS_RETIRE_PAGE_FAULT, 139}; 140 141atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0); 142 143static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 144 uint64_t addr); 145static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 146 uint64_t addr); 147 148static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev); 149static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev); 150 151#ifdef CONFIG_X86_MCE_AMD 152static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev); 153static void 154amdgpu_unregister_bad_pages_mca_notifier(struct amdgpu_device *adev); 155struct mce_notifier_adev_list { 156 struct amdgpu_device *devs[MAX_GPU_INSTANCE]; 157 int num_gpu; 158}; 159static struct mce_notifier_adev_list mce_adev_list; 160#endif 161 162void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready) 163{ 164 if (adev && amdgpu_ras_get_context(adev)) 165 amdgpu_ras_get_context(adev)->error_query_ready = ready; 166} 167 168static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev) 169{ 170 if (adev && amdgpu_ras_get_context(adev)) 171 return amdgpu_ras_get_context(adev)->error_query_ready; 172 173 return false; 174} 175 176static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address) 177{ 178 struct ras_err_data err_data; 179 struct eeprom_table_record err_rec; 180 int ret; 181 182 ret = amdgpu_ras_check_bad_page(adev, address); 183 if (ret == -EINVAL) { 184 dev_warn(adev->dev, 185 "RAS WARN: input address 0x%llx is invalid.\n", 186 address); 187 return -EINVAL; 188 } else if (ret == 1) { 189 dev_warn(adev->dev, 190 "RAS WARN: 0x%llx has already been marked as bad page!\n", 191 address); 192 return 0; 193 } 194 195 ret = amdgpu_ras_error_data_init(&err_data); 196 if (ret) 197 return ret; 198 199 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); 200 err_data.err_addr = &err_rec; 201 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0); 202 203 if (amdgpu_bad_page_threshold != 0) { 204 amdgpu_ras_add_bad_pages(adev, err_data.err_addr, 205 err_data.err_addr_cnt, false); 206 amdgpu_ras_save_bad_pages(adev, NULL); 207 } 208 209 amdgpu_ras_error_data_fini(&err_data); 210 211 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n"); 212 dev_warn(adev->dev, "Clear EEPROM:\n"); 213 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n"); 214 215 return 0; 216} 217 218static int amdgpu_check_address_validity(struct amdgpu_device *adev, 219 uint64_t address, uint64_t flags) 220{ 221 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 222 struct amdgpu_vram_block_info blk_info; 223 uint64_t page_pfns[32] = {0}; 224 int i, ret, count; 225 bool hit = false; 226 227 if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0)) 228 return 0; 229 230 if (amdgpu_sriov_vf(adev)) { 231 if (amdgpu_virt_check_vf_critical_region(adev, address, &hit)) 232 return -EPERM; 233 return hit ? -EACCES : 0; 234 } 235 236 if ((address >= adev->gmc.mc_vram_size) || 237 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) 238 return -EFAULT; 239 240 count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, 241 address, page_pfns, ARRAY_SIZE(page_pfns)); 242 if (count <= 0) 243 return -EPERM; 244 245 for (i = 0; i < count; i++) { 246 memset(&blk_info, 0, sizeof(blk_info)); 247 ret = amdgpu_vram_mgr_query_address_block_info(&adev->mman.vram_mgr, 248 page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, &blk_info); 249 if (!ret) { 250 /* The input address that needs to be checked is allocated by 251 * current calling process, so it is necessary to exclude 252 * the calling process. 253 */ 254 if ((flags == BYPASS_ALLOCATED_ADDRESS) && 255 ((blk_info.task.pid != task_pid_nr(current)) || 256 strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) 257 return -EACCES; 258 else if ((flags == BYPASS_INITIALIZATION_ADDRESS) && 259 (blk_info.task.pid == con->init_task_pid) && 260 !strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) 261 return -EACCES; 262 } 263 } 264 265 return 0; 266} 267 268static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf, 269 size_t size, loff_t *pos) 270{ 271 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private; 272 struct ras_query_if info = { 273 .head = obj->head, 274 }; 275 ssize_t s; 276 char val[128]; 277 278 if (amdgpu_ras_query_error_status(obj->adev, &info)) 279 return -EINVAL; 280 281 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */ 282 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 283 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 284 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 285 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 286 } 287 288 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n", 289 "ue", info.ue_count, 290 "ce", info.ce_count); 291 if (*pos >= s) 292 return 0; 293 294 s -= *pos; 295 s = min_t(u64, s, size); 296 297 298 if (copy_to_user(buf, &val[*pos], s)) 299 return -EINVAL; 300 301 *pos += s; 302 303 return s; 304} 305 306static const struct file_operations amdgpu_ras_debugfs_ops = { 307 .owner = THIS_MODULE, 308 .read = amdgpu_ras_debugfs_read, 309 .write = NULL, 310 .llseek = default_llseek 311}; 312 313static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id) 314{ 315 int i; 316 317 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) { 318 *block_id = i; 319 if (strcmp(name, ras_block_string[i]) == 0) 320 return 0; 321 } 322 return -EINVAL; 323} 324 325static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, 326 const char __user *buf, size_t size, 327 loff_t *pos, struct ras_debug_if *data) 328{ 329 ssize_t s = min_t(u64, 64, size); 330 char str[65]; 331 char block_name[33]; 332 char err[9] = "ue"; 333 int op = -1; 334 int block_id; 335 uint32_t sub_block; 336 u64 address, value; 337 /* default value is 0 if the mask is not set by user */ 338 u32 instance_mask = 0; 339 340 if (*pos) 341 return -EINVAL; 342 *pos = size; 343 344 memset(str, 0, sizeof(str)); 345 memset(data, 0, sizeof(*data)); 346 347 if (copy_from_user(str, buf, s)) 348 return -EINVAL; 349 350 if (sscanf(str, "disable %32s", block_name) == 1) 351 op = 0; 352 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2) 353 op = 1; 354 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2) 355 op = 2; 356 else if (strstr(str, "retire_page") != NULL) 357 op = 3; 358 else if (strstr(str, "check_address") != NULL) 359 op = 4; 360 else if (str[0] && str[1] && str[2] && str[3]) 361 /* ascii string, but commands are not matched. */ 362 return -EINVAL; 363 364 if (op != -1) { 365 if (op == 3) { 366 if (sscanf(str, "%*s 0x%llx", &address) != 1 && 367 sscanf(str, "%*s %llu", &address) != 1) 368 return -EINVAL; 369 370 data->op = op; 371 data->inject.address = address; 372 373 return 0; 374 } else if (op == 4) { 375 if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 && 376 sscanf(str, "%*s %llu %llu", &address, &value) != 2) 377 return -EINVAL; 378 379 data->op = op; 380 data->inject.address = address; 381 data->inject.value = value; 382 return 0; 383 } 384 385 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id)) 386 return -EINVAL; 387 388 data->head.block = block_id; 389 /* only ue, ce and poison errors are supported */ 390 if (!memcmp("ue", err, 2)) 391 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 392 else if (!memcmp("ce", err, 2)) 393 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE; 394 else if (!memcmp("poison", err, 6)) 395 data->head.type = AMDGPU_RAS_ERROR__POISON; 396 else 397 return -EINVAL; 398 399 data->op = op; 400 401 if (op == 2) { 402 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x", 403 &sub_block, &address, &value, &instance_mask) != 4 && 404 sscanf(str, "%*s %*s %*s %u %llu %llu %u", 405 &sub_block, &address, &value, &instance_mask) != 4 && 406 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx", 407 &sub_block, &address, &value) != 3 && 408 sscanf(str, "%*s %*s %*s %u %llu %llu", 409 &sub_block, &address, &value) != 3) 410 return -EINVAL; 411 data->head.sub_block_index = sub_block; 412 data->inject.address = address; 413 data->inject.value = value; 414 data->inject.instance_mask = instance_mask; 415 } 416 } else { 417 if (size < sizeof(*data)) 418 return -EINVAL; 419 420 if (copy_from_user(data, buf, sizeof(*data))) 421 return -EINVAL; 422 } 423 424 return 0; 425} 426 427static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev, 428 struct ras_debug_if *data) 429{ 430 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; 431 uint32_t mask, inst_mask = data->inject.instance_mask; 432 433 /* no need to set instance mask if there is only one instance */ 434 if (num_xcc <= 1 && inst_mask) { 435 data->inject.instance_mask = 0; 436 dev_dbg(adev->dev, 437 "RAS inject mask(0x%x) isn't supported and force it to 0.\n", 438 inst_mask); 439 440 return; 441 } 442 443 switch (data->head.block) { 444 case AMDGPU_RAS_BLOCK__GFX: 445 mask = GENMASK(num_xcc - 1, 0); 446 break; 447 case AMDGPU_RAS_BLOCK__SDMA: 448 mask = GENMASK(adev->sdma.num_instances - 1, 0); 449 break; 450 case AMDGPU_RAS_BLOCK__VCN: 451 case AMDGPU_RAS_BLOCK__JPEG: 452 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0); 453 break; 454 default: 455 mask = inst_mask; 456 break; 457 } 458 459 /* remove invalid bits in instance mask */ 460 data->inject.instance_mask &= mask; 461 if (inst_mask != data->inject.instance_mask) 462 dev_dbg(adev->dev, 463 "Adjust RAS inject mask 0x%x to 0x%x\n", 464 inst_mask, data->inject.instance_mask); 465} 466 467/** 468 * DOC: AMDGPU RAS debugfs control interface 469 * 470 * The control interface accepts struct ras_debug_if which has two members. 471 * 472 * First member: ras_debug_if::head or ras_debug_if::inject. 473 * 474 * head is used to indicate which IP block will be under control. 475 * 476 * head has four members, they are block, type, sub_block_index, name. 477 * block: which IP will be under control. 478 * type: what kind of error will be enabled/disabled/injected. 479 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA. 480 * name: the name of IP. 481 * 482 * inject has three more members than head, they are address, value and mask. 483 * As their names indicate, inject operation will write the 484 * value to the address. 485 * 486 * The second member: struct ras_debug_if::op. 487 * It has three kinds of operations. 488 * 489 * - 0: disable RAS on the block. Take ::head as its data. 490 * - 1: enable RAS on the block. Take ::head as its data. 491 * - 2: inject errors on the block. Take ::inject as its data. 492 * 493 * How to use the interface? 494 * 495 * In a program 496 * 497 * Copy the struct ras_debug_if in your code and initialize it. 498 * Write the struct to the control interface. 499 * 500 * From shell 501 * 502 * .. code-block:: bash 503 * 504 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 505 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 506 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl 507 * 508 * Where N, is the card which you want to affect. 509 * 510 * "disable" requires only the block. 511 * "enable" requires the block and error type. 512 * "inject" requires the block, error type, address, and value. 513 * 514 * The block is one of: umc, sdma, gfx, etc. 515 * see ras_block_string[] for details 516 * 517 * The error type is one of: ue, ce and poison where, 518 * ue is multi-uncorrectable 519 * ce is single-correctable 520 * poison is poison 521 * 522 * The sub-block is a the sub-block index, pass 0 if there is no sub-block. 523 * The address and value are hexadecimal numbers, leading 0x is optional. 524 * The mask means instance mask, is optional, default value is 0x1. 525 * 526 * For instance, 527 * 528 * .. code-block:: bash 529 * 530 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl 531 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl 532 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl 533 * 534 * How to check the result of the operation? 535 * 536 * To check disable/enable, see "ras" features at, 537 * /sys/class/drm/card[0/1/2...]/device/ras/features 538 * 539 * To check inject, see the corresponding error count at, 540 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count 541 * 542 * .. note:: 543 * Operations are only allowed on blocks which are supported. 544 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask 545 * to see which blocks support RAS on a particular asic. 546 * 547 */ 548static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, 549 const char __user *buf, 550 size_t size, loff_t *pos) 551{ 552 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; 553 struct ras_debug_if data; 554 int ret = 0; 555 556 if (!amdgpu_ras_get_error_query_ready(adev)) { 557 dev_warn(adev->dev, "RAS WARN: error injection " 558 "currently inaccessible\n"); 559 return size; 560 } 561 562 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data); 563 if (ret) 564 return ret; 565 566 if (data.op == 3) { 567 ret = amdgpu_reserve_page_direct(adev, data.inject.address); 568 if (!ret) 569 return size; 570 else 571 return ret; 572 } else if (data.op == 4) { 573 ret = amdgpu_check_address_validity(adev, data.inject.address, data.inject.value); 574 return ret ? ret : size; 575 } 576 577 if (!amdgpu_ras_is_supported(adev, data.head.block)) 578 return -EINVAL; 579 580 switch (data.op) { 581 case 0: 582 ret = amdgpu_ras_feature_enable(adev, &data.head, 0); 583 break; 584 case 1: 585 ret = amdgpu_ras_feature_enable(adev, &data.head, 1); 586 break; 587 case 2: 588 /* umc ce/ue error injection for a bad page is not allowed */ 589 if (data.head.block == AMDGPU_RAS_BLOCK__UMC) 590 ret = amdgpu_ras_check_bad_page(adev, data.inject.address); 591 if (ret == -EINVAL) { 592 dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.", 593 data.inject.address); 594 break; 595 } else if (ret == 1) { 596 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n", 597 data.inject.address); 598 break; 599 } 600 601 amdgpu_ras_instance_mask_check(adev, &data); 602 603 /* data.inject.address is offset instead of absolute gpu address */ 604 ret = amdgpu_ras_error_inject(adev, &data.inject); 605 break; 606 default: 607 ret = -EINVAL; 608 break; 609 } 610 611 if (ret) 612 return ret; 613 614 return size; 615} 616 617static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev); 618 619/** 620 * DOC: AMDGPU RAS debugfs EEPROM table reset interface 621 * 622 * Some boards contain an EEPROM which is used to persistently store a list of 623 * bad pages which experiences ECC errors in vram. This interface provides 624 * a way to reset the EEPROM, e.g., after testing error injection. 625 * 626 * Usage: 627 * 628 * .. code-block:: bash 629 * 630 * echo 1 > ../ras/ras_eeprom_reset 631 * 632 * will reset EEPROM table to 0 entries. 633 * 634 */ 635static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f, 636 const char __user *buf, 637 size_t size, loff_t *pos) 638{ 639 struct amdgpu_device *adev = 640 (struct amdgpu_device *)file_inode(f)->i_private; 641 int ret; 642 643 if (amdgpu_uniras_enabled(adev)) { 644 ret = amdgpu_uniras_clear_badpages_info(adev); 645 return ret ? ret : size; 646 } 647 648 ret = amdgpu_ras_eeprom_reset_table( 649 &(amdgpu_ras_get_context(adev)->eeprom_control)); 650 651 if (!ret) { 652 /* Something was written to EEPROM. 653 */ 654 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS; 655 return size; 656 } else { 657 return ret; 658 } 659} 660 661static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = { 662 .owner = THIS_MODULE, 663 .read = NULL, 664 .write = amdgpu_ras_debugfs_ctrl_write, 665 .llseek = default_llseek 666}; 667 668static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = { 669 .owner = THIS_MODULE, 670 .read = NULL, 671 .write = amdgpu_ras_debugfs_eeprom_write, 672 .llseek = default_llseek 673}; 674 675/** 676 * DOC: AMDGPU RAS sysfs Error Count Interface 677 * 678 * It allows the user to read the error count for each IP block on the gpu through 679 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count 680 * 681 * It outputs the multiple lines which report the uncorrected (ue) and corrected 682 * (ce) error counts. 683 * 684 * The format of one line is below, 685 * 686 * [ce|ue]: count 687 * 688 * Example: 689 * 690 * .. code-block:: bash 691 * 692 * ue: 0 693 * ce: 1 694 * 695 */ 696static ssize_t amdgpu_ras_sysfs_read(struct device *dev, 697 struct device_attribute *attr, char *buf) 698{ 699 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr); 700 struct ras_query_if info = { 701 .head = obj->head, 702 }; 703 704 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 705 return sysfs_emit(buf, "Query currently inaccessible\n"); 706 707 if (amdgpu_ras_query_error_status(obj->adev, &info)) 708 return -EINVAL; 709 710 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 711 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 712 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block)) 713 dev_warn(obj->adev->dev, "Failed to reset error counter and error status"); 714 } 715 716 if (info.head.block == AMDGPU_RAS_BLOCK__UMC) 717 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 718 "ce", info.ce_count, "de", info.de_count); 719 else 720 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count, 721 "ce", info.ce_count); 722} 723 724/* obj begin */ 725 726#define get_obj(obj) do { (obj)->use++; } while (0) 727#define alive_obj(obj) ((obj)->use) 728 729static inline void put_obj(struct ras_manager *obj) 730{ 731 if (obj && (--obj->use == 0)) { 732 list_del(&obj->node); 733 amdgpu_ras_error_data_fini(&obj->err_data); 734 } 735 736 if (obj && (obj->use < 0)) 737 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head)); 738} 739 740/* make one obj and return it. */ 741static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev, 742 struct ras_common_if *head) 743{ 744 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 745 struct ras_manager *obj; 746 747 if (!adev->ras_enabled || !con) 748 return NULL; 749 750 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 751 return NULL; 752 753 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 754 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 755 return NULL; 756 757 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 758 } else 759 obj = &con->objs[head->block]; 760 761 /* already exist. return obj? */ 762 if (alive_obj(obj)) 763 return NULL; 764 765 if (amdgpu_ras_error_data_init(&obj->err_data)) 766 return NULL; 767 768 obj->head = *head; 769 obj->adev = adev; 770 list_add(&obj->node, &con->head); 771 get_obj(obj); 772 773 return obj; 774} 775 776/* return an obj equal to head, or the first when head is NULL */ 777struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev, 778 struct ras_common_if *head) 779{ 780 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 781 struct ras_manager *obj; 782 int i; 783 784 if (!adev->ras_enabled || !con) 785 return NULL; 786 787 if (head) { 788 if (head->block >= AMDGPU_RAS_BLOCK_COUNT) 789 return NULL; 790 791 if (head->block == AMDGPU_RAS_BLOCK__MCA) { 792 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST) 793 return NULL; 794 795 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index]; 796 } else 797 obj = &con->objs[head->block]; 798 799 if (alive_obj(obj)) 800 return obj; 801 } else { 802 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 803 obj = &con->objs[i]; 804 if (alive_obj(obj)) 805 return obj; 806 } 807 } 808 809 return NULL; 810} 811/* obj end */ 812 813/* feature ctl begin */ 814static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev, 815 struct ras_common_if *head) 816{ 817 return adev->ras_hw_enabled & BIT(head->block); 818} 819 820static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev, 821 struct ras_common_if *head) 822{ 823 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 824 825 return con->features & BIT(head->block); 826} 827 828/* 829 * if obj is not created, then create one. 830 * set feature enable flag. 831 */ 832static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, 833 struct ras_common_if *head, int enable) 834{ 835 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 836 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 837 838 /* If hardware does not support ras, then do not create obj. 839 * But if hardware support ras, we can create the obj. 840 * Ras framework checks con->hw_supported to see if it need do 841 * corresponding initialization. 842 * IP checks con->support to see if it need disable ras. 843 */ 844 if (!amdgpu_ras_is_feature_allowed(adev, head)) 845 return 0; 846 847 if (enable) { 848 if (!obj) { 849 obj = amdgpu_ras_create_obj(adev, head); 850 if (!obj) 851 return -EINVAL; 852 } else { 853 /* In case we create obj somewhere else */ 854 get_obj(obj); 855 } 856 con->features |= BIT(head->block); 857 } else { 858 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) { 859 con->features &= ~BIT(head->block); 860 put_obj(obj); 861 } 862 } 863 864 return 0; 865} 866 867/* wrapper of psp_ras_enable_features */ 868int amdgpu_ras_feature_enable(struct amdgpu_device *adev, 869 struct ras_common_if *head, bool enable) 870{ 871 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 872 union ta_ras_cmd_input *info; 873 int ret; 874 875 if (!con) 876 return -EINVAL; 877 878 /* For non-gfx ip, do not enable ras feature if it is not allowed */ 879 /* For gfx ip, regardless of feature support status, */ 880 /* Force issue enable or disable ras feature commands */ 881 if (head->block != AMDGPU_RAS_BLOCK__GFX && 882 !amdgpu_ras_is_feature_allowed(adev, head)) 883 return 0; 884 885 /* Only enable gfx ras feature from host side */ 886 if (head->block == AMDGPU_RAS_BLOCK__GFX && 887 !amdgpu_sriov_vf(adev) && 888 !amdgpu_ras_intr_triggered()) { 889 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL); 890 if (!info) 891 return -ENOMEM; 892 893 if (!enable) { 894 info->disable_features = (struct ta_ras_disable_features_input) { 895 .block_id = amdgpu_ras_block_to_ta(head->block), 896 .error_type = amdgpu_ras_error_to_ta(head->type), 897 }; 898 } else { 899 info->enable_features = (struct ta_ras_enable_features_input) { 900 .block_id = amdgpu_ras_block_to_ta(head->block), 901 .error_type = amdgpu_ras_error_to_ta(head->type), 902 }; 903 } 904 905 ret = psp_ras_enable_features(&adev->psp, info, enable); 906 if (ret) { 907 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n", 908 enable ? "enable":"disable", 909 get_ras_block_str(head), 910 amdgpu_ras_is_poison_mode_supported(adev), ret); 911 kfree(info); 912 return ret; 913 } 914 915 kfree(info); 916 } 917 918 /* setup the obj */ 919 __amdgpu_ras_feature_enable(adev, head, enable); 920 921 return 0; 922} 923 924/* Only used in device probe stage and called only once. */ 925int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, 926 struct ras_common_if *head, bool enable) 927{ 928 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 929 int ret; 930 931 if (!con) 932 return -EINVAL; 933 934 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 935 if (enable) { 936 /* There is no harm to issue a ras TA cmd regardless of 937 * the currecnt ras state. 938 * If current state == target state, it will do nothing 939 * But sometimes it requests driver to reset and repost 940 * with error code -EAGAIN. 941 */ 942 ret = amdgpu_ras_feature_enable(adev, head, 1); 943 /* With old ras TA, we might fail to enable ras. 944 * Log it and just setup the object. 945 * TODO need remove this WA in the future. 946 */ 947 if (ret == -EINVAL) { 948 ret = __amdgpu_ras_feature_enable(adev, head, 1); 949 if (!ret) 950 dev_info(adev->dev, 951 "RAS INFO: %s setup object\n", 952 get_ras_block_str(head)); 953 } 954 } else { 955 /* setup the object then issue a ras TA disable cmd.*/ 956 ret = __amdgpu_ras_feature_enable(adev, head, 1); 957 if (ret) 958 return ret; 959 960 /* gfx block ras disable cmd must send to ras-ta */ 961 if (head->block == AMDGPU_RAS_BLOCK__GFX) 962 con->features |= BIT(head->block); 963 964 ret = amdgpu_ras_feature_enable(adev, head, 0); 965 966 /* clean gfx block ras features flag */ 967 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX) 968 con->features &= ~BIT(head->block); 969 } 970 } else 971 ret = amdgpu_ras_feature_enable(adev, head, enable); 972 973 return ret; 974} 975 976static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev, 977 bool bypass) 978{ 979 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 980 struct ras_manager *obj, *tmp; 981 982 list_for_each_entry_safe(obj, tmp, &con->head, node) { 983 /* bypass psp. 984 * aka just release the obj and corresponding flags 985 */ 986 if (bypass) { 987 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0)) 988 break; 989 } else { 990 if (amdgpu_ras_feature_enable(adev, &obj->head, 0)) 991 break; 992 } 993 } 994 995 return con->features; 996} 997 998static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev, 999 bool bypass) 1000{ 1001 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1002 int i; 1003 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE; 1004 1005 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) { 1006 struct ras_common_if head = { 1007 .block = i, 1008 .type = default_ras_type, 1009 .sub_block_index = 0, 1010 }; 1011 1012 if (i == AMDGPU_RAS_BLOCK__MCA) 1013 continue; 1014 1015 if (bypass) { 1016 /* 1017 * bypass psp. vbios enable ras for us. 1018 * so just create the obj 1019 */ 1020 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1021 break; 1022 } else { 1023 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1024 break; 1025 } 1026 } 1027 1028 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) { 1029 struct ras_common_if head = { 1030 .block = AMDGPU_RAS_BLOCK__MCA, 1031 .type = default_ras_type, 1032 .sub_block_index = i, 1033 }; 1034 1035 if (bypass) { 1036 /* 1037 * bypass psp. vbios enable ras for us. 1038 * so just create the obj 1039 */ 1040 if (__amdgpu_ras_feature_enable(adev, &head, 1)) 1041 break; 1042 } else { 1043 if (amdgpu_ras_feature_enable(adev, &head, 1)) 1044 break; 1045 } 1046 } 1047 1048 return con->features; 1049} 1050/* feature ctl end */ 1051 1052static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj, 1053 enum amdgpu_ras_block block) 1054{ 1055 if (!block_obj) 1056 return -EINVAL; 1057 1058 if (block_obj->ras_comm.block == block) 1059 return 0; 1060 1061 return -EINVAL; 1062} 1063 1064static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev, 1065 enum amdgpu_ras_block block, uint32_t sub_block_index) 1066{ 1067 struct amdgpu_ras_block_list *node, *tmp; 1068 struct amdgpu_ras_block_object *obj; 1069 1070 if (block >= AMDGPU_RAS_BLOCK__LAST) 1071 return NULL; 1072 1073 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 1074 if (!node->ras_obj) { 1075 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 1076 continue; 1077 } 1078 1079 obj = node->ras_obj; 1080 if (obj->ras_block_match) { 1081 if (obj->ras_block_match(obj, block, sub_block_index) == 0) 1082 return obj; 1083 } else { 1084 if (amdgpu_ras_block_match_default(obj, block) == 0) 1085 return obj; 1086 } 1087 } 1088 1089 return NULL; 1090} 1091 1092static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data) 1093{ 1094 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 1095 int ret = 0; 1096 1097 /* 1098 * choosing right query method according to 1099 * whether smu support query error information 1100 */ 1101 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc)); 1102 if (ret == -EOPNOTSUPP) { 1103 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1104 adev->umc.ras->ras_block.hw_ops->query_ras_error_count) 1105 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data); 1106 1107 /* umc query_ras_error_address is also responsible for clearing 1108 * error status 1109 */ 1110 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && 1111 adev->umc.ras->ras_block.hw_ops->query_ras_error_address) 1112 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data); 1113 } else if (!ret) { 1114 if (adev->umc.ras && 1115 adev->umc.ras->ecc_info_query_ras_error_count) 1116 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data); 1117 1118 if (adev->umc.ras && 1119 adev->umc.ras->ecc_info_query_ras_error_address) 1120 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data); 1121 } 1122} 1123 1124static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev, 1125 struct ras_manager *ras_mgr, 1126 struct ras_err_data *err_data, 1127 struct ras_query_context *qctx, 1128 const char *blk_name, 1129 bool is_ue, 1130 bool is_de) 1131{ 1132 struct amdgpu_smuio_mcm_config_info *mcm_info; 1133 struct ras_err_node *err_node; 1134 struct ras_err_info *err_info; 1135 u64 event_id = qctx->evid.event_id; 1136 1137 if (is_ue) { 1138 for_each_ras_error(err_node, err_data) { 1139 err_info = &err_node->err_info; 1140 mcm_info = &err_info->mcm_info; 1141 if (err_info->ue_count) { 1142 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1143 "%lld new uncorrectable hardware errors detected in %s block\n", 1144 mcm_info->socket_id, 1145 mcm_info->die_id, 1146 err_info->ue_count, 1147 blk_name); 1148 } 1149 } 1150 1151 for_each_ras_error(err_node, &ras_mgr->err_data) { 1152 err_info = &err_node->err_info; 1153 mcm_info = &err_info->mcm_info; 1154 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1155 "%lld uncorrectable hardware errors detected in total in %s block\n", 1156 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name); 1157 } 1158 1159 } else { 1160 if (is_de) { 1161 for_each_ras_error(err_node, err_data) { 1162 err_info = &err_node->err_info; 1163 mcm_info = &err_info->mcm_info; 1164 if (err_info->de_count) { 1165 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1166 "%lld new deferred hardware errors detected in %s block\n", 1167 mcm_info->socket_id, 1168 mcm_info->die_id, 1169 err_info->de_count, 1170 blk_name); 1171 } 1172 } 1173 1174 for_each_ras_error(err_node, &ras_mgr->err_data) { 1175 err_info = &err_node->err_info; 1176 mcm_info = &err_info->mcm_info; 1177 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1178 "%lld deferred hardware errors detected in total in %s block\n", 1179 mcm_info->socket_id, mcm_info->die_id, 1180 err_info->de_count, blk_name); 1181 } 1182 } else { 1183 if (adev->debug_disable_ce_logs) 1184 return; 1185 1186 for_each_ras_error(err_node, err_data) { 1187 err_info = &err_node->err_info; 1188 mcm_info = &err_info->mcm_info; 1189 if (err_info->ce_count) { 1190 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1191 "%lld new correctable hardware errors detected in %s block\n", 1192 mcm_info->socket_id, 1193 mcm_info->die_id, 1194 err_info->ce_count, 1195 blk_name); 1196 } 1197 } 1198 1199 for_each_ras_error(err_node, &ras_mgr->err_data) { 1200 err_info = &err_node->err_info; 1201 mcm_info = &err_info->mcm_info; 1202 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, " 1203 "%lld correctable hardware errors detected in total in %s block\n", 1204 mcm_info->socket_id, mcm_info->die_id, 1205 err_info->ce_count, blk_name); 1206 } 1207 } 1208 } 1209} 1210 1211static inline bool err_data_has_source_info(struct ras_err_data *data) 1212{ 1213 return !list_empty(&data->err_node_list); 1214} 1215 1216static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev, 1217 struct ras_query_if *query_if, 1218 struct ras_err_data *err_data, 1219 struct ras_query_context *qctx) 1220{ 1221 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head); 1222 const char *blk_name = get_ras_block_str(&query_if->head); 1223 u64 event_id = qctx->evid.event_id; 1224 1225 if (err_data->ce_count) { 1226 if (err_data_has_source_info(err_data)) { 1227 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1228 blk_name, false, false); 1229 } else if (!adev->aid_mask && 1230 adev->smuio.funcs && 1231 adev->smuio.funcs->get_socket_id && 1232 adev->smuio.funcs->get_die_id) { 1233 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1234 "%ld correctable hardware errors " 1235 "detected in %s block\n", 1236 adev->smuio.funcs->get_socket_id(adev), 1237 adev->smuio.funcs->get_die_id(adev), 1238 ras_mgr->err_data.ce_count, 1239 blk_name); 1240 } else { 1241 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors " 1242 "detected in %s block\n", 1243 ras_mgr->err_data.ce_count, 1244 blk_name); 1245 } 1246 } 1247 1248 if (err_data->ue_count) { 1249 if (err_data_has_source_info(err_data)) { 1250 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1251 blk_name, true, false); 1252 } else if (!adev->aid_mask && 1253 adev->smuio.funcs && 1254 adev->smuio.funcs->get_socket_id && 1255 adev->smuio.funcs->get_die_id) { 1256 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1257 "%ld uncorrectable hardware errors " 1258 "detected in %s block\n", 1259 adev->smuio.funcs->get_socket_id(adev), 1260 adev->smuio.funcs->get_die_id(adev), 1261 ras_mgr->err_data.ue_count, 1262 blk_name); 1263 } else { 1264 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors " 1265 "detected in %s block\n", 1266 ras_mgr->err_data.ue_count, 1267 blk_name); 1268 } 1269 } 1270 1271 if (err_data->de_count) { 1272 if (err_data_has_source_info(err_data)) { 1273 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, 1274 blk_name, false, true); 1275 } else if (!adev->aid_mask && 1276 adev->smuio.funcs && 1277 adev->smuio.funcs->get_socket_id && 1278 adev->smuio.funcs->get_die_id) { 1279 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d " 1280 "%ld deferred hardware errors " 1281 "detected in %s block\n", 1282 adev->smuio.funcs->get_socket_id(adev), 1283 adev->smuio.funcs->get_die_id(adev), 1284 ras_mgr->err_data.de_count, 1285 blk_name); 1286 } else { 1287 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors " 1288 "detected in %s block\n", 1289 ras_mgr->err_data.de_count, 1290 blk_name); 1291 } 1292 } 1293} 1294 1295static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev, 1296 struct ras_query_if *query_if, 1297 struct ras_err_data *err_data, 1298 struct ras_query_context *qctx) 1299{ 1300 unsigned long new_ue, new_ce, new_de; 1301 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head); 1302 const char *blk_name = get_ras_block_str(&query_if->head); 1303 u64 event_id = qctx->evid.event_id; 1304 1305 new_ce = err_data->ce_count - obj->err_data.ce_count; 1306 new_ue = err_data->ue_count - obj->err_data.ue_count; 1307 new_de = err_data->de_count - obj->err_data.de_count; 1308 1309 if (new_ce) { 1310 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors " 1311 "detected in %s block\n", 1312 new_ce, 1313 blk_name); 1314 } 1315 1316 if (new_ue) { 1317 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors " 1318 "detected in %s block\n", 1319 new_ue, 1320 blk_name); 1321 } 1322 1323 if (new_de) { 1324 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors " 1325 "detected in %s block\n", 1326 new_de, 1327 blk_name); 1328 } 1329} 1330 1331static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data) 1332{ 1333 struct ras_err_node *err_node; 1334 struct ras_err_info *err_info; 1335 1336 if (err_data_has_source_info(err_data)) { 1337 for_each_ras_error(err_node, err_data) { 1338 err_info = &err_node->err_info; 1339 amdgpu_ras_error_statistic_de_count(&obj->err_data, 1340 &err_info->mcm_info, err_info->de_count); 1341 amdgpu_ras_error_statistic_ce_count(&obj->err_data, 1342 &err_info->mcm_info, err_info->ce_count); 1343 amdgpu_ras_error_statistic_ue_count(&obj->err_data, 1344 &err_info->mcm_info, err_info->ue_count); 1345 } 1346 } else { 1347 /* for legacy asic path which doesn't has error source info */ 1348 obj->err_data.ue_count += err_data->ue_count; 1349 obj->err_data.ce_count += err_data->ce_count; 1350 obj->err_data.de_count += err_data->de_count; 1351 } 1352} 1353 1354static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj, 1355 struct ras_err_data *err_data) 1356{ 1357 /* Host reports absolute counts */ 1358 obj->err_data.ue_count = err_data->ue_count; 1359 obj->err_data.ce_count = err_data->ce_count; 1360 obj->err_data.de_count = err_data->de_count; 1361} 1362 1363static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1364{ 1365 struct ras_common_if head; 1366 1367 memset(&head, 0, sizeof(head)); 1368 head.block = blk; 1369 1370 return amdgpu_ras_find_obj(adev, &head); 1371} 1372 1373int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1374 const struct aca_info *aca_info, void *data) 1375{ 1376 struct ras_manager *obj; 1377 1378 /* in resume phase, no need to create aca fs node */ 1379 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 1380 return 0; 1381 1382 obj = get_ras_manager(adev, blk); 1383 if (!obj) 1384 return -EINVAL; 1385 1386 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); 1387} 1388 1389int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) 1390{ 1391 struct ras_manager *obj; 1392 1393 obj = get_ras_manager(adev, blk); 1394 if (!obj) 1395 return -EINVAL; 1396 1397 amdgpu_aca_remove_handle(&obj->aca_handle); 1398 1399 return 0; 1400} 1401 1402static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, 1403 enum aca_error_type type, struct ras_err_data *err_data, 1404 struct ras_query_context *qctx) 1405{ 1406 struct ras_manager *obj; 1407 1408 obj = get_ras_manager(adev, blk); 1409 if (!obj) 1410 return -EINVAL; 1411 1412 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); 1413} 1414 1415ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, 1416 struct aca_handle *handle, char *buf, void *data) 1417{ 1418 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); 1419 struct ras_query_if info = { 1420 .head = obj->head, 1421 }; 1422 1423 if (!amdgpu_ras_get_error_query_ready(obj->adev)) 1424 return sysfs_emit(buf, "Query currently inaccessible\n"); 1425 1426 if (amdgpu_ras_query_error_status(obj->adev, &info)) 1427 return -EINVAL; 1428 1429 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, 1430 "ce", info.ce_count, "de", info.de_count); 1431} 1432 1433static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, 1434 struct ras_query_if *info, 1435 struct ras_err_data *err_data, 1436 struct ras_query_context *qctx, 1437 unsigned int error_query_mode) 1438{ 1439 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; 1440 struct amdgpu_ras_block_object *block_obj = NULL; 1441 int ret; 1442 1443 if (blk == AMDGPU_RAS_BLOCK_COUNT) 1444 return -EINVAL; 1445 1446 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY) 1447 return -EINVAL; 1448 1449 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1450 return amdgpu_virt_req_ras_err_count(adev, blk, err_data); 1451 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { 1452 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { 1453 amdgpu_ras_get_ecc_info(adev, err_data); 1454 } else { 1455 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0); 1456 if (!block_obj || !block_obj->hw_ops) { 1457 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1458 get_ras_block_str(&info->head)); 1459 return -EINVAL; 1460 } 1461 1462 if (block_obj->hw_ops->query_ras_error_count) 1463 block_obj->hw_ops->query_ras_error_count(adev, err_data); 1464 1465 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) || 1466 (info->head.block == AMDGPU_RAS_BLOCK__GFX) || 1467 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) { 1468 if (block_obj->hw_ops->query_ras_error_status) 1469 block_obj->hw_ops->query_ras_error_status(adev); 1470 } 1471 } 1472 } else { 1473 if (amdgpu_aca_is_enabled(adev)) { 1474 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); 1475 if (ret) 1476 return ret; 1477 1478 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); 1479 if (ret) 1480 return ret; 1481 1482 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); 1483 if (ret) 1484 return ret; 1485 } else { 1486 /* FIXME: add code to check return value later */ 1487 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); 1488 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); 1489 } 1490 } 1491 1492 return 0; 1493} 1494 1495/* query/inject/cure begin */ 1496static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev, 1497 struct ras_query_if *info, 1498 enum ras_event_type type) 1499{ 1500 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1501 struct ras_err_data err_data; 1502 struct ras_query_context qctx; 1503 unsigned int error_query_mode; 1504 int ret; 1505 1506 if (!obj) 1507 return -EINVAL; 1508 1509 ret = amdgpu_ras_error_data_init(&err_data); 1510 if (ret) 1511 return ret; 1512 1513 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) 1514 return -EINVAL; 1515 1516 memset(&qctx, 0, sizeof(qctx)); 1517 qctx.evid.type = type; 1518 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type); 1519 1520 if (!down_read_trylock(&adev->reset_domain->sem)) { 1521 ret = -EIO; 1522 goto out_fini_err_data; 1523 } 1524 1525 ret = amdgpu_ras_query_error_status_helper(adev, info, 1526 &err_data, 1527 &qctx, 1528 error_query_mode); 1529 up_read(&adev->reset_domain->sem); 1530 if (ret) 1531 goto out_fini_err_data; 1532 1533 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { 1534 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data); 1535 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx); 1536 } else { 1537 /* Host provides absolute error counts. First generate the report 1538 * using the previous VF internal count against new host count. 1539 * Then Update VF internal count. 1540 */ 1541 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx); 1542 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data); 1543 } 1544 1545 info->ue_count = obj->err_data.ue_count; 1546 info->ce_count = obj->err_data.ce_count; 1547 info->de_count = obj->err_data.de_count; 1548 1549out_fini_err_data: 1550 amdgpu_ras_error_data_fini(&err_data); 1551 1552 return ret; 1553} 1554 1555static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev) 1556{ 1557 struct ras_cmd_dev_handle req = {0}; 1558 int ret; 1559 1560 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CLEAR_BAD_PAGE_INFO, 1561 &req, sizeof(req), NULL, 0); 1562 if (ret) { 1563 dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret); 1564 return ret; 1565 } 1566 1567 return 0; 1568} 1569 1570static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev, 1571 struct ras_query_if *info) 1572{ 1573 struct ras_cmd_block_ecc_info_req req = {0}; 1574 struct ras_cmd_block_ecc_info_rsp rsp = {0}; 1575 int ret; 1576 1577 if (!info) 1578 return -EINVAL; 1579 1580 req.block_id = info->head.block; 1581 req.subblock_id = info->head.sub_block_index; 1582 1583 ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BLOCK_ECC_STATUS, 1584 &req, sizeof(req), &rsp, sizeof(rsp)); 1585 if (!ret) { 1586 info->ce_count = rsp.ce_count; 1587 info->ue_count = rsp.ue_count; 1588 info->de_count = rsp.de_count; 1589 } 1590 1591 return ret; 1592} 1593 1594int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info) 1595{ 1596 if (amdgpu_uniras_enabled(adev)) 1597 return amdgpu_uniras_query_block_ecc(adev, info); 1598 else 1599 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID); 1600} 1601 1602int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, 1603 enum amdgpu_ras_block block) 1604{ 1605 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1606 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 1607 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 1608 1609 if (!block_obj || !block_obj->hw_ops) { 1610 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1611 ras_block_str(block)); 1612 return -EOPNOTSUPP; 1613 } 1614 1615 if (!amdgpu_ras_is_supported(adev, block) || 1616 !amdgpu_ras_get_aca_debug_mode(adev)) 1617 return -EOPNOTSUPP; 1618 1619 if (amdgpu_sriov_vf(adev)) 1620 return -EOPNOTSUPP; 1621 1622 /* skip ras error reset in gpu reset */ 1623 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && 1624 ((smu_funcs && smu_funcs->set_debug_mode) || 1625 (mca_funcs && mca_funcs->mca_set_debug_mode))) 1626 return -EOPNOTSUPP; 1627 1628 if (block_obj->hw_ops->reset_ras_error_count) 1629 block_obj->hw_ops->reset_ras_error_count(adev); 1630 1631 return 0; 1632} 1633 1634int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, 1635 enum amdgpu_ras_block block) 1636{ 1637 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); 1638 1639 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP) 1640 return 0; 1641 1642 if ((block == AMDGPU_RAS_BLOCK__GFX) || 1643 (block == AMDGPU_RAS_BLOCK__MMHUB)) { 1644 if (block_obj->hw_ops->reset_ras_error_status) 1645 block_obj->hw_ops->reset_ras_error_status(adev); 1646 } 1647 1648 return 0; 1649} 1650 1651static int amdgpu_uniras_error_inject(struct amdgpu_device *adev, 1652 struct ras_inject_if *info) 1653{ 1654 struct ras_cmd_inject_error_req inject_req; 1655 struct ras_cmd_inject_error_rsp rsp; 1656 1657 if (!info) 1658 return -EINVAL; 1659 1660 memset(&inject_req, 0, sizeof(inject_req)); 1661 inject_req.block_id = info->head.block; 1662 inject_req.subblock_id = info->head.sub_block_index; 1663 inject_req.address = info->address; 1664 inject_req.error_type = info->head.type; 1665 inject_req.instance_mask = info->instance_mask; 1666 inject_req.method = info->value; 1667 1668 return amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__INJECT_ERROR, 1669 &inject_req, sizeof(inject_req), &rsp, sizeof(rsp)); 1670} 1671 1672/* wrapper of psp_ras_trigger_error */ 1673int amdgpu_ras_error_inject(struct amdgpu_device *adev, 1674 struct ras_inject_if *info) 1675{ 1676 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head); 1677 struct ta_ras_trigger_error_input block_info = { 1678 .block_id = amdgpu_ras_block_to_ta(info->head.block), 1679 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type), 1680 .sub_block_index = info->head.sub_block_index, 1681 .address = info->address, 1682 .value = info->value, 1683 }; 1684 int ret = -EINVAL; 1685 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, 1686 info->head.block, 1687 info->head.sub_block_index); 1688 1689 if (amdgpu_uniras_enabled(adev)) 1690 return amdgpu_uniras_error_inject(adev, info); 1691 1692 /* inject on guest isn't allowed, return success directly */ 1693 if (amdgpu_sriov_vf(adev)) 1694 return 0; 1695 1696 if (!obj) 1697 return -EINVAL; 1698 1699 if (!block_obj || !block_obj->hw_ops) { 1700 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 1701 get_ras_block_str(&info->head)); 1702 return -EINVAL; 1703 } 1704 1705 /* Calculate XGMI relative offset */ 1706 if (adev->gmc.xgmi.num_physical_nodes > 1 && 1707 info->head.block != AMDGPU_RAS_BLOCK__GFX) { 1708 block_info.address = 1709 amdgpu_xgmi_get_relative_phy_addr(adev, 1710 block_info.address); 1711 } 1712 1713 if (block_obj->hw_ops->ras_error_inject) { 1714 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) 1715 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask); 1716 else /* Special ras_error_inject is defined (e.g: xgmi) */ 1717 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info, 1718 info->instance_mask); 1719 } else { 1720 /* default path */ 1721 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask); 1722 } 1723 1724 if (ret) 1725 dev_err(adev->dev, "ras inject %s failed %d\n", 1726 get_ras_block_str(&info->head), ret); 1727 1728 return ret; 1729} 1730 1731/** 1732 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP 1733 * @adev: pointer to AMD GPU device 1734 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1735 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors. 1736 * @query_info: pointer to ras_query_if 1737 * 1738 * Return 0 for query success or do nothing, otherwise return an error 1739 * on failures 1740 */ 1741static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev, 1742 unsigned long *ce_count, 1743 unsigned long *ue_count, 1744 struct ras_query_if *query_info) 1745{ 1746 int ret; 1747 1748 if (!query_info) 1749 /* do nothing if query_info is not specified */ 1750 return 0; 1751 1752 ret = amdgpu_ras_query_error_status(adev, query_info); 1753 if (ret) 1754 return ret; 1755 1756 *ce_count += query_info->ce_count; 1757 *ue_count += query_info->ue_count; 1758 1759 /* some hardware/IP supports read to clear 1760 * no need to explictly reset the err status after the query call */ 1761 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) && 1762 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) { 1763 if (amdgpu_ras_reset_error_status(adev, query_info->head.block)) 1764 dev_warn(adev->dev, 1765 "Failed to reset error counter and error status\n"); 1766 } 1767 1768 return 0; 1769} 1770 1771/** 1772 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP 1773 * @adev: pointer to AMD GPU device 1774 * @ce_count: pointer to an integer to be set to the count of correctible errors. 1775 * @ue_count: pointer to an integer to be set to the count of uncorrectible 1776 * errors. 1777 * @query_info: pointer to ras_query_if if the query request is only for 1778 * specific ip block; if info is NULL, then the qurey request is for 1779 * all the ip blocks that support query ras error counters/status 1780 * 1781 * If set, @ce_count or @ue_count, count and return the corresponding 1782 * error counts in those integer pointers. Return 0 if the device 1783 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS. 1784 */ 1785int amdgpu_ras_query_error_count(struct amdgpu_device *adev, 1786 unsigned long *ce_count, 1787 unsigned long *ue_count, 1788 struct ras_query_if *query_info) 1789{ 1790 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1791 struct ras_manager *obj; 1792 unsigned long ce, ue; 1793 int ret; 1794 1795 if (!adev->ras_enabled || !con) 1796 return -EOPNOTSUPP; 1797 1798 /* Don't count since no reporting. 1799 */ 1800 if (!ce_count && !ue_count) 1801 return 0; 1802 1803 ce = 0; 1804 ue = 0; 1805 if (!query_info) { 1806 /* query all the ip blocks that support ras query interface */ 1807 list_for_each_entry(obj, &con->head, node) { 1808 struct ras_query_if info = { 1809 .head = obj->head, 1810 }; 1811 1812 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info); 1813 } 1814 } else { 1815 /* query specific ip block */ 1816 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info); 1817 } 1818 1819 if (ret) 1820 return ret; 1821 1822 if (ce_count) 1823 *ce_count = ce; 1824 1825 if (ue_count) 1826 *ue_count = ue; 1827 1828 return 0; 1829} 1830/* query/inject/cure end */ 1831 1832 1833/* sysfs begin */ 1834 1835static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 1836 struct ras_badpage *bps, uint32_t count, uint32_t start); 1837static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 1838 struct ras_badpage *bps, uint32_t count, uint32_t start); 1839 1840static char *amdgpu_ras_badpage_flags_str(unsigned int flags) 1841{ 1842 switch (flags) { 1843 case AMDGPU_RAS_RETIRE_PAGE_RESERVED: 1844 return "R"; 1845 case AMDGPU_RAS_RETIRE_PAGE_PENDING: 1846 return "P"; 1847 case AMDGPU_RAS_RETIRE_PAGE_FAULT: 1848 default: 1849 return "F"; 1850 } 1851} 1852 1853/** 1854 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface 1855 * 1856 * It allows user to read the bad pages of vram on the gpu through 1857 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages 1858 * 1859 * It outputs multiple lines, and each line stands for one gpu page. 1860 * 1861 * The format of one line is below, 1862 * gpu pfn : gpu page size : flags 1863 * 1864 * gpu pfn and gpu page size are printed in hex format. 1865 * flags can be one of below character, 1866 * 1867 * R: reserved, this gpu page is reserved and not able to use. 1868 * 1869 * P: pending for reserve, this gpu page is marked as bad, will be reserved 1870 * in next window of page_reserve. 1871 * 1872 * F: unable to reserve. this gpu page can't be reserved due to some reasons. 1873 * 1874 * Examples: 1875 * 1876 * .. code-block:: bash 1877 * 1878 * 0x00000001 : 0x00001000 : R 1879 * 0x00000002 : 0x00001000 : P 1880 * 1881 */ 1882 1883static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f, 1884 struct kobject *kobj, const struct bin_attribute *attr, 1885 char *buf, loff_t ppos, size_t count) 1886{ 1887 struct amdgpu_ras *con = 1888 container_of(attr, struct amdgpu_ras, badpages_attr); 1889 struct amdgpu_device *adev = con->adev; 1890 const unsigned int element_size = 1891 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1; 1892 unsigned int start = div64_ul(ppos + element_size - 1, element_size); 1893 unsigned int end = div64_ul(ppos + count - 1, element_size); 1894 ssize_t s = 0; 1895 struct ras_badpage *bps = NULL; 1896 int bps_count = 0, i, status; 1897 uint64_t address; 1898 1899 memset(buf, 0, count); 1900 1901 bps_count = end - start; 1902 bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL); 1903 if (!bps) 1904 return 0; 1905 1906 memset(bps, 0, sizeof(*bps) * bps_count); 1907 1908 if (amdgpu_uniras_enabled(adev)) 1909 bps_count = amdgpu_uniras_badpages_read(adev, bps, bps_count, start); 1910 else 1911 bps_count = amdgpu_ras_badpages_read(adev, bps, bps_count, start); 1912 1913 if (bps_count <= 0) { 1914 kfree(bps); 1915 return 0; 1916 } 1917 1918 for (i = 0; i < bps_count; i++) { 1919 address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT; 1920 if (amdgpu_ras_check_critical_address(adev, address)) 1921 continue; 1922 1923 bps[i].size = AMDGPU_GPU_PAGE_SIZE; 1924 1925 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr, 1926 address); 1927 if (status == -EBUSY) 1928 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING; 1929 else if (status == -ENOENT) 1930 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT; 1931 else 1932 bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED; 1933 1934 s += scnprintf(&buf[s], element_size + 1, 1935 "0x%08x : 0x%08x : %1s\n", 1936 bps[i].bp, 1937 bps[i].size, 1938 amdgpu_ras_badpage_flags_str(bps[i].flags)); 1939 } 1940 1941 kfree(bps); 1942 1943 return s; 1944} 1945 1946static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev, 1947 struct device_attribute *attr, char *buf) 1948{ 1949 struct amdgpu_ras *con = 1950 container_of(attr, struct amdgpu_ras, features_attr); 1951 1952 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features); 1953} 1954 1955static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major, 1956 u32 *minor, u32 *rev) 1957{ 1958 int i; 1959 1960 if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev)) 1961 return false; 1962 1963 for (i = 0; i < adev->num_ip_blocks; i++) { 1964 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) { 1965 *major = adev->ip_blocks[i].version->major; 1966 *minor = adev->ip_blocks[i].version->minor; 1967 *rev = adev->ip_blocks[i].version->rev; 1968 return true; 1969 } 1970 } 1971 1972 return false; 1973} 1974 1975static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev, 1976 struct device_attribute *attr, char *buf) 1977{ 1978 struct amdgpu_ras *con = 1979 container_of(attr, struct amdgpu_ras, version_attr); 1980 u32 major, minor, rev; 1981 ssize_t size = 0; 1982 1983 size += sysfs_emit_at(buf, size, "table version: 0x%x\n", 1984 con->eeprom_control.tbl_hdr.version); 1985 1986 if (amdgpu_ras_get_version_info(con->adev, &major, &minor, &rev)) 1987 size += sysfs_emit_at(buf, size, "ras version: %u.%u.%u\n", 1988 major, minor, rev); 1989 1990 return size; 1991} 1992 1993static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev, 1994 struct device_attribute *attr, char *buf) 1995{ 1996 struct amdgpu_ras *con = 1997 container_of(attr, struct amdgpu_ras, schema_attr); 1998 return sysfs_emit(buf, "schema: 0x%x\n", con->schema); 1999} 2000 2001static struct { 2002 enum ras_event_type type; 2003 const char *name; 2004} dump_event[] = { 2005 {RAS_EVENT_TYPE_FATAL, "Fatal Error"}, 2006 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"}, 2007 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"}, 2008}; 2009 2010static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev, 2011 struct device_attribute *attr, char *buf) 2012{ 2013 struct amdgpu_ras *con = 2014 container_of(attr, struct amdgpu_ras, event_state_attr); 2015 struct ras_event_manager *event_mgr = con->event_mgr; 2016 struct ras_event_state *event_state; 2017 int i, size = 0; 2018 2019 if (!event_mgr) 2020 return -EINVAL; 2021 2022 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno)); 2023 for (i = 0; i < ARRAY_SIZE(dump_event); i++) { 2024 event_state = &event_mgr->event_state[dump_event[i].type]; 2025 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n", 2026 dump_event[i].name, 2027 atomic64_read(&event_state->count), 2028 event_state->last_seqno); 2029 } 2030 2031 return (ssize_t)size; 2032} 2033 2034static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev) 2035{ 2036 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2037 2038 if (adev->dev->kobj.sd) 2039 sysfs_remove_file_from_group(&adev->dev->kobj, 2040 &con->badpages_attr.attr, 2041 RAS_FS_NAME); 2042} 2043 2044static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev) 2045{ 2046 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2047 struct attribute *attrs[] = { 2048 &con->features_attr.attr, 2049 &con->version_attr.attr, 2050 &con->schema_attr.attr, 2051 &con->event_state_attr.attr, 2052 NULL 2053 }; 2054 struct attribute_group group = { 2055 .name = RAS_FS_NAME, 2056 .attrs = attrs, 2057 }; 2058 2059 if (adev->dev->kobj.sd) 2060 sysfs_remove_group(&adev->dev->kobj, &group); 2061 2062 return 0; 2063} 2064 2065int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, 2066 struct ras_common_if *head) 2067{ 2068 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2069 2070 if (amdgpu_aca_is_enabled(adev)) 2071 return 0; 2072 2073 if (!obj || obj->attr_inuse) 2074 return -EINVAL; 2075 2076 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block)) 2077 return 0; 2078 2079 get_obj(obj); 2080 2081 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name), 2082 "%s_err_count", head->name); 2083 2084 obj->sysfs_attr = (struct device_attribute){ 2085 .attr = { 2086 .name = obj->fs_data.sysfs_name, 2087 .mode = S_IRUGO, 2088 }, 2089 .show = amdgpu_ras_sysfs_read, 2090 }; 2091 sysfs_attr_init(&obj->sysfs_attr.attr); 2092 2093 if (sysfs_add_file_to_group(&adev->dev->kobj, 2094 &obj->sysfs_attr.attr, 2095 RAS_FS_NAME)) { 2096 put_obj(obj); 2097 return -EINVAL; 2098 } 2099 2100 obj->attr_inuse = 1; 2101 2102 return 0; 2103} 2104 2105int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, 2106 struct ras_common_if *head) 2107{ 2108 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2109 2110 if (amdgpu_aca_is_enabled(adev)) 2111 return 0; 2112 2113 if (!obj || !obj->attr_inuse) 2114 return -EINVAL; 2115 2116 if (adev->dev->kobj.sd) 2117 sysfs_remove_file_from_group(&adev->dev->kobj, 2118 &obj->sysfs_attr.attr, 2119 RAS_FS_NAME); 2120 obj->attr_inuse = 0; 2121 put_obj(obj); 2122 2123 return 0; 2124} 2125 2126static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) 2127{ 2128 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2129 struct ras_manager *obj, *tmp; 2130 2131 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2132 amdgpu_ras_sysfs_remove(adev, &obj->head); 2133 } 2134 2135 if (amdgpu_bad_page_threshold != 0) 2136 amdgpu_ras_sysfs_remove_bad_page_node(adev); 2137 2138 amdgpu_ras_sysfs_remove_dev_attr_node(adev); 2139 2140 return 0; 2141} 2142/* sysfs end */ 2143 2144/** 2145 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors 2146 * 2147 * Normally when there is an uncorrectable error, the driver will reset 2148 * the GPU to recover. However, in the event of an unrecoverable error, 2149 * the driver provides an interface to reboot the system automatically 2150 * in that event. 2151 * 2152 * The following file in debugfs provides that interface: 2153 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot 2154 * 2155 * Usage: 2156 * 2157 * .. code-block:: bash 2158 * 2159 * echo true > .../ras/auto_reboot 2160 * 2161 */ 2162/* debugfs begin */ 2163static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev) 2164{ 2165 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2166 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control; 2167 struct drm_minor *minor = adev_to_drm(adev)->primary; 2168 struct dentry *dir; 2169 2170 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root); 2171 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev, 2172 &amdgpu_ras_debugfs_ctrl_ops); 2173 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev, 2174 &amdgpu_ras_debugfs_eeprom_ops); 2175 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir, 2176 &con->bad_page_cnt_threshold); 2177 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs); 2178 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled); 2179 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled); 2180 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev, 2181 &amdgpu_ras_debugfs_eeprom_size_ops); 2182 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table", 2183 S_IRUGO, dir, adev, 2184 &amdgpu_ras_debugfs_eeprom_table_ops); 2185 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control); 2186 2187 /* 2188 * After one uncorrectable error happens, usually GPU recovery will 2189 * be scheduled. But due to the known problem in GPU recovery failing 2190 * to bring GPU back, below interface provides one direct way to 2191 * user to reboot system automatically in such case within 2192 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine 2193 * will never be called. 2194 */ 2195 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot); 2196 2197 /* 2198 * User could set this not to clean up hardware's error count register 2199 * of RAS IPs during ras recovery. 2200 */ 2201 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir, 2202 &con->disable_ras_err_cnt_harvest); 2203 return dir; 2204} 2205 2206static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, 2207 struct ras_fs_if *head, 2208 struct dentry *dir) 2209{ 2210 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head); 2211 2212 if (!obj || !dir) 2213 return; 2214 2215 get_obj(obj); 2216 2217 memcpy(obj->fs_data.debugfs_name, 2218 head->debugfs_name, 2219 sizeof(obj->fs_data.debugfs_name)); 2220 2221 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir, 2222 obj, &amdgpu_ras_debugfs_ops); 2223} 2224 2225static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) 2226{ 2227 bool ret; 2228 2229 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 2230 case IP_VERSION(13, 0, 6): 2231 case IP_VERSION(13, 0, 12): 2232 case IP_VERSION(13, 0, 14): 2233 ret = true; 2234 break; 2235 default: 2236 ret = false; 2237 break; 2238 } 2239 2240 return ret; 2241} 2242 2243void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) 2244{ 2245 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2246 struct dentry *dir; 2247 struct ras_manager *obj; 2248 struct ras_fs_if fs_info; 2249 2250 /* 2251 * it won't be called in resume path, no need to check 2252 * suspend and gpu reset status 2253 */ 2254 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con) 2255 return; 2256 2257 dir = amdgpu_ras_debugfs_create_ctrl_node(adev); 2258 2259 list_for_each_entry(obj, &con->head, node) { 2260 if (amdgpu_ras_is_supported(adev, obj->head.block) && 2261 (obj->attr_inuse == 1)) { 2262 sprintf(fs_info.debugfs_name, "%s_err_inject", 2263 get_ras_block_str(&obj->head)); 2264 fs_info.head = obj->head; 2265 amdgpu_ras_debugfs_create(adev, &fs_info, dir); 2266 } 2267 } 2268 2269 if (amdgpu_ras_aca_is_supported(adev)) { 2270 if (amdgpu_aca_is_enabled(adev)) 2271 amdgpu_aca_smu_debugfs_init(adev, dir); 2272 else 2273 amdgpu_mca_smu_debugfs_init(adev, dir); 2274 } 2275} 2276 2277/* debugfs end */ 2278 2279/* ras fs */ 2280static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO, 2281 amdgpu_ras_sysfs_badpages_read, NULL, 0); 2282static DEVICE_ATTR(features, S_IRUGO, 2283 amdgpu_ras_sysfs_features_read, NULL); 2284static DEVICE_ATTR(version, 0444, 2285 amdgpu_ras_sysfs_version_show, NULL); 2286static DEVICE_ATTR(schema, 0444, 2287 amdgpu_ras_sysfs_schema_show, NULL); 2288static DEVICE_ATTR(event_state, 0444, 2289 amdgpu_ras_sysfs_event_state_show, NULL); 2290static int amdgpu_ras_fs_init(struct amdgpu_device *adev) 2291{ 2292 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2293 struct attribute_group group = { 2294 .name = RAS_FS_NAME, 2295 }; 2296 struct attribute *attrs[] = { 2297 &con->features_attr.attr, 2298 &con->version_attr.attr, 2299 &con->schema_attr.attr, 2300 &con->event_state_attr.attr, 2301 NULL 2302 }; 2303 const struct bin_attribute *bin_attrs[] = { 2304 NULL, 2305 NULL, 2306 }; 2307 int r; 2308 2309 group.attrs = attrs; 2310 2311 /* add features entry */ 2312 con->features_attr = dev_attr_features; 2313 sysfs_attr_init(attrs[0]); 2314 2315 /* add version entry */ 2316 con->version_attr = dev_attr_version; 2317 sysfs_attr_init(attrs[1]); 2318 2319 /* add schema entry */ 2320 con->schema_attr = dev_attr_schema; 2321 sysfs_attr_init(attrs[2]); 2322 2323 /* add event_state entry */ 2324 con->event_state_attr = dev_attr_event_state; 2325 sysfs_attr_init(attrs[3]); 2326 2327 if (amdgpu_bad_page_threshold != 0) { 2328 /* add bad_page_features entry */ 2329 con->badpages_attr = bin_attr_gpu_vram_bad_pages; 2330 sysfs_bin_attr_init(&con->badpages_attr); 2331 bin_attrs[0] = &con->badpages_attr; 2332 group.bin_attrs = bin_attrs; 2333 } 2334 2335 r = sysfs_create_group(&adev->dev->kobj, &group); 2336 if (r) 2337 dev_err(adev->dev, "Failed to create RAS sysfs group!"); 2338 2339 return 0; 2340} 2341 2342static int amdgpu_ras_fs_fini(struct amdgpu_device *adev) 2343{ 2344 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2345 struct ras_manager *con_obj, *ip_obj, *tmp; 2346 2347 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2348 list_for_each_entry_safe(con_obj, tmp, &con->head, node) { 2349 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head); 2350 if (ip_obj) 2351 put_obj(ip_obj); 2352 } 2353 } 2354 2355 amdgpu_ras_sysfs_remove_all(adev); 2356 return 0; 2357} 2358/* ras fs end */ 2359 2360/* ih begin */ 2361 2362/* For the hardware that cannot enable bif ring for both ras_controller_irq 2363 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status 2364 * register to check whether the interrupt is triggered or not, and properly 2365 * ack the interrupt if it is there 2366 */ 2367void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev) 2368{ 2369 /* Fatal error events are handled on host side */ 2370 if (amdgpu_sriov_vf(adev)) 2371 return; 2372 /* 2373 * If the current interrupt is caused by a non-fatal RAS error, skip 2374 * check for fatal error. For fatal errors, FED status of all devices 2375 * in XGMI hive gets set when the first device gets fatal error 2376 * interrupt. The error gets propagated to other devices as well, so 2377 * make sure to ack the interrupt regardless of FED status. 2378 */ 2379 if (!amdgpu_ras_get_fed_status(adev) && 2380 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY)) 2381 return; 2382 2383 if (amdgpu_uniras_enabled(adev)) { 2384 amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL); 2385 return; 2386 } 2387 2388 if (adev->nbio.ras && 2389 adev->nbio.ras->handle_ras_controller_intr_no_bifring) 2390 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev); 2391 2392 if (adev->nbio.ras && 2393 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring) 2394 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev); 2395} 2396 2397static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj, 2398 struct amdgpu_iv_entry *entry) 2399{ 2400 bool poison_stat = false; 2401 struct amdgpu_device *adev = obj->adev; 2402 struct amdgpu_ras_block_object *block_obj = 2403 amdgpu_ras_get_ras_block(adev, obj->head.block, 0); 2404 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2405 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION; 2406 u64 event_id; 2407 int ret; 2408 2409 if (!block_obj || !con) 2410 return; 2411 2412 ret = amdgpu_ras_mark_ras_event(adev, type); 2413 if (ret) 2414 return; 2415 2416 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block); 2417 /* both query_poison_status and handle_poison_consumption are optional, 2418 * but at least one of them should be implemented if we need poison 2419 * consumption handler 2420 */ 2421 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) { 2422 poison_stat = block_obj->hw_ops->query_poison_status(adev); 2423 if (!poison_stat) { 2424 /* Not poison consumption interrupt, no need to handle it */ 2425 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n", 2426 block_obj->ras_comm.name); 2427 2428 return; 2429 } 2430 } 2431 2432 amdgpu_umc_poison_handler(adev, obj->head.block, 0); 2433 2434 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption) 2435 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); 2436 2437 /* gpu reset is fallback for failed and default cases. 2438 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset. 2439 */ 2440 if (poison_stat && !amdgpu_ras_is_rma(adev)) { 2441 event_id = amdgpu_ras_acquire_event_id(adev, type); 2442 RAS_EVENT_LOG(adev, event_id, 2443 "GPU reset for %s RAS poison consumption is issued!\n", 2444 block_obj->ras_comm.name); 2445 amdgpu_ras_reset_gpu(adev); 2446 } 2447 2448 if (!poison_stat) 2449 amdgpu_gfx_poison_consumption_handler(adev, entry); 2450} 2451 2452static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj, 2453 struct amdgpu_iv_entry *entry) 2454{ 2455 struct amdgpu_device *adev = obj->adev; 2456 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 2457 u64 event_id; 2458 int ret; 2459 2460 ret = amdgpu_ras_mark_ras_event(adev, type); 2461 if (ret) 2462 return; 2463 2464 event_id = amdgpu_ras_acquire_event_id(adev, type); 2465 RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); 2466 2467 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { 2468 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); 2469 2470 atomic_inc(&con->page_retirement_req_cnt); 2471 atomic_inc(&con->poison_creation_count); 2472 2473 wake_up(&con->page_retirement_wq); 2474 } 2475} 2476 2477static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, 2478 struct amdgpu_iv_entry *entry) 2479{ 2480 struct ras_ih_data *data = &obj->ih_data; 2481 struct ras_err_data err_data; 2482 int ret; 2483 2484 if (!data->cb) 2485 return; 2486 2487 ret = amdgpu_ras_error_data_init(&err_data); 2488 if (ret) 2489 return; 2490 2491 /* Let IP handle its data, maybe we need get the output 2492 * from the callback to update the error type/count, etc 2493 */ 2494 amdgpu_ras_set_fed(obj->adev, true); 2495 ret = data->cb(obj->adev, &err_data, entry); 2496 /* ue will trigger an interrupt, and in that case 2497 * we need do a reset to recovery the whole system. 2498 * But leave IP do that recovery, here we just dispatch 2499 * the error. 2500 */ 2501 if (ret == AMDGPU_RAS_SUCCESS) { 2502 /* these counts could be left as 0 if 2503 * some blocks do not count error number 2504 */ 2505 obj->err_data.ue_count += err_data.ue_count; 2506 obj->err_data.ce_count += err_data.ce_count; 2507 obj->err_data.de_count += err_data.de_count; 2508 } 2509 2510 amdgpu_ras_error_data_fini(&err_data); 2511} 2512 2513static void amdgpu_ras_interrupt_handler(struct ras_manager *obj) 2514{ 2515 struct ras_ih_data *data = &obj->ih_data; 2516 struct amdgpu_iv_entry entry; 2517 2518 while (data->rptr != data->wptr) { 2519 rmb(); 2520 memcpy(&entry, &data->ring[data->rptr], 2521 data->element_size); 2522 2523 wmb(); 2524 data->rptr = (data->aligned_element_size + 2525 data->rptr) % data->ring_size; 2526 2527 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) { 2528 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2529 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry); 2530 else 2531 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry); 2532 } else { 2533 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC) 2534 amdgpu_ras_interrupt_umc_handler(obj, &entry); 2535 else 2536 dev_warn(obj->adev->dev, 2537 "No RAS interrupt handler for non-UMC block with poison disabled.\n"); 2538 } 2539 } 2540} 2541 2542static void amdgpu_ras_interrupt_process_handler(struct work_struct *work) 2543{ 2544 struct ras_ih_data *data = 2545 container_of(work, struct ras_ih_data, ih_work); 2546 struct ras_manager *obj = 2547 container_of(data, struct ras_manager, ih_data); 2548 2549 amdgpu_ras_interrupt_handler(obj); 2550} 2551 2552int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev, 2553 struct ras_dispatch_if *info) 2554{ 2555 struct ras_manager *obj; 2556 struct ras_ih_data *data; 2557 2558 if (amdgpu_uniras_enabled(adev)) { 2559 struct ras_ih_info ih_info; 2560 2561 memset(&ih_info, 0, sizeof(ih_info)); 2562 ih_info.block = info->head.block; 2563 memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry)); 2564 2565 return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info); 2566 } 2567 2568 obj = amdgpu_ras_find_obj(adev, &info->head); 2569 if (!obj) 2570 return -EINVAL; 2571 2572 data = &obj->ih_data; 2573 2574 if (data->inuse == 0) 2575 return 0; 2576 2577 /* Might be overflow... */ 2578 memcpy(&data->ring[data->wptr], info->entry, 2579 data->element_size); 2580 2581 wmb(); 2582 data->wptr = (data->aligned_element_size + 2583 data->wptr) % data->ring_size; 2584 2585 schedule_work(&data->ih_work); 2586 2587 return 0; 2588} 2589 2590int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev, 2591 struct ras_common_if *head) 2592{ 2593 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2594 struct ras_ih_data *data; 2595 2596 if (!obj) 2597 return -EINVAL; 2598 2599 data = &obj->ih_data; 2600 if (data->inuse == 0) 2601 return 0; 2602 2603 cancel_work_sync(&data->ih_work); 2604 2605 kfree(data->ring); 2606 memset(data, 0, sizeof(*data)); 2607 put_obj(obj); 2608 2609 return 0; 2610} 2611 2612int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev, 2613 struct ras_common_if *head) 2614{ 2615 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); 2616 struct ras_ih_data *data; 2617 struct amdgpu_ras_block_object *ras_obj; 2618 2619 if (!obj) { 2620 /* in case we registe the IH before enable ras feature */ 2621 obj = amdgpu_ras_create_obj(adev, head); 2622 if (!obj) 2623 return -EINVAL; 2624 } else 2625 get_obj(obj); 2626 2627 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm); 2628 2629 data = &obj->ih_data; 2630 /* add the callback.etc */ 2631 *data = (struct ras_ih_data) { 2632 .inuse = 0, 2633 .cb = ras_obj->ras_cb, 2634 .element_size = sizeof(struct amdgpu_iv_entry), 2635 .rptr = 0, 2636 .wptr = 0, 2637 }; 2638 2639 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler); 2640 2641 data->aligned_element_size = ALIGN(data->element_size, 8); 2642 /* the ring can store 64 iv entries. */ 2643 data->ring_size = 64 * data->aligned_element_size; 2644 data->ring = kmalloc(data->ring_size, GFP_KERNEL); 2645 if (!data->ring) { 2646 put_obj(obj); 2647 return -ENOMEM; 2648 } 2649 2650 /* IH is ready */ 2651 data->inuse = 1; 2652 2653 return 0; 2654} 2655 2656static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev) 2657{ 2658 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2659 struct ras_manager *obj, *tmp; 2660 2661 list_for_each_entry_safe(obj, tmp, &con->head, node) { 2662 amdgpu_ras_interrupt_remove_handler(adev, &obj->head); 2663 } 2664 2665 return 0; 2666} 2667/* ih end */ 2668 2669/* traversal all IPs except NBIO to query error counter */ 2670static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type) 2671{ 2672 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2673 struct ras_manager *obj; 2674 2675 if (!adev->ras_enabled || !con) 2676 return; 2677 2678 list_for_each_entry(obj, &con->head, node) { 2679 struct ras_query_if info = { 2680 .head = obj->head, 2681 }; 2682 2683 /* 2684 * PCIE_BIF IP has one different isr by ras controller 2685 * interrupt, the specific ras counter query will be 2686 * done in that isr. So skip such block from common 2687 * sync flood interrupt isr calling. 2688 */ 2689 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF) 2690 continue; 2691 2692 /* 2693 * this is a workaround for aldebaran, skip send msg to 2694 * smu to get ecc_info table due to smu handle get ecc 2695 * info table failed temporarily. 2696 * should be removed until smu fix handle ecc_info table. 2697 */ 2698 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) && 2699 (amdgpu_ip_version(adev, MP1_HWIP, 0) == 2700 IP_VERSION(13, 0, 2))) 2701 continue; 2702 2703 amdgpu_ras_query_error_status_with_event(adev, &info, type); 2704 2705 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != 2706 IP_VERSION(11, 0, 2) && 2707 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2708 IP_VERSION(11, 0, 4) && 2709 amdgpu_ip_version(adev, MP0_HWIP, 0) != 2710 IP_VERSION(13, 0, 0)) { 2711 if (amdgpu_ras_reset_error_status(adev, info.head.block)) 2712 dev_warn(adev->dev, "Failed to reset error counter and error status"); 2713 } 2714 } 2715} 2716 2717/* Parse RdRspStatus and WrRspStatus */ 2718static void amdgpu_ras_error_status_query(struct amdgpu_device *adev, 2719 struct ras_query_if *info) 2720{ 2721 struct amdgpu_ras_block_object *block_obj; 2722 /* 2723 * Only two block need to query read/write 2724 * RspStatus at current state 2725 */ 2726 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) && 2727 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB)) 2728 return; 2729 2730 block_obj = amdgpu_ras_get_ras_block(adev, 2731 info->head.block, 2732 info->head.sub_block_index); 2733 2734 if (!block_obj || !block_obj->hw_ops) { 2735 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", 2736 get_ras_block_str(&info->head)); 2737 return; 2738 } 2739 2740 if (block_obj->hw_ops->query_ras_error_status) 2741 block_obj->hw_ops->query_ras_error_status(adev); 2742 2743} 2744 2745static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) 2746{ 2747 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2748 struct ras_manager *obj; 2749 2750 if (!adev->ras_enabled || !con) 2751 return; 2752 2753 list_for_each_entry(obj, &con->head, node) { 2754 struct ras_query_if info = { 2755 .head = obj->head, 2756 }; 2757 2758 amdgpu_ras_error_status_query(adev, &info); 2759 } 2760} 2761 2762static int amdgpu_ras_badpages_read(struct amdgpu_device *adev, 2763 struct ras_badpage *bps, uint32_t count, uint32_t start) 2764{ 2765 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 2766 struct ras_err_handler_data *data; 2767 int r = 0; 2768 uint32_t i; 2769 2770 if (!con || !con->eh_data || !bps || !count) 2771 return -EINVAL; 2772 2773 mutex_lock(&con->recovery_lock); 2774 data = con->eh_data; 2775 if (start < data->count) { 2776 for (i = start; i < data->count; i++) { 2777 if (!data->bps[i].ts) 2778 continue; 2779 2780 /* U64_MAX is used to mark the record as invalid */ 2781 if (data->bps[i].retired_page == U64_MAX) 2782 continue; 2783 2784 bps[r].bp = data->bps[i].retired_page; 2785 r++; 2786 if (r >= count) 2787 break; 2788 } 2789 } 2790 mutex_unlock(&con->recovery_lock); 2791 2792 return r; 2793} 2794 2795static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev, 2796 struct ras_badpage *bps, uint32_t count, uint32_t start) 2797{ 2798 struct ras_cmd_bad_pages_info_req cmd_input; 2799 struct ras_cmd_bad_pages_info_rsp *output; 2800 uint32_t group, start_group, end_group; 2801 uint32_t pos, pos_in_group; 2802 int r = 0, i; 2803 2804 if (!bps || !count) 2805 return -EINVAL; 2806 2807 output = kmalloc(sizeof(*output), GFP_KERNEL); 2808 if (!output) 2809 return -ENOMEM; 2810 2811 memset(&cmd_input, 0, sizeof(cmd_input)); 2812 2813 start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2814 end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) / 2815 RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2816 2817 pos = start; 2818 for (group = start_group; group < end_group; group++) { 2819 memset(output, 0, sizeof(*output)); 2820 cmd_input.group_index = group; 2821 if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_BAD_PAGES, 2822 &cmd_input, sizeof(cmd_input), output, sizeof(*output))) 2823 goto out; 2824 2825 if (pos >= output->bp_total_cnt) 2826 goto out; 2827 2828 pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; 2829 for (i = pos_in_group; i < output->bp_in_group; i++, pos++) { 2830 if (!output->records[i].ts) 2831 continue; 2832 2833 bps[r].bp = output->records[i].retired_page; 2834 r++; 2835 if (r >= count) 2836 goto out; 2837 } 2838 } 2839 2840out: 2841 kfree(output); 2842 return r; 2843} 2844 2845static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev, 2846 struct amdgpu_hive_info *hive, bool status) 2847{ 2848 struct amdgpu_device *tmp_adev; 2849 2850 if (hive) { 2851 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 2852 amdgpu_ras_set_fed(tmp_adev, status); 2853 } else { 2854 amdgpu_ras_set_fed(adev, status); 2855 } 2856} 2857 2858bool amdgpu_ras_in_recovery(struct amdgpu_device *adev) 2859{ 2860 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2861 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 2862 int hive_ras_recovery = 0; 2863 2864 if (hive) { 2865 hive_ras_recovery = atomic_read(&hive->ras_recovery); 2866 amdgpu_put_xgmi_hive(hive); 2867 } 2868 2869 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery)) 2870 return true; 2871 2872 return false; 2873} 2874 2875static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev) 2876{ 2877 if (amdgpu_ras_intr_triggered()) 2878 return RAS_EVENT_TYPE_FATAL; 2879 else 2880 return RAS_EVENT_TYPE_POISON_CONSUMPTION; 2881} 2882 2883static void amdgpu_ras_do_recovery(struct work_struct *work) 2884{ 2885 struct amdgpu_ras *ras = 2886 container_of(work, struct amdgpu_ras, recovery_work); 2887 struct amdgpu_device *remote_adev = NULL; 2888 struct amdgpu_device *adev = ras->adev; 2889 struct list_head device_list, *device_list_handle = NULL; 2890 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2891 unsigned int error_query_mode; 2892 enum ras_event_type type; 2893 2894 if (hive) { 2895 atomic_set(&hive->ras_recovery, 1); 2896 2897 /* If any device which is part of the hive received RAS fatal 2898 * error interrupt, set fatal error status on all. This 2899 * condition will need a recovery, and flag will be cleared 2900 * as part of recovery. 2901 */ 2902 list_for_each_entry(remote_adev, &hive->device_list, 2903 gmc.xgmi.head) 2904 if (amdgpu_ras_get_fed_status(remote_adev)) { 2905 amdgpu_ras_set_fed_all(adev, hive, true); 2906 break; 2907 } 2908 } 2909 if (!ras->disable_ras_err_cnt_harvest) { 2910 2911 /* Build list of devices to query RAS related errors */ 2912 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) { 2913 device_list_handle = &hive->device_list; 2914 } else { 2915 INIT_LIST_HEAD(&device_list); 2916 list_add_tail(&adev->gmc.xgmi.head, &device_list); 2917 device_list_handle = &device_list; 2918 } 2919 2920 if (amdgpu_ras_get_error_query_mode(adev, &error_query_mode)) { 2921 if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) { 2922 /* wait 500ms to ensure pmfw polling mca bank info done */ 2923 msleep(500); 2924 } 2925 } 2926 2927 type = amdgpu_ras_get_fatal_error_event(adev); 2928 list_for_each_entry(remote_adev, 2929 device_list_handle, gmc.xgmi.head) { 2930 if (amdgpu_uniras_enabled(remote_adev)) { 2931 amdgpu_ras_mgr_update_ras_ecc(remote_adev); 2932 } else { 2933 amdgpu_ras_query_err_status(remote_adev); 2934 amdgpu_ras_log_on_err_counter(remote_adev, type); 2935 } 2936 } 2937 2938 } 2939 2940 if (amdgpu_device_should_recover_gpu(ras->adev)) { 2941 struct amdgpu_reset_context reset_context; 2942 memset(&reset_context, 0, sizeof(reset_context)); 2943 2944 reset_context.method = AMD_RESET_METHOD_NONE; 2945 reset_context.reset_req_dev = adev; 2946 reset_context.src = AMDGPU_RESET_SRC_RAS; 2947 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 2948 2949 /* Perform full reset in fatal error mode */ 2950 if (!amdgpu_ras_is_poison_mode_supported(ras->adev)) 2951 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2952 else { 2953 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2954 2955 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) { 2956 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET; 2957 reset_context.method = AMD_RESET_METHOD_MODE2; 2958 } 2959 2960 /* Fatal error occurs in poison mode, mode1 reset is used to 2961 * recover gpu. 2962 */ 2963 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) { 2964 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET; 2965 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 2966 2967 psp_fatal_error_recovery_quirk(&adev->psp); 2968 } 2969 } 2970 2971 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); 2972 } 2973 atomic_set(&ras->in_recovery, 0); 2974 if (hive) { 2975 atomic_set(&hive->ras_recovery, 0); 2976 amdgpu_put_xgmi_hive(hive); 2977 } 2978} 2979 2980/* alloc/realloc bps array */ 2981static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, 2982 struct ras_err_handler_data *data, int pages) 2983{ 2984 unsigned int old_space = data->count + data->space_left; 2985 unsigned int new_space = old_space + pages; 2986 unsigned int align_space = ALIGN(new_space, 512); 2987 void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL); 2988 2989 if (!bps) { 2990 return -ENOMEM; 2991 } 2992 2993 if (data->bps) { 2994 memcpy(bps, data->bps, 2995 data->count * sizeof(*data->bps)); 2996 kfree(data->bps); 2997 } 2998 2999 data->bps = bps; 3000 data->space_left += align_space - old_space; 3001 return 0; 3002} 3003 3004static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, 3005 struct eeprom_table_record *bps, 3006 struct ras_err_data *err_data) 3007{ 3008 struct ta_ras_query_address_input addr_in; 3009 uint32_t socket = 0; 3010 int ret = 0; 3011 3012 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3013 socket = adev->smuio.funcs->get_socket_id(adev); 3014 3015 /* reinit err_data */ 3016 err_data->err_addr_cnt = 0; 3017 err_data->err_addr_len = adev->umc.retire_unit; 3018 3019 memset(&addr_in, 0, sizeof(addr_in)); 3020 addr_in.ma.err_addr = bps->address; 3021 addr_in.ma.socket_id = socket; 3022 addr_in.ma.ch_inst = bps->mem_channel; 3023 if (!amdgpu_ras_smu_eeprom_supported(adev)) { 3024 /* tell RAS TA the node instance is not used */ 3025 addr_in.ma.node_inst = TA_RAS_INV_NODE; 3026 } else { 3027 addr_in.ma.umc_inst = bps->mcumc_id; 3028 addr_in.ma.node_inst = bps->cu; 3029 } 3030 3031 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3032 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, 3033 &addr_in, NULL, false); 3034 3035 return ret; 3036} 3037 3038static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, 3039 struct eeprom_table_record *bps, 3040 struct ras_err_data *err_data) 3041{ 3042 struct ta_ras_query_address_input addr_in; 3043 uint32_t die_id, socket = 0; 3044 3045 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) 3046 socket = adev->smuio.funcs->get_socket_id(adev); 3047 3048 /* although die id is gotten from PA in nps1 mode, the id is 3049 * fitable for any nps mode 3050 */ 3051 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) 3052 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, 3053 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); 3054 else 3055 return -EINVAL; 3056 3057 /* reinit err_data */ 3058 err_data->err_addr_cnt = 0; 3059 err_data->err_addr_len = adev->umc.retire_unit; 3060 3061 memset(&addr_in, 0, sizeof(addr_in)); 3062 addr_in.ma.err_addr = bps->address; 3063 addr_in.ma.ch_inst = bps->mem_channel; 3064 addr_in.ma.umc_inst = bps->mcumc_id; 3065 addr_in.ma.node_inst = die_id; 3066 addr_in.ma.socket_id = socket; 3067 3068 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) 3069 return adev->umc.ras->convert_ras_err_addr(adev, err_data, 3070 &addr_in, NULL, false); 3071 else 3072 return -EINVAL; 3073} 3074 3075static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, 3076 struct eeprom_table_record *bps, int count) 3077{ 3078 int j; 3079 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3080 struct ras_err_handler_data *data = con->eh_data; 3081 3082 for (j = 0; j < count; j++) { 3083 if (!data->space_left && 3084 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { 3085 return -ENOMEM; 3086 } 3087 3088 if (amdgpu_ras_check_bad_page_unlock(con, 3089 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { 3090 /* set to U64_MAX to mark it as invalid */ 3091 data->bps[data->count].retired_page = U64_MAX; 3092 data->count++; 3093 data->space_left--; 3094 continue; 3095 } 3096 3097 amdgpu_ras_reserve_page(adev, bps[j].retired_page); 3098 3099 memcpy(&data->bps[data->count], &(bps[j]), 3100 sizeof(struct eeprom_table_record)); 3101 data->count++; 3102 data->space_left--; 3103 con->bad_page_num++; 3104 } 3105 3106 return 0; 3107} 3108 3109static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, 3110 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3111 enum amdgpu_memory_partition nps) 3112{ 3113 int i = 0; 3114 enum amdgpu_memory_partition save_nps; 3115 3116 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3117 3118 /*old asics just have pa in eeprom*/ 3119 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3120 memcpy(err_data->err_addr, bps, 3121 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3122 goto out; 3123 } 3124 3125 for (i = 0; i < adev->umc.retire_unit; i++) 3126 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3127 3128 if (save_nps) { 3129 if (save_nps == nps) { 3130 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3131 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3132 return -EINVAL; 3133 for (i = 0; i < adev->umc.retire_unit; i++) { 3134 err_data->err_addr[i].address = bps[0].address; 3135 err_data->err_addr[i].mem_channel = bps[0].mem_channel; 3136 err_data->err_addr[i].bank = bps[0].bank; 3137 err_data->err_addr[i].err_type = bps[0].err_type; 3138 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; 3139 } 3140 } else { 3141 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) 3142 return -EINVAL; 3143 } 3144 } else { 3145 if (bps[0].address == 0) { 3146 /* for specific old eeprom data, mca address is not stored, 3147 * calc it from pa 3148 */ 3149 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, 3150 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) 3151 return -EINVAL; 3152 } 3153 3154 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { 3155 if (nps == AMDGPU_NPS1_PARTITION_MODE) 3156 memcpy(err_data->err_addr, bps, 3157 sizeof(struct eeprom_table_record) * adev->umc.retire_unit); 3158 else 3159 return -EOPNOTSUPP; 3160 } 3161 } 3162 3163out: 3164 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); 3165} 3166 3167static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, 3168 struct eeprom_table_record *bps, struct ras_err_data *err_data, 3169 enum amdgpu_memory_partition nps) 3170{ 3171 int i = 0; 3172 enum amdgpu_memory_partition save_nps; 3173 3174 if (!amdgpu_ras_smu_eeprom_supported(adev)) { 3175 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; 3176 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); 3177 } else { 3178 /* if pmfw manages eeprom, save_nps is not stored on eeprom, 3179 * we should always convert mca address into physical address, 3180 * make save_nps different from nps 3181 */ 3182 save_nps = nps + 1; 3183 } 3184 3185 if (save_nps == nps) { 3186 if (amdgpu_umc_pages_in_a_row(adev, err_data, 3187 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) 3188 return -EINVAL; 3189 for (i = 0; i < adev->umc.retire_unit; i++) { 3190 err_data->err_addr[i].address = bps->address; 3191 err_data->err_addr[i].mem_channel = bps->mem_channel; 3192 err_data->err_addr[i].bank = bps->bank; 3193 err_data->err_addr[i].err_type = bps->err_type; 3194 err_data->err_addr[i].mcumc_id = bps->mcumc_id; 3195 } 3196 } else { 3197 if (bps->address) { 3198 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) 3199 return -EINVAL; 3200 } else { 3201 /* for specific old eeprom data, mca address is not stored, 3202 * calc it from pa 3203 */ 3204 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, 3205 &(bps->address), AMDGPU_NPS1_PARTITION_MODE)) 3206 return -EINVAL; 3207 3208 if (amdgpu_ras_mca2pa(adev, bps, err_data)) 3209 return -EOPNOTSUPP; 3210 } 3211 } 3212 3213 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, 3214 adev->umc.retire_unit); 3215} 3216 3217/* it deal with vram only. */ 3218int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, 3219 struct eeprom_table_record *bps, int pages, bool from_rom) 3220{ 3221 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3222 struct ras_err_data err_data; 3223 struct amdgpu_ras_eeprom_control *control = 3224 &adev->psp.ras_context.ras->eeprom_control; 3225 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; 3226 int ret = 0; 3227 uint32_t i = 0; 3228 3229 if (!con || !con->eh_data || !bps || pages <= 0) 3230 return 0; 3231 3232 if (from_rom) { 3233 err_data.err_addr = 3234 kcalloc(adev->umc.retire_unit, 3235 sizeof(struct eeprom_table_record), GFP_KERNEL); 3236 if (!err_data.err_addr) { 3237 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n"); 3238 return -ENOMEM; 3239 } 3240 3241 if (adev->gmc.gmc_funcs->query_mem_partition_mode) 3242 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 3243 } 3244 3245 mutex_lock(&con->recovery_lock); 3246 3247 if (from_rom) { 3248 /* there is no pa recs in V3, so skip pa recs processing */ 3249 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3250 !amdgpu_ras_smu_eeprom_supported(adev)) { 3251 for (i = 0; i < pages; i++) { 3252 if (control->ras_num_recs - i >= adev->umc.retire_unit) { 3253 if ((bps[i].address == bps[i + 1].address) && 3254 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3255 /* deal with retire_unit records a time */ 3256 ret = __amdgpu_ras_convert_rec_array_from_rom(adev, 3257 &bps[i], &err_data, nps); 3258 i += (adev->umc.retire_unit - 1); 3259 } else { 3260 break; 3261 } 3262 } else { 3263 break; 3264 } 3265 } 3266 } 3267 for (; i < pages; i++) { 3268 ret = __amdgpu_ras_convert_rec_from_rom(adev, 3269 &bps[i], &err_data, nps); 3270 } 3271 3272 con->eh_data->count_saved = con->eh_data->count; 3273 } else { 3274 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages); 3275 } 3276 3277 if (from_rom) 3278 kfree(err_data.err_addr); 3279 mutex_unlock(&con->recovery_lock); 3280 3281 return ret; 3282} 3283 3284/* 3285 * write error record array to eeprom, the function should be 3286 * protected by recovery_lock 3287 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL 3288 */ 3289int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, 3290 unsigned long *new_cnt) 3291{ 3292 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3293 struct ras_err_handler_data *data; 3294 struct amdgpu_ras_eeprom_control *control; 3295 int save_count, unit_num, i; 3296 3297 if (!con || !con->eh_data) { 3298 if (new_cnt) 3299 *new_cnt = 0; 3300 3301 return 0; 3302 } 3303 3304 if (!con->eeprom_control.is_eeprom_valid) { 3305 dev_warn(adev->dev, 3306 "Failed to save EEPROM table data because of EEPROM data corruption!"); 3307 if (new_cnt) 3308 *new_cnt = 0; 3309 3310 return 0; 3311 } 3312 3313 mutex_lock(&con->recovery_lock); 3314 control = &con->eeprom_control; 3315 data = con->eh_data; 3316 if (amdgpu_ras_smu_eeprom_supported(adev)) 3317 unit_num = control->ras_num_recs - 3318 control->ras_num_recs_old; 3319 else 3320 unit_num = data->count / adev->umc.retire_unit - 3321 control->ras_num_recs; 3322 3323 save_count = con->bad_page_num - control->ras_num_bad_pages; 3324 mutex_unlock(&con->recovery_lock); 3325 3326 if (new_cnt) 3327 *new_cnt = unit_num; 3328 3329 /* only new entries are saved */ 3330 if (unit_num && save_count) { 3331 /*old asics only save pa to eeprom like before*/ 3332 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { 3333 if (amdgpu_ras_eeprom_append(control, 3334 &data->bps[data->count_saved], unit_num)) { 3335 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3336 return -EIO; 3337 } 3338 } else { 3339 for (i = 0; i < unit_num; i++) { 3340 if (amdgpu_ras_eeprom_append(control, 3341 &data->bps[data->count_saved + 3342 i * adev->umc.retire_unit], 1)) { 3343 dev_err(adev->dev, "Failed to save EEPROM table data!"); 3344 return -EIO; 3345 } 3346 } 3347 } 3348 3349 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); 3350 data->count_saved = data->count; 3351 } 3352 3353 return 0; 3354} 3355 3356/* 3357 * read error record array in eeprom and reserve enough space for 3358 * storing new bad pages 3359 */ 3360static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) 3361{ 3362 struct amdgpu_ras_eeprom_control *control = 3363 &adev->psp.ras_context.ras->eeprom_control; 3364 struct eeprom_table_record *bps; 3365 int ret, i = 0; 3366 3367 /* no bad page record, skip eeprom access */ 3368 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) 3369 return 0; 3370 3371 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL); 3372 if (!bps) 3373 return -ENOMEM; 3374 3375 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs); 3376 if (ret) { 3377 dev_err(adev->dev, "Failed to load EEPROM table records!"); 3378 } else { 3379 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { 3380 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed 3381 as pa recs, so add verion check to avoid it. 3382 */ 3383 if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && 3384 !amdgpu_ras_smu_eeprom_supported(adev)) { 3385 for (i = 0; i < control->ras_num_recs; i++) { 3386 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { 3387 if ((bps[i].address == bps[i + 1].address) && 3388 (bps[i].mem_channel == bps[i + 1].mem_channel)) { 3389 control->ras_num_pa_recs += adev->umc.retire_unit; 3390 i += (adev->umc.retire_unit - 1); 3391 } else { 3392 control->ras_num_mca_recs += 3393 (control->ras_num_recs - i); 3394 break; 3395 } 3396 } else { 3397 control->ras_num_mca_recs += (control->ras_num_recs - i); 3398 break; 3399 } 3400 } 3401 } else { 3402 control->ras_num_mca_recs = control->ras_num_recs; 3403 } 3404 } 3405 3406 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); 3407 if (ret) 3408 goto out; 3409 3410 ret = amdgpu_ras_eeprom_check(control); 3411 if (ret) 3412 goto out; 3413 3414 /* HW not usable */ 3415 if (amdgpu_ras_is_rma(adev)) 3416 ret = -EHWPOISON; 3417 } 3418 3419out: 3420 kfree(bps); 3421 return ret; 3422} 3423 3424static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con, 3425 uint64_t addr) 3426{ 3427 struct ras_err_handler_data *data = con->eh_data; 3428 struct amdgpu_device *adev = con->adev; 3429 int i; 3430 3431 if ((addr >= adev->gmc.mc_vram_size && 3432 adev->gmc.mc_vram_size) || 3433 (addr >= RAS_UMC_INJECT_ADDR_LIMIT)) 3434 return -EINVAL; 3435 3436 addr >>= AMDGPU_GPU_PAGE_SHIFT; 3437 for (i = 0; i < data->count; i++) 3438 if (addr == data->bps[i].retired_page) 3439 return 1; 3440 3441 return 0; 3442} 3443 3444/* 3445 * check if an address belongs to bad page 3446 * 3447 * Note: this check is only for umc block 3448 */ 3449static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev, 3450 uint64_t addr) 3451{ 3452 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3453 int ret = 0; 3454 3455 if (!con || !con->eh_data) 3456 return ret; 3457 3458 mutex_lock(&con->recovery_lock); 3459 ret = amdgpu_ras_check_bad_page_unlock(con, addr); 3460 mutex_unlock(&con->recovery_lock); 3461 return ret; 3462} 3463 3464static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, 3465 uint32_t max_count) 3466{ 3467 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3468 3469 /* 3470 * amdgpu_bad_page_threshold is used to config 3471 * the threshold for the number of bad pages. 3472 * -1: Threshold is set to default value 3473 * Driver will issue a warning message when threshold is reached 3474 * and continue runtime services. 3475 * 0: Disable bad page retirement 3476 * Driver will not retire bad pages 3477 * which is intended for debugging purpose. 3478 * -2: Threshold is determined by a formula 3479 * that assumes 1 bad page per 100M of local memory. 3480 * Driver will continue runtime services when threhold is reached. 3481 * 0 < threshold < max number of bad page records in EEPROM, 3482 * A user-defined threshold is set 3483 * Driver will halt runtime services when this custom threshold is reached. 3484 */ 3485 if (amdgpu_bad_page_threshold == -2) { 3486 u64 val = adev->gmc.mc_vram_size; 3487 3488 do_div(val, RAS_BAD_PAGE_COVER); 3489 con->bad_page_cnt_threshold = min(lower_32_bits(val), 3490 max_count); 3491 } else if (amdgpu_bad_page_threshold == -1) { 3492 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4; 3493 } else { 3494 con->bad_page_cnt_threshold = min_t(int, max_count, 3495 amdgpu_bad_page_threshold); 3496 } 3497} 3498 3499int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, 3500 enum amdgpu_ras_block block, uint16_t pasid, 3501 pasid_notify pasid_fn, void *data, uint32_t reset) 3502{ 3503 int ret = 0; 3504 struct ras_poison_msg poison_msg; 3505 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3506 3507 memset(&poison_msg, 0, sizeof(poison_msg)); 3508 poison_msg.block = block; 3509 poison_msg.pasid = pasid; 3510 poison_msg.reset = reset; 3511 poison_msg.pasid_fn = pasid_fn; 3512 poison_msg.data = data; 3513 3514 ret = kfifo_put(&con->poison_fifo, poison_msg); 3515 if (!ret) { 3516 dev_err(adev->dev, "Poison message fifo is full!\n"); 3517 return -ENOSPC; 3518 } 3519 3520 return 0; 3521} 3522 3523static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, 3524 struct ras_poison_msg *poison_msg) 3525{ 3526 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3527 3528 return kfifo_get(&con->poison_fifo, poison_msg); 3529} 3530 3531static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) 3532{ 3533 mutex_init(&ecc_log->lock); 3534 3535 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); 3536 ecc_log->de_queried_count = 0; 3537 ecc_log->consumption_q_count = 0; 3538} 3539 3540static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) 3541{ 3542 struct radix_tree_iter iter; 3543 void __rcu **slot; 3544 struct ras_ecc_err *ecc_err; 3545 3546 mutex_lock(&ecc_log->lock); 3547 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { 3548 ecc_err = radix_tree_deref_slot(slot); 3549 kfree(ecc_err->err_pages.pfn); 3550 kfree(ecc_err); 3551 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); 3552 } 3553 mutex_unlock(&ecc_log->lock); 3554 3555 mutex_destroy(&ecc_log->lock); 3556 ecc_log->de_queried_count = 0; 3557 ecc_log->consumption_q_count = 0; 3558} 3559 3560static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, 3561 uint32_t delayed_ms) 3562{ 3563 int ret; 3564 3565 mutex_lock(&con->umc_ecc_log.lock); 3566 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, 3567 UMC_ECC_NEW_DETECTED_TAG); 3568 mutex_unlock(&con->umc_ecc_log.lock); 3569 3570 if (ret) 3571 schedule_delayed_work(&con->page_retirement_dwork, 3572 msecs_to_jiffies(delayed_ms)); 3573 3574 return ret ? true : false; 3575} 3576 3577static void amdgpu_ras_do_page_retirement(struct work_struct *work) 3578{ 3579 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 3580 page_retirement_dwork.work); 3581 struct amdgpu_device *adev = con->adev; 3582 struct ras_err_data err_data; 3583 3584 /* If gpu reset is ongoing, delay retiring the bad pages */ 3585 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { 3586 amdgpu_ras_schedule_retirement_dwork(con, 3587 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); 3588 return; 3589 } 3590 3591 amdgpu_ras_error_data_init(&err_data); 3592 3593 amdgpu_umc_handle_bad_pages(adev, &err_data); 3594 3595 amdgpu_ras_error_data_fini(&err_data); 3596 3597 amdgpu_ras_schedule_retirement_dwork(con, 3598 AMDGPU_RAS_RETIRE_PAGE_INTERVAL); 3599} 3600 3601static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, 3602 uint32_t poison_creation_count) 3603{ 3604 int ret = 0; 3605 struct ras_ecc_log_info *ecc_log; 3606 struct ras_query_if info; 3607 u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; 3608 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 3609 u64 de_queried_count; 3610 u64 consumption_q_count; 3611 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; 3612 3613 memset(&info, 0, sizeof(info)); 3614 info.head.block = AMDGPU_RAS_BLOCK__UMC; 3615 3616 ecc_log = &ras->umc_ecc_log; 3617 ecc_log->de_queried_count = 0; 3618 ecc_log->consumption_q_count = 0; 3619 3620 do { 3621 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); 3622 if (ret) 3623 return ret; 3624 3625 de_queried_count = ecc_log->de_queried_count; 3626 consumption_q_count = ecc_log->consumption_q_count; 3627 3628 if (de_queried_count && consumption_q_count) 3629 break; 3630 3631 msleep(100); 3632 } while (--timeout); 3633 3634 if (de_queried_count) 3635 schedule_delayed_work(&ras->page_retirement_dwork, 0); 3636 3637 if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) 3638 amdgpu_ras_reset_gpu(adev); 3639 3640 return 0; 3641} 3642 3643static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) 3644{ 3645 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3646 struct ras_poison_msg msg; 3647 int ret; 3648 3649 do { 3650 ret = kfifo_get(&con->poison_fifo, &msg); 3651 } while (ret); 3652} 3653 3654static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, 3655 uint32_t msg_count, uint32_t *gpu_reset) 3656{ 3657 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3658 uint32_t reset_flags = 0, reset = 0; 3659 struct ras_poison_msg msg; 3660 int ret, i; 3661 3662 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 3663 3664 for (i = 0; i < msg_count; i++) { 3665 ret = amdgpu_ras_get_poison_req(adev, &msg); 3666 if (!ret) 3667 continue; 3668 3669 if (msg.pasid_fn) 3670 msg.pasid_fn(adev, msg.pasid, msg.data); 3671 3672 reset_flags |= msg.reset; 3673 } 3674 3675 /* 3676 * Try to ensure poison creation handler is completed first 3677 * to set rma if bad page exceed threshold. 3678 */ 3679 flush_delayed_work(&con->page_retirement_dwork); 3680 3681 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ 3682 if (reset_flags && !amdgpu_ras_is_rma(adev)) { 3683 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) 3684 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; 3685 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) 3686 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 3687 else 3688 reset = reset_flags; 3689 3690 con->gpu_reset_flags |= reset; 3691 amdgpu_ras_reset_gpu(adev); 3692 3693 *gpu_reset = reset; 3694 3695 /* Wait for gpu recovery to complete */ 3696 flush_work(&con->recovery_work); 3697 } 3698 3699 return 0; 3700} 3701 3702static int amdgpu_ras_page_retirement_thread(void *param) 3703{ 3704 struct amdgpu_device *adev = (struct amdgpu_device *)param; 3705 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3706 uint32_t poison_creation_count, msg_count; 3707 uint32_t gpu_reset; 3708 int ret; 3709 3710 while (!kthread_should_stop()) { 3711 3712 wait_event_interruptible(con->page_retirement_wq, 3713 kthread_should_stop() || 3714 atomic_read(&con->page_retirement_req_cnt)); 3715 3716 if (kthread_should_stop()) 3717 break; 3718 3719 mutex_lock(&con->poison_lock); 3720 gpu_reset = 0; 3721 3722 do { 3723 poison_creation_count = atomic_read(&con->poison_creation_count); 3724 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); 3725 if (ret == -EIO) 3726 break; 3727 3728 if (poison_creation_count) { 3729 atomic_sub(poison_creation_count, &con->poison_creation_count); 3730 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); 3731 } 3732 } while (atomic_read(&con->poison_creation_count) && 3733 !atomic_read(&con->poison_consumption_count)); 3734 3735 if (ret != -EIO) { 3736 msg_count = kfifo_len(&con->poison_fifo); 3737 if (msg_count) { 3738 ret = amdgpu_ras_poison_consumption_handler(adev, 3739 msg_count, &gpu_reset); 3740 if ((ret != -EIO) && 3741 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) 3742 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3743 } 3744 } 3745 3746 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { 3747 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ 3748 /* Clear poison creation request */ 3749 atomic_set(&con->poison_creation_count, 0); 3750 atomic_set(&con->poison_consumption_count, 0); 3751 3752 /* Clear poison fifo */ 3753 amdgpu_ras_clear_poison_fifo(adev); 3754 3755 /* Clear all poison requests */ 3756 atomic_set(&con->page_retirement_req_cnt, 0); 3757 3758 if (ret == -EIO) { 3759 /* Wait for mode-1 reset to complete */ 3760 down_read(&adev->reset_domain->sem); 3761 up_read(&adev->reset_domain->sem); 3762 } 3763 3764 /* Wake up work to save bad pages to eeprom */ 3765 schedule_delayed_work(&con->page_retirement_dwork, 0); 3766 } else if (gpu_reset) { 3767 /* gpu just completed mode-2 reset or other reset */ 3768 /* Clear poison consumption messages cached in fifo */ 3769 msg_count = kfifo_len(&con->poison_fifo); 3770 if (msg_count) { 3771 amdgpu_ras_clear_poison_fifo(adev); 3772 atomic_sub(msg_count, &con->page_retirement_req_cnt); 3773 } 3774 3775 atomic_set(&con->poison_consumption_count, 0); 3776 3777 /* Wake up work to save bad pages to eeprom */ 3778 schedule_delayed_work(&con->page_retirement_dwork, 0); 3779 } 3780 mutex_unlock(&con->poison_lock); 3781 } 3782 3783 return 0; 3784} 3785 3786int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) 3787{ 3788 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3789 struct amdgpu_ras_eeprom_control *control; 3790 int ret; 3791 3792 if (!con || amdgpu_sriov_vf(adev)) 3793 return 0; 3794 3795 if (amdgpu_uniras_enabled(adev)) 3796 return 0; 3797 3798 control = &con->eeprom_control; 3799 con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev); 3800 3801 ret = amdgpu_ras_eeprom_init(control); 3802 control->is_eeprom_valid = !ret; 3803 3804 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) 3805 control->ras_num_pa_recs = control->ras_num_recs; 3806 3807 if (adev->umc.ras && 3808 adev->umc.ras->get_retire_flip_bits) 3809 adev->umc.ras->get_retire_flip_bits(adev); 3810 3811 if (control->ras_num_recs && control->is_eeprom_valid) { 3812 ret = amdgpu_ras_load_bad_pages(adev); 3813 if (ret) { 3814 control->is_eeprom_valid = false; 3815 return 0; 3816 } 3817 3818 amdgpu_dpm_send_hbm_bad_pages_num( 3819 adev, control->ras_num_bad_pages); 3820 3821 if (con->update_channel_flag == true) { 3822 amdgpu_dpm_send_hbm_bad_channel_flag( 3823 adev, control->bad_channel_bitmap); 3824 con->update_channel_flag = false; 3825 } 3826 3827 /* The format action is only applied to new ASICs */ 3828 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && 3829 control->tbl_hdr.version < RAS_TABLE_VER_V3) 3830 if (!amdgpu_ras_eeprom_reset_table(control)) 3831 if (amdgpu_ras_save_bad_pages(adev, NULL)) 3832 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); 3833 } 3834 3835 return 0; 3836} 3837 3838int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) 3839{ 3840 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3841 struct ras_err_handler_data **data; 3842 u32 max_eeprom_records_count = 0; 3843 int ret; 3844 3845 if (!con || amdgpu_sriov_vf(adev)) 3846 return 0; 3847 3848 /* Allow access to RAS EEPROM via debugfs, when the ASIC 3849 * supports RAS and debugfs is enabled, but when 3850 * adev->ras_enabled is unset, i.e. when "ras_enable" 3851 * module parameter is set to 0. 3852 */ 3853 con->adev = adev; 3854 3855 if (!adev->ras_enabled) 3856 return 0; 3857 3858 data = &con->eh_data; 3859 *data = kzalloc(sizeof(**data), GFP_KERNEL); 3860 if (!*data) { 3861 ret = -ENOMEM; 3862 goto out; 3863 } 3864 3865 mutex_init(&con->recovery_lock); 3866 mutex_init(&con->poison_lock); 3867 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); 3868 atomic_set(&con->in_recovery, 0); 3869 atomic_set(&con->rma_in_recovery, 0); 3870 con->eeprom_control.bad_channel_bitmap = 0; 3871 3872 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); 3873 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count); 3874 3875 if (init_bp_info) { 3876 ret = amdgpu_ras_init_badpage_info(adev); 3877 if (ret) 3878 goto free; 3879 } 3880 3881 mutex_init(&con->page_rsv_lock); 3882 INIT_KFIFO(con->poison_fifo); 3883 mutex_init(&con->page_retirement_lock); 3884 init_waitqueue_head(&con->page_retirement_wq); 3885 atomic_set(&con->page_retirement_req_cnt, 0); 3886 atomic_set(&con->poison_creation_count, 0); 3887 atomic_set(&con->poison_consumption_count, 0); 3888 con->page_retirement_thread = 3889 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); 3890 if (IS_ERR(con->page_retirement_thread)) { 3891 con->page_retirement_thread = NULL; 3892 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); 3893 } 3894 3895 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); 3896 amdgpu_ras_ecc_log_init(&con->umc_ecc_log); 3897#ifdef CONFIG_X86_MCE_AMD 3898 if ((adev->asic_type == CHIP_ALDEBARAN) && 3899 (adev->gmc.xgmi.connected_to_cpu)) 3900 amdgpu_register_bad_pages_mca_notifier(adev); 3901#endif 3902 return 0; 3903 3904free: 3905 kfree((*data)->bps); 3906 kfree(*data); 3907 con->eh_data = NULL; 3908out: 3909 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret); 3910 3911 /* 3912 * Except error threshold exceeding case, other failure cases in this 3913 * function would not fail amdgpu driver init. 3914 */ 3915 if (!amdgpu_ras_is_rma(adev)) 3916 ret = 0; 3917 else 3918 ret = -EINVAL; 3919 3920 return ret; 3921} 3922 3923static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) 3924{ 3925 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 3926 struct ras_err_handler_data *data = con->eh_data; 3927 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; 3928 bool ret; 3929 3930 /* recovery_init failed to init it, fini is useless */ 3931 if (!data) 3932 return 0; 3933 3934 /* Save all cached bad pages to eeprom */ 3935 do { 3936 flush_delayed_work(&con->page_retirement_dwork); 3937 ret = amdgpu_ras_schedule_retirement_dwork(con, 0); 3938 } while (ret && max_flush_timeout--); 3939 3940 if (con->page_retirement_thread) 3941 kthread_stop(con->page_retirement_thread); 3942 3943 atomic_set(&con->page_retirement_req_cnt, 0); 3944 atomic_set(&con->poison_creation_count, 0); 3945 3946 mutex_destroy(&con->page_rsv_lock); 3947 3948 cancel_work_sync(&con->recovery_work); 3949 3950 cancel_delayed_work_sync(&con->page_retirement_dwork); 3951 3952 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); 3953 3954 mutex_lock(&con->recovery_lock); 3955 con->eh_data = NULL; 3956 kfree(data->bps); 3957 kfree(data); 3958 mutex_unlock(&con->recovery_lock); 3959 3960 amdgpu_ras_critical_region_init(adev); 3961#ifdef CONFIG_X86_MCE_AMD 3962 amdgpu_unregister_bad_pages_mca_notifier(adev); 3963#endif 3964 return 0; 3965} 3966/* recovery end */ 3967 3968static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) 3969{ 3970 if (amdgpu_sriov_vf(adev)) { 3971 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3972 case IP_VERSION(13, 0, 2): 3973 case IP_VERSION(13, 0, 6): 3974 case IP_VERSION(13, 0, 12): 3975 case IP_VERSION(13, 0, 14): 3976 return true; 3977 default: 3978 return false; 3979 } 3980 } 3981 3982 if (adev->asic_type == CHIP_IP_DISCOVERY) { 3983 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 3984 case IP_VERSION(13, 0, 0): 3985 case IP_VERSION(13, 0, 6): 3986 case IP_VERSION(13, 0, 10): 3987 case IP_VERSION(13, 0, 12): 3988 case IP_VERSION(13, 0, 14): 3989 case IP_VERSION(14, 0, 3): 3990 return true; 3991 default: 3992 return false; 3993 } 3994 } 3995 3996 return adev->asic_type == CHIP_VEGA10 || 3997 adev->asic_type == CHIP_VEGA20 || 3998 adev->asic_type == CHIP_ARCTURUS || 3999 adev->asic_type == CHIP_ALDEBARAN || 4000 adev->asic_type == CHIP_SIENNA_CICHLID; 4001} 4002 4003/* 4004 * this is workaround for vega20 workstation sku, 4005 * force enable gfx ras, ignore vbios gfx ras flag 4006 * due to GC EDC can not write 4007 */ 4008static void amdgpu_ras_get_quirks(struct amdgpu_device *adev) 4009{ 4010 struct atom_context *ctx = adev->mode_info.atom_context; 4011 4012 if (!ctx) 4013 return; 4014 4015 if (strnstr(ctx->vbios_pn, "D16406", 4016 sizeof(ctx->vbios_pn)) || 4017 strnstr(ctx->vbios_pn, "D36002", 4018 sizeof(ctx->vbios_pn))) 4019 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX); 4020} 4021 4022/* Query ras capablity via atomfirmware interface */ 4023static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev) 4024{ 4025 /* mem_ecc cap */ 4026 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { 4027 dev_info(adev->dev, "MEM ECC is active.\n"); 4028 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC | 4029 1 << AMDGPU_RAS_BLOCK__DF); 4030 } else { 4031 dev_info(adev->dev, "MEM ECC is not presented.\n"); 4032 } 4033 4034 /* sram_ecc cap */ 4035 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) { 4036 dev_info(adev->dev, "SRAM ECC is active.\n"); 4037 if (!amdgpu_sriov_vf(adev)) 4038 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC | 4039 1 << AMDGPU_RAS_BLOCK__DF); 4040 else 4041 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF | 4042 1 << AMDGPU_RAS_BLOCK__SDMA | 4043 1 << AMDGPU_RAS_BLOCK__GFX); 4044 4045 /* 4046 * VCN/JPEG RAS can be supported on both bare metal and 4047 * SRIOV environment 4048 */ 4049 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) || 4050 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) || 4051 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) || 4052 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1)) 4053 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN | 4054 1 << AMDGPU_RAS_BLOCK__JPEG); 4055 else 4056 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN | 4057 1 << AMDGPU_RAS_BLOCK__JPEG); 4058 4059 /* 4060 * XGMI RAS is not supported if xgmi num physical nodes 4061 * is zero 4062 */ 4063 if (!adev->gmc.xgmi.num_physical_nodes) 4064 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL); 4065 } else { 4066 dev_info(adev->dev, "SRAM ECC is not presented.\n"); 4067 } 4068} 4069 4070/* Query poison mode from umc/df IP callbacks */ 4071static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev) 4072{ 4073 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4074 bool df_poison, umc_poison; 4075 4076 /* poison setting is useless on SRIOV guest */ 4077 if (amdgpu_sriov_vf(adev) || !con) 4078 return; 4079 4080 /* Init poison supported flag, the default value is false */ 4081 if (adev->gmc.xgmi.connected_to_cpu || 4082 adev->gmc.is_app_apu) { 4083 /* enabled by default when GPU is connected to CPU */ 4084 con->poison_supported = true; 4085 } else if (adev->df.funcs && 4086 adev->df.funcs->query_ras_poison_mode && 4087 adev->umc.ras && 4088 adev->umc.ras->query_ras_poison_mode) { 4089 df_poison = 4090 adev->df.funcs->query_ras_poison_mode(adev); 4091 umc_poison = 4092 adev->umc.ras->query_ras_poison_mode(adev); 4093 4094 /* Only poison is set in both DF and UMC, we can support it */ 4095 if (df_poison && umc_poison) 4096 con->poison_supported = true; 4097 else if (df_poison != umc_poison) 4098 dev_warn(adev->dev, 4099 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n", 4100 df_poison, umc_poison); 4101 } 4102} 4103 4104/* 4105 * check hardware's ras ability which will be saved in hw_supported. 4106 * if hardware does not support ras, we can skip some ras initializtion and 4107 * forbid some ras operations from IP. 4108 * if software itself, say boot parameter, limit the ras ability. We still 4109 * need allow IP do some limited operations, like disable. In such case, 4110 * we have to initialize ras as normal. but need check if operation is 4111 * allowed or not in each function. 4112 */ 4113static void amdgpu_ras_check_supported(struct amdgpu_device *adev) 4114{ 4115 adev->ras_hw_enabled = adev->ras_enabled = 0; 4116 4117 if (!amdgpu_ras_asic_supported(adev)) 4118 return; 4119 4120 if (amdgpu_sriov_vf(adev)) { 4121 if (amdgpu_virt_get_ras_capability(adev)) 4122 goto init_ras_enabled_flag; 4123 } 4124 4125 /* query ras capability from psp */ 4126 if (amdgpu_psp_get_ras_capability(&adev->psp)) 4127 goto init_ras_enabled_flag; 4128 4129 /* query ras capablity from bios */ 4130 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4131 amdgpu_ras_query_ras_capablity_from_vbios(adev); 4132 } else { 4133 /* driver only manages a few IP blocks RAS feature 4134 * when GPU is connected cpu through XGMI */ 4135 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX | 4136 1 << AMDGPU_RAS_BLOCK__SDMA | 4137 1 << AMDGPU_RAS_BLOCK__MMHUB); 4138 } 4139 4140 /* apply asic specific settings (vega20 only for now) */ 4141 amdgpu_ras_get_quirks(adev); 4142 4143 /* query poison mode from umc/df ip callback */ 4144 amdgpu_ras_query_poison_mode(adev); 4145 4146init_ras_enabled_flag: 4147 /* hw_supported needs to be aligned with RAS block mask. */ 4148 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK; 4149 4150 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : 4151 adev->ras_hw_enabled & amdgpu_ras_mask; 4152 4153 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ 4154 if (!amdgpu_sriov_vf(adev)) { 4155 adev->aca.is_enabled = 4156 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || 4157 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || 4158 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)); 4159 } 4160 4161 /* bad page feature is not applicable to specific app platform */ 4162 if (adev->gmc.is_app_apu && 4163 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) 4164 amdgpu_bad_page_threshold = 0; 4165} 4166 4167static void amdgpu_ras_counte_dw(struct work_struct *work) 4168{ 4169 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, 4170 ras_counte_delay_work.work); 4171 struct amdgpu_device *adev = con->adev; 4172 struct drm_device *dev = adev_to_drm(adev); 4173 unsigned long ce_count, ue_count; 4174 int res; 4175 4176 res = pm_runtime_get_sync(dev->dev); 4177 if (res < 0) 4178 goto Out; 4179 4180 /* Cache new values. 4181 */ 4182 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) { 4183 atomic_set(&con->ras_ce_count, ce_count); 4184 atomic_set(&con->ras_ue_count, ue_count); 4185 } 4186 4187Out: 4188 pm_runtime_put_autosuspend(dev->dev); 4189} 4190 4191static int amdgpu_get_ras_schema(struct amdgpu_device *adev) 4192{ 4193 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 | 4194 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE | 4195 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE | 4196 AMDGPU_RAS_ERROR__PARITY; 4197} 4198 4199static void ras_event_mgr_init(struct ras_event_manager *mgr) 4200{ 4201 struct ras_event_state *event_state; 4202 int i; 4203 4204 memset(mgr, 0, sizeof(*mgr)); 4205 atomic64_set(&mgr->seqno, 0); 4206 4207 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) { 4208 event_state = &mgr->event_state[i]; 4209 event_state->last_seqno = RAS_EVENT_INVALID_ID; 4210 atomic64_set(&event_state->count, 0); 4211 } 4212} 4213 4214static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev) 4215{ 4216 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4217 struct amdgpu_hive_info *hive; 4218 4219 if (!ras) 4220 return; 4221 4222 hive = amdgpu_get_xgmi_hive(adev); 4223 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr; 4224 4225 /* init event manager with node 0 on xgmi system */ 4226 if (!amdgpu_reset_in_recovery(adev)) { 4227 if (!hive || adev->gmc.xgmi.node_id == 0) 4228 ras_event_mgr_init(ras->event_mgr); 4229 } 4230 4231 if (hive) 4232 amdgpu_put_xgmi_hive(hive); 4233} 4234 4235static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev) 4236{ 4237 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4238 4239 if (!con || (adev->flags & AMD_IS_APU)) 4240 return; 4241 4242 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { 4243 case IP_VERSION(13, 0, 2): 4244 case IP_VERSION(13, 0, 6): 4245 case IP_VERSION(13, 0, 12): 4246 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT; 4247 break; 4248 case IP_VERSION(13, 0, 14): 4249 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1); 4250 break; 4251 default: 4252 break; 4253 } 4254} 4255 4256int amdgpu_ras_init(struct amdgpu_device *adev) 4257{ 4258 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4259 int r; 4260 4261 if (con) 4262 return 0; 4263 4264 con = kzalloc(sizeof(*con) + 4265 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT + 4266 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT, 4267 GFP_KERNEL); 4268 if (!con) 4269 return -ENOMEM; 4270 4271 con->adev = adev; 4272 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw); 4273 atomic_set(&con->ras_ce_count, 0); 4274 atomic_set(&con->ras_ue_count, 0); 4275 4276 con->objs = (struct ras_manager *)(con + 1); 4277 4278 amdgpu_ras_set_context(adev, con); 4279 4280 amdgpu_ras_check_supported(adev); 4281 4282 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) { 4283 /* set gfx block ras context feature for VEGA20 Gaming 4284 * send ras disable cmd to ras ta during ras late init. 4285 */ 4286 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) { 4287 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX); 4288 4289 return 0; 4290 } 4291 4292 r = 0; 4293 goto release_con; 4294 } 4295 4296 con->update_channel_flag = false; 4297 con->features = 0; 4298 con->schema = 0; 4299 INIT_LIST_HEAD(&con->head); 4300 /* Might need get this flag from vbios. */ 4301 con->flags = RAS_DEFAULT_FLAGS; 4302 4303 /* initialize nbio ras function ahead of any other 4304 * ras functions so hardware fatal error interrupt 4305 * can be enabled as early as possible */ 4306 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) { 4307 case IP_VERSION(7, 4, 0): 4308 case IP_VERSION(7, 4, 1): 4309 case IP_VERSION(7, 4, 4): 4310 if (!adev->gmc.xgmi.connected_to_cpu) 4311 adev->nbio.ras = &nbio_v7_4_ras; 4312 break; 4313 case IP_VERSION(4, 3, 0): 4314 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4315 /* unlike other generation of nbio ras, 4316 * nbio v4_3 only support fatal error interrupt 4317 * to inform software that DF is freezed due to 4318 * system fatal error event. driver should not 4319 * enable nbio ras in such case. Instead, 4320 * check DF RAS */ 4321 adev->nbio.ras = &nbio_v4_3_ras; 4322 break; 4323 case IP_VERSION(6, 3, 1): 4324 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF)) 4325 /* unlike other generation of nbio ras, 4326 * nbif v6_3_1 only support fatal error interrupt 4327 * to inform software that DF is freezed due to 4328 * system fatal error event. driver should not 4329 * enable nbio ras in such case. Instead, 4330 * check DF RAS 4331 */ 4332 adev->nbio.ras = &nbif_v6_3_1_ras; 4333 break; 4334 case IP_VERSION(7, 9, 0): 4335 case IP_VERSION(7, 9, 1): 4336 if (!adev->gmc.is_app_apu) 4337 adev->nbio.ras = &nbio_v7_9_ras; 4338 break; 4339 default: 4340 /* nbio ras is not available */ 4341 break; 4342 } 4343 4344 /* nbio ras block needs to be enabled ahead of other ras blocks 4345 * to handle fatal error */ 4346 r = amdgpu_nbio_ras_sw_init(adev); 4347 if (r) 4348 goto release_con; 4349 4350 if (adev->nbio.ras && 4351 adev->nbio.ras->init_ras_controller_interrupt) { 4352 r = adev->nbio.ras->init_ras_controller_interrupt(adev); 4353 if (r) 4354 goto release_con; 4355 } 4356 4357 if (adev->nbio.ras && 4358 adev->nbio.ras->init_ras_err_event_athub_interrupt) { 4359 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev); 4360 if (r) 4361 goto release_con; 4362 } 4363 4364 /* Packed socket_id to ras feature mask bits[31:29] */ 4365 if (adev->smuio.funcs && 4366 adev->smuio.funcs->get_socket_id) 4367 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) << 4368 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT); 4369 4370 /* Get RAS schema for particular SOC */ 4371 con->schema = amdgpu_get_ras_schema(adev); 4372 4373 amdgpu_ras_init_reserved_vram_size(adev); 4374 4375 if (amdgpu_ras_fs_init(adev)) { 4376 r = -EINVAL; 4377 goto release_con; 4378 } 4379 4380 if (amdgpu_ras_aca_is_supported(adev)) { 4381 if (amdgpu_aca_is_enabled(adev)) 4382 r = amdgpu_aca_init(adev); 4383 else 4384 r = amdgpu_mca_init(adev); 4385 if (r) 4386 goto release_con; 4387 } 4388 4389 con->init_task_pid = task_pid_nr(current); 4390 get_task_comm(con->init_task_comm, current); 4391 4392 mutex_init(&con->critical_region_lock); 4393 INIT_LIST_HEAD(&con->critical_region_head); 4394 4395 dev_info(adev->dev, "RAS INFO: ras initialized successfully, " 4396 "hardware ability[%x] ras_mask[%x]\n", 4397 adev->ras_hw_enabled, adev->ras_enabled); 4398 4399 return 0; 4400release_con: 4401 amdgpu_ras_set_context(adev, NULL); 4402 kfree(con); 4403 4404 return r; 4405} 4406 4407int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev) 4408{ 4409 if (adev->gmc.xgmi.connected_to_cpu || 4410 adev->gmc.is_app_apu) 4411 return 1; 4412 return 0; 4413} 4414 4415static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev, 4416 struct ras_common_if *ras_block) 4417{ 4418 struct ras_query_if info = { 4419 .head = *ras_block, 4420 }; 4421 4422 if (!amdgpu_persistent_edc_harvesting_supported(adev)) 4423 return 0; 4424 4425 if (amdgpu_ras_query_error_status(adev, &info) != 0) 4426 DRM_WARN("RAS init harvest failure"); 4427 4428 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0) 4429 DRM_WARN("RAS init harvest reset failure"); 4430 4431 return 0; 4432} 4433 4434bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev) 4435{ 4436 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4437 4438 if (!con) 4439 return false; 4440 4441 return con->poison_supported; 4442} 4443 4444/* helper function to handle common stuff in ip late init phase */ 4445int amdgpu_ras_block_late_init(struct amdgpu_device *adev, 4446 struct ras_common_if *ras_block) 4447{ 4448 struct amdgpu_ras_block_object *ras_obj = NULL; 4449 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4450 struct ras_query_if *query_info; 4451 unsigned long ue_count, ce_count; 4452 int r; 4453 4454 /* disable RAS feature per IP block if it is not supported */ 4455 if (!amdgpu_ras_is_supported(adev, ras_block->block)) { 4456 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0); 4457 return 0; 4458 } 4459 4460 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1); 4461 if (r) { 4462 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) { 4463 /* in resume phase, if fail to enable ras, 4464 * clean up all ras fs nodes, and disable ras */ 4465 goto cleanup; 4466 } else 4467 return r; 4468 } 4469 4470 /* check for errors on warm reset edc persisant supported ASIC */ 4471 amdgpu_persistent_edc_harvesting(adev, ras_block); 4472 4473 /* in resume phase, no need to create ras fs node */ 4474 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) 4475 return 0; 4476 4477 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4478 if (ras_obj->ras_cb || (ras_obj->hw_ops && 4479 (ras_obj->hw_ops->query_poison_status || 4480 ras_obj->hw_ops->handle_poison_consumption))) { 4481 r = amdgpu_ras_interrupt_add_handler(adev, ras_block); 4482 if (r) 4483 goto cleanup; 4484 } 4485 4486 if (ras_obj->hw_ops && 4487 (ras_obj->hw_ops->query_ras_error_count || 4488 ras_obj->hw_ops->query_ras_error_status)) { 4489 r = amdgpu_ras_sysfs_create(adev, ras_block); 4490 if (r) 4491 goto interrupt; 4492 4493 /* Those are the cached values at init. 4494 */ 4495 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL); 4496 if (!query_info) 4497 return -ENOMEM; 4498 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if)); 4499 4500 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) { 4501 atomic_set(&con->ras_ce_count, ce_count); 4502 atomic_set(&con->ras_ue_count, ue_count); 4503 } 4504 4505 kfree(query_info); 4506 } 4507 4508 return 0; 4509 4510interrupt: 4511 if (ras_obj->ras_cb) 4512 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4513cleanup: 4514 amdgpu_ras_feature_enable(adev, ras_block, 0); 4515 return r; 4516} 4517 4518static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev, 4519 struct ras_common_if *ras_block) 4520{ 4521 return amdgpu_ras_block_late_init(adev, ras_block); 4522} 4523 4524/* helper function to remove ras fs node and interrupt handler */ 4525void amdgpu_ras_block_late_fini(struct amdgpu_device *adev, 4526 struct ras_common_if *ras_block) 4527{ 4528 struct amdgpu_ras_block_object *ras_obj; 4529 if (!ras_block) 4530 return; 4531 4532 amdgpu_ras_sysfs_remove(adev, ras_block); 4533 4534 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm); 4535 if (ras_obj->ras_cb) 4536 amdgpu_ras_interrupt_remove_handler(adev, ras_block); 4537} 4538 4539static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev, 4540 struct ras_common_if *ras_block) 4541{ 4542 return amdgpu_ras_block_late_fini(adev, ras_block); 4543} 4544 4545/* do some init work after IP late init as dependence. 4546 * and it runs in resume/gpu reset/booting up cases. 4547 */ 4548void amdgpu_ras_resume(struct amdgpu_device *adev) 4549{ 4550 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4551 struct ras_manager *obj, *tmp; 4552 4553 if (!adev->ras_enabled || !con) { 4554 /* clean ras context for VEGA20 Gaming after send ras disable cmd */ 4555 amdgpu_release_ras_context(adev); 4556 4557 return; 4558 } 4559 4560 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) { 4561 /* Set up all other IPs which are not implemented. There is a 4562 * tricky thing that IP's actual ras error type should be 4563 * MULTI_UNCORRECTABLE, but as driver does not handle it, so 4564 * ERROR_NONE make sense anyway. 4565 */ 4566 amdgpu_ras_enable_all_features(adev, 1); 4567 4568 /* We enable ras on all hw_supported block, but as boot 4569 * parameter might disable some of them and one or more IP has 4570 * not implemented yet. So we disable them on behalf. 4571 */ 4572 list_for_each_entry_safe(obj, tmp, &con->head, node) { 4573 if (!amdgpu_ras_is_supported(adev, obj->head.block)) { 4574 amdgpu_ras_feature_enable(adev, &obj->head, 0); 4575 /* there should be no any reference. */ 4576 WARN_ON(alive_obj(obj)); 4577 } 4578 } 4579 } 4580} 4581 4582void amdgpu_ras_suspend(struct amdgpu_device *adev) 4583{ 4584 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4585 4586 if (!adev->ras_enabled || !con) 4587 return; 4588 4589 amdgpu_ras_disable_all_features(adev, 0); 4590 /* Make sure all ras objects are disabled. */ 4591 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4592 amdgpu_ras_disable_all_features(adev, 1); 4593} 4594 4595int amdgpu_ras_late_init(struct amdgpu_device *adev) 4596{ 4597 struct amdgpu_ras_block_list *node, *tmp; 4598 struct amdgpu_ras_block_object *obj; 4599 int r; 4600 4601 amdgpu_ras_event_mgr_init(adev); 4602 4603 if (amdgpu_ras_aca_is_supported(adev)) { 4604 if (amdgpu_reset_in_recovery(adev)) { 4605 if (amdgpu_aca_is_enabled(adev)) 4606 r = amdgpu_aca_reset(adev); 4607 else 4608 r = amdgpu_mca_reset(adev); 4609 if (r) 4610 return r; 4611 } 4612 4613 if (!amdgpu_sriov_vf(adev)) { 4614 if (amdgpu_aca_is_enabled(adev)) 4615 amdgpu_ras_set_aca_debug_mode(adev, false); 4616 else 4617 amdgpu_ras_set_mca_debug_mode(adev, false); 4618 } 4619 } 4620 4621 /* Guest side doesn't need init ras feature */ 4622 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) 4623 return 0; 4624 4625 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { 4626 obj = node->ras_obj; 4627 if (!obj) { 4628 dev_warn(adev->dev, "Warning: abnormal ras list node.\n"); 4629 continue; 4630 } 4631 4632 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block)) 4633 continue; 4634 4635 if (obj->ras_late_init) { 4636 r = obj->ras_late_init(adev, &obj->ras_comm); 4637 if (r) { 4638 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n", 4639 obj->ras_comm.name, r); 4640 return r; 4641 } 4642 } else 4643 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); 4644 } 4645 4646 return 0; 4647} 4648 4649/* do some fini work before IP fini as dependence */ 4650int amdgpu_ras_pre_fini(struct amdgpu_device *adev) 4651{ 4652 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4653 4654 if (!adev->ras_enabled || !con) 4655 return 0; 4656 4657 4658 /* Need disable ras on all IPs here before ip [hw/sw]fini */ 4659 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4660 amdgpu_ras_disable_all_features(adev, 0); 4661 amdgpu_ras_recovery_fini(adev); 4662 return 0; 4663} 4664 4665int amdgpu_ras_fini(struct amdgpu_device *adev) 4666{ 4667 struct amdgpu_ras_block_list *ras_node, *tmp; 4668 struct amdgpu_ras_block_object *obj = NULL; 4669 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4670 4671 if (!adev->ras_enabled || !con) 4672 return 0; 4673 4674 amdgpu_ras_critical_region_fini(adev); 4675 mutex_destroy(&con->critical_region_lock); 4676 4677 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { 4678 if (ras_node->ras_obj) { 4679 obj = ras_node->ras_obj; 4680 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && 4681 obj->ras_fini) 4682 obj->ras_fini(adev, &obj->ras_comm); 4683 else 4684 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); 4685 } 4686 4687 /* Clear ras blocks from ras_list and free ras block list node */ 4688 list_del(&ras_node->node); 4689 kfree(ras_node); 4690 } 4691 4692 amdgpu_ras_fs_fini(adev); 4693 amdgpu_ras_interrupt_remove_all(adev); 4694 4695 if (amdgpu_ras_aca_is_supported(adev)) { 4696 if (amdgpu_aca_is_enabled(adev)) 4697 amdgpu_aca_fini(adev); 4698 else 4699 amdgpu_mca_fini(adev); 4700 } 4701 4702 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); 4703 4704 if (AMDGPU_RAS_GET_FEATURES(con->features)) 4705 amdgpu_ras_disable_all_features(adev, 0); 4706 4707 cancel_delayed_work_sync(&con->ras_counte_delay_work); 4708 4709 amdgpu_ras_set_context(adev, NULL); 4710 kfree(con); 4711 4712 return 0; 4713} 4714 4715bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev) 4716{ 4717 struct amdgpu_ras *ras; 4718 4719 ras = amdgpu_ras_get_context(adev); 4720 if (!ras) 4721 return false; 4722 4723 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4724} 4725 4726void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status) 4727{ 4728 struct amdgpu_ras *ras; 4729 4730 ras = amdgpu_ras_get_context(adev); 4731 if (ras) { 4732 if (status) 4733 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4734 else 4735 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state); 4736 } 4737} 4738 4739void amdgpu_ras_clear_err_state(struct amdgpu_device *adev) 4740{ 4741 struct amdgpu_ras *ras; 4742 4743 ras = amdgpu_ras_get_context(adev); 4744 if (ras) { 4745 ras->ras_err_state = 0; 4746 ras->gpu_reset_flags = 0; 4747 } 4748} 4749 4750void amdgpu_ras_set_err_poison(struct amdgpu_device *adev, 4751 enum amdgpu_ras_block block) 4752{ 4753 struct amdgpu_ras *ras; 4754 4755 ras = amdgpu_ras_get_context(adev); 4756 if (ras) 4757 set_bit(block, &ras->ras_err_state); 4758} 4759 4760bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block) 4761{ 4762 struct amdgpu_ras *ras; 4763 4764 ras = amdgpu_ras_get_context(adev); 4765 if (ras) { 4766 if (block == AMDGPU_RAS_BLOCK__ANY) 4767 return (ras->ras_err_state != 0); 4768 else 4769 return test_bit(block, &ras->ras_err_state) || 4770 test_bit(AMDGPU_RAS_BLOCK__LAST, 4771 &ras->ras_err_state); 4772 } 4773 4774 return false; 4775} 4776 4777static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev) 4778{ 4779 struct amdgpu_ras *ras; 4780 4781 ras = amdgpu_ras_get_context(adev); 4782 if (!ras) 4783 return NULL; 4784 4785 return ras->event_mgr; 4786} 4787 4788int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type, 4789 const void *caller) 4790{ 4791 struct ras_event_manager *event_mgr; 4792 struct ras_event_state *event_state; 4793 int ret = 0; 4794 4795 if (amdgpu_uniras_enabled(adev)) 4796 return 0; 4797 4798 if (type >= RAS_EVENT_TYPE_COUNT) { 4799 ret = -EINVAL; 4800 goto out; 4801 } 4802 4803 event_mgr = __get_ras_event_mgr(adev); 4804 if (!event_mgr) { 4805 ret = -EINVAL; 4806 goto out; 4807 } 4808 4809 event_state = &event_mgr->event_state[type]; 4810 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno); 4811 atomic64_inc(&event_state->count); 4812 4813out: 4814 if (ret && caller) 4815 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n", 4816 (int)type, caller, ret); 4817 4818 return ret; 4819} 4820 4821u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type) 4822{ 4823 struct ras_event_manager *event_mgr; 4824 u64 id; 4825 4826 if (type >= RAS_EVENT_TYPE_COUNT) 4827 return RAS_EVENT_INVALID_ID; 4828 4829 switch (type) { 4830 case RAS_EVENT_TYPE_FATAL: 4831 case RAS_EVENT_TYPE_POISON_CREATION: 4832 case RAS_EVENT_TYPE_POISON_CONSUMPTION: 4833 event_mgr = __get_ras_event_mgr(adev); 4834 if (!event_mgr) 4835 return RAS_EVENT_INVALID_ID; 4836 4837 id = event_mgr->event_state[type].last_seqno; 4838 break; 4839 case RAS_EVENT_TYPE_INVALID: 4840 default: 4841 id = RAS_EVENT_INVALID_ID; 4842 break; 4843 } 4844 4845 return id; 4846} 4847 4848int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev) 4849{ 4850 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) { 4851 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4852 enum ras_event_type type = RAS_EVENT_TYPE_FATAL; 4853 u64 event_id = RAS_EVENT_INVALID_ID; 4854 4855 if (amdgpu_uniras_enabled(adev)) 4856 return 0; 4857 4858 if (!amdgpu_ras_mark_ras_event(adev, type)) 4859 event_id = amdgpu_ras_acquire_event_id(adev, type); 4860 4861 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error" 4862 "(ERREVENT_ATHUB_INTERRUPT) detected!\n"); 4863 4864 amdgpu_ras_set_fed(adev, true); 4865 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 4866 amdgpu_ras_reset_gpu(adev); 4867 } 4868 4869 return -EBUSY; 4870} 4871 4872bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev) 4873{ 4874 if (adev->asic_type == CHIP_VEGA20 && 4875 adev->pm.fw_version <= 0x283400) { 4876 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) && 4877 amdgpu_ras_intr_triggered(); 4878 } 4879 4880 return false; 4881} 4882 4883void amdgpu_release_ras_context(struct amdgpu_device *adev) 4884{ 4885 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 4886 4887 if (!con) 4888 return; 4889 4890 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) { 4891 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX); 4892 amdgpu_ras_set_context(adev, NULL); 4893 kfree(con); 4894 } 4895} 4896 4897#ifdef CONFIG_X86_MCE_AMD 4898static struct amdgpu_device *find_adev(uint32_t node_id) 4899{ 4900 int i; 4901 struct amdgpu_device *adev = NULL; 4902 4903 for (i = 0; i < mce_adev_list.num_gpu; i++) { 4904 adev = mce_adev_list.devs[i]; 4905 4906 if (adev && adev->gmc.xgmi.connected_to_cpu && 4907 adev->gmc.xgmi.physical_node_id == node_id) 4908 break; 4909 adev = NULL; 4910 } 4911 4912 return adev; 4913} 4914 4915#define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF) 4916#define GET_UMC_INST(m) (((m) >> 21) & 0x7) 4917#define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4)) 4918#define GPU_ID_OFFSET 8 4919 4920static int amdgpu_bad_page_notifier(struct notifier_block *nb, 4921 unsigned long val, void *data) 4922{ 4923 struct mce *m = (struct mce *)data; 4924 struct amdgpu_device *adev = NULL; 4925 uint32_t gpu_id = 0; 4926 uint32_t umc_inst = 0, ch_inst = 0; 4927 4928 /* 4929 * If the error was generated in UMC_V2, which belongs to GPU UMCs, 4930 * and error occurred in DramECC (Extended error code = 0) then only 4931 * process the error, else bail out. 4932 */ 4933 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) && 4934 (XEC(m->status, 0x3f) == 0x0))) 4935 return NOTIFY_DONE; 4936 4937 /* 4938 * If it is correctable error, return. 4939 */ 4940 if (mce_is_correctable(m)) 4941 return NOTIFY_OK; 4942 4943 /* 4944 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register. 4945 */ 4946 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET; 4947 4948 adev = find_adev(gpu_id); 4949 if (!adev) { 4950 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__, 4951 gpu_id); 4952 return NOTIFY_DONE; 4953 } 4954 4955 /* 4956 * If it is uncorrectable error, then find out UMC instance and 4957 * channel index. 4958 */ 4959 umc_inst = GET_UMC_INST(m->ipid); 4960 ch_inst = GET_CHAN_INDEX(m->ipid); 4961 4962 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", 4963 umc_inst, ch_inst); 4964 4965 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) 4966 return NOTIFY_OK; 4967 else 4968 return NOTIFY_DONE; 4969} 4970 4971static struct notifier_block amdgpu_bad_page_nb = { 4972 .notifier_call = amdgpu_bad_page_notifier, 4973 .priority = MCE_PRIO_UC, 4974}; 4975 4976static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) 4977{ 4978 /* 4979 * Add the adev to the mce_adev_list. 4980 * During mode2 reset, amdgpu device is temporarily 4981 * removed from the mgpu_info list which can cause 4982 * page retirement to fail. 4983 * Use this list instead of mgpu_info to find the amdgpu 4984 * device on which the UMC error was reported. 4985 */ 4986 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; 4987 4988 /* 4989 * Register the x86 notifier only once 4990 * with MCE subsystem. 4991 */ 4992 if (notifier_registered == false) { 4993 mce_register_decode_chain(&amdgpu_bad_page_nb); 4994 notifier_registered = true; 4995 } 4996} 4997static void amdgpu_unregister_bad_pages_mca_notifier(struct amdgpu_device *adev) 4998{ 4999 int i, j; 5000 5001 if (!notifier_registered && !mce_adev_list.num_gpu) 5002 return; 5003 for (i = 0, j = 0; i < mce_adev_list.num_gpu; i++) { 5004 if (mce_adev_list.devs[i] == adev) 5005 mce_adev_list.devs[i] = NULL; 5006 if (!mce_adev_list.devs[i]) 5007 ++j; 5008 } 5009 5010 if (j == mce_adev_list.num_gpu) { 5011 mce_adev_list.num_gpu = 0; 5012 /* Unregister x86 notifier with MCE subsystem. */ 5013 if (notifier_registered) { 5014 mce_unregister_decode_chain(&amdgpu_bad_page_nb); 5015 notifier_registered = false; 5016 } 5017 } 5018} 5019#endif 5020 5021struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev) 5022{ 5023 if (!adev) 5024 return NULL; 5025 5026 return adev->psp.ras_context.ras; 5027} 5028 5029int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con) 5030{ 5031 if (!adev) 5032 return -EINVAL; 5033 5034 adev->psp.ras_context.ras = ras_con; 5035 return 0; 5036} 5037 5038/* check if ras is supported on block, say, sdma, gfx */ 5039int amdgpu_ras_is_supported(struct amdgpu_device *adev, 5040 unsigned int block) 5041{ 5042 int ret = 0; 5043 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5044 5045 if (block >= AMDGPU_RAS_BLOCK_COUNT) 5046 return 0; 5047 5048 ret = ras && (adev->ras_enabled & (1 << block)); 5049 5050 /* For the special asic with mem ecc enabled but sram ecc 5051 * not enabled, even if the ras block is not supported on 5052 * .ras_enabled, if the asic supports poison mode and the 5053 * ras block has ras configuration, it can be considered 5054 * that the ras block supports ras function. 5055 */ 5056 if (!ret && 5057 (block == AMDGPU_RAS_BLOCK__GFX || 5058 block == AMDGPU_RAS_BLOCK__SDMA || 5059 block == AMDGPU_RAS_BLOCK__VCN || 5060 block == AMDGPU_RAS_BLOCK__JPEG) && 5061 (amdgpu_ras_mask & (1 << block)) && 5062 amdgpu_ras_is_poison_mode_supported(adev) && 5063 amdgpu_ras_get_ras_block(adev, block, 0)) 5064 ret = 1; 5065 5066 return ret; 5067} 5068 5069int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) 5070{ 5071 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5072 5073 /* mode1 is the only selection for RMA status */ 5074 if (amdgpu_ras_is_rma(adev)) { 5075 ras->gpu_reset_flags = 0; 5076 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET; 5077 } 5078 5079 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) { 5080 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 5081 int hive_ras_recovery = 0; 5082 5083 if (hive) { 5084 hive_ras_recovery = atomic_read(&hive->ras_recovery); 5085 amdgpu_put_xgmi_hive(hive); 5086 } 5087 /* In the case of multiple GPUs, after a GPU has started 5088 * resetting all GPUs on hive, other GPUs do not need to 5089 * trigger GPU reset again. 5090 */ 5091 if (!hive_ras_recovery) 5092 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5093 else 5094 atomic_set(&ras->in_recovery, 0); 5095 } else { 5096 flush_work(&ras->recovery_work); 5097 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work); 5098 } 5099 5100 return 0; 5101} 5102 5103int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) 5104{ 5105 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5106 int ret = 0; 5107 5108 if (con) { 5109 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5110 if (!ret) 5111 con->is_aca_debug_mode = enable; 5112 } 5113 5114 return ret; 5115} 5116 5117int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) 5118{ 5119 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5120 int ret = 0; 5121 5122 if (con) { 5123 if (amdgpu_aca_is_enabled(adev)) 5124 ret = amdgpu_aca_smu_set_debug_mode(adev, enable); 5125 else 5126 ret = amdgpu_mca_smu_set_debug_mode(adev, enable); 5127 if (!ret) 5128 con->is_aca_debug_mode = enable; 5129 } 5130 5131 return ret; 5132} 5133 5134bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) 5135{ 5136 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5137 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5138 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5139 5140 if (!con) 5141 return false; 5142 5143 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || 5144 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) 5145 return con->is_aca_debug_mode; 5146 else 5147 return true; 5148} 5149 5150bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, 5151 unsigned int *error_query_mode) 5152{ 5153 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5154 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; 5155 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; 5156 5157 if (!con) { 5158 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; 5159 return false; 5160 } 5161 5162 if (amdgpu_sriov_vf(adev)) { 5163 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; 5164 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { 5165 *error_query_mode = 5166 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; 5167 } else { 5168 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; 5169 } 5170 5171 return true; 5172} 5173 5174/* Register each ip ras block into amdgpu ras */ 5175int amdgpu_ras_register_ras_block(struct amdgpu_device *adev, 5176 struct amdgpu_ras_block_object *ras_block_obj) 5177{ 5178 struct amdgpu_ras_block_list *ras_node; 5179 if (!adev || !ras_block_obj) 5180 return -EINVAL; 5181 5182 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL); 5183 if (!ras_node) 5184 return -ENOMEM; 5185 5186 INIT_LIST_HEAD(&ras_node->node); 5187 ras_node->ras_obj = ras_block_obj; 5188 list_add_tail(&ras_node->node, &adev->ras_list); 5189 5190 return 0; 5191} 5192 5193void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name) 5194{ 5195 if (!err_type_name) 5196 return; 5197 5198 switch (err_type) { 5199 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE: 5200 sprintf(err_type_name, "correctable"); 5201 break; 5202 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE: 5203 sprintf(err_type_name, "uncorrectable"); 5204 break; 5205 default: 5206 sprintf(err_type_name, "unknown"); 5207 break; 5208 } 5209} 5210 5211bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev, 5212 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5213 uint32_t instance, 5214 uint32_t *memory_id) 5215{ 5216 uint32_t err_status_lo_data, err_status_lo_offset; 5217 5218 if (!reg_entry) 5219 return false; 5220 5221 err_status_lo_offset = 5222 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5223 reg_entry->seg_lo, reg_entry->reg_lo); 5224 err_status_lo_data = RREG32(err_status_lo_offset); 5225 5226 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) && 5227 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG)) 5228 return false; 5229 5230 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID); 5231 5232 return true; 5233} 5234 5235bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev, 5236 const struct amdgpu_ras_err_status_reg_entry *reg_entry, 5237 uint32_t instance, 5238 unsigned long *err_cnt) 5239{ 5240 uint32_t err_status_hi_data, err_status_hi_offset; 5241 5242 if (!reg_entry) 5243 return false; 5244 5245 err_status_hi_offset = 5246 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance, 5247 reg_entry->seg_hi, reg_entry->reg_hi); 5248 err_status_hi_data = RREG32(err_status_hi_offset); 5249 5250 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) && 5251 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG)) 5252 /* keep the check here in case we need to refer to the result later */ 5253 dev_dbg(adev->dev, "Invalid err_info field\n"); 5254 5255 /* read err count */ 5256 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT); 5257 5258 return true; 5259} 5260 5261void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev, 5262 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5263 uint32_t reg_list_size, 5264 const struct amdgpu_ras_memory_id_entry *mem_list, 5265 uint32_t mem_list_size, 5266 uint32_t instance, 5267 uint32_t err_type, 5268 unsigned long *err_count) 5269{ 5270 uint32_t memory_id; 5271 unsigned long err_cnt; 5272 char err_type_name[16]; 5273 uint32_t i, j; 5274 5275 for (i = 0; i < reg_list_size; i++) { 5276 /* query memory_id from err_status_lo */ 5277 if (!amdgpu_ras_inst_get_memory_id_field(adev, ®_list[i], 5278 instance, &memory_id)) 5279 continue; 5280 5281 /* query err_cnt from err_status_hi */ 5282 if (!amdgpu_ras_inst_get_err_cnt_field(adev, ®_list[i], 5283 instance, &err_cnt) || 5284 !err_cnt) 5285 continue; 5286 5287 *err_count += err_cnt; 5288 5289 /* log the errors */ 5290 amdgpu_ras_get_error_type_name(err_type, err_type_name); 5291 if (!mem_list) { 5292 /* memory_list is not supported */ 5293 dev_info(adev->dev, 5294 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n", 5295 err_cnt, err_type_name, 5296 reg_list[i].block_name, 5297 instance, memory_id); 5298 } else { 5299 for (j = 0; j < mem_list_size; j++) { 5300 if (memory_id == mem_list[j].memory_id) { 5301 dev_info(adev->dev, 5302 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n", 5303 err_cnt, err_type_name, 5304 reg_list[i].block_name, 5305 instance, mem_list[j].name); 5306 break; 5307 } 5308 } 5309 } 5310 } 5311} 5312 5313void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev, 5314 const struct amdgpu_ras_err_status_reg_entry *reg_list, 5315 uint32_t reg_list_size, 5316 uint32_t instance) 5317{ 5318 uint32_t err_status_lo_offset, err_status_hi_offset; 5319 uint32_t i; 5320 5321 for (i = 0; i < reg_list_size; i++) { 5322 err_status_lo_offset = 5323 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5324 reg_list[i].seg_lo, reg_list[i].reg_lo); 5325 err_status_hi_offset = 5326 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance, 5327 reg_list[i].seg_hi, reg_list[i].reg_hi); 5328 WREG32(err_status_lo_offset, 0); 5329 WREG32(err_status_hi_offset, 0); 5330 } 5331} 5332 5333int amdgpu_ras_error_data_init(struct ras_err_data *err_data) 5334{ 5335 memset(err_data, 0, sizeof(*err_data)); 5336 5337 INIT_LIST_HEAD(&err_data->err_node_list); 5338 5339 return 0; 5340} 5341 5342static void amdgpu_ras_error_node_release(struct ras_err_node *err_node) 5343{ 5344 if (!err_node) 5345 return; 5346 5347 list_del(&err_node->node); 5348 kvfree(err_node); 5349} 5350 5351void amdgpu_ras_error_data_fini(struct ras_err_data *err_data) 5352{ 5353 struct ras_err_node *err_node, *tmp; 5354 5355 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node) 5356 amdgpu_ras_error_node_release(err_node); 5357} 5358 5359static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data, 5360 struct amdgpu_smuio_mcm_config_info *mcm_info) 5361{ 5362 struct ras_err_node *err_node; 5363 struct amdgpu_smuio_mcm_config_info *ref_id; 5364 5365 if (!err_data || !mcm_info) 5366 return NULL; 5367 5368 for_each_ras_error(err_node, err_data) { 5369 ref_id = &err_node->err_info.mcm_info; 5370 5371 if (mcm_info->socket_id == ref_id->socket_id && 5372 mcm_info->die_id == ref_id->die_id) 5373 return err_node; 5374 } 5375 5376 return NULL; 5377} 5378 5379static struct ras_err_node *amdgpu_ras_error_node_new(void) 5380{ 5381 struct ras_err_node *err_node; 5382 5383 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL); 5384 if (!err_node) 5385 return NULL; 5386 5387 INIT_LIST_HEAD(&err_node->node); 5388 5389 return err_node; 5390} 5391 5392static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b) 5393{ 5394 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node); 5395 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node); 5396 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info; 5397 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info; 5398 5399 if (unlikely(infoa->socket_id != infob->socket_id)) 5400 return infoa->socket_id - infob->socket_id; 5401 else 5402 return infoa->die_id - infob->die_id; 5403 5404 return 0; 5405} 5406 5407static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data, 5408 struct amdgpu_smuio_mcm_config_info *mcm_info) 5409{ 5410 struct ras_err_node *err_node; 5411 5412 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info); 5413 if (err_node) 5414 return &err_node->err_info; 5415 5416 err_node = amdgpu_ras_error_node_new(); 5417 if (!err_node) 5418 return NULL; 5419 5420 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info)); 5421 5422 err_data->err_list_count++; 5423 list_add_tail(&err_node->node, &err_data->err_node_list); 5424 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp); 5425 5426 return &err_node->err_info; 5427} 5428 5429int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data, 5430 struct amdgpu_smuio_mcm_config_info *mcm_info, 5431 u64 count) 5432{ 5433 struct ras_err_info *err_info; 5434 5435 if (!err_data || !mcm_info) 5436 return -EINVAL; 5437 5438 if (!count) 5439 return 0; 5440 5441 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5442 if (!err_info) 5443 return -EINVAL; 5444 5445 err_info->ue_count += count; 5446 err_data->ue_count += count; 5447 5448 return 0; 5449} 5450 5451int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data, 5452 struct amdgpu_smuio_mcm_config_info *mcm_info, 5453 u64 count) 5454{ 5455 struct ras_err_info *err_info; 5456 5457 if (!err_data || !mcm_info) 5458 return -EINVAL; 5459 5460 if (!count) 5461 return 0; 5462 5463 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5464 if (!err_info) 5465 return -EINVAL; 5466 5467 err_info->ce_count += count; 5468 err_data->ce_count += count; 5469 5470 return 0; 5471} 5472 5473int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, 5474 struct amdgpu_smuio_mcm_config_info *mcm_info, 5475 u64 count) 5476{ 5477 struct ras_err_info *err_info; 5478 5479 if (!err_data || !mcm_info) 5480 return -EINVAL; 5481 5482 if (!count) 5483 return 0; 5484 5485 err_info = amdgpu_ras_error_get_info(err_data, mcm_info); 5486 if (!err_info) 5487 return -EINVAL; 5488 5489 err_info->de_count += count; 5490 err_data->de_count += count; 5491 5492 return 0; 5493} 5494 5495#define mmMP0_SMN_C2PMSG_92 0x1609C 5496#define mmMP0_SMN_C2PMSG_126 0x160BE 5497static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev, 5498 u32 instance) 5499{ 5500 u32 socket_id, aid_id, hbm_id; 5501 u32 fw_status; 5502 u32 boot_error; 5503 u64 reg_addr; 5504 5505 /* The pattern for smn addressing in other SOC could be different from 5506 * the one for aqua_vanjaram. We should revisit the code if the pattern 5507 * is changed. In such case, replace the aqua_vanjaram implementation 5508 * with more common helper */ 5509 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5510 aqua_vanjaram_encode_ext_smn_addressing(instance); 5511 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5512 5513 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) + 5514 aqua_vanjaram_encode_ext_smn_addressing(instance); 5515 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5516 5517 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error); 5518 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error); 5519 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1); 5520 5521 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error)) 5522 dev_info(adev->dev, 5523 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n", 5524 socket_id, aid_id, hbm_id, fw_status); 5525 5526 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error)) 5527 dev_info(adev->dev, 5528 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n", 5529 socket_id, aid_id, fw_status); 5530 5531 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error)) 5532 dev_info(adev->dev, 5533 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n", 5534 socket_id, aid_id, fw_status); 5535 5536 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error)) 5537 dev_info(adev->dev, 5538 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n", 5539 socket_id, aid_id, fw_status); 5540 5541 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error)) 5542 dev_info(adev->dev, 5543 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n", 5544 socket_id, aid_id, fw_status); 5545 5546 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error)) 5547 dev_info(adev->dev, 5548 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n", 5549 socket_id, aid_id, fw_status); 5550 5551 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error)) 5552 dev_info(adev->dev, 5553 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n", 5554 socket_id, aid_id, hbm_id, fw_status); 5555 5556 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error)) 5557 dev_info(adev->dev, 5558 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n", 5559 socket_id, aid_id, hbm_id, fw_status); 5560 5561 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error)) 5562 dev_info(adev->dev, 5563 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n", 5564 socket_id, aid_id, fw_status); 5565 5566 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error)) 5567 dev_info(adev->dev, 5568 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n", 5569 socket_id, aid_id, fw_status); 5570} 5571 5572static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev, 5573 u32 instance) 5574{ 5575 u64 reg_addr; 5576 u32 reg_data; 5577 int retry_loop; 5578 5579 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) + 5580 aqua_vanjaram_encode_ext_smn_addressing(instance); 5581 5582 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) { 5583 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr); 5584 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS) 5585 return false; 5586 else 5587 msleep(1); 5588 } 5589 5590 return true; 5591} 5592 5593void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances) 5594{ 5595 u32 i; 5596 5597 for (i = 0; i < num_instances; i++) { 5598 if (amdgpu_ras_boot_error_detected(adev, i)) 5599 amdgpu_ras_boot_time_error_reporting(adev, i); 5600 } 5601} 5602 5603int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) 5604{ 5605 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5606 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 5607 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; 5608 int ret = 0; 5609 5610 if (amdgpu_ras_check_critical_address(adev, start)) 5611 return 0; 5612 5613 mutex_lock(&con->page_rsv_lock); 5614 ret = amdgpu_vram_mgr_query_page_status(mgr, start); 5615 if (ret == -ENOENT) 5616 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE); 5617 mutex_unlock(&con->page_rsv_lock); 5618 5619 return ret; 5620} 5621 5622void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id, 5623 const char *fmt, ...) 5624{ 5625 struct va_format vaf; 5626 va_list args; 5627 5628 va_start(args, fmt); 5629 vaf.fmt = fmt; 5630 vaf.va = &args; 5631 5632 if (RAS_EVENT_ID_IS_VALID(event_id)) 5633 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf); 5634 else 5635 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf); 5636 5637 va_end(args); 5638} 5639 5640bool amdgpu_ras_is_rma(struct amdgpu_device *adev) 5641{ 5642 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5643 5644 if (amdgpu_uniras_enabled(adev)) 5645 return amdgpu_ras_mgr_is_rma(adev); 5646 5647 if (!con) 5648 return false; 5649 5650 return con->is_rma; 5651} 5652 5653int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, 5654 struct amdgpu_bo *bo) 5655{ 5656 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5657 struct amdgpu_vram_mgr_resource *vres; 5658 struct ras_critical_region *region; 5659 struct drm_buddy_block *block; 5660 int ret = 0; 5661 5662 if (!bo || !bo->tbo.resource) 5663 return -EINVAL; 5664 5665 vres = to_amdgpu_vram_mgr_resource(bo->tbo.resource); 5666 5667 mutex_lock(&con->critical_region_lock); 5668 5669 /* Check if the bo had been recorded */ 5670 list_for_each_entry(region, &con->critical_region_head, node) 5671 if (region->bo == bo) 5672 goto out; 5673 5674 /* Record new critical amdgpu bo */ 5675 list_for_each_entry(block, &vres->blocks, link) { 5676 region = kzalloc(sizeof(*region), GFP_KERNEL); 5677 if (!region) { 5678 ret = -ENOMEM; 5679 goto out; 5680 } 5681 region->bo = bo; 5682 region->start = amdgpu_vram_mgr_block_start(block); 5683 region->size = amdgpu_vram_mgr_block_size(block); 5684 list_add_tail(®ion->node, &con->critical_region_head); 5685 } 5686 5687out: 5688 mutex_unlock(&con->critical_region_lock); 5689 5690 return ret; 5691} 5692 5693static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev) 5694{ 5695 amdgpu_ras_add_critical_region(adev, adev->mman.fw_reserved_memory); 5696} 5697 5698static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev) 5699{ 5700 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5701 struct ras_critical_region *region, *tmp; 5702 5703 mutex_lock(&con->critical_region_lock); 5704 list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) { 5705 list_del(®ion->node); 5706 kfree(region); 5707 } 5708 mutex_unlock(&con->critical_region_lock); 5709} 5710 5711bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr) 5712{ 5713 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5714 struct ras_critical_region *region; 5715 bool ret = false; 5716 5717 mutex_lock(&con->critical_region_lock); 5718 list_for_each_entry(region, &con->critical_region_head, node) { 5719 if ((region->start <= addr) && 5720 (addr < (region->start + region->size))) { 5721 ret = true; 5722 break; 5723 } 5724 } 5725 mutex_unlock(&con->critical_region_lock); 5726 5727 return ret; 5728} 5729 5730void amdgpu_ras_pre_reset(struct amdgpu_device *adev, 5731 struct list_head *device_list) 5732{ 5733 struct amdgpu_device *tmp_adev = NULL; 5734 5735 list_for_each_entry(tmp_adev, device_list, reset_list) { 5736 if (amdgpu_uniras_enabled(tmp_adev)) 5737 amdgpu_ras_mgr_pre_reset(tmp_adev); 5738 } 5739} 5740 5741void amdgpu_ras_post_reset(struct amdgpu_device *adev, 5742 struct list_head *device_list) 5743{ 5744 struct amdgpu_device *tmp_adev = NULL; 5745 5746 list_for_each_entry(tmp_adev, device_list, reset_list) { 5747 if (amdgpu_uniras_enabled(tmp_adev)) 5748 amdgpu_ras_mgr_post_reset(tmp_adev); 5749 } 5750}
~/
Read-only file preview.