FILE
amdgpu_device.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
1/* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29#include <linux/aperture.h> 30#include <linux/power_supply.h> 31#include <linux/kthread.h> 32#include <linux/module.h> 33#include <linux/console.h> 34#include <linux/slab.h> 35#include <linux/iommu.h> 36#include <linux/pci.h> 37#include <linux/pci-p2pdma.h> 38#include <linux/apple-gmux.h> 39 40#include <drm/drm_atomic_helper.h> 41#include <drm/drm_client_event.h> 42#include <drm/drm_crtc_helper.h> 43#include <drm/drm_probe_helper.h> 44#include <drm/amdgpu_drm.h> 45#include <linux/device.h> 46#include <linux/vgaarb.h> 47#include <linux/vga_switcheroo.h> 48#include <linux/efi.h> 49#include "amdgpu.h" 50#include "amdgpu_trace.h" 51#include "amdgpu_i2c.h" 52#include "atom.h" 53#include "amdgpu_atombios.h" 54#include "amdgpu_atomfirmware.h" 55#include "amd_pcie.h" 56#ifdef CONFIG_DRM_AMDGPU_SI 57#include "si.h" 58#endif 59#ifdef CONFIG_DRM_AMDGPU_CIK 60#include "cik.h" 61#endif 62#include "vi.h" 63#include "soc15.h" 64#include "nv.h" 65#include "bif/bif_4_1_d.h" 66#include <linux/firmware.h> 67#include "amdgpu_vf_error.h" 68 69#include "amdgpu_amdkfd.h" 70#include "amdgpu_pm.h" 71 72#include "amdgpu_xgmi.h" 73#include "amdgpu_ras.h" 74#include "amdgpu_ras_mgr.h" 75#include "amdgpu_pmu.h" 76#include "amdgpu_fru_eeprom.h" 77#include "amdgpu_reset.h" 78#include "amdgpu_virt.h" 79#include "amdgpu_dev_coredump.h" 80 81#include <linux/suspend.h> 82#include <drm/task_barrier.h> 83#include <linux/pm_runtime.h> 84 85#include <drm/drm_drv.h> 86 87#if IS_ENABLED(CONFIG_X86) 88#include <asm/intel-family.h> 89#include <asm/cpu_device_id.h> 90#endif 91 92MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 93MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 94MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 95MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 96MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 97MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 98MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 99MODULE_FIRMWARE("amdgpu/cyan_skillfish_gpu_info.bin"); 100 101#define AMDGPU_RESUME_MS 2000 102#define AMDGPU_MAX_RETRY_LIMIT 2 103#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL) 104#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2) 105#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2) 106#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2) 107 108#define AMDGPU_VBIOS_SKIP (1U << 0) 109#define AMDGPU_VBIOS_OPTIONAL (1U << 1) 110 111static const struct drm_driver amdgpu_kms_driver; 112 113const char *amdgpu_asic_name[] = { 114 "TAHITI", 115 "PITCAIRN", 116 "VERDE", 117 "OLAND", 118 "HAINAN", 119 "BONAIRE", 120 "KAVERI", 121 "KABINI", 122 "HAWAII", 123 "MULLINS", 124 "TOPAZ", 125 "TONGA", 126 "FIJI", 127 "CARRIZO", 128 "STONEY", 129 "POLARIS10", 130 "POLARIS11", 131 "POLARIS12", 132 "VEGAM", 133 "VEGA10", 134 "VEGA12", 135 "VEGA20", 136 "RAVEN", 137 "ARCTURUS", 138 "RENOIR", 139 "ALDEBARAN", 140 "NAVI10", 141 "CYAN_SKILLFISH", 142 "NAVI14", 143 "NAVI12", 144 "SIENNA_CICHLID", 145 "NAVY_FLOUNDER", 146 "VANGOGH", 147 "DIMGREY_CAVEFISH", 148 "BEIGE_GOBY", 149 "YELLOW_CARP", 150 "IP DISCOVERY", 151 "LAST", 152}; 153 154#define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0) 155/* 156 * Default init level where all blocks are expected to be initialized. This is 157 * the level of initialization expected by default and also after a full reset 158 * of the device. 159 */ 160struct amdgpu_init_level amdgpu_init_default = { 161 .level = AMDGPU_INIT_LEVEL_DEFAULT, 162 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 163}; 164 165struct amdgpu_init_level amdgpu_init_recovery = { 166 .level = AMDGPU_INIT_LEVEL_RESET_RECOVERY, 167 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 168}; 169 170/* 171 * Minimal blocks needed to be initialized before a XGMI hive can be reset. This 172 * is used for cases like reset on initialization where the entire hive needs to 173 * be reset before first use. 174 */ 175struct amdgpu_init_level amdgpu_init_minimal_xgmi = { 176 .level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI, 177 .hwini_ip_block_mask = 178 BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) | 179 BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) | 180 BIT(AMD_IP_BLOCK_TYPE_PSP) 181}; 182 183static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev); 184static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev); 185static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev); 186 187static void amdgpu_device_load_switch_state(struct amdgpu_device *adev); 188 189static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev, 190 enum amd_ip_block_type block) 191{ 192 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0; 193} 194 195void amdgpu_set_init_level(struct amdgpu_device *adev, 196 enum amdgpu_init_lvl_id lvl) 197{ 198 switch (lvl) { 199 case AMDGPU_INIT_LEVEL_MINIMAL_XGMI: 200 adev->init_lvl = &amdgpu_init_minimal_xgmi; 201 break; 202 case AMDGPU_INIT_LEVEL_RESET_RECOVERY: 203 adev->init_lvl = &amdgpu_init_recovery; 204 break; 205 case AMDGPU_INIT_LEVEL_DEFAULT: 206 fallthrough; 207 default: 208 adev->init_lvl = &amdgpu_init_default; 209 break; 210 } 211} 212 213static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev); 214static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 215 void *data); 216 217/** 218 * DOC: pcie_replay_count 219 * 220 * The amdgpu driver provides a sysfs API for reporting the total number 221 * of PCIe replays (NAKs). 222 * The file pcie_replay_count is used for this and returns the total 223 * number of replays as a sum of the NAKs generated and NAKs received. 224 */ 225 226static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 227 struct device_attribute *attr, char *buf) 228{ 229 struct drm_device *ddev = dev_get_drvdata(dev); 230 struct amdgpu_device *adev = drm_to_adev(ddev); 231 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 232 233 return sysfs_emit(buf, "%llu\n", cnt); 234} 235 236static DEVICE_ATTR(pcie_replay_count, 0444, 237 amdgpu_device_get_pcie_replay_count, NULL); 238 239static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev) 240{ 241 int ret = 0; 242 243 if (amdgpu_nbio_is_replay_cnt_supported(adev)) 244 ret = sysfs_create_file(&adev->dev->kobj, 245 &dev_attr_pcie_replay_count.attr); 246 247 return ret; 248} 249 250static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev) 251{ 252 if (amdgpu_nbio_is_replay_cnt_supported(adev)) 253 sysfs_remove_file(&adev->dev->kobj, 254 &dev_attr_pcie_replay_count.attr); 255} 256 257static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj, 258 const struct bin_attribute *attr, char *buf, 259 loff_t ppos, size_t count) 260{ 261 struct device *dev = kobj_to_dev(kobj); 262 struct drm_device *ddev = dev_get_drvdata(dev); 263 struct amdgpu_device *adev = drm_to_adev(ddev); 264 ssize_t bytes_read; 265 266 switch (ppos) { 267 case AMDGPU_SYS_REG_STATE_XGMI: 268 bytes_read = amdgpu_asic_get_reg_state( 269 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count); 270 break; 271 case AMDGPU_SYS_REG_STATE_WAFL: 272 bytes_read = amdgpu_asic_get_reg_state( 273 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count); 274 break; 275 case AMDGPU_SYS_REG_STATE_PCIE: 276 bytes_read = amdgpu_asic_get_reg_state( 277 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count); 278 break; 279 case AMDGPU_SYS_REG_STATE_USR: 280 bytes_read = amdgpu_asic_get_reg_state( 281 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count); 282 break; 283 case AMDGPU_SYS_REG_STATE_USR_1: 284 bytes_read = amdgpu_asic_get_reg_state( 285 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count); 286 break; 287 default: 288 return -EINVAL; 289 } 290 291 return bytes_read; 292} 293 294static const BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL, 295 AMDGPU_SYS_REG_STATE_END); 296 297int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev) 298{ 299 int ret; 300 301 if (!amdgpu_asic_get_reg_state_supported(adev)) 302 return 0; 303 304 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 305 306 return ret; 307} 308 309void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev) 310{ 311 if (!amdgpu_asic_get_reg_state_supported(adev)) 312 return; 313 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 314} 315 316int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block) 317{ 318 int r; 319 320 if (ip_block->version->funcs->suspend) { 321 r = ip_block->version->funcs->suspend(ip_block); 322 if (r) { 323 dev_err(ip_block->adev->dev, 324 "suspend of IP block <%s> failed %d\n", 325 ip_block->version->funcs->name, r); 326 return r; 327 } 328 } 329 330 ip_block->status.hw = false; 331 return 0; 332} 333 334int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block) 335{ 336 int r; 337 338 if (ip_block->version->funcs->resume) { 339 r = ip_block->version->funcs->resume(ip_block); 340 if (r) { 341 dev_err(ip_block->adev->dev, 342 "resume of IP block <%s> failed %d\n", 343 ip_block->version->funcs->name, r); 344 return r; 345 } 346 } 347 348 ip_block->status.hw = true; 349 return 0; 350} 351 352/** 353 * DOC: board_info 354 * 355 * The amdgpu driver provides a sysfs API for giving board related information. 356 * It provides the form factor information in the format 357 * 358 * type : form factor 359 * 360 * Possible form factor values 361 * 362 * - "cem" - PCIE CEM card 363 * - "oam" - Open Compute Accelerator Module 364 * - "unknown" - Not known 365 * 366 */ 367 368static ssize_t amdgpu_device_get_board_info(struct device *dev, 369 struct device_attribute *attr, 370 char *buf) 371{ 372 struct drm_device *ddev = dev_get_drvdata(dev); 373 struct amdgpu_device *adev = drm_to_adev(ddev); 374 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM; 375 const char *pkg; 376 377 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) 378 pkg_type = adev->smuio.funcs->get_pkg_type(adev); 379 380 switch (pkg_type) { 381 case AMDGPU_PKG_TYPE_CEM: 382 pkg = "cem"; 383 break; 384 case AMDGPU_PKG_TYPE_OAM: 385 pkg = "oam"; 386 break; 387 default: 388 pkg = "unknown"; 389 break; 390 } 391 392 return sysfs_emit(buf, "%s : %s\n", "type", pkg); 393} 394 395static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL); 396 397static struct attribute *amdgpu_board_attrs[] = { 398 &dev_attr_board_info.attr, 399 NULL, 400}; 401 402static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj, 403 struct attribute *attr, int n) 404{ 405 struct device *dev = kobj_to_dev(kobj); 406 struct drm_device *ddev = dev_get_drvdata(dev); 407 struct amdgpu_device *adev = drm_to_adev(ddev); 408 409 if (adev->flags & AMD_IS_APU) 410 return 0; 411 412 return attr->mode; 413} 414 415static const struct attribute_group amdgpu_board_attrs_group = { 416 .attrs = amdgpu_board_attrs, 417 .is_visible = amdgpu_board_attrs_is_visible 418}; 419 420static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 421 422/** 423 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control 424 * 425 * @adev: amdgpu device pointer 426 * 427 * Returns true if the device is a dGPU with ATPX power control, 428 * otherwise return false. 429 */ 430bool amdgpu_device_supports_px(struct amdgpu_device *adev) 431{ 432 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) 433 return true; 434 return false; 435} 436 437/** 438 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources 439 * 440 * @adev: amdgpu device pointer 441 * 442 * Returns true if the device is a dGPU with ACPI power control, 443 * otherwise return false. 444 */ 445bool amdgpu_device_supports_boco(struct amdgpu_device *adev) 446{ 447 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) 448 return false; 449 450 if (adev->has_pr3 || 451 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) 452 return true; 453 return false; 454} 455 456/** 457 * amdgpu_device_supports_baco - Does the device support BACO 458 * 459 * @adev: amdgpu device pointer 460 * 461 * Return: 462 * 1 if the device supports BACO; 463 * 3 if the device supports MACO (only works if BACO is supported) 464 * otherwise return 0. 465 */ 466int amdgpu_device_supports_baco(struct amdgpu_device *adev) 467{ 468 return amdgpu_asic_supports_baco(adev); 469} 470 471void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev) 472{ 473 int bamaco_support; 474 475 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; 476 bamaco_support = amdgpu_device_supports_baco(adev); 477 478 switch (amdgpu_runtime_pm) { 479 case 2: 480 if (bamaco_support & MACO_SUPPORT) { 481 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 482 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n"); 483 } else if (bamaco_support == BACO_SUPPORT) { 484 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 485 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n"); 486 } 487 break; 488 case 1: 489 if (bamaco_support & BACO_SUPPORT) { 490 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 491 dev_info(adev->dev, "Forcing BACO for runtime pm\n"); 492 } 493 break; 494 case -1: 495 case -2: 496 if (amdgpu_device_supports_px(adev)) { 497 /* enable PX as runtime mode */ 498 adev->pm.rpm_mode = AMDGPU_RUNPM_PX; 499 dev_info(adev->dev, "Using ATPX for runtime pm\n"); 500 } else if (amdgpu_device_supports_boco(adev)) { 501 /* enable boco as runtime mode */ 502 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO; 503 dev_info(adev->dev, "Using BOCO for runtime pm\n"); 504 } else { 505 if (!bamaco_support) 506 goto no_runtime_pm; 507 508 switch (adev->asic_type) { 509 case CHIP_VEGA20: 510 case CHIP_ARCTURUS: 511 /* BACO are not supported on vega20 and arctrus */ 512 break; 513 case CHIP_VEGA10: 514 /* enable BACO as runpm mode if noretry=0 */ 515 if (!adev->gmc.noretry && !amdgpu_passthrough(adev)) 516 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 517 break; 518 default: 519 /* enable BACO as runpm mode on CI+ */ 520 if (!amdgpu_passthrough(adev)) 521 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 522 break; 523 } 524 525 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) { 526 if (bamaco_support & MACO_SUPPORT) { 527 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 528 dev_info(adev->dev, "Using BAMACO for runtime pm\n"); 529 } else { 530 dev_info(adev->dev, "Using BACO for runtime pm\n"); 531 } 532 } 533 } 534 break; 535 case 0: 536 dev_info(adev->dev, "runtime pm is manually disabled\n"); 537 break; 538 default: 539 break; 540 } 541 542no_runtime_pm: 543 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) 544 dev_info(adev->dev, "Runtime PM not available\n"); 545} 546/** 547 * amdgpu_device_supports_smart_shift - Is the device dGPU with 548 * smart shift support 549 * 550 * @adev: amdgpu device pointer 551 * 552 * Returns true if the device is a dGPU with Smart Shift support, 553 * otherwise returns false. 554 */ 555bool amdgpu_device_supports_smart_shift(struct amdgpu_device *adev) 556{ 557 return (amdgpu_device_supports_boco(adev) && 558 amdgpu_acpi_is_power_shift_control_supported()); 559} 560 561/* 562 * VRAM access helper functions 563 */ 564 565/** 566 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA 567 * 568 * @adev: amdgpu_device pointer 569 * @pos: offset of the buffer in vram 570 * @buf: virtual address of the buffer in system memory 571 * @size: read/write size, sizeof(@buf) must > @size 572 * @write: true - write to vram, otherwise - read from vram 573 */ 574void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, 575 void *buf, size_t size, bool write) 576{ 577 unsigned long flags; 578 uint32_t hi = ~0, tmp = 0; 579 uint32_t *data = buf; 580 uint64_t last; 581 int idx; 582 583 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 584 return; 585 586 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)); 587 588 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 589 for (last = pos + size; pos < last; pos += 4) { 590 tmp = pos >> 31; 591 592 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 593 if (tmp != hi) { 594 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 595 hi = tmp; 596 } 597 if (write) 598 WREG32_NO_KIQ(mmMM_DATA, *data++); 599 else 600 *data++ = RREG32_NO_KIQ(mmMM_DATA); 601 } 602 603 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 604 drm_dev_exit(idx); 605} 606 607/** 608 * amdgpu_device_aper_access - access vram by vram aperture 609 * 610 * @adev: amdgpu_device pointer 611 * @pos: offset of the buffer in vram 612 * @buf: virtual address of the buffer in system memory 613 * @size: read/write size, sizeof(@buf) must > @size 614 * @write: true - write to vram, otherwise - read from vram 615 * 616 * The return value means how many bytes have been transferred. 617 */ 618size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, 619 void *buf, size_t size, bool write) 620{ 621#ifdef CONFIG_64BIT 622 void __iomem *addr; 623 size_t count = 0; 624 uint64_t last; 625 626 if (!adev->mman.aper_base_kaddr) 627 return 0; 628 629 last = min(pos + size, adev->gmc.visible_vram_size); 630 if (last > pos) { 631 addr = adev->mman.aper_base_kaddr + pos; 632 count = last - pos; 633 634 if (write) { 635 memcpy_toio(addr, buf, count); 636 /* Make sure HDP write cache flush happens without any reordering 637 * after the system memory contents are sent over PCIe device 638 */ 639 mb(); 640 amdgpu_device_flush_hdp(adev, NULL); 641 } else { 642 amdgpu_device_invalidate_hdp(adev, NULL); 643 /* Make sure HDP read cache is invalidated before issuing a read 644 * to the PCIe device 645 */ 646 mb(); 647 memcpy_fromio(buf, addr, count); 648 } 649 650 } 651 652 return count; 653#else 654 return 0; 655#endif 656} 657 658/** 659 * amdgpu_device_vram_access - read/write a buffer in vram 660 * 661 * @adev: amdgpu_device pointer 662 * @pos: offset of the buffer in vram 663 * @buf: virtual address of the buffer in system memory 664 * @size: read/write size, sizeof(@buf) must > @size 665 * @write: true - write to vram, otherwise - read from vram 666 */ 667void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 668 void *buf, size_t size, bool write) 669{ 670 size_t count; 671 672 /* try to using vram apreature to access vram first */ 673 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 674 size -= count; 675 if (size) { 676 /* using MM to access rest vram */ 677 pos += count; 678 buf += count; 679 amdgpu_device_mm_access(adev, pos, buf, size, write); 680 } 681} 682 683/* 684 * register access helper functions. 685 */ 686 687/* Check if hw access should be skipped because of hotplug or device error */ 688bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) 689{ 690 if (adev->no_hw_access) 691 return true; 692 693#ifdef CONFIG_LOCKDEP 694 /* 695 * This is a bit complicated to understand, so worth a comment. What we assert 696 * here is that the GPU reset is not running on another thread in parallel. 697 * 698 * For this we trylock the read side of the reset semaphore, if that succeeds 699 * we know that the reset is not running in parallel. 700 * 701 * If the trylock fails we assert that we are either already holding the read 702 * side of the lock or are the reset thread itself and hold the write side of 703 * the lock. 704 */ 705 if (in_task()) { 706 if (down_read_trylock(&adev->reset_domain->sem)) 707 up_read(&adev->reset_domain->sem); 708 else 709 lockdep_assert_held(&adev->reset_domain->sem); 710 } 711#endif 712 return false; 713} 714 715/** 716 * amdgpu_device_rreg - read a memory mapped IO or indirect register 717 * 718 * @adev: amdgpu_device pointer 719 * @reg: dword aligned register offset 720 * @acc_flags: access flags which require special behavior 721 * 722 * Returns the 32 bit value from the offset specified. 723 */ 724uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, 725 uint32_t reg, uint32_t acc_flags) 726{ 727 uint32_t ret; 728 729 if (amdgpu_device_skip_hw_access(adev)) 730 return 0; 731 732 if ((reg * 4) < adev->rmmio_size) { 733 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 734 amdgpu_sriov_runtime(adev) && 735 down_read_trylock(&adev->reset_domain->sem)) { 736 ret = amdgpu_kiq_rreg(adev, reg, 0); 737 up_read(&adev->reset_domain->sem); 738 } else { 739 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 740 } 741 } else { 742 ret = adev->pcie_rreg(adev, reg * 4); 743 } 744 745 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 746 747 return ret; 748} 749 750/* 751 * MMIO register read with bytes helper functions 752 * @offset:bytes offset from MMIO start 753 */ 754 755/** 756 * amdgpu_mm_rreg8 - read a memory mapped IO register 757 * 758 * @adev: amdgpu_device pointer 759 * @offset: byte aligned register offset 760 * 761 * Returns the 8 bit value from the offset specified. 762 */ 763uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 764{ 765 if (amdgpu_device_skip_hw_access(adev)) 766 return 0; 767 768 if (offset < adev->rmmio_size) 769 return (readb(adev->rmmio + offset)); 770 BUG(); 771} 772 773 774/** 775 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC 776 * 777 * @adev: amdgpu_device pointer 778 * @reg: dword aligned register offset 779 * @acc_flags: access flags which require special behavior 780 * @xcc_id: xcc accelerated compute core id 781 * 782 * Returns the 32 bit value from the offset specified. 783 */ 784uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, 785 uint32_t reg, uint32_t acc_flags, 786 uint32_t xcc_id) 787{ 788 uint32_t ret, rlcg_flag; 789 790 if (amdgpu_device_skip_hw_access(adev)) 791 return 0; 792 793 if ((reg * 4) < adev->rmmio_size) { 794 if (amdgpu_sriov_vf(adev) && 795 !amdgpu_sriov_runtime(adev) && 796 adev->gfx.rlc.rlcg_reg_access_supported && 797 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, 798 GC_HWIP, false, 799 &rlcg_flag)) { 800 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id)); 801 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 802 amdgpu_sriov_runtime(adev) && 803 down_read_trylock(&adev->reset_domain->sem)) { 804 ret = amdgpu_kiq_rreg(adev, reg, xcc_id); 805 up_read(&adev->reset_domain->sem); 806 } else { 807 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 808 } 809 } else { 810 ret = adev->pcie_rreg(adev, reg * 4); 811 } 812 813 return ret; 814} 815 816/* 817 * MMIO register write with bytes helper functions 818 * @offset:bytes offset from MMIO start 819 * @value: the value want to be written to the register 820 */ 821 822/** 823 * amdgpu_mm_wreg8 - read a memory mapped IO register 824 * 825 * @adev: amdgpu_device pointer 826 * @offset: byte aligned register offset 827 * @value: 8 bit value to write 828 * 829 * Writes the value specified to the offset specified. 830 */ 831void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 832{ 833 if (amdgpu_device_skip_hw_access(adev)) 834 return; 835 836 if (offset < adev->rmmio_size) 837 writeb(value, adev->rmmio + offset); 838 else 839 BUG(); 840} 841 842/** 843 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 844 * 845 * @adev: amdgpu_device pointer 846 * @reg: dword aligned register offset 847 * @v: 32 bit value to write to the register 848 * @acc_flags: access flags which require special behavior 849 * 850 * Writes the value specified to the offset specified. 851 */ 852void amdgpu_device_wreg(struct amdgpu_device *adev, 853 uint32_t reg, uint32_t v, 854 uint32_t acc_flags) 855{ 856 if (amdgpu_device_skip_hw_access(adev)) 857 return; 858 859 if ((reg * 4) < adev->rmmio_size) { 860 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 861 amdgpu_sriov_runtime(adev) && 862 down_read_trylock(&adev->reset_domain->sem)) { 863 amdgpu_kiq_wreg(adev, reg, v, 0); 864 up_read(&adev->reset_domain->sem); 865 } else { 866 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 867 } 868 } else { 869 adev->pcie_wreg(adev, reg * 4, v); 870 } 871 872 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 873} 874 875/** 876 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range 877 * 878 * @adev: amdgpu_device pointer 879 * @reg: mmio/rlc register 880 * @v: value to write 881 * @xcc_id: xcc accelerated compute core id 882 * 883 * this function is invoked only for the debugfs register access 884 */ 885void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, 886 uint32_t reg, uint32_t v, 887 uint32_t xcc_id) 888{ 889 if (amdgpu_device_skip_hw_access(adev)) 890 return; 891 892 if (amdgpu_sriov_fullaccess(adev) && 893 adev->gfx.rlc.funcs && 894 adev->gfx.rlc.funcs->is_rlcg_access_range) { 895 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 896 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id); 897 } else if ((reg * 4) >= adev->rmmio_size) { 898 adev->pcie_wreg(adev, reg * 4, v); 899 } else { 900 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 901 } 902} 903 904/** 905 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC 906 * 907 * @adev: amdgpu_device pointer 908 * @reg: dword aligned register offset 909 * @v: 32 bit value to write to the register 910 * @acc_flags: access flags which require special behavior 911 * @xcc_id: xcc accelerated compute core id 912 * 913 * Writes the value specified to the offset specified. 914 */ 915void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, 916 uint32_t reg, uint32_t v, 917 uint32_t acc_flags, uint32_t xcc_id) 918{ 919 uint32_t rlcg_flag; 920 921 if (amdgpu_device_skip_hw_access(adev)) 922 return; 923 924 if ((reg * 4) < adev->rmmio_size) { 925 if (amdgpu_sriov_vf(adev) && 926 !amdgpu_sriov_runtime(adev) && 927 adev->gfx.rlc.rlcg_reg_access_supported && 928 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, 929 GC_HWIP, true, 930 &rlcg_flag)) { 931 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id)); 932 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 933 amdgpu_sriov_runtime(adev) && 934 down_read_trylock(&adev->reset_domain->sem)) { 935 amdgpu_kiq_wreg(adev, reg, v, xcc_id); 936 up_read(&adev->reset_domain->sem); 937 } else { 938 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 939 } 940 } else { 941 adev->pcie_wreg(adev, reg * 4, v); 942 } 943} 944 945/** 946 * amdgpu_device_indirect_rreg - read an indirect register 947 * 948 * @adev: amdgpu_device pointer 949 * @reg_addr: indirect register address to read from 950 * 951 * Returns the value of indirect register @reg_addr 952 */ 953u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, 954 u32 reg_addr) 955{ 956 unsigned long flags, pcie_index, pcie_data; 957 void __iomem *pcie_index_offset; 958 void __iomem *pcie_data_offset; 959 u32 r; 960 961 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 962 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 963 964 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 965 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 966 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 967 968 writel(reg_addr, pcie_index_offset); 969 readl(pcie_index_offset); 970 r = readl(pcie_data_offset); 971 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 972 973 return r; 974} 975 976u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, 977 u64 reg_addr) 978{ 979 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 980 u32 r; 981 void __iomem *pcie_index_offset; 982 void __iomem *pcie_index_hi_offset; 983 void __iomem *pcie_data_offset; 984 985 if (unlikely(!adev->nbio.funcs)) { 986 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK; 987 pcie_data = AMDGPU_PCIE_DATA_FALLBACK; 988 } else { 989 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 990 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 991 } 992 993 if (reg_addr >> 32) { 994 if (unlikely(!adev->nbio.funcs)) 995 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK; 996 else 997 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 998 } else { 999 pcie_index_hi = 0; 1000 } 1001 1002 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1003 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1004 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1005 if (pcie_index_hi != 0) 1006 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1007 pcie_index_hi * 4; 1008 1009 writel(reg_addr, pcie_index_offset); 1010 readl(pcie_index_offset); 1011 if (pcie_index_hi != 0) { 1012 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1013 readl(pcie_index_hi_offset); 1014 } 1015 r = readl(pcie_data_offset); 1016 1017 /* clear the high bits */ 1018 if (pcie_index_hi != 0) { 1019 writel(0, pcie_index_hi_offset); 1020 readl(pcie_index_hi_offset); 1021 } 1022 1023 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1024 1025 return r; 1026} 1027 1028/** 1029 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 1030 * 1031 * @adev: amdgpu_device pointer 1032 * @reg_addr: indirect register address to read from 1033 * 1034 * Returns the value of indirect register @reg_addr 1035 */ 1036u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, 1037 u32 reg_addr) 1038{ 1039 unsigned long flags, pcie_index, pcie_data; 1040 void __iomem *pcie_index_offset; 1041 void __iomem *pcie_data_offset; 1042 u64 r; 1043 1044 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1045 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1046 1047 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1048 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1049 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1050 1051 /* read low 32 bits */ 1052 writel(reg_addr, pcie_index_offset); 1053 readl(pcie_index_offset); 1054 r = readl(pcie_data_offset); 1055 /* read high 32 bits */ 1056 writel(reg_addr + 4, pcie_index_offset); 1057 readl(pcie_index_offset); 1058 r |= ((u64)readl(pcie_data_offset) << 32); 1059 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1060 1061 return r; 1062} 1063 1064u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, 1065 u64 reg_addr) 1066{ 1067 unsigned long flags, pcie_index, pcie_data; 1068 unsigned long pcie_index_hi = 0; 1069 void __iomem *pcie_index_offset; 1070 void __iomem *pcie_index_hi_offset; 1071 void __iomem *pcie_data_offset; 1072 u64 r; 1073 1074 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1075 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1076 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1077 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1078 1079 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1080 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1081 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1082 if (pcie_index_hi != 0) 1083 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1084 pcie_index_hi * 4; 1085 1086 /* read low 32 bits */ 1087 writel(reg_addr, pcie_index_offset); 1088 readl(pcie_index_offset); 1089 if (pcie_index_hi != 0) { 1090 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1091 readl(pcie_index_hi_offset); 1092 } 1093 r = readl(pcie_data_offset); 1094 /* read high 32 bits */ 1095 writel(reg_addr + 4, pcie_index_offset); 1096 readl(pcie_index_offset); 1097 if (pcie_index_hi != 0) { 1098 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1099 readl(pcie_index_hi_offset); 1100 } 1101 r |= ((u64)readl(pcie_data_offset) << 32); 1102 1103 /* clear the high bits */ 1104 if (pcie_index_hi != 0) { 1105 writel(0, pcie_index_hi_offset); 1106 readl(pcie_index_hi_offset); 1107 } 1108 1109 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1110 1111 return r; 1112} 1113 1114/** 1115 * amdgpu_device_indirect_wreg - write an indirect register address 1116 * 1117 * @adev: amdgpu_device pointer 1118 * @reg_addr: indirect register offset 1119 * @reg_data: indirect register data 1120 * 1121 */ 1122void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, 1123 u32 reg_addr, u32 reg_data) 1124{ 1125 unsigned long flags, pcie_index, pcie_data; 1126 void __iomem *pcie_index_offset; 1127 void __iomem *pcie_data_offset; 1128 1129 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1130 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1131 1132 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1133 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1134 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1135 1136 writel(reg_addr, pcie_index_offset); 1137 readl(pcie_index_offset); 1138 writel(reg_data, pcie_data_offset); 1139 readl(pcie_data_offset); 1140 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1141} 1142 1143void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, 1144 u64 reg_addr, u32 reg_data) 1145{ 1146 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 1147 void __iomem *pcie_index_offset; 1148 void __iomem *pcie_index_hi_offset; 1149 void __iomem *pcie_data_offset; 1150 1151 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1152 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1153 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1154 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1155 else 1156 pcie_index_hi = 0; 1157 1158 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1159 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1160 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1161 if (pcie_index_hi != 0) 1162 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1163 pcie_index_hi * 4; 1164 1165 writel(reg_addr, pcie_index_offset); 1166 readl(pcie_index_offset); 1167 if (pcie_index_hi != 0) { 1168 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1169 readl(pcie_index_hi_offset); 1170 } 1171 writel(reg_data, pcie_data_offset); 1172 readl(pcie_data_offset); 1173 1174 /* clear the high bits */ 1175 if (pcie_index_hi != 0) { 1176 writel(0, pcie_index_hi_offset); 1177 readl(pcie_index_hi_offset); 1178 } 1179 1180 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1181} 1182 1183/** 1184 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 1185 * 1186 * @adev: amdgpu_device pointer 1187 * @reg_addr: indirect register offset 1188 * @reg_data: indirect register data 1189 * 1190 */ 1191void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, 1192 u32 reg_addr, u64 reg_data) 1193{ 1194 unsigned long flags, pcie_index, pcie_data; 1195 void __iomem *pcie_index_offset; 1196 void __iomem *pcie_data_offset; 1197 1198 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1199 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1200 1201 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1202 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1203 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1204 1205 /* write low 32 bits */ 1206 writel(reg_addr, pcie_index_offset); 1207 readl(pcie_index_offset); 1208 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 1209 readl(pcie_data_offset); 1210 /* write high 32 bits */ 1211 writel(reg_addr + 4, pcie_index_offset); 1212 readl(pcie_index_offset); 1213 writel((u32)(reg_data >> 32), pcie_data_offset); 1214 readl(pcie_data_offset); 1215 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1216} 1217 1218void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, 1219 u64 reg_addr, u64 reg_data) 1220{ 1221 unsigned long flags, pcie_index, pcie_data; 1222 unsigned long pcie_index_hi = 0; 1223 void __iomem *pcie_index_offset; 1224 void __iomem *pcie_index_hi_offset; 1225 void __iomem *pcie_data_offset; 1226 1227 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1228 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1229 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1230 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1231 1232 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1233 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1234 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1235 if (pcie_index_hi != 0) 1236 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1237 pcie_index_hi * 4; 1238 1239 /* write low 32 bits */ 1240 writel(reg_addr, pcie_index_offset); 1241 readl(pcie_index_offset); 1242 if (pcie_index_hi != 0) { 1243 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1244 readl(pcie_index_hi_offset); 1245 } 1246 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 1247 readl(pcie_data_offset); 1248 /* write high 32 bits */ 1249 writel(reg_addr + 4, pcie_index_offset); 1250 readl(pcie_index_offset); 1251 if (pcie_index_hi != 0) { 1252 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1253 readl(pcie_index_hi_offset); 1254 } 1255 writel((u32)(reg_data >> 32), pcie_data_offset); 1256 readl(pcie_data_offset); 1257 1258 /* clear the high bits */ 1259 if (pcie_index_hi != 0) { 1260 writel(0, pcie_index_hi_offset); 1261 readl(pcie_index_hi_offset); 1262 } 1263 1264 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1265} 1266 1267/** 1268 * amdgpu_device_get_rev_id - query device rev_id 1269 * 1270 * @adev: amdgpu_device pointer 1271 * 1272 * Return device rev_id 1273 */ 1274u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev) 1275{ 1276 return adev->nbio.funcs->get_rev_id(adev); 1277} 1278 1279/** 1280 * amdgpu_invalid_rreg - dummy reg read function 1281 * 1282 * @adev: amdgpu_device pointer 1283 * @reg: offset of register 1284 * 1285 * Dummy register read function. Used for register blocks 1286 * that certain asics don't have (all asics). 1287 * Returns the value in the register. 1288 */ 1289static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 1290{ 1291 dev_err(adev->dev, "Invalid callback to read register 0x%04X\n", reg); 1292 BUG(); 1293 return 0; 1294} 1295 1296static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg) 1297{ 1298 dev_err(adev->dev, "Invalid callback to read register 0x%llX\n", reg); 1299 BUG(); 1300 return 0; 1301} 1302 1303/** 1304 * amdgpu_invalid_wreg - dummy reg write function 1305 * 1306 * @adev: amdgpu_device pointer 1307 * @reg: offset of register 1308 * @v: value to write to the register 1309 * 1310 * Dummy register read function. Used for register blocks 1311 * that certain asics don't have (all asics). 1312 */ 1313static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 1314{ 1315 dev_err(adev->dev, 1316 "Invalid callback to write register 0x%04X with 0x%08X\n", reg, 1317 v); 1318 BUG(); 1319} 1320 1321static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v) 1322{ 1323 dev_err(adev->dev, 1324 "Invalid callback to write register 0x%llX with 0x%08X\n", reg, 1325 v); 1326 BUG(); 1327} 1328 1329/** 1330 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function 1331 * 1332 * @adev: amdgpu_device pointer 1333 * @reg: offset of register 1334 * 1335 * Dummy register read function. Used for register blocks 1336 * that certain asics don't have (all asics). 1337 * Returns the value in the register. 1338 */ 1339static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) 1340{ 1341 dev_err(adev->dev, "Invalid callback to read 64 bit register 0x%04X\n", 1342 reg); 1343 BUG(); 1344 return 0; 1345} 1346 1347static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg) 1348{ 1349 dev_err(adev->dev, "Invalid callback to read register 0x%llX\n", reg); 1350 BUG(); 1351 return 0; 1352} 1353 1354/** 1355 * amdgpu_invalid_wreg64 - dummy reg write function 1356 * 1357 * @adev: amdgpu_device pointer 1358 * @reg: offset of register 1359 * @v: value to write to the register 1360 * 1361 * Dummy register read function. Used for register blocks 1362 * that certain asics don't have (all asics). 1363 */ 1364static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 1365{ 1366 dev_err(adev->dev, 1367 "Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n", 1368 reg, v); 1369 BUG(); 1370} 1371 1372static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v) 1373{ 1374 dev_err(adev->dev, 1375 "Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n", 1376 reg, v); 1377 BUG(); 1378} 1379 1380/** 1381 * amdgpu_block_invalid_rreg - dummy reg read function 1382 * 1383 * @adev: amdgpu_device pointer 1384 * @block: offset of instance 1385 * @reg: offset of register 1386 * 1387 * Dummy register read function. Used for register blocks 1388 * that certain asics don't have (all asics). 1389 * Returns the value in the register. 1390 */ 1391static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 1392 uint32_t block, uint32_t reg) 1393{ 1394 dev_err(adev->dev, 1395 "Invalid callback to read register 0x%04X in block 0x%04X\n", 1396 reg, block); 1397 BUG(); 1398 return 0; 1399} 1400 1401/** 1402 * amdgpu_block_invalid_wreg - dummy reg write function 1403 * 1404 * @adev: amdgpu_device pointer 1405 * @block: offset of instance 1406 * @reg: offset of register 1407 * @v: value to write to the register 1408 * 1409 * Dummy register read function. Used for register blocks 1410 * that certain asics don't have (all asics). 1411 */ 1412static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 1413 uint32_t block, 1414 uint32_t reg, uint32_t v) 1415{ 1416 dev_err(adev->dev, 1417 "Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 1418 reg, block, v); 1419 BUG(); 1420} 1421 1422static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev) 1423{ 1424 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) 1425 return AMDGPU_VBIOS_SKIP; 1426 1427 if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev)) 1428 return AMDGPU_VBIOS_OPTIONAL; 1429 1430 return 0; 1431} 1432 1433/** 1434 * amdgpu_device_asic_init - Wrapper for atom asic_init 1435 * 1436 * @adev: amdgpu_device pointer 1437 * 1438 * Does any asic specific work and then calls atom asic init. 1439 */ 1440static int amdgpu_device_asic_init(struct amdgpu_device *adev) 1441{ 1442 uint32_t flags; 1443 bool optional; 1444 int ret; 1445 1446 amdgpu_asic_pre_asic_init(adev); 1447 flags = amdgpu_device_get_vbios_flags(adev); 1448 optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP)); 1449 1450 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 1451 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 1452 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) || 1453 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { 1454 amdgpu_psp_wait_for_bootloader(adev); 1455 if (optional && !adev->bios) 1456 return 0; 1457 1458 ret = amdgpu_atomfirmware_asic_init(adev, true); 1459 return ret; 1460 } else { 1461 if (optional && !adev->bios) 1462 return 0; 1463 1464 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 1465 } 1466 1467 return 0; 1468} 1469 1470/** 1471 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page 1472 * 1473 * @adev: amdgpu_device pointer 1474 * 1475 * Allocates a scratch page of VRAM for use by various things in the 1476 * driver. 1477 */ 1478static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev) 1479{ 1480 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE, 1481 AMDGPU_GEM_DOMAIN_VRAM | 1482 AMDGPU_GEM_DOMAIN_GTT, 1483 &adev->mem_scratch.robj, 1484 &adev->mem_scratch.gpu_addr, 1485 (void **)&adev->mem_scratch.ptr); 1486} 1487 1488/** 1489 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page 1490 * 1491 * @adev: amdgpu_device pointer 1492 * 1493 * Frees the VRAM scratch page. 1494 */ 1495static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev) 1496{ 1497 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); 1498} 1499 1500/** 1501 * amdgpu_device_program_register_sequence - program an array of registers. 1502 * 1503 * @adev: amdgpu_device pointer 1504 * @registers: pointer to the register array 1505 * @array_size: size of the register array 1506 * 1507 * Programs an array or registers with and or masks. 1508 * This is a helper for setting golden registers. 1509 */ 1510void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 1511 const u32 *registers, 1512 const u32 array_size) 1513{ 1514 u32 tmp, reg, and_mask, or_mask; 1515 int i; 1516 1517 if (array_size % 3) 1518 return; 1519 1520 for (i = 0; i < array_size; i += 3) { 1521 reg = registers[i + 0]; 1522 and_mask = registers[i + 1]; 1523 or_mask = registers[i + 2]; 1524 1525 if (and_mask == 0xffffffff) { 1526 tmp = or_mask; 1527 } else { 1528 tmp = RREG32(reg); 1529 tmp &= ~and_mask; 1530 if (adev->family >= AMDGPU_FAMILY_AI) 1531 tmp |= (or_mask & and_mask); 1532 else 1533 tmp |= or_mask; 1534 } 1535 WREG32(reg, tmp); 1536 } 1537} 1538 1539/** 1540 * amdgpu_device_pci_config_reset - reset the GPU 1541 * 1542 * @adev: amdgpu_device pointer 1543 * 1544 * Resets the GPU using the pci config reset sequence. 1545 * Only applicable to asics prior to vega10. 1546 */ 1547void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 1548{ 1549 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 1550} 1551 1552/** 1553 * amdgpu_device_pci_reset - reset the GPU using generic PCI means 1554 * 1555 * @adev: amdgpu_device pointer 1556 * 1557 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). 1558 */ 1559int amdgpu_device_pci_reset(struct amdgpu_device *adev) 1560{ 1561 return pci_reset_function(adev->pdev); 1562} 1563 1564/* 1565 * amdgpu_device_wb_*() 1566 * Writeback is the method by which the GPU updates special pages in memory 1567 * with the status of certain GPU events (fences, ring pointers,etc.). 1568 */ 1569 1570/** 1571 * amdgpu_device_wb_fini - Disable Writeback and free memory 1572 * 1573 * @adev: amdgpu_device pointer 1574 * 1575 * Disables Writeback and frees the Writeback memory (all asics). 1576 * Used at driver shutdown. 1577 */ 1578static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 1579{ 1580 if (adev->wb.wb_obj) { 1581 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 1582 &adev->wb.gpu_addr, 1583 (void **)&adev->wb.wb); 1584 adev->wb.wb_obj = NULL; 1585 } 1586} 1587 1588/** 1589 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory 1590 * 1591 * @adev: amdgpu_device pointer 1592 * 1593 * Initializes writeback and allocates writeback memory (all asics). 1594 * Used at driver startup. 1595 * Returns 0 on success or an -error on failure. 1596 */ 1597static int amdgpu_device_wb_init(struct amdgpu_device *adev) 1598{ 1599 int r; 1600 1601 if (adev->wb.wb_obj == NULL) { 1602 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 1603 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 1604 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1605 &adev->wb.wb_obj, &adev->wb.gpu_addr, 1606 (void **)&adev->wb.wb); 1607 if (r) { 1608 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 1609 return r; 1610 } 1611 1612 adev->wb.num_wb = AMDGPU_MAX_WB; 1613 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 1614 1615 /* clear wb memory */ 1616 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 1617 } 1618 1619 return 0; 1620} 1621 1622/** 1623 * amdgpu_device_wb_get - Allocate a wb entry 1624 * 1625 * @adev: amdgpu_device pointer 1626 * @wb: wb index 1627 * 1628 * Allocate a wb slot for use by the driver (all asics). 1629 * Returns 0 on success or -EINVAL on failure. 1630 */ 1631int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 1632{ 1633 unsigned long flags, offset; 1634 1635 spin_lock_irqsave(&adev->wb.lock, flags); 1636 offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 1637 if (offset < adev->wb.num_wb) { 1638 __set_bit(offset, adev->wb.used); 1639 spin_unlock_irqrestore(&adev->wb.lock, flags); 1640 *wb = offset << 3; /* convert to dw offset */ 1641 return 0; 1642 } else { 1643 spin_unlock_irqrestore(&adev->wb.lock, flags); 1644 return -EINVAL; 1645 } 1646} 1647 1648/** 1649 * amdgpu_device_wb_free - Free a wb entry 1650 * 1651 * @adev: amdgpu_device pointer 1652 * @wb: wb index 1653 * 1654 * Free a wb slot allocated for use by the driver (all asics) 1655 */ 1656void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 1657{ 1658 unsigned long flags; 1659 1660 wb >>= 3; 1661 spin_lock_irqsave(&adev->wb.lock, flags); 1662 if (wb < adev->wb.num_wb) 1663 __clear_bit(wb, adev->wb.used); 1664 spin_unlock_irqrestore(&adev->wb.lock, flags); 1665} 1666 1667/** 1668 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1669 * 1670 * @adev: amdgpu_device pointer 1671 * 1672 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1673 * to fail, but if any of the BARs is not accessible after the size we abort 1674 * driver loading by returning -ENODEV. 1675 */ 1676int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1677{ 1678 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); 1679 struct pci_bus *root; 1680 struct resource *res; 1681 int max_size, r; 1682 unsigned int i; 1683 u16 cmd; 1684 1685 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) 1686 return 0; 1687 1688 /* Bypass for VF */ 1689 if (amdgpu_sriov_vf(adev)) 1690 return 0; 1691 1692 if (!amdgpu_rebar) 1693 return 0; 1694 1695 /* resizing on Dell G5 SE platforms causes problems with runtime pm */ 1696 if ((amdgpu_runtime_pm != 0) && 1697 adev->pdev->vendor == PCI_VENDOR_ID_ATI && 1698 adev->pdev->device == 0x731f && 1699 adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL) 1700 return 0; 1701 1702 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */ 1703 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR)) 1704 dev_warn( 1705 adev->dev, 1706 "System can't access extended configuration space, please check!!\n"); 1707 1708 /* skip if the bios has already enabled large BAR */ 1709 if (adev->gmc.real_vram_size && 1710 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1711 return 0; 1712 1713 /* Check if the root BUS has 64bit memory resources */ 1714 root = adev->pdev->bus; 1715 while (root->parent) 1716 root = root->parent; 1717 1718 pci_bus_for_each_resource(root, res, i) { 1719 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1720 res->start > 0x100000000ull) 1721 break; 1722 } 1723 1724 /* Trying to resize is pointless without a root hub window above 4GB */ 1725 if (!res) 1726 return 0; 1727 1728 /* Limit the BAR size to what is available */ 1729 max_size = pci_rebar_get_max_size(adev->pdev, 0); 1730 if (max_size < 0) 1731 return 0; 1732 rbar_size = min(max_size, rbar_size); 1733 1734 /* Disable memory decoding while we change the BAR addresses and size */ 1735 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1736 pci_write_config_word(adev->pdev, PCI_COMMAND, 1737 cmd & ~PCI_COMMAND_MEMORY); 1738 1739 /* Tear down doorbell as resizing will release BARs */ 1740 amdgpu_doorbell_fini(adev); 1741 1742 r = pci_resize_resource(adev->pdev, 0, rbar_size, 1743 (adev->asic_type >= CHIP_BONAIRE) ? 1 << 5 1744 : 1 << 2); 1745 if (r == -ENOSPC) 1746 dev_info(adev->dev, 1747 "Not enough PCI address space for a large BAR."); 1748 else if (r && r != -ENOTSUPP) 1749 dev_err(adev->dev, "Problem resizing BAR0 (%d).", r); 1750 1751 /* When the doorbell or fb BAR isn't available we have no chance of 1752 * using the device. 1753 */ 1754 r = amdgpu_doorbell_init(adev); 1755 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1756 return -ENODEV; 1757 1758 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1759 1760 return 0; 1761} 1762 1763/* 1764 * GPU helpers function. 1765 */ 1766/** 1767 * amdgpu_device_need_post - check if the hw need post or not 1768 * 1769 * @adev: amdgpu_device pointer 1770 * 1771 * Check if the asic has been initialized (all asics) at driver startup 1772 * or post is needed if hw reset is performed. 1773 * Returns true if need or false if not. 1774 */ 1775bool amdgpu_device_need_post(struct amdgpu_device *adev) 1776{ 1777 uint32_t reg, flags; 1778 1779 if (amdgpu_sriov_vf(adev)) 1780 return false; 1781 1782 flags = amdgpu_device_get_vbios_flags(adev); 1783 if (flags & AMDGPU_VBIOS_SKIP) 1784 return false; 1785 if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios) 1786 return false; 1787 1788 if (amdgpu_passthrough(adev)) { 1789 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1790 * some old smc fw still need driver do vPost otherwise gpu hang, while 1791 * those smc fw version above 22.15 doesn't have this flaw, so we force 1792 * vpost executed for smc version below 22.15 1793 */ 1794 if (adev->asic_type == CHIP_FIJI) { 1795 int err; 1796 uint32_t fw_ver; 1797 1798 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1799 /* force vPost if error occurred */ 1800 if (err) 1801 return true; 1802 1803 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1804 release_firmware(adev->pm.fw); 1805 if (fw_ver < 0x00160e00) 1806 return true; 1807 } 1808 } 1809 1810 /* Don't post if we need to reset whole hive on init */ 1811 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 1812 return false; 1813 1814 if (adev->has_hw_reset) { 1815 adev->has_hw_reset = false; 1816 return true; 1817 } 1818 1819 /* bios scratch used on CIK+ */ 1820 if (adev->asic_type >= CHIP_BONAIRE) 1821 return amdgpu_atombios_scratch_need_asic_init(adev); 1822 1823 /* check MEM_SIZE for older asics */ 1824 reg = amdgpu_asic_get_config_memsize(adev); 1825 1826 if ((reg != 0) && (reg != 0xffffffff)) 1827 return false; 1828 1829 return true; 1830} 1831 1832/* 1833 * Check whether seamless boot is supported. 1834 * 1835 * So far we only support seamless boot on DCE 3.0 or later. 1836 * If users report that it works on older ASICS as well, we may 1837 * loosen this. 1838 */ 1839bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev) 1840{ 1841 switch (amdgpu_seamless) { 1842 case -1: 1843 break; 1844 case 1: 1845 return true; 1846 case 0: 1847 return false; 1848 default: 1849 dev_err(adev->dev, "Invalid value for amdgpu.seamless: %d\n", 1850 amdgpu_seamless); 1851 return false; 1852 } 1853 1854 if (!(adev->flags & AMD_IS_APU)) 1855 return false; 1856 1857 if (adev->mman.keep_stolen_vga_memory) 1858 return false; 1859 1860 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0); 1861} 1862 1863/* 1864 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids 1865 * don't support dynamic speed switching. Until we have confirmation from Intel 1866 * that a specific host supports it, it's safer that we keep it disabled for all. 1867 * 1868 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/ 1869 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663 1870 */ 1871static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev) 1872{ 1873#if IS_ENABLED(CONFIG_X86) 1874 struct cpuinfo_x86 *c = &cpu_data(0); 1875 1876 /* eGPU change speeds based on USB4 fabric conditions */ 1877 if (dev_is_removable(adev->dev)) 1878 return true; 1879 1880 if (c->x86_vendor == X86_VENDOR_INTEL) 1881 return false; 1882#endif 1883 return true; 1884} 1885 1886static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) 1887{ 1888 /* Enabling ASPM causes randoms hangs on Tahiti and Oland on Zen4. 1889 * It's unclear if this is a platform-specific or GPU-specific issue. 1890 * Disable ASPM on SI for the time being. 1891 */ 1892 if (adev->family == AMDGPU_FAMILY_SI) 1893 return true; 1894 1895#if IS_ENABLED(CONFIG_X86) 1896 struct cpuinfo_x86 *c = &cpu_data(0); 1897 1898 if (!(amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 0, 0) || 1899 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 0, 1))) 1900 return false; 1901 1902 if (c->x86 == 6 && 1903 adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5) { 1904 switch (c->x86_model) { 1905 case VFM_MODEL(INTEL_ALDERLAKE): 1906 case VFM_MODEL(INTEL_ALDERLAKE_L): 1907 case VFM_MODEL(INTEL_RAPTORLAKE): 1908 case VFM_MODEL(INTEL_RAPTORLAKE_P): 1909 case VFM_MODEL(INTEL_RAPTORLAKE_S): 1910 return true; 1911 default: 1912 return false; 1913 } 1914 } else { 1915 return false; 1916 } 1917#else 1918 return false; 1919#endif 1920} 1921 1922/** 1923 * amdgpu_device_should_use_aspm - check if the device should program ASPM 1924 * 1925 * @adev: amdgpu_device pointer 1926 * 1927 * Confirm whether the module parameter and pcie bridge agree that ASPM should 1928 * be set for this device. 1929 * 1930 * Returns true if it should be used or false if not. 1931 */ 1932bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) 1933{ 1934 switch (amdgpu_aspm) { 1935 case -1: 1936 break; 1937 case 0: 1938 return false; 1939 case 1: 1940 return true; 1941 default: 1942 return false; 1943 } 1944 if (adev->flags & AMD_IS_APU) 1945 return false; 1946 if (amdgpu_device_aspm_support_quirk(adev)) 1947 return false; 1948 return pcie_aspm_enabled(adev->pdev); 1949} 1950 1951/* if we get transitioned to only one device, take VGA back */ 1952/** 1953 * amdgpu_device_vga_set_decode - enable/disable vga decode 1954 * 1955 * @pdev: PCI device pointer 1956 * @state: enable/disable vga decode 1957 * 1958 * Enable/disable vga decode (all asics). 1959 * Returns VGA resource flags. 1960 */ 1961static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev, 1962 bool state) 1963{ 1964 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev)); 1965 1966 amdgpu_asic_set_vga_state(adev, state); 1967 if (state) 1968 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1969 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1970 else 1971 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1972} 1973 1974/** 1975 * amdgpu_device_check_block_size - validate the vm block size 1976 * 1977 * @adev: amdgpu_device pointer 1978 * 1979 * Validates the vm block size specified via module parameter. 1980 * The vm block size defines number of bits in page table versus page directory, 1981 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1982 * page table and the remaining bits are in the page directory. 1983 */ 1984static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1985{ 1986 /* defines number of bits in page table versus page directory, 1987 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1988 * page table and the remaining bits are in the page directory 1989 */ 1990 if (amdgpu_vm_block_size == -1) 1991 return; 1992 1993 if (amdgpu_vm_block_size < 9) { 1994 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1995 amdgpu_vm_block_size); 1996 amdgpu_vm_block_size = -1; 1997 } 1998} 1999 2000/** 2001 * amdgpu_device_check_vm_size - validate the vm size 2002 * 2003 * @adev: amdgpu_device pointer 2004 * 2005 * Validates the vm size in GB specified via module parameter. 2006 * The VM size is the size of the GPU virtual memory space in GB. 2007 */ 2008static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 2009{ 2010 /* no need to check the default value */ 2011 if (amdgpu_vm_size == -1) 2012 return; 2013 2014 if (amdgpu_vm_size < 1) { 2015 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 2016 amdgpu_vm_size); 2017 amdgpu_vm_size = -1; 2018 } 2019} 2020 2021static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 2022{ 2023 struct sysinfo si; 2024 bool is_os_64 = (sizeof(void *) == 8); 2025 uint64_t total_memory; 2026 uint64_t dram_size_seven_GB = 0x1B8000000; 2027 uint64_t dram_size_three_GB = 0xB8000000; 2028 2029 if (amdgpu_smu_memory_pool_size == 0) 2030 return; 2031 2032 if (!is_os_64) { 2033 dev_warn(adev->dev, "Not 64-bit OS, feature not supported\n"); 2034 goto def_value; 2035 } 2036 si_meminfo(&si); 2037 total_memory = (uint64_t)si.totalram * si.mem_unit; 2038 2039 if ((amdgpu_smu_memory_pool_size == 1) || 2040 (amdgpu_smu_memory_pool_size == 2)) { 2041 if (total_memory < dram_size_three_GB) 2042 goto def_value1; 2043 } else if ((amdgpu_smu_memory_pool_size == 4) || 2044 (amdgpu_smu_memory_pool_size == 8)) { 2045 if (total_memory < dram_size_seven_GB) 2046 goto def_value1; 2047 } else { 2048 dev_warn(adev->dev, "Smu memory pool size not supported\n"); 2049 goto def_value; 2050 } 2051 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 2052 2053 return; 2054 2055def_value1: 2056 dev_warn(adev->dev, "No enough system memory\n"); 2057def_value: 2058 adev->pm.smu_prv_buffer_size = 0; 2059} 2060 2061static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev) 2062{ 2063 if (!(adev->flags & AMD_IS_APU) || 2064 adev->asic_type < CHIP_RAVEN) 2065 return 0; 2066 2067 switch (adev->asic_type) { 2068 case CHIP_RAVEN: 2069 if (adev->pdev->device == 0x15dd) 2070 adev->apu_flags |= AMD_APU_IS_RAVEN; 2071 if (adev->pdev->device == 0x15d8) 2072 adev->apu_flags |= AMD_APU_IS_PICASSO; 2073 break; 2074 case CHIP_RENOIR: 2075 if ((adev->pdev->device == 0x1636) || 2076 (adev->pdev->device == 0x164c)) 2077 adev->apu_flags |= AMD_APU_IS_RENOIR; 2078 else 2079 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; 2080 break; 2081 case CHIP_VANGOGH: 2082 adev->apu_flags |= AMD_APU_IS_VANGOGH; 2083 break; 2084 case CHIP_YELLOW_CARP: 2085 break; 2086 case CHIP_CYAN_SKILLFISH: 2087 if ((adev->pdev->device == 0x13FE) || 2088 (adev->pdev->device == 0x143F)) 2089 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; 2090 break; 2091 default: 2092 break; 2093 } 2094 2095 return 0; 2096} 2097 2098/** 2099 * amdgpu_device_check_arguments - validate module params 2100 * 2101 * @adev: amdgpu_device pointer 2102 * 2103 * Validates certain module parameters and updates 2104 * the associated values used by the driver (all asics). 2105 */ 2106static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 2107{ 2108 int i; 2109 2110 if (amdgpu_sched_jobs < 4) { 2111 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 2112 amdgpu_sched_jobs); 2113 amdgpu_sched_jobs = 4; 2114 } else if (!is_power_of_2(amdgpu_sched_jobs)) { 2115 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 2116 amdgpu_sched_jobs); 2117 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 2118 } 2119 2120 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 2121 /* gart size must be greater or equal to 32M */ 2122 dev_warn(adev->dev, "gart size (%d) too small\n", 2123 amdgpu_gart_size); 2124 amdgpu_gart_size = -1; 2125 } 2126 2127 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 2128 /* gtt size must be greater or equal to 32M */ 2129 dev_warn(adev->dev, "gtt size (%d) too small\n", 2130 amdgpu_gtt_size); 2131 amdgpu_gtt_size = -1; 2132 } 2133 2134 /* valid range is between 4 and 9 inclusive */ 2135 if (amdgpu_vm_fragment_size != -1 && 2136 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 2137 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 2138 amdgpu_vm_fragment_size = -1; 2139 } 2140 2141 if (amdgpu_sched_hw_submission < 2) { 2142 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 2143 amdgpu_sched_hw_submission); 2144 amdgpu_sched_hw_submission = 2; 2145 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 2146 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 2147 amdgpu_sched_hw_submission); 2148 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 2149 } 2150 2151 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { 2152 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); 2153 amdgpu_reset_method = -1; 2154 } 2155 2156 amdgpu_device_check_smu_prv_buffer_size(adev); 2157 2158 amdgpu_device_check_vm_size(adev); 2159 2160 amdgpu_device_check_block_size(adev); 2161 2162 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 2163 2164 for (i = 0; i < MAX_XCP; i++) { 2165 switch (amdgpu_enforce_isolation) { 2166 case -1: 2167 case 0: 2168 default: 2169 /* disable */ 2170 adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE; 2171 break; 2172 case 1: 2173 /* enable */ 2174 adev->enforce_isolation[i] = 2175 AMDGPU_ENFORCE_ISOLATION_ENABLE; 2176 break; 2177 case 2: 2178 /* enable legacy mode */ 2179 adev->enforce_isolation[i] = 2180 AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY; 2181 break; 2182 case 3: 2183 /* enable only process isolation without submitting cleaner shader */ 2184 adev->enforce_isolation[i] = 2185 AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER; 2186 break; 2187 } 2188 } 2189 2190 return 0; 2191} 2192 2193/** 2194 * amdgpu_switcheroo_set_state - set switcheroo state 2195 * 2196 * @pdev: pci dev pointer 2197 * @state: vga_switcheroo state 2198 * 2199 * Callback for the switcheroo driver. Suspends or resumes 2200 * the asics before or after it is powered up using ACPI methods. 2201 */ 2202static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 2203 enum vga_switcheroo_state state) 2204{ 2205 struct drm_device *dev = pci_get_drvdata(pdev); 2206 int r; 2207 2208 if (amdgpu_device_supports_px(drm_to_adev(dev)) && 2209 state == VGA_SWITCHEROO_OFF) 2210 return; 2211 2212 if (state == VGA_SWITCHEROO_ON) { 2213 pr_info("switched on\n"); 2214 /* don't suspend or resume card normally */ 2215 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 2216 2217 pci_set_power_state(pdev, PCI_D0); 2218 amdgpu_device_load_pci_state(pdev); 2219 r = pci_enable_device(pdev); 2220 if (r) 2221 dev_warn(&pdev->dev, "pci_enable_device failed (%d)\n", 2222 r); 2223 amdgpu_device_resume(dev, true); 2224 2225 dev->switch_power_state = DRM_SWITCH_POWER_ON; 2226 } else { 2227 dev_info(&pdev->dev, "switched off\n"); 2228 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 2229 amdgpu_device_prepare(dev); 2230 amdgpu_device_suspend(dev, true); 2231 amdgpu_device_cache_pci_state(pdev); 2232 /* Shut down the device */ 2233 pci_disable_device(pdev); 2234 pci_set_power_state(pdev, PCI_D3cold); 2235 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 2236 } 2237} 2238 2239/** 2240 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 2241 * 2242 * @pdev: pci dev pointer 2243 * 2244 * Callback for the switcheroo driver. Check of the switcheroo 2245 * state can be changed. 2246 * Returns true if the state can be changed, false if not. 2247 */ 2248static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 2249{ 2250 struct drm_device *dev = pci_get_drvdata(pdev); 2251 2252 /* 2253 * FIXME: open_count is protected by drm_global_mutex but that would lead to 2254 * locking inversion with the driver load path. And the access here is 2255 * completely racy anyway. So don't bother with locking for now. 2256 */ 2257 return atomic_read(&dev->open_count) == 0; 2258} 2259 2260static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 2261 .set_gpu_state = amdgpu_switcheroo_set_state, 2262 .reprobe = NULL, 2263 .can_switch = amdgpu_switcheroo_can_switch, 2264}; 2265 2266/** 2267 * amdgpu_device_ip_set_clockgating_state - set the CG state 2268 * 2269 * @dev: amdgpu_device pointer 2270 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2271 * @state: clockgating state (gate or ungate) 2272 * 2273 * Sets the requested clockgating state for all instances of 2274 * the hardware IP specified. 2275 * Returns the error code from the last instance. 2276 */ 2277int amdgpu_device_ip_set_clockgating_state(void *dev, 2278 enum amd_ip_block_type block_type, 2279 enum amd_clockgating_state state) 2280{ 2281 struct amdgpu_device *adev = dev; 2282 int i, r = 0; 2283 2284 for (i = 0; i < adev->num_ip_blocks; i++) { 2285 if (!adev->ip_blocks[i].status.valid) 2286 continue; 2287 if (adev->ip_blocks[i].version->type != block_type) 2288 continue; 2289 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 2290 continue; 2291 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 2292 &adev->ip_blocks[i], state); 2293 if (r) 2294 dev_err(adev->dev, 2295 "set_clockgating_state of IP block <%s> failed %d\n", 2296 adev->ip_blocks[i].version->funcs->name, r); 2297 } 2298 return r; 2299} 2300 2301/** 2302 * amdgpu_device_ip_set_powergating_state - set the PG state 2303 * 2304 * @dev: amdgpu_device pointer 2305 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2306 * @state: powergating state (gate or ungate) 2307 * 2308 * Sets the requested powergating state for all instances of 2309 * the hardware IP specified. 2310 * Returns the error code from the last instance. 2311 */ 2312int amdgpu_device_ip_set_powergating_state(void *dev, 2313 enum amd_ip_block_type block_type, 2314 enum amd_powergating_state state) 2315{ 2316 struct amdgpu_device *adev = dev; 2317 int i, r = 0; 2318 2319 for (i = 0; i < adev->num_ip_blocks; i++) { 2320 if (!adev->ip_blocks[i].status.valid) 2321 continue; 2322 if (adev->ip_blocks[i].version->type != block_type) 2323 continue; 2324 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 2325 continue; 2326 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 2327 &adev->ip_blocks[i], state); 2328 if (r) 2329 dev_err(adev->dev, 2330 "set_powergating_state of IP block <%s> failed %d\n", 2331 adev->ip_blocks[i].version->funcs->name, r); 2332 } 2333 return r; 2334} 2335 2336/** 2337 * amdgpu_device_ip_get_clockgating_state - get the CG state 2338 * 2339 * @adev: amdgpu_device pointer 2340 * @flags: clockgating feature flags 2341 * 2342 * Walks the list of IPs on the device and updates the clockgating 2343 * flags for each IP. 2344 * Updates @flags with the feature flags for each hardware IP where 2345 * clockgating is enabled. 2346 */ 2347void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 2348 u64 *flags) 2349{ 2350 int i; 2351 2352 for (i = 0; i < adev->num_ip_blocks; i++) { 2353 if (!adev->ip_blocks[i].status.valid) 2354 continue; 2355 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 2356 adev->ip_blocks[i].version->funcs->get_clockgating_state( 2357 &adev->ip_blocks[i], flags); 2358 } 2359} 2360 2361/** 2362 * amdgpu_device_ip_wait_for_idle - wait for idle 2363 * 2364 * @adev: amdgpu_device pointer 2365 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2366 * 2367 * Waits for the request hardware IP to be idle. 2368 * Returns 0 for success or a negative error code on failure. 2369 */ 2370int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 2371 enum amd_ip_block_type block_type) 2372{ 2373 int i, r; 2374 2375 for (i = 0; i < adev->num_ip_blocks; i++) { 2376 if (!adev->ip_blocks[i].status.valid) 2377 continue; 2378 if (adev->ip_blocks[i].version->type == block_type) { 2379 if (adev->ip_blocks[i].version->funcs->wait_for_idle) { 2380 r = adev->ip_blocks[i].version->funcs->wait_for_idle( 2381 &adev->ip_blocks[i]); 2382 if (r) 2383 return r; 2384 } 2385 break; 2386 } 2387 } 2388 return 0; 2389 2390} 2391 2392/** 2393 * amdgpu_device_ip_is_hw - is the hardware IP enabled 2394 * 2395 * @adev: amdgpu_device pointer 2396 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2397 * 2398 * Check if the hardware IP is enable or not. 2399 * Returns true if it the IP is enable, false if not. 2400 */ 2401bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev, 2402 enum amd_ip_block_type block_type) 2403{ 2404 int i; 2405 2406 for (i = 0; i < adev->num_ip_blocks; i++) { 2407 if (adev->ip_blocks[i].version->type == block_type) 2408 return adev->ip_blocks[i].status.hw; 2409 } 2410 return false; 2411} 2412 2413/** 2414 * amdgpu_device_ip_is_valid - is the hardware IP valid 2415 * 2416 * @adev: amdgpu_device pointer 2417 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2418 * 2419 * Check if the hardware IP is valid or not. 2420 * Returns true if it the IP is valid, false if not. 2421 */ 2422bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev, 2423 enum amd_ip_block_type block_type) 2424{ 2425 int i; 2426 2427 for (i = 0; i < adev->num_ip_blocks; i++) { 2428 if (adev->ip_blocks[i].version->type == block_type) 2429 return adev->ip_blocks[i].status.valid; 2430 } 2431 return false; 2432 2433} 2434 2435/** 2436 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 2437 * 2438 * @adev: amdgpu_device pointer 2439 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 2440 * 2441 * Returns a pointer to the hardware IP block structure 2442 * if it exists for the asic, otherwise NULL. 2443 */ 2444struct amdgpu_ip_block * 2445amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 2446 enum amd_ip_block_type type) 2447{ 2448 int i; 2449 2450 for (i = 0; i < adev->num_ip_blocks; i++) 2451 if (adev->ip_blocks[i].version->type == type) 2452 return &adev->ip_blocks[i]; 2453 2454 return NULL; 2455} 2456 2457/** 2458 * amdgpu_device_ip_block_version_cmp 2459 * 2460 * @adev: amdgpu_device pointer 2461 * @type: enum amd_ip_block_type 2462 * @major: major version 2463 * @minor: minor version 2464 * 2465 * return 0 if equal or greater 2466 * return 1 if smaller or the ip_block doesn't exist 2467 */ 2468int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 2469 enum amd_ip_block_type type, 2470 u32 major, u32 minor) 2471{ 2472 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 2473 2474 if (ip_block && ((ip_block->version->major > major) || 2475 ((ip_block->version->major == major) && 2476 (ip_block->version->minor >= minor)))) 2477 return 0; 2478 2479 return 1; 2480} 2481 2482static const char *ip_block_names[] = { 2483 [AMD_IP_BLOCK_TYPE_COMMON] = "common", 2484 [AMD_IP_BLOCK_TYPE_GMC] = "gmc", 2485 [AMD_IP_BLOCK_TYPE_IH] = "ih", 2486 [AMD_IP_BLOCK_TYPE_SMC] = "smu", 2487 [AMD_IP_BLOCK_TYPE_PSP] = "psp", 2488 [AMD_IP_BLOCK_TYPE_DCE] = "dce", 2489 [AMD_IP_BLOCK_TYPE_GFX] = "gfx", 2490 [AMD_IP_BLOCK_TYPE_SDMA] = "sdma", 2491 [AMD_IP_BLOCK_TYPE_UVD] = "uvd", 2492 [AMD_IP_BLOCK_TYPE_VCE] = "vce", 2493 [AMD_IP_BLOCK_TYPE_ACP] = "acp", 2494 [AMD_IP_BLOCK_TYPE_VCN] = "vcn", 2495 [AMD_IP_BLOCK_TYPE_MES] = "mes", 2496 [AMD_IP_BLOCK_TYPE_JPEG] = "jpeg", 2497 [AMD_IP_BLOCK_TYPE_VPE] = "vpe", 2498 [AMD_IP_BLOCK_TYPE_UMSCH_MM] = "umsch_mm", 2499 [AMD_IP_BLOCK_TYPE_ISP] = "isp", 2500 [AMD_IP_BLOCK_TYPE_RAS] = "ras", 2501}; 2502 2503static const char *ip_block_name(struct amdgpu_device *adev, enum amd_ip_block_type type) 2504{ 2505 int idx = (int)type; 2506 2507 return idx < ARRAY_SIZE(ip_block_names) ? ip_block_names[idx] : "unknown"; 2508} 2509 2510/** 2511 * amdgpu_device_ip_block_add 2512 * 2513 * @adev: amdgpu_device pointer 2514 * @ip_block_version: pointer to the IP to add 2515 * 2516 * Adds the IP block driver information to the collection of IPs 2517 * on the asic. 2518 */ 2519int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 2520 const struct amdgpu_ip_block_version *ip_block_version) 2521{ 2522 if (!ip_block_version) 2523 return -EINVAL; 2524 2525 switch (ip_block_version->type) { 2526 case AMD_IP_BLOCK_TYPE_VCN: 2527 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) 2528 return 0; 2529 break; 2530 case AMD_IP_BLOCK_TYPE_JPEG: 2531 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) 2532 return 0; 2533 break; 2534 default: 2535 break; 2536 } 2537 2538 dev_info(adev->dev, "detected ip block number %d <%s_v%d_%d_%d> (%s)\n", 2539 adev->num_ip_blocks, 2540 ip_block_name(adev, ip_block_version->type), 2541 ip_block_version->major, 2542 ip_block_version->minor, 2543 ip_block_version->rev, 2544 ip_block_version->funcs->name); 2545 2546 adev->ip_blocks[adev->num_ip_blocks].adev = adev; 2547 2548 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 2549 2550 return 0; 2551} 2552 2553/** 2554 * amdgpu_device_enable_virtual_display - enable virtual display feature 2555 * 2556 * @adev: amdgpu_device pointer 2557 * 2558 * Enabled the virtual display feature if the user has enabled it via 2559 * the module parameter virtual_display. This feature provides a virtual 2560 * display hardware on headless boards or in virtualized environments. 2561 * This function parses and validates the configuration string specified by 2562 * the user and configures the virtual display configuration (number of 2563 * virtual connectors, crtcs, etc.) specified. 2564 */ 2565static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 2566{ 2567 adev->enable_virtual_display = false; 2568 2569 if (amdgpu_virtual_display) { 2570 const char *pci_address_name = pci_name(adev->pdev); 2571 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 2572 2573 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 2574 pciaddstr_tmp = pciaddstr; 2575 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 2576 pciaddname = strsep(&pciaddname_tmp, ","); 2577 if (!strcmp("all", pciaddname) 2578 || !strcmp(pci_address_name, pciaddname)) { 2579 long num_crtc; 2580 int res = -1; 2581 2582 adev->enable_virtual_display = true; 2583 2584 if (pciaddname_tmp) 2585 res = kstrtol(pciaddname_tmp, 10, 2586 &num_crtc); 2587 2588 if (!res) { 2589 if (num_crtc < 1) 2590 num_crtc = 1; 2591 if (num_crtc > 6) 2592 num_crtc = 6; 2593 adev->mode_info.num_crtc = num_crtc; 2594 } else { 2595 adev->mode_info.num_crtc = 1; 2596 } 2597 break; 2598 } 2599 } 2600 2601 dev_info( 2602 adev->dev, 2603 "virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 2604 amdgpu_virtual_display, pci_address_name, 2605 adev->enable_virtual_display, adev->mode_info.num_crtc); 2606 2607 kfree(pciaddstr); 2608 } 2609} 2610 2611void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev) 2612{ 2613 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { 2614 adev->mode_info.num_crtc = 1; 2615 adev->enable_virtual_display = true; 2616 dev_info(adev->dev, "virtual_display:%d, num_crtc:%d\n", 2617 adev->enable_virtual_display, 2618 adev->mode_info.num_crtc); 2619 } 2620} 2621 2622/** 2623 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 2624 * 2625 * @adev: amdgpu_device pointer 2626 * 2627 * Parses the asic configuration parameters specified in the gpu info 2628 * firmware and makes them available to the driver for use in configuring 2629 * the asic. 2630 * Returns 0 on success, -EINVAL on failure. 2631 */ 2632static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 2633{ 2634 const char *chip_name; 2635 int err; 2636 const struct gpu_info_firmware_header_v1_0 *hdr; 2637 2638 adev->firmware.gpu_info_fw = NULL; 2639 2640 switch (adev->asic_type) { 2641 default: 2642 return 0; 2643 case CHIP_VEGA10: 2644 chip_name = "vega10"; 2645 break; 2646 case CHIP_VEGA12: 2647 chip_name = "vega12"; 2648 break; 2649 case CHIP_RAVEN: 2650 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2651 chip_name = "raven2"; 2652 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 2653 chip_name = "picasso"; 2654 else 2655 chip_name = "raven"; 2656 break; 2657 case CHIP_ARCTURUS: 2658 chip_name = "arcturus"; 2659 break; 2660 case CHIP_NAVI12: 2661 if (adev->discovery.bin) 2662 return 0; 2663 chip_name = "navi12"; 2664 break; 2665 case CHIP_CYAN_SKILLFISH: 2666 if (adev->discovery.bin) 2667 return 0; 2668 chip_name = "cyan_skillfish"; 2669 break; 2670 } 2671 2672 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, 2673 AMDGPU_UCODE_OPTIONAL, 2674 "amdgpu/%s_gpu_info.bin", chip_name); 2675 if (err) { 2676 dev_err(adev->dev, 2677 "Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n", 2678 chip_name); 2679 goto out; 2680 } 2681 2682 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 2683 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 2684 2685 switch (hdr->version_major) { 2686 case 1: 2687 { 2688 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 2689 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 2690 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2691 2692 /* 2693 * Should be dropped when DAL no longer needs it. 2694 */ 2695 if (adev->asic_type == CHIP_NAVI12) 2696 goto parse_soc_bounding_box; 2697 2698 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 2699 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 2700 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 2701 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 2702 adev->gfx.config.max_texture_channel_caches = 2703 le32_to_cpu(gpu_info_fw->gc_num_tccs); 2704 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 2705 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 2706 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 2707 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 2708 adev->gfx.config.double_offchip_lds_buf = 2709 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 2710 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 2711 adev->gfx.cu_info.max_waves_per_simd = 2712 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 2713 adev->gfx.cu_info.max_scratch_slots_per_cu = 2714 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 2715 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 2716 if (hdr->version_minor >= 1) { 2717 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 2718 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 2719 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2720 adev->gfx.config.num_sc_per_sh = 2721 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 2722 adev->gfx.config.num_packer_per_sc = 2723 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 2724 } 2725 2726parse_soc_bounding_box: 2727 /* 2728 * soc bounding box info is not integrated in disocovery table, 2729 * we always need to parse it from gpu info firmware if needed. 2730 */ 2731 if (hdr->version_minor == 2) { 2732 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 2733 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 2734 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2735 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 2736 } 2737 break; 2738 } 2739 default: 2740 dev_err(adev->dev, 2741 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 2742 err = -EINVAL; 2743 goto out; 2744 } 2745out: 2746 return err; 2747} 2748 2749static void amdgpu_uid_init(struct amdgpu_device *adev) 2750{ 2751 /* Initialize the UID for the device */ 2752 adev->uid_info = kzalloc(sizeof(struct amdgpu_uid), GFP_KERNEL); 2753 if (!adev->uid_info) { 2754 dev_warn(adev->dev, "Failed to allocate memory for UID\n"); 2755 return; 2756 } 2757 adev->uid_info->adev = adev; 2758} 2759 2760static void amdgpu_uid_fini(struct amdgpu_device *adev) 2761{ 2762 /* Free the UID memory */ 2763 kfree(adev->uid_info); 2764 adev->uid_info = NULL; 2765} 2766 2767/** 2768 * amdgpu_device_ip_early_init - run early init for hardware IPs 2769 * 2770 * @adev: amdgpu_device pointer 2771 * 2772 * Early initialization pass for hardware IPs. The hardware IPs that make 2773 * up each asic are discovered each IP's early_init callback is run. This 2774 * is the first stage in initializing the asic. 2775 * Returns 0 on success, negative error code on failure. 2776 */ 2777static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 2778{ 2779 struct amdgpu_ip_block *ip_block; 2780 struct pci_dev *parent; 2781 bool total, skip_bios; 2782 uint32_t bios_flags; 2783 int i, r; 2784 2785 amdgpu_device_enable_virtual_display(adev); 2786 2787 if (amdgpu_sriov_vf(adev)) { 2788 r = amdgpu_virt_request_full_gpu(adev, true); 2789 if (r) 2790 return r; 2791 2792 r = amdgpu_virt_init_critical_region(adev); 2793 if (r) 2794 return r; 2795 } 2796 2797 switch (adev->asic_type) { 2798#ifdef CONFIG_DRM_AMDGPU_SI 2799 case CHIP_VERDE: 2800 case CHIP_TAHITI: 2801 case CHIP_PITCAIRN: 2802 case CHIP_OLAND: 2803 case CHIP_HAINAN: 2804 adev->family = AMDGPU_FAMILY_SI; 2805 r = si_set_ip_blocks(adev); 2806 if (r) 2807 return r; 2808 break; 2809#endif 2810#ifdef CONFIG_DRM_AMDGPU_CIK 2811 case CHIP_BONAIRE: 2812 case CHIP_HAWAII: 2813 case CHIP_KAVERI: 2814 case CHIP_KABINI: 2815 case CHIP_MULLINS: 2816 if (adev->flags & AMD_IS_APU) 2817 adev->family = AMDGPU_FAMILY_KV; 2818 else 2819 adev->family = AMDGPU_FAMILY_CI; 2820 2821 r = cik_set_ip_blocks(adev); 2822 if (r) 2823 return r; 2824 break; 2825#endif 2826 case CHIP_TOPAZ: 2827 case CHIP_TONGA: 2828 case CHIP_FIJI: 2829 case CHIP_POLARIS10: 2830 case CHIP_POLARIS11: 2831 case CHIP_POLARIS12: 2832 case CHIP_VEGAM: 2833 case CHIP_CARRIZO: 2834 case CHIP_STONEY: 2835 if (adev->flags & AMD_IS_APU) 2836 adev->family = AMDGPU_FAMILY_CZ; 2837 else 2838 adev->family = AMDGPU_FAMILY_VI; 2839 2840 r = vi_set_ip_blocks(adev); 2841 if (r) 2842 return r; 2843 break; 2844 default: 2845 r = amdgpu_discovery_set_ip_blocks(adev); 2846 if (r) { 2847 adev->num_ip_blocks = 0; 2848 return r; 2849 } 2850 break; 2851 } 2852 2853 /* Check for IP version 9.4.3 with A0 hardware */ 2854 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) && 2855 !amdgpu_device_get_rev_id(adev)) { 2856 dev_err(adev->dev, "Unsupported A0 hardware\n"); 2857 return -ENODEV; /* device unsupported - no device error */ 2858 } 2859 2860 if (amdgpu_has_atpx() && 2861 (amdgpu_is_atpx_hybrid() || 2862 amdgpu_has_atpx_dgpu_power_cntl()) && 2863 ((adev->flags & AMD_IS_APU) == 0) && 2864 !dev_is_removable(&adev->pdev->dev)) 2865 adev->flags |= AMD_IS_PX; 2866 2867 if (!(adev->flags & AMD_IS_APU)) { 2868 parent = pcie_find_root_port(adev->pdev); 2869 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; 2870 } 2871 2872 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2873 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2874 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2875 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) 2876 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; 2877 if (!amdgpu_device_pcie_dynamic_switching_supported(adev)) 2878 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; 2879 2880 adev->virt.is_xgmi_node_migrate_enabled = false; 2881 if (amdgpu_sriov_vf(adev)) { 2882 adev->virt.is_xgmi_node_migrate_enabled = 2883 amdgpu_ip_version((adev), GC_HWIP, 0) == IP_VERSION(9, 4, 4); 2884 } 2885 2886 total = true; 2887 for (i = 0; i < adev->num_ip_blocks; i++) { 2888 ip_block = &adev->ip_blocks[i]; 2889 2890 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2891 dev_warn(adev->dev, "disabled ip block: %d <%s>\n", i, 2892 adev->ip_blocks[i].version->funcs->name); 2893 adev->ip_blocks[i].status.valid = false; 2894 } else if (ip_block->version->funcs->early_init) { 2895 r = ip_block->version->funcs->early_init(ip_block); 2896 if (r == -ENOENT) { 2897 adev->ip_blocks[i].status.valid = false; 2898 } else if (r) { 2899 dev_err(adev->dev, 2900 "early_init of IP block <%s> failed %d\n", 2901 adev->ip_blocks[i].version->funcs->name, 2902 r); 2903 total = false; 2904 } else { 2905 adev->ip_blocks[i].status.valid = true; 2906 } 2907 } else { 2908 adev->ip_blocks[i].status.valid = true; 2909 } 2910 /* get the vbios after the asic_funcs are set up */ 2911 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2912 r = amdgpu_device_parse_gpu_info_fw(adev); 2913 if (r) 2914 return r; 2915 2916 bios_flags = amdgpu_device_get_vbios_flags(adev); 2917 skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP); 2918 /* Read BIOS */ 2919 if (!skip_bios) { 2920 bool optional = 2921 !!(bios_flags & AMDGPU_VBIOS_OPTIONAL); 2922 if (!amdgpu_get_bios(adev) && !optional) 2923 return -EINVAL; 2924 2925 if (optional && !adev->bios) 2926 dev_info( 2927 adev->dev, 2928 "VBIOS image optional, proceeding without VBIOS image"); 2929 2930 if (adev->bios) { 2931 r = amdgpu_atombios_init(adev); 2932 if (r) { 2933 dev_err(adev->dev, 2934 "amdgpu_atombios_init failed\n"); 2935 amdgpu_vf_error_put( 2936 adev, 2937 AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 2938 0, 0); 2939 return r; 2940 } 2941 } 2942 } 2943 2944 /*get pf2vf msg info at it's earliest time*/ 2945 if (amdgpu_sriov_vf(adev)) 2946 amdgpu_virt_init_data_exchange(adev); 2947 2948 } 2949 } 2950 if (!total) 2951 return -ENODEV; 2952 2953 if (adev->gmc.xgmi.supported) 2954 amdgpu_xgmi_early_init(adev); 2955 2956 if (amdgpu_is_multi_aid(adev)) 2957 amdgpu_uid_init(adev); 2958 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 2959 if (ip_block->status.valid != false) 2960 amdgpu_amdkfd_device_probe(adev); 2961 2962 adev->cg_flags &= amdgpu_cg_mask; 2963 adev->pg_flags &= amdgpu_pg_mask; 2964 2965 return 0; 2966} 2967 2968static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2969{ 2970 int i, r; 2971 2972 for (i = 0; i < adev->num_ip_blocks; i++) { 2973 if (!adev->ip_blocks[i].status.sw) 2974 continue; 2975 if (adev->ip_blocks[i].status.hw) 2976 continue; 2977 if (!amdgpu_ip_member_of_hwini( 2978 adev, adev->ip_blocks[i].version->type)) 2979 continue; 2980 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2981 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2982 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2983 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2984 if (r) { 2985 dev_err(adev->dev, 2986 "hw_init of IP block <%s> failed %d\n", 2987 adev->ip_blocks[i].version->funcs->name, 2988 r); 2989 return r; 2990 } 2991 adev->ip_blocks[i].status.hw = true; 2992 } 2993 } 2994 2995 return 0; 2996} 2997 2998static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2999{ 3000 int i, r; 3001 3002 for (i = 0; i < adev->num_ip_blocks; i++) { 3003 if (!adev->ip_blocks[i].status.sw) 3004 continue; 3005 if (adev->ip_blocks[i].status.hw) 3006 continue; 3007 if (!amdgpu_ip_member_of_hwini( 3008 adev, adev->ip_blocks[i].version->type)) 3009 continue; 3010 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 3011 if (r) { 3012 dev_err(adev->dev, 3013 "hw_init of IP block <%s> failed %d\n", 3014 adev->ip_blocks[i].version->funcs->name, r); 3015 return r; 3016 } 3017 adev->ip_blocks[i].status.hw = true; 3018 } 3019 3020 return 0; 3021} 3022 3023static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 3024{ 3025 int r = 0; 3026 int i; 3027 uint32_t smu_version; 3028 3029 if (adev->asic_type >= CHIP_VEGA10) { 3030 for (i = 0; i < adev->num_ip_blocks; i++) { 3031 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 3032 continue; 3033 3034 if (!amdgpu_ip_member_of_hwini(adev, 3035 AMD_IP_BLOCK_TYPE_PSP)) 3036 break; 3037 3038 if (!adev->ip_blocks[i].status.sw) 3039 continue; 3040 3041 /* no need to do the fw loading again if already done*/ 3042 if (adev->ip_blocks[i].status.hw == true) 3043 break; 3044 3045 if (amdgpu_in_reset(adev) || adev->in_suspend) { 3046 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3047 if (r) 3048 return r; 3049 } else { 3050 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 3051 if (r) { 3052 dev_err(adev->dev, 3053 "hw_init of IP block <%s> failed %d\n", 3054 adev->ip_blocks[i] 3055 .version->funcs->name, 3056 r); 3057 return r; 3058 } 3059 adev->ip_blocks[i].status.hw = true; 3060 } 3061 break; 3062 } 3063 } 3064 3065 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 3066 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 3067 3068 return r; 3069} 3070 3071static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) 3072{ 3073 struct drm_sched_init_args args = { 3074 .ops = &amdgpu_sched_ops, 3075 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 3076 .timeout_wq = adev->reset_domain->wq, 3077 .dev = adev->dev, 3078 }; 3079 long timeout; 3080 int r, i; 3081 3082 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 3083 struct amdgpu_ring *ring = adev->rings[i]; 3084 3085 /* No need to setup the GPU scheduler for rings that don't need it */ 3086 if (!ring || ring->no_scheduler) 3087 continue; 3088 3089 switch (ring->funcs->type) { 3090 case AMDGPU_RING_TYPE_GFX: 3091 timeout = adev->gfx_timeout; 3092 break; 3093 case AMDGPU_RING_TYPE_COMPUTE: 3094 timeout = adev->compute_timeout; 3095 break; 3096 case AMDGPU_RING_TYPE_SDMA: 3097 timeout = adev->sdma_timeout; 3098 break; 3099 default: 3100 timeout = adev->video_timeout; 3101 break; 3102 } 3103 3104 args.timeout = timeout; 3105 args.credit_limit = ring->num_hw_submission; 3106 args.score = ring->sched_score; 3107 args.name = ring->name; 3108 3109 r = drm_sched_init(&ring->sched, &args); 3110 if (r) { 3111 dev_err(adev->dev, 3112 "Failed to create scheduler on ring %s.\n", 3113 ring->name); 3114 return r; 3115 } 3116 r = amdgpu_uvd_entity_init(adev, ring); 3117 if (r) { 3118 dev_err(adev->dev, 3119 "Failed to create UVD scheduling entity on ring %s.\n", 3120 ring->name); 3121 return r; 3122 } 3123 r = amdgpu_vce_entity_init(adev, ring); 3124 if (r) { 3125 dev_err(adev->dev, 3126 "Failed to create VCE scheduling entity on ring %s.\n", 3127 ring->name); 3128 return r; 3129 } 3130 } 3131 3132 if (adev->xcp_mgr) 3133 amdgpu_xcp_update_partition_sched_list(adev); 3134 3135 return 0; 3136} 3137 3138 3139/** 3140 * amdgpu_device_ip_init - run init for hardware IPs 3141 * 3142 * @adev: amdgpu_device pointer 3143 * 3144 * Main initialization pass for hardware IPs. The list of all the hardware 3145 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 3146 * are run. sw_init initializes the software state associated with each IP 3147 * and hw_init initializes the hardware associated with each IP. 3148 * Returns 0 on success, negative error code on failure. 3149 */ 3150static int amdgpu_device_ip_init(struct amdgpu_device *adev) 3151{ 3152 bool init_badpage; 3153 int i, r; 3154 3155 r = amdgpu_ras_init(adev); 3156 if (r) 3157 return r; 3158 3159 for (i = 0; i < adev->num_ip_blocks; i++) { 3160 if (!adev->ip_blocks[i].status.valid) 3161 continue; 3162 if (adev->ip_blocks[i].version->funcs->sw_init) { 3163 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]); 3164 if (r) { 3165 dev_err(adev->dev, 3166 "sw_init of IP block <%s> failed %d\n", 3167 adev->ip_blocks[i].version->funcs->name, 3168 r); 3169 goto init_failed; 3170 } 3171 } 3172 adev->ip_blocks[i].status.sw = true; 3173 3174 if (!amdgpu_ip_member_of_hwini( 3175 adev, adev->ip_blocks[i].version->type)) 3176 continue; 3177 3178 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 3179 /* need to do common hw init early so everything is set up for gmc */ 3180 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 3181 if (r) { 3182 dev_err(adev->dev, "hw_init %d failed %d\n", i, 3183 r); 3184 goto init_failed; 3185 } 3186 adev->ip_blocks[i].status.hw = true; 3187 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 3188 /* need to do gmc hw init early so we can allocate gpu mem */ 3189 /* Try to reserve bad pages early */ 3190 if (amdgpu_sriov_vf(adev)) 3191 amdgpu_virt_exchange_data(adev); 3192 3193 r = amdgpu_device_mem_scratch_init(adev); 3194 if (r) { 3195 dev_err(adev->dev, 3196 "amdgpu_mem_scratch_init failed %d\n", 3197 r); 3198 goto init_failed; 3199 } 3200 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 3201 if (r) { 3202 dev_err(adev->dev, "hw_init %d failed %d\n", i, 3203 r); 3204 goto init_failed; 3205 } 3206 r = amdgpu_device_wb_init(adev); 3207 if (r) { 3208 dev_err(adev->dev, 3209 "amdgpu_device_wb_init failed %d\n", r); 3210 goto init_failed; 3211 } 3212 adev->ip_blocks[i].status.hw = true; 3213 3214 /* right after GMC hw init, we create CSA */ 3215 if (adev->gfx.mcbp) { 3216 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 3217 AMDGPU_GEM_DOMAIN_VRAM | 3218 AMDGPU_GEM_DOMAIN_GTT, 3219 AMDGPU_CSA_SIZE); 3220 if (r) { 3221 dev_err(adev->dev, 3222 "allocate CSA failed %d\n", r); 3223 goto init_failed; 3224 } 3225 } 3226 3227 r = amdgpu_seq64_init(adev); 3228 if (r) { 3229 dev_err(adev->dev, "allocate seq64 failed %d\n", 3230 r); 3231 goto init_failed; 3232 } 3233 } 3234 } 3235 3236 if (amdgpu_sriov_vf(adev)) 3237 amdgpu_virt_init_data_exchange(adev); 3238 3239 r = amdgpu_ib_pool_init(adev); 3240 if (r) { 3241 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 3242 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 3243 goto init_failed; 3244 } 3245 3246 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 3247 if (r) 3248 goto init_failed; 3249 3250 r = amdgpu_device_ip_hw_init_phase1(adev); 3251 if (r) 3252 goto init_failed; 3253 3254 r = amdgpu_device_fw_loading(adev); 3255 if (r) 3256 goto init_failed; 3257 3258 r = amdgpu_device_ip_hw_init_phase2(adev); 3259 if (r) 3260 goto init_failed; 3261 3262 /* 3263 * retired pages will be loaded from eeprom and reserved here, 3264 * it should be called after amdgpu_device_ip_hw_init_phase2 since 3265 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 3266 * for I2C communication which only true at this point. 3267 * 3268 * amdgpu_ras_recovery_init may fail, but the upper only cares the 3269 * failure from bad gpu situation and stop amdgpu init process 3270 * accordingly. For other failed cases, it will still release all 3271 * the resource and print error message, rather than returning one 3272 * negative value to upper level. 3273 * 3274 * Note: theoretically, this should be called before all vram allocations 3275 * to protect retired page from abusing 3276 */ 3277 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 3278 r = amdgpu_ras_recovery_init(adev, init_badpage); 3279 if (r) 3280 goto init_failed; 3281 3282 /** 3283 * In case of XGMI grab extra reference for reset domain for this device 3284 */ 3285 if (adev->gmc.xgmi.num_physical_nodes > 1) { 3286 if (amdgpu_xgmi_add_device(adev) == 0) { 3287 if (!amdgpu_sriov_vf(adev)) { 3288 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3289 3290 if (WARN_ON(!hive)) { 3291 r = -ENOENT; 3292 goto init_failed; 3293 } 3294 3295 if (!hive->reset_domain || 3296 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { 3297 r = -ENOENT; 3298 amdgpu_put_xgmi_hive(hive); 3299 goto init_failed; 3300 } 3301 3302 /* Drop the early temporary reset domain we created for device */ 3303 amdgpu_reset_put_reset_domain(adev->reset_domain); 3304 adev->reset_domain = hive->reset_domain; 3305 amdgpu_put_xgmi_hive(hive); 3306 } 3307 } 3308 } 3309 3310 r = amdgpu_device_init_schedulers(adev); 3311 if (r) 3312 goto init_failed; 3313 3314 if (adev->mman.buffer_funcs_ring && 3315 adev->mman.buffer_funcs_ring->sched.ready) 3316 amdgpu_ttm_set_buffer_funcs_status(adev, true); 3317 3318 /* Don't init kfd if whole hive need to be reset during init */ 3319 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 3320 kgd2kfd_init_zone_device(adev); 3321 amdgpu_amdkfd_device_init(adev); 3322 } 3323 3324 amdgpu_fru_get_product_info(adev); 3325 3326 if (!amdgpu_sriov_vf(adev) || amdgpu_sriov_ras_cper_en(adev)) 3327 r = amdgpu_cper_init(adev); 3328 3329init_failed: 3330 3331 return r; 3332} 3333 3334/** 3335 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 3336 * 3337 * @adev: amdgpu_device pointer 3338 * 3339 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 3340 * this function before a GPU reset. If the value is retained after a 3341 * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents. 3342 */ 3343static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 3344{ 3345 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 3346} 3347 3348/** 3349 * amdgpu_device_check_vram_lost - check if vram is valid 3350 * 3351 * @adev: amdgpu_device pointer 3352 * 3353 * Checks the reset magic value written to the gart pointer in VRAM. 3354 * The driver calls this after a GPU reset to see if the contents of 3355 * VRAM is lost or now. 3356 * returns true if vram is lost, false if not. 3357 */ 3358static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 3359{ 3360 if (memcmp(adev->gart.ptr, adev->reset_magic, 3361 AMDGPU_RESET_MAGIC_NUM)) 3362 return true; 3363 3364 if (!amdgpu_in_reset(adev)) 3365 return false; 3366 3367 /* 3368 * For all ASICs with baco/mode1 reset, the VRAM is 3369 * always assumed to be lost. 3370 */ 3371 switch (amdgpu_asic_reset_method(adev)) { 3372 case AMD_RESET_METHOD_LEGACY: 3373 case AMD_RESET_METHOD_LINK: 3374 case AMD_RESET_METHOD_BACO: 3375 case AMD_RESET_METHOD_MODE1: 3376 return true; 3377 default: 3378 return false; 3379 } 3380} 3381 3382/** 3383 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 3384 * 3385 * @adev: amdgpu_device pointer 3386 * @state: clockgating state (gate or ungate) 3387 * 3388 * The list of all the hardware IPs that make up the asic is walked and the 3389 * set_clockgating_state callbacks are run. 3390 * Late initialization pass enabling clockgating for hardware IPs. 3391 * Fini or suspend, pass disabling clockgating for hardware IPs. 3392 * Returns 0 on success, negative error code on failure. 3393 */ 3394 3395int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 3396 enum amd_clockgating_state state) 3397{ 3398 int i, j, r; 3399 3400 if (amdgpu_emu_mode == 1) 3401 return 0; 3402 3403 for (j = 0; j < adev->num_ip_blocks; j++) { 3404 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 3405 if (!adev->ip_blocks[i].status.late_initialized) 3406 continue; 3407 if (!adev->ip_blocks[i].version) 3408 continue; 3409 /* skip CG for GFX, SDMA on S0ix */ 3410 if (adev->in_s0ix && 3411 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3412 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 3413 continue; 3414 /* skip CG for VCE/UVD, it's handled specially */ 3415 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 3416 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 3417 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 3418 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 3419 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 3420 /* enable clockgating to save power */ 3421 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i], 3422 state); 3423 if (r) { 3424 dev_err(adev->dev, 3425 "set_clockgating_state(gate) of IP block <%s> failed %d\n", 3426 adev->ip_blocks[i].version->funcs->name, 3427 r); 3428 return r; 3429 } 3430 } 3431 } 3432 3433 return 0; 3434} 3435 3436int amdgpu_device_set_pg_state(struct amdgpu_device *adev, 3437 enum amd_powergating_state state) 3438{ 3439 int i, j, r; 3440 3441 if (amdgpu_emu_mode == 1) 3442 return 0; 3443 3444 for (j = 0; j < adev->num_ip_blocks; j++) { 3445 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 3446 if (!adev->ip_blocks[i].status.late_initialized) 3447 continue; 3448 if (!adev->ip_blocks[i].version) 3449 continue; 3450 /* skip PG for GFX, SDMA on S0ix */ 3451 if (adev->in_s0ix && 3452 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3453 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 3454 continue; 3455 /* skip CG for VCE/UVD, it's handled specially */ 3456 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 3457 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 3458 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 3459 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 3460 adev->ip_blocks[i].version->funcs->set_powergating_state) { 3461 /* enable powergating to save power */ 3462 r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i], 3463 state); 3464 if (r) { 3465 dev_err(adev->dev, 3466 "set_powergating_state(gate) of IP block <%s> failed %d\n", 3467 adev->ip_blocks[i].version->funcs->name, 3468 r); 3469 return r; 3470 } 3471 } 3472 } 3473 return 0; 3474} 3475 3476static int amdgpu_device_enable_mgpu_fan_boost(void) 3477{ 3478 struct amdgpu_gpu_instance *gpu_ins; 3479 struct amdgpu_device *adev; 3480 int i, ret = 0; 3481 3482 mutex_lock(&mgpu_info.mutex); 3483 3484 /* 3485 * MGPU fan boost feature should be enabled 3486 * only when there are two or more dGPUs in 3487 * the system 3488 */ 3489 if (mgpu_info.num_dgpu < 2) 3490 goto out; 3491 3492 for (i = 0; i < mgpu_info.num_dgpu; i++) { 3493 gpu_ins = &(mgpu_info.gpu_ins[i]); 3494 adev = gpu_ins->adev; 3495 if (!(adev->flags & AMD_IS_APU || amdgpu_sriov_multi_vf_mode(adev)) && 3496 !gpu_ins->mgpu_fan_enabled) { 3497 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 3498 if (ret) 3499 break; 3500 3501 gpu_ins->mgpu_fan_enabled = 1; 3502 } 3503 } 3504 3505out: 3506 mutex_unlock(&mgpu_info.mutex); 3507 3508 return ret; 3509} 3510 3511/** 3512 * amdgpu_device_ip_late_init - run late init for hardware IPs 3513 * 3514 * @adev: amdgpu_device pointer 3515 * 3516 * Late initialization pass for hardware IPs. The list of all the hardware 3517 * IPs that make up the asic is walked and the late_init callbacks are run. 3518 * late_init covers any special initialization that an IP requires 3519 * after all of the have been initialized or something that needs to happen 3520 * late in the init process. 3521 * Returns 0 on success, negative error code on failure. 3522 */ 3523static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 3524{ 3525 struct amdgpu_gpu_instance *gpu_instance; 3526 int i = 0, r; 3527 3528 for (i = 0; i < adev->num_ip_blocks; i++) { 3529 if (!adev->ip_blocks[i].status.hw) 3530 continue; 3531 if (adev->ip_blocks[i].version->funcs->late_init) { 3532 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]); 3533 if (r) { 3534 dev_err(adev->dev, 3535 "late_init of IP block <%s> failed %d\n", 3536 adev->ip_blocks[i].version->funcs->name, 3537 r); 3538 return r; 3539 } 3540 } 3541 adev->ip_blocks[i].status.late_initialized = true; 3542 } 3543 3544 r = amdgpu_ras_late_init(adev); 3545 if (r) { 3546 dev_err(adev->dev, "amdgpu_ras_late_init failed %d", r); 3547 return r; 3548 } 3549 3550 if (!amdgpu_reset_in_recovery(adev)) 3551 amdgpu_ras_set_error_query_ready(adev, true); 3552 3553 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 3554 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 3555 3556 amdgpu_device_fill_reset_magic(adev); 3557 3558 r = amdgpu_device_enable_mgpu_fan_boost(); 3559 if (r) 3560 dev_err(adev->dev, "enable mgpu fan boost failed (%d).\n", r); 3561 3562 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */ 3563 if (amdgpu_passthrough(adev) && 3564 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || 3565 adev->asic_type == CHIP_ALDEBARAN)) 3566 amdgpu_dpm_handle_passthrough_sbr(adev, true); 3567 3568 if (adev->gmc.xgmi.num_physical_nodes > 1) { 3569 mutex_lock(&mgpu_info.mutex); 3570 3571 /* 3572 * Reset device p-state to low as this was booted with high. 3573 * 3574 * This should be performed only after all devices from the same 3575 * hive get initialized. 3576 * 3577 * However, it's unknown how many device in the hive in advance. 3578 * As this is counted one by one during devices initializations. 3579 * 3580 * So, we wait for all XGMI interlinked devices initialized. 3581 * This may bring some delays as those devices may come from 3582 * different hives. But that should be OK. 3583 */ 3584 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 3585 for (i = 0; i < mgpu_info.num_gpu; i++) { 3586 gpu_instance = &(mgpu_info.gpu_ins[i]); 3587 if (gpu_instance->adev->flags & AMD_IS_APU) 3588 continue; 3589 3590 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 3591 AMDGPU_XGMI_PSTATE_MIN); 3592 if (r) { 3593 dev_err(adev->dev, 3594 "pstate setting failed (%d).\n", 3595 r); 3596 break; 3597 } 3598 } 3599 } 3600 3601 mutex_unlock(&mgpu_info.mutex); 3602 } 3603 3604 return 0; 3605} 3606 3607static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block) 3608{ 3609 struct amdgpu_device *adev = ip_block->adev; 3610 int r; 3611 3612 if (!ip_block->version->funcs->hw_fini) { 3613 dev_err(adev->dev, "hw_fini of IP block <%s> not defined\n", 3614 ip_block->version->funcs->name); 3615 } else { 3616 r = ip_block->version->funcs->hw_fini(ip_block); 3617 /* XXX handle errors */ 3618 if (r) { 3619 dev_dbg(adev->dev, 3620 "hw_fini of IP block <%s> failed %d\n", 3621 ip_block->version->funcs->name, r); 3622 } 3623 } 3624 3625 ip_block->status.hw = false; 3626} 3627 3628/** 3629 * amdgpu_device_smu_fini_early - smu hw_fini wrapper 3630 * 3631 * @adev: amdgpu_device pointer 3632 * 3633 * For ASICs need to disable SMC first 3634 */ 3635static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev) 3636{ 3637 int i; 3638 3639 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) 3640 return; 3641 3642 for (i = 0; i < adev->num_ip_blocks; i++) { 3643 if (!adev->ip_blocks[i].status.hw) 3644 continue; 3645 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3646 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 3647 break; 3648 } 3649 } 3650} 3651 3652static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) 3653{ 3654 int i, r; 3655 3656 for (i = 0; i < adev->num_ip_blocks; i++) { 3657 if (!adev->ip_blocks[i].version) 3658 continue; 3659 if (!adev->ip_blocks[i].version->funcs->early_fini) 3660 continue; 3661 3662 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]); 3663 if (r) { 3664 dev_dbg(adev->dev, 3665 "early_fini of IP block <%s> failed %d\n", 3666 adev->ip_blocks[i].version->funcs->name, r); 3667 } 3668 } 3669 3670 amdgpu_amdkfd_suspend(adev, true); 3671 amdgpu_userq_suspend(adev); 3672 3673 /* Workaround for ASICs need to disable SMC first */ 3674 amdgpu_device_smu_fini_early(adev); 3675 3676 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3677 if (!adev->ip_blocks[i].status.hw) 3678 continue; 3679 3680 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 3681 } 3682 3683 if (amdgpu_sriov_vf(adev)) { 3684 if (amdgpu_virt_release_full_gpu(adev, false)) 3685 dev_err(adev->dev, 3686 "failed to release exclusive mode on fini\n"); 3687 } 3688 3689 /* 3690 * Driver reload on the APU can fail due to firmware validation because 3691 * the PSP is always running, as it is shared across the whole SoC. 3692 * This same issue does not occur on dGPU because it has a mechanism 3693 * that checks whether the PSP is running. A solution for those issues 3694 * in the APU is to trigger a GPU reset, but this should be done during 3695 * the unload phase to avoid adding boot latency and screen flicker. 3696 */ 3697 if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu) { 3698 r = amdgpu_asic_reset(adev); 3699 if (r) 3700 dev_err(adev->dev, "asic reset on %s failed\n", __func__); 3701 } 3702 3703 return 0; 3704} 3705 3706/** 3707 * amdgpu_device_ip_fini - run fini for hardware IPs 3708 * 3709 * @adev: amdgpu_device pointer 3710 * 3711 * Main teardown pass for hardware IPs. The list of all the hardware 3712 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 3713 * are run. hw_fini tears down the hardware associated with each IP 3714 * and sw_fini tears down any software state associated with each IP. 3715 * Returns 0 on success, negative error code on failure. 3716 */ 3717static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 3718{ 3719 int i, r; 3720 3721 amdgpu_cper_fini(adev); 3722 3723 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 3724 amdgpu_virt_release_ras_err_handler_data(adev); 3725 3726 if (adev->gmc.xgmi.num_physical_nodes > 1) 3727 amdgpu_xgmi_remove_device(adev); 3728 3729 amdgpu_amdkfd_device_fini_sw(adev); 3730 3731 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3732 if (!adev->ip_blocks[i].status.sw) 3733 continue; 3734 3735 if (!adev->ip_blocks[i].version) 3736 continue; 3737 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 3738 amdgpu_ucode_free_bo(adev); 3739 amdgpu_free_static_csa(&adev->virt.csa_obj); 3740 amdgpu_device_wb_fini(adev); 3741 amdgpu_device_mem_scratch_fini(adev); 3742 amdgpu_ib_pool_fini(adev); 3743 amdgpu_seq64_fini(adev); 3744 amdgpu_doorbell_fini(adev); 3745 } 3746 if (adev->ip_blocks[i].version->funcs->sw_fini) { 3747 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]); 3748 /* XXX handle errors */ 3749 if (r) { 3750 dev_dbg(adev->dev, 3751 "sw_fini of IP block <%s> failed %d\n", 3752 adev->ip_blocks[i].version->funcs->name, 3753 r); 3754 } 3755 } 3756 adev->ip_blocks[i].status.sw = false; 3757 adev->ip_blocks[i].status.valid = false; 3758 } 3759 3760 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3761 if (!adev->ip_blocks[i].status.late_initialized) 3762 continue; 3763 if (!adev->ip_blocks[i].version) 3764 continue; 3765 if (adev->ip_blocks[i].version->funcs->late_fini) 3766 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); 3767 adev->ip_blocks[i].status.late_initialized = false; 3768 } 3769 3770 amdgpu_ras_fini(adev); 3771 amdgpu_uid_fini(adev); 3772 3773 return 0; 3774} 3775 3776/** 3777 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 3778 * 3779 * @work: work_struct. 3780 */ 3781static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 3782{ 3783 struct amdgpu_device *adev = 3784 container_of(work, struct amdgpu_device, delayed_init_work.work); 3785 int r; 3786 3787 r = amdgpu_ib_ring_tests(adev); 3788 if (r) 3789 dev_err(adev->dev, "ib ring test failed (%d).\n", r); 3790} 3791 3792static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 3793{ 3794 struct amdgpu_device *adev = 3795 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 3796 3797 WARN_ON_ONCE(adev->gfx.gfx_off_state); 3798 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); 3799 3800 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0)) 3801 adev->gfx.gfx_off_state = true; 3802} 3803 3804/** 3805 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 3806 * 3807 * @adev: amdgpu_device pointer 3808 * 3809 * Main suspend function for hardware IPs. The list of all the hardware 3810 * IPs that make up the asic is walked, clockgating is disabled and the 3811 * suspend callbacks are run. suspend puts the hardware and software state 3812 * in each IP into a state suitable for suspend. 3813 * Returns 0 on success, negative error code on failure. 3814 */ 3815static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 3816{ 3817 int i, r, rec; 3818 3819 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 3820 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 3821 3822 /* 3823 * Per PMFW team's suggestion, driver needs to handle gfxoff 3824 * and df cstate features disablement for gpu reset(e.g. Mode1Reset) 3825 * scenario. Add the missing df cstate disablement here. 3826 */ 3827 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) 3828 dev_warn(adev->dev, "Failed to disallow df cstate"); 3829 3830 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3831 if (!adev->ip_blocks[i].status.valid) 3832 continue; 3833 3834 /* displays are handled separately */ 3835 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 3836 continue; 3837 3838 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3839 if (r) 3840 goto unwind; 3841 } 3842 3843 return 0; 3844unwind: 3845 rec = amdgpu_device_ip_resume_phase3(adev); 3846 if (rec) 3847 dev_err(adev->dev, 3848 "amdgpu_device_ip_resume_phase3 failed during unwind: %d\n", 3849 rec); 3850 3851 amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW); 3852 3853 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 3854 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 3855 3856 return r; 3857} 3858 3859/** 3860 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 3861 * 3862 * @adev: amdgpu_device pointer 3863 * 3864 * Main suspend function for hardware IPs. The list of all the hardware 3865 * IPs that make up the asic is walked, clockgating is disabled and the 3866 * suspend callbacks are run. suspend puts the hardware and software state 3867 * in each IP into a state suitable for suspend. 3868 * Returns 0 on success, negative error code on failure. 3869 */ 3870static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 3871{ 3872 int i, r, rec; 3873 3874 if (adev->in_s0ix) 3875 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); 3876 3877 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3878 if (!adev->ip_blocks[i].status.valid) 3879 continue; 3880 /* displays are handled in phase1 */ 3881 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 3882 continue; 3883 /* PSP lost connection when err_event_athub occurs */ 3884 if (amdgpu_ras_intr_triggered() && 3885 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 3886 adev->ip_blocks[i].status.hw = false; 3887 continue; 3888 } 3889 3890 /* skip unnecessary suspend if we do not initialize them yet */ 3891 if (!amdgpu_ip_member_of_hwini( 3892 adev, adev->ip_blocks[i].version->type)) 3893 continue; 3894 3895 /* Since we skip suspend for S0i3, we need to cancel the delayed 3896 * idle work here as the suspend callback never gets called. 3897 */ 3898 if (adev->in_s0ix && 3899 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX && 3900 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0)) 3901 cancel_delayed_work_sync(&adev->gfx.idle_work); 3902 /* skip suspend of gfx/mes and psp for S0ix 3903 * gfx is in gfxoff state, so on resume it will exit gfxoff just 3904 * like at runtime. PSP is also part of the always on hardware 3905 * so no need to suspend it. 3906 */ 3907 if (adev->in_s0ix && 3908 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || 3909 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3910 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) 3911 continue; 3912 3913 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */ 3914 if (adev->in_s0ix && 3915 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= 3916 IP_VERSION(5, 0, 0)) && 3917 (adev->ip_blocks[i].version->type == 3918 AMD_IP_BLOCK_TYPE_SDMA)) 3919 continue; 3920 3921 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. 3922 * These are in TMR, hence are expected to be reused by PSP-TOS to reload 3923 * from this location and RLC Autoload automatically also gets loaded 3924 * from here based on PMFW -> PSP message during re-init sequence. 3925 * Therefore, the psp suspend & resume should be skipped to avoid destroy 3926 * the TMR and reload FWs again for IMU enabled APU ASICs. 3927 */ 3928 if (amdgpu_in_reset(adev) && 3929 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && 3930 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3931 continue; 3932 3933 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3934 if (r) 3935 goto unwind; 3936 3937 /* handle putting the SMC in the appropriate state */ 3938 if (!amdgpu_sriov_vf(adev)) { 3939 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3940 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 3941 if (r) { 3942 dev_err(adev->dev, 3943 "SMC failed to set mp1 state %d, %d\n", 3944 adev->mp1_state, r); 3945 goto unwind; 3946 } 3947 } 3948 } 3949 } 3950 3951 return 0; 3952unwind: 3953 /* suspend phase 2 = resume phase 1 + resume phase 2 */ 3954 rec = amdgpu_device_ip_resume_phase1(adev); 3955 if (rec) { 3956 dev_err(adev->dev, 3957 "amdgpu_device_ip_resume_phase1 failed during unwind: %d\n", 3958 rec); 3959 return r; 3960 } 3961 3962 rec = amdgpu_device_fw_loading(adev); 3963 if (rec) { 3964 dev_err(adev->dev, 3965 "amdgpu_device_fw_loading failed during unwind: %d\n", 3966 rec); 3967 return r; 3968 } 3969 3970 rec = amdgpu_device_ip_resume_phase2(adev); 3971 if (rec) { 3972 dev_err(adev->dev, 3973 "amdgpu_device_ip_resume_phase2 failed during unwind: %d\n", 3974 rec); 3975 return r; 3976 } 3977 3978 return r; 3979} 3980 3981/** 3982 * amdgpu_device_ip_suspend - run suspend for hardware IPs 3983 * 3984 * @adev: amdgpu_device pointer 3985 * 3986 * Main suspend function for hardware IPs. The list of all the hardware 3987 * IPs that make up the asic is walked, clockgating is disabled and the 3988 * suspend callbacks are run. suspend puts the hardware and software state 3989 * in each IP into a state suitable for suspend. 3990 * Returns 0 on success, negative error code on failure. 3991 */ 3992static int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 3993{ 3994 int r; 3995 3996 if (amdgpu_sriov_vf(adev)) { 3997 amdgpu_virt_fini_data_exchange(adev); 3998 amdgpu_virt_request_full_gpu(adev, false); 3999 } 4000 4001 amdgpu_ttm_set_buffer_funcs_status(adev, false); 4002 4003 r = amdgpu_device_ip_suspend_phase1(adev); 4004 if (r) 4005 return r; 4006 r = amdgpu_device_ip_suspend_phase2(adev); 4007 4008 if (amdgpu_sriov_vf(adev)) 4009 amdgpu_virt_release_full_gpu(adev, false); 4010 4011 return r; 4012} 4013 4014static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 4015{ 4016 int i, r; 4017 4018 static enum amd_ip_block_type ip_order[] = { 4019 AMD_IP_BLOCK_TYPE_COMMON, 4020 AMD_IP_BLOCK_TYPE_GMC, 4021 AMD_IP_BLOCK_TYPE_PSP, 4022 AMD_IP_BLOCK_TYPE_IH, 4023 }; 4024 4025 for (i = 0; i < adev->num_ip_blocks; i++) { 4026 int j; 4027 struct amdgpu_ip_block *block; 4028 4029 block = &adev->ip_blocks[i]; 4030 block->status.hw = false; 4031 4032 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 4033 4034 if (block->version->type != ip_order[j] || 4035 !block->status.valid) 4036 continue; 4037 4038 r = block->version->funcs->hw_init(&adev->ip_blocks[i]); 4039 if (r) { 4040 dev_err(adev->dev, "RE-INIT-early: %s failed\n", 4041 block->version->funcs->name); 4042 return r; 4043 } 4044 block->status.hw = true; 4045 } 4046 } 4047 4048 return 0; 4049} 4050 4051static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 4052{ 4053 struct amdgpu_ip_block *block; 4054 int i, r = 0; 4055 4056 static enum amd_ip_block_type ip_order[] = { 4057 AMD_IP_BLOCK_TYPE_SMC, 4058 AMD_IP_BLOCK_TYPE_DCE, 4059 AMD_IP_BLOCK_TYPE_GFX, 4060 AMD_IP_BLOCK_TYPE_SDMA, 4061 AMD_IP_BLOCK_TYPE_MES, 4062 AMD_IP_BLOCK_TYPE_UVD, 4063 AMD_IP_BLOCK_TYPE_VCE, 4064 AMD_IP_BLOCK_TYPE_VCN, 4065 AMD_IP_BLOCK_TYPE_JPEG 4066 }; 4067 4068 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 4069 block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]); 4070 4071 if (!block) 4072 continue; 4073 4074 if (block->status.valid && !block->status.hw) { 4075 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) { 4076 r = amdgpu_ip_block_resume(block); 4077 } else { 4078 r = block->version->funcs->hw_init(block); 4079 } 4080 4081 if (r) { 4082 dev_err(adev->dev, "RE-INIT-late: %s failed\n", 4083 block->version->funcs->name); 4084 break; 4085 } 4086 block->status.hw = true; 4087 } 4088 } 4089 4090 return r; 4091} 4092 4093/** 4094 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 4095 * 4096 * @adev: amdgpu_device pointer 4097 * 4098 * First resume function for hardware IPs. The list of all the hardware 4099 * IPs that make up the asic is walked and the resume callbacks are run for 4100 * COMMON, GMC, and IH. resume puts the hardware into a functional state 4101 * after a suspend and updates the software state as necessary. This 4102 * function is also used for restoring the GPU after a GPU reset. 4103 * Returns 0 on success, negative error code on failure. 4104 */ 4105static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 4106{ 4107 int i, r; 4108 4109 for (i = 0; i < adev->num_ip_blocks; i++) { 4110 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 4111 continue; 4112 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 4113 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 4114 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 4115 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { 4116 4117 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 4118 if (r) 4119 return r; 4120 } 4121 } 4122 4123 return 0; 4124} 4125 4126/** 4127 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 4128 * 4129 * @adev: amdgpu_device pointer 4130 * 4131 * Second resume function for hardware IPs. The list of all the hardware 4132 * IPs that make up the asic is walked and the resume callbacks are run for 4133 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 4134 * functional state after a suspend and updates the software state as 4135 * necessary. This function is also used for restoring the GPU after a GPU 4136 * reset. 4137 * Returns 0 on success, negative error code on failure. 4138 */ 4139static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 4140{ 4141 int i, r; 4142 4143 for (i = 0; i < adev->num_ip_blocks; i++) { 4144 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 4145 continue; 4146 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 4147 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 4148 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 4149 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE || 4150 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 4151 continue; 4152 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 4153 if (r) 4154 return r; 4155 } 4156 4157 return 0; 4158} 4159 4160/** 4161 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs 4162 * 4163 * @adev: amdgpu_device pointer 4164 * 4165 * Third resume function for hardware IPs. The list of all the hardware 4166 * IPs that make up the asic is walked and the resume callbacks are run for 4167 * all DCE. resume puts the hardware into a functional state after a suspend 4168 * and updates the software state as necessary. This function is also used 4169 * for restoring the GPU after a GPU reset. 4170 * 4171 * Returns 0 on success, negative error code on failure. 4172 */ 4173static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev) 4174{ 4175 int i, r; 4176 4177 for (i = 0; i < adev->num_ip_blocks; i++) { 4178 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 4179 continue; 4180 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { 4181 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 4182 if (r) 4183 return r; 4184 } 4185 } 4186 4187 return 0; 4188} 4189 4190/** 4191 * amdgpu_device_ip_resume - run resume for hardware IPs 4192 * 4193 * @adev: amdgpu_device pointer 4194 * 4195 * Main resume function for hardware IPs. The hardware IPs 4196 * are split into two resume functions because they are 4197 * also used in recovering from a GPU reset and some additional 4198 * steps need to be take between them. In this case (S3/S4) they are 4199 * run sequentially. 4200 * Returns 0 on success, negative error code on failure. 4201 */ 4202static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 4203{ 4204 int r; 4205 4206 r = amdgpu_device_ip_resume_phase1(adev); 4207 if (r) 4208 return r; 4209 4210 r = amdgpu_device_fw_loading(adev); 4211 if (r) 4212 return r; 4213 4214 r = amdgpu_device_ip_resume_phase2(adev); 4215 4216 if (adev->mman.buffer_funcs_ring->sched.ready) 4217 amdgpu_ttm_set_buffer_funcs_status(adev, true); 4218 4219 if (r) 4220 return r; 4221 4222 amdgpu_fence_driver_hw_init(adev); 4223 4224 r = amdgpu_device_ip_resume_phase3(adev); 4225 4226 return r; 4227} 4228 4229/** 4230 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 4231 * 4232 * @adev: amdgpu_device pointer 4233 * 4234 * Query the VBIOS data tables to determine if the board supports SR-IOV. 4235 */ 4236static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 4237{ 4238 if (amdgpu_sriov_vf(adev)) { 4239 if (adev->is_atom_fw) { 4240 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev)) 4241 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 4242 } else { 4243 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 4244 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 4245 } 4246 4247 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 4248 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 4249 } 4250} 4251 4252/** 4253 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 4254 * 4255 * @pdev : pci device context 4256 * @asic_type: AMD asic type 4257 * 4258 * Check if there is DC (new modesetting infrastructre) support for an asic. 4259 * returns true if DC has support, false if not. 4260 */ 4261bool amdgpu_device_asic_has_dc_support(struct pci_dev *pdev, 4262 enum amd_asic_type asic_type) 4263{ 4264 switch (asic_type) { 4265#ifdef CONFIG_DRM_AMDGPU_SI 4266 case CHIP_HAINAN: 4267#endif 4268 case CHIP_TOPAZ: 4269 /* chips with no display hardware */ 4270 return false; 4271#if defined(CONFIG_DRM_AMD_DC) 4272 case CHIP_TAHITI: 4273 case CHIP_PITCAIRN: 4274 case CHIP_VERDE: 4275 case CHIP_OLAND: 4276 return amdgpu_dc != 0 && IS_ENABLED(CONFIG_DRM_AMD_DC_SI); 4277 case CHIP_KAVERI: 4278 case CHIP_KABINI: 4279 case CHIP_MULLINS: 4280 /* 4281 * We have systems in the wild with these ASICs that require 4282 * TRAVIS and NUTMEG support which is not supported with DC. 4283 * 4284 * Fallback to the non-DC driver here by default so as not to 4285 * cause regressions. 4286 */ 4287 return amdgpu_dc > 0; 4288 default: 4289 return amdgpu_dc != 0; 4290#else 4291 default: 4292 if (amdgpu_dc > 0) 4293 dev_info_once( 4294 &pdev->dev, 4295 "Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n"); 4296 return false; 4297#endif 4298 } 4299} 4300 4301/** 4302 * amdgpu_device_has_dc_support - check if dc is supported 4303 * 4304 * @adev: amdgpu_device pointer 4305 * 4306 * Returns true for supported, false for not supported 4307 */ 4308bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 4309{ 4310 if (adev->enable_virtual_display || 4311 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 4312 return false; 4313 4314 return amdgpu_device_asic_has_dc_support(adev->pdev, adev->asic_type); 4315} 4316 4317static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 4318{ 4319 struct amdgpu_device *adev = 4320 container_of(__work, struct amdgpu_device, xgmi_reset_work); 4321 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4322 4323 /* It's a bug to not have a hive within this function */ 4324 if (WARN_ON(!hive)) 4325 return; 4326 4327 /* 4328 * Use task barrier to synchronize all xgmi reset works across the 4329 * hive. task_barrier_enter and task_barrier_exit will block 4330 * until all the threads running the xgmi reset works reach 4331 * those points. task_barrier_full will do both blocks. 4332 */ 4333 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 4334 4335 task_barrier_enter(&hive->tb); 4336 adev->asic_reset_res = amdgpu_device_baco_enter(adev); 4337 4338 if (adev->asic_reset_res) 4339 goto fail; 4340 4341 task_barrier_exit(&hive->tb); 4342 adev->asic_reset_res = amdgpu_device_baco_exit(adev); 4343 4344 if (adev->asic_reset_res) 4345 goto fail; 4346 4347 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB); 4348 } else { 4349 4350 task_barrier_full(&hive->tb); 4351 adev->asic_reset_res = amdgpu_asic_reset(adev); 4352 } 4353 4354fail: 4355 if (adev->asic_reset_res) 4356 dev_warn(adev->dev, 4357 "ASIC reset failed with error, %d for drm dev, %s", 4358 adev->asic_reset_res, adev_to_drm(adev)->unique); 4359 amdgpu_put_xgmi_hive(hive); 4360} 4361 4362static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 4363{ 4364 char buf[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH]; 4365 char *input = buf; 4366 char *timeout_setting = NULL; 4367 int index = 0; 4368 long timeout; 4369 int ret = 0; 4370 4371 /* By default timeout for all queues is 2 sec */ 4372 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = 4373 adev->video_timeout = msecs_to_jiffies(2000); 4374 4375 if (!strnlen(amdgpu_lockup_timeout, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) 4376 return 0; 4377 4378 /* 4379 * strsep() destructively modifies its input by replacing delimiters 4380 * with '\0'. Use a stack copy so the global module parameter buffer 4381 * remains intact for multi-GPU systems where this function is called 4382 * once per device. 4383 */ 4384 strscpy(buf, amdgpu_lockup_timeout, sizeof(buf)); 4385 4386 while ((timeout_setting = strsep(&input, ",")) && 4387 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 4388 ret = kstrtol(timeout_setting, 0, &timeout); 4389 if (ret) 4390 return ret; 4391 4392 if (timeout == 0) { 4393 index++; 4394 continue; 4395 } else if (timeout < 0) { 4396 timeout = MAX_SCHEDULE_TIMEOUT; 4397 dev_warn(adev->dev, "lockup timeout disabled"); 4398 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 4399 } else { 4400 timeout = msecs_to_jiffies(timeout); 4401 } 4402 4403 switch (index++) { 4404 case 0: 4405 adev->gfx_timeout = timeout; 4406 break; 4407 case 1: 4408 adev->compute_timeout = timeout; 4409 break; 4410 case 2: 4411 adev->sdma_timeout = timeout; 4412 break; 4413 case 3: 4414 adev->video_timeout = timeout; 4415 break; 4416 default: 4417 break; 4418 } 4419 } 4420 4421 /* When only one value specified apply it to all queues. */ 4422 if (index == 1) 4423 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = 4424 adev->video_timeout = timeout; 4425 4426 return ret; 4427} 4428 4429/** 4430 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU 4431 * 4432 * @adev: amdgpu_device pointer 4433 * 4434 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode 4435 */ 4436static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev) 4437{ 4438 struct iommu_domain *domain; 4439 4440 domain = iommu_get_domain_for_dev(adev->dev); 4441 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) 4442 adev->ram_is_direct_mapped = true; 4443} 4444 4445#if defined(CONFIG_HSA_AMD_P2P) 4446/** 4447 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled. 4448 * 4449 * @adev: amdgpu_device pointer 4450 * 4451 * return if IOMMU remapping bar address 4452 */ 4453static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev) 4454{ 4455 struct iommu_domain *domain; 4456 4457 domain = iommu_get_domain_for_dev(adev->dev); 4458 if (domain && (domain->type == IOMMU_DOMAIN_DMA || 4459 domain->type == IOMMU_DOMAIN_DMA_FQ)) 4460 return true; 4461 4462 return false; 4463} 4464#endif 4465 4466static void amdgpu_device_set_mcbp(struct amdgpu_device *adev) 4467{ 4468 if (amdgpu_mcbp == 1) 4469 adev->gfx.mcbp = true; 4470 else if (amdgpu_mcbp == 0) 4471 adev->gfx.mcbp = false; 4472 4473 if (amdgpu_sriov_vf(adev)) 4474 adev->gfx.mcbp = true; 4475 4476 if (adev->gfx.mcbp) 4477 dev_info(adev->dev, "MCBP is enabled\n"); 4478} 4479 4480static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev) 4481{ 4482 int r; 4483 4484 r = amdgpu_atombios_sysfs_init(adev); 4485 if (r) 4486 drm_err(&adev->ddev, 4487 "registering atombios sysfs failed (%d).\n", r); 4488 4489 r = amdgpu_pm_sysfs_init(adev); 4490 if (r) 4491 dev_err(adev->dev, "registering pm sysfs failed (%d).\n", r); 4492 4493 r = amdgpu_ucode_sysfs_init(adev); 4494 if (r) { 4495 adev->ucode_sysfs_en = false; 4496 dev_err(adev->dev, "Creating firmware sysfs failed (%d).\n", r); 4497 } else 4498 adev->ucode_sysfs_en = true; 4499 4500 r = amdgpu_device_attr_sysfs_init(adev); 4501 if (r) 4502 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 4503 4504 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); 4505 if (r) 4506 dev_err(adev->dev, 4507 "Could not create amdgpu board attributes\n"); 4508 4509 amdgpu_fru_sysfs_init(adev); 4510 amdgpu_reg_state_sysfs_init(adev); 4511 amdgpu_xcp_sysfs_init(adev); 4512 4513 return r; 4514} 4515 4516static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev) 4517{ 4518 if (adev->pm.sysfs_initialized) 4519 amdgpu_pm_sysfs_fini(adev); 4520 if (adev->ucode_sysfs_en) 4521 amdgpu_ucode_sysfs_fini(adev); 4522 amdgpu_device_attr_sysfs_fini(adev); 4523 amdgpu_fru_sysfs_fini(adev); 4524 4525 amdgpu_reg_state_sysfs_fini(adev); 4526 amdgpu_xcp_sysfs_fini(adev); 4527} 4528 4529/** 4530 * amdgpu_device_init - initialize the driver 4531 * 4532 * @adev: amdgpu_device pointer 4533 * @flags: driver flags 4534 * 4535 * Initializes the driver info and hw (all asics). 4536 * Returns 0 for success or an error on failure. 4537 * Called at driver startup. 4538 */ 4539int amdgpu_device_init(struct amdgpu_device *adev, 4540 uint32_t flags) 4541{ 4542 struct pci_dev *pdev = adev->pdev; 4543 int r, i; 4544 bool px = false; 4545 u32 max_MBps; 4546 int tmp; 4547 4548 adev->shutdown = false; 4549 adev->flags = flags; 4550 4551 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 4552 adev->asic_type = amdgpu_force_asic_type; 4553 else 4554 adev->asic_type = flags & AMD_ASIC_MASK; 4555 4556 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 4557 if (amdgpu_emu_mode == 1) 4558 adev->usec_timeout *= 10; 4559 adev->gmc.gart_size = 512 * 1024 * 1024; 4560 adev->accel_working = false; 4561 adev->num_rings = 0; 4562 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); 4563 adev->mman.buffer_funcs = NULL; 4564 adev->mman.buffer_funcs_ring = NULL; 4565 adev->vm_manager.vm_pte_funcs = NULL; 4566 adev->vm_manager.vm_pte_num_scheds = 0; 4567 adev->gmc.gmc_funcs = NULL; 4568 adev->harvest_ip_mask = 0x0; 4569 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 4570 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 4571 4572 adev->smc_rreg = &amdgpu_invalid_rreg; 4573 adev->smc_wreg = &amdgpu_invalid_wreg; 4574 adev->pcie_rreg = &amdgpu_invalid_rreg; 4575 adev->pcie_wreg = &amdgpu_invalid_wreg; 4576 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext; 4577 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext; 4578 adev->pciep_rreg = &amdgpu_invalid_rreg; 4579 adev->pciep_wreg = &amdgpu_invalid_wreg; 4580 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; 4581 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; 4582 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext; 4583 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext; 4584 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 4585 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 4586 adev->didt_rreg = &amdgpu_invalid_rreg; 4587 adev->didt_wreg = &amdgpu_invalid_wreg; 4588 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 4589 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 4590 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 4591 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 4592 4593 dev_info( 4594 adev->dev, 4595 "initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 4596 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 4597 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 4598 4599 /* mutex initialization are all done here so we 4600 * can recall function without having locking issues 4601 */ 4602 mutex_init(&adev->firmware.mutex); 4603 mutex_init(&adev->pm.mutex); 4604 mutex_init(&adev->gfx.gpu_clock_mutex); 4605 mutex_init(&adev->srbm_mutex); 4606 mutex_init(&adev->gfx.pipe_reserve_mutex); 4607 mutex_init(&adev->gfx.gfx_off_mutex); 4608 mutex_init(&adev->gfx.partition_mutex); 4609 mutex_init(&adev->grbm_idx_mutex); 4610 mutex_init(&adev->mn_lock); 4611 mutex_init(&adev->virt.vf_errors.lock); 4612 hash_init(adev->mn_hash); 4613 mutex_init(&adev->psp.mutex); 4614 mutex_init(&adev->notifier_lock); 4615 mutex_init(&adev->pm.stable_pstate_ctx_lock); 4616 mutex_init(&adev->benchmark_mutex); 4617 mutex_init(&adev->gfx.reset_sem_mutex); 4618 /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */ 4619 mutex_init(&adev->enforce_isolation_mutex); 4620 for (i = 0; i < MAX_XCP; ++i) { 4621 adev->isolation[i].spearhead = dma_fence_get_stub(); 4622 amdgpu_sync_create(&adev->isolation[i].active); 4623 amdgpu_sync_create(&adev->isolation[i].prev); 4624 } 4625 mutex_init(&adev->gfx.userq_sch_mutex); 4626 mutex_init(&adev->gfx.workload_profile_mutex); 4627 mutex_init(&adev->vcn.workload_profile_mutex); 4628 4629 amdgpu_device_init_apu_flags(adev); 4630 4631 r = amdgpu_device_check_arguments(adev); 4632 if (r) 4633 return r; 4634 4635 spin_lock_init(&adev->mmio_idx_lock); 4636 spin_lock_init(&adev->smc_idx_lock); 4637 spin_lock_init(&adev->pcie_idx_lock); 4638 spin_lock_init(&adev->uvd_ctx_idx_lock); 4639 spin_lock_init(&adev->didt_idx_lock); 4640 spin_lock_init(&adev->gc_cac_idx_lock); 4641 spin_lock_init(&adev->se_cac_idx_lock); 4642 spin_lock_init(&adev->audio_endpt_idx_lock); 4643 spin_lock_init(&adev->mm_stats.lock); 4644 spin_lock_init(&adev->virt.rlcg_reg_lock); 4645 spin_lock_init(&adev->wb.lock); 4646 4647 xa_init_flags(&adev->userq_xa, XA_FLAGS_LOCK_IRQ); 4648 4649 INIT_LIST_HEAD(&adev->reset_list); 4650 4651 INIT_LIST_HEAD(&adev->ras_list); 4652 4653 INIT_LIST_HEAD(&adev->pm.od_kobj_list); 4654 4655 xa_init(&adev->userq_doorbell_xa); 4656 4657 INIT_DELAYED_WORK(&adev->delayed_init_work, 4658 amdgpu_device_delayed_init_work_handler); 4659 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 4660 amdgpu_device_delay_enable_gfx_off); 4661 /* 4662 * Initialize the enforce_isolation work structures for each XCP 4663 * partition. This work handler is responsible for enforcing shader 4664 * isolation on AMD GPUs. It counts the number of emitted fences for 4665 * each GFX and compute ring. If there are any fences, it schedules 4666 * the `enforce_isolation_work` to be run after a delay. If there are 4667 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the 4668 * runqueue. 4669 */ 4670 for (i = 0; i < MAX_XCP; i++) { 4671 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work, 4672 amdgpu_gfx_enforce_isolation_handler); 4673 adev->gfx.enforce_isolation[i].adev = adev; 4674 adev->gfx.enforce_isolation[i].xcp_id = i; 4675 } 4676 4677 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 4678 INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work); 4679 4680 adev->gfx.gfx_off_req_count = 1; 4681 adev->gfx.gfx_off_residency = 0; 4682 adev->gfx.gfx_off_entrycount = 0; 4683 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 4684 4685 atomic_set(&adev->throttling_logging_enabled, 1); 4686 /* 4687 * If throttling continues, logging will be performed every minute 4688 * to avoid log flooding. "-1" is subtracted since the thermal 4689 * throttling interrupt comes every second. Thus, the total logging 4690 * interval is 59 seconds(retelimited printk interval) + 1(waiting 4691 * for throttling interrupt) = 60 seconds. 4692 */ 4693 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 4694 4695 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 4696 4697 /* Registers mapping */ 4698 /* TODO: block userspace mapping of io register */ 4699 if (adev->asic_type >= CHIP_BONAIRE) { 4700 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 4701 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 4702 } else { 4703 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 4704 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 4705 } 4706 4707 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) 4708 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); 4709 4710 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 4711 if (!adev->rmmio) 4712 return -ENOMEM; 4713 4714 dev_info(adev->dev, "register mmio base: 0x%08X\n", 4715 (uint32_t)adev->rmmio_base); 4716 dev_info(adev->dev, "register mmio size: %u\n", 4717 (unsigned int)adev->rmmio_size); 4718 4719 /* 4720 * Reset domain needs to be present early, before XGMI hive discovered 4721 * (if any) and initialized to use reset sem and in_gpu reset flag 4722 * early on during init and before calling to RREG32. 4723 */ 4724 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); 4725 if (!adev->reset_domain) 4726 return -ENOMEM; 4727 4728 /* detect hw virtualization here */ 4729 amdgpu_virt_init(adev); 4730 4731 amdgpu_device_get_pcie_info(adev); 4732 4733 r = amdgpu_device_get_job_timeout_settings(adev); 4734 if (r) { 4735 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 4736 return r; 4737 } 4738 4739 amdgpu_device_set_mcbp(adev); 4740 4741 /* 4742 * By default, use default mode where all blocks are expected to be 4743 * initialized. At present a 'swinit' of blocks is required to be 4744 * completed before the need for a different level is detected. 4745 */ 4746 amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT); 4747 /* early init functions */ 4748 r = amdgpu_device_ip_early_init(adev); 4749 if (r) 4750 return r; 4751 4752 /* 4753 * No need to remove conflicting FBs for non-display class devices. 4754 * This prevents the sysfb from being freed accidently. 4755 */ 4756 if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA || 4757 (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) { 4758 /* Get rid of things like offb */ 4759 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name); 4760 if (r) 4761 return r; 4762 } 4763 4764 /* Enable TMZ based on IP_VERSION */ 4765 amdgpu_gmc_tmz_set(adev); 4766 4767 if (amdgpu_sriov_vf(adev) && 4768 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0)) 4769 /* VF MMIO access (except mailbox range) from CPU 4770 * will be blocked during sriov runtime 4771 */ 4772 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; 4773 4774 amdgpu_gmc_noretry_set(adev); 4775 /* Need to get xgmi info early to decide the reset behavior*/ 4776 if (adev->gmc.xgmi.supported) { 4777 r = adev->gfxhub.funcs->get_xgmi_info(adev); 4778 if (r) 4779 return r; 4780 } 4781 4782 /* enable PCIE atomic ops */ 4783 if (amdgpu_sriov_vf(adev)) { 4784 if (adev->virt.fw_reserve.p_pf2vf) 4785 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) 4786 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == 4787 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4788 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a 4789 * internal path natively support atomics, set have_atomics_support to true. 4790 */ 4791 } else if ((adev->flags & AMD_IS_APU) && 4792 (amdgpu_ip_version(adev, GC_HWIP, 0) > 4793 IP_VERSION(9, 0, 0))) { 4794 adev->have_atomics_support = true; 4795 } else { 4796 adev->have_atomics_support = 4797 !pci_enable_atomic_ops_to_root(adev->pdev, 4798 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 4799 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4800 } 4801 4802 if (!adev->have_atomics_support) 4803 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); 4804 4805 /* doorbell bar mapping and doorbell index init*/ 4806 amdgpu_doorbell_init(adev); 4807 4808 if (amdgpu_emu_mode == 1) { 4809 /* post the asic on emulation mode */ 4810 emu_soc_asic_init(adev); 4811 goto fence_driver_init; 4812 } 4813 4814 amdgpu_reset_init(adev); 4815 4816 /* detect if we are with an SRIOV vbios */ 4817 if (adev->bios) 4818 amdgpu_device_detect_sriov_bios(adev); 4819 4820 /* check if we need to reset the asic 4821 * E.g., driver was not cleanly unloaded previously, etc. 4822 */ 4823 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 4824 if (adev->gmc.xgmi.num_physical_nodes) { 4825 dev_info(adev->dev, "Pending hive reset.\n"); 4826 amdgpu_set_init_level(adev, 4827 AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 4828 } else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) && 4829 !amdgpu_device_has_display_hardware(adev)) { 4830 r = psp_gpu_reset(adev); 4831 } else { 4832 tmp = amdgpu_reset_method; 4833 /* It should do a default reset when loading or reloading the driver, 4834 * regardless of the module parameter reset_method. 4835 */ 4836 amdgpu_reset_method = AMD_RESET_METHOD_NONE; 4837 r = amdgpu_asic_reset(adev); 4838 amdgpu_reset_method = tmp; 4839 } 4840 4841 if (r) { 4842 dev_err(adev->dev, "asic reset on init failed\n"); 4843 goto failed; 4844 } 4845 } 4846 4847 /* Post card if necessary */ 4848 if (amdgpu_device_need_post(adev)) { 4849 if (!adev->bios) { 4850 dev_err(adev->dev, "no vBIOS found\n"); 4851 r = -EINVAL; 4852 goto failed; 4853 } 4854 dev_info(adev->dev, "GPU posting now...\n"); 4855 r = amdgpu_device_asic_init(adev); 4856 if (r) { 4857 dev_err(adev->dev, "gpu post error!\n"); 4858 goto failed; 4859 } 4860 } 4861 4862 if (adev->bios) { 4863 if (adev->is_atom_fw) { 4864 /* Initialize clocks */ 4865 r = amdgpu_atomfirmware_get_clock_info(adev); 4866 if (r) { 4867 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 4868 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4869 goto failed; 4870 } 4871 } else { 4872 /* Initialize clocks */ 4873 r = amdgpu_atombios_get_clock_info(adev); 4874 if (r) { 4875 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 4876 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4877 goto failed; 4878 } 4879 /* init i2c buses */ 4880 amdgpu_i2c_init(adev); 4881 } 4882 } 4883 4884fence_driver_init: 4885 /* Fence driver */ 4886 r = amdgpu_fence_driver_sw_init(adev); 4887 if (r) { 4888 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); 4889 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 4890 goto failed; 4891 } 4892 4893 /* init the mode config */ 4894 drm_mode_config_init(adev_to_drm(adev)); 4895 4896 r = amdgpu_device_ip_init(adev); 4897 if (r) { 4898 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 4899 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 4900 goto release_ras_con; 4901 } 4902 4903 amdgpu_fence_driver_hw_init(adev); 4904 4905 dev_info(adev->dev, 4906 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 4907 adev->gfx.config.max_shader_engines, 4908 adev->gfx.config.max_sh_per_se, 4909 adev->gfx.config.max_cu_per_sh, 4910 adev->gfx.cu_info.number); 4911 4912 adev->accel_working = true; 4913 4914 amdgpu_vm_check_compute_bug(adev); 4915 4916 /* Initialize the buffer migration limit. */ 4917 if (amdgpu_moverate >= 0) 4918 max_MBps = amdgpu_moverate; 4919 else 4920 max_MBps = 8; /* Allow 8 MB/s. */ 4921 /* Get a log2 for easy divisions. */ 4922 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 4923 4924 /* 4925 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 4926 * Otherwise the mgpu fan boost feature will be skipped due to the 4927 * gpu instance is counted less. 4928 */ 4929 amdgpu_register_gpu_instance(adev); 4930 4931 /* enable clockgating, etc. after ib tests, etc. since some blocks require 4932 * explicit gating rather than handling it automatically. 4933 */ 4934 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 4935 r = amdgpu_device_ip_late_init(adev); 4936 if (r) { 4937 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 4938 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 4939 goto release_ras_con; 4940 } 4941 /* must succeed. */ 4942 amdgpu_ras_resume(adev); 4943 queue_delayed_work(system_wq, &adev->delayed_init_work, 4944 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4945 } 4946 4947 if (amdgpu_sriov_vf(adev)) { 4948 amdgpu_virt_release_full_gpu(adev, true); 4949 flush_delayed_work(&adev->delayed_init_work); 4950 } 4951 4952 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 4953 amdgpu_xgmi_reset_on_init(adev); 4954 /* 4955 * Place those sysfs registering after `late_init`. As some of those 4956 * operations performed in `late_init` might affect the sysfs 4957 * interfaces creating. 4958 */ 4959 r = amdgpu_device_sys_interface_init(adev); 4960 4961 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4962 r = amdgpu_pmu_init(adev); 4963 if (r) 4964 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 4965 4966 /* Have stored pci confspace at hand for restore in sudden PCI error */ 4967 if (amdgpu_device_cache_pci_state(adev->pdev)) 4968 pci_restore_state(pdev); 4969 4970 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 4971 /* this will fail for cards that aren't VGA class devices, just 4972 * ignore it 4973 */ 4974 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4975 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); 4976 4977 px = amdgpu_device_supports_px(adev); 4978 4979 if (px || (!dev_is_removable(&adev->pdev->dev) && 4980 apple_gmux_detect(NULL, NULL))) 4981 vga_switcheroo_register_client(adev->pdev, 4982 &amdgpu_switcheroo_ops, px); 4983 4984 if (px) 4985 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 4986 4987 amdgpu_device_check_iommu_direct_map(adev); 4988 4989 adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; 4990 r = register_pm_notifier(&adev->pm_nb); 4991 if (r) 4992 goto failed; 4993 4994 return 0; 4995 4996release_ras_con: 4997 if (amdgpu_sriov_vf(adev)) 4998 amdgpu_virt_release_full_gpu(adev, true); 4999 5000 /* failed in exclusive mode due to timeout */ 5001 if (amdgpu_sriov_vf(adev) && 5002 !amdgpu_sriov_runtime(adev) && 5003 amdgpu_virt_mmio_blocked(adev) && 5004 !amdgpu_virt_wait_reset(adev)) { 5005 dev_err(adev->dev, "VF exclusive mode timeout\n"); 5006 /* Don't send request since VF is inactive. */ 5007 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 5008 adev->virt.ops = NULL; 5009 r = -EAGAIN; 5010 } 5011 amdgpu_release_ras_context(adev); 5012 5013failed: 5014 amdgpu_vf_error_trans_all(adev); 5015 5016 return r; 5017} 5018 5019static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) 5020{ 5021 5022 /* Clear all CPU mappings pointing to this device */ 5023 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 5024 5025 /* Unmap all mapped bars - Doorbell, registers and VRAM */ 5026 amdgpu_doorbell_fini(adev); 5027 5028 iounmap(adev->rmmio); 5029 adev->rmmio = NULL; 5030 if (adev->mman.aper_base_kaddr) 5031 iounmap(adev->mman.aper_base_kaddr); 5032 adev->mman.aper_base_kaddr = NULL; 5033 5034 /* Memory manager related */ 5035 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 5036 arch_phys_wc_del(adev->gmc.vram_mtrr); 5037 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 5038 } 5039} 5040 5041/** 5042 * amdgpu_device_fini_hw - tear down the driver 5043 * 5044 * @adev: amdgpu_device pointer 5045 * 5046 * Tear down the driver info (all asics). 5047 * Called at driver shutdown. 5048 */ 5049void amdgpu_device_fini_hw(struct amdgpu_device *adev) 5050{ 5051 dev_info(adev->dev, "amdgpu: finishing device.\n"); 5052 flush_delayed_work(&adev->delayed_init_work); 5053 5054 if (adev->mman.initialized) 5055 drain_workqueue(adev->mman.bdev.wq); 5056 adev->shutdown = true; 5057 5058 unregister_pm_notifier(&adev->pm_nb); 5059 5060 /* make sure IB test finished before entering exclusive mode 5061 * to avoid preemption on IB test 5062 */ 5063 if (amdgpu_sriov_vf(adev)) { 5064 amdgpu_virt_request_full_gpu(adev, false); 5065 amdgpu_virt_fini_data_exchange(adev); 5066 } 5067 5068 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 5069 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 5070 5071 /* disable all interrupts */ 5072 amdgpu_irq_disable_all(adev); 5073 if (adev->mode_info.mode_config_initialized) { 5074 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev))) 5075 drm_helper_force_disable_all(adev_to_drm(adev)); 5076 else 5077 drm_atomic_helper_shutdown(adev_to_drm(adev)); 5078 } 5079 amdgpu_fence_driver_hw_fini(adev); 5080 5081 amdgpu_device_sys_interface_fini(adev); 5082 5083 /* disable ras feature must before hw fini */ 5084 amdgpu_ras_pre_fini(adev); 5085 5086 amdgpu_ttm_set_buffer_funcs_status(adev, false); 5087 5088 /* 5089 * device went through surprise hotplug; we need to destroy topology 5090 * before ip_fini_early to prevent kfd locking refcount issues by calling 5091 * amdgpu_amdkfd_suspend() 5092 */ 5093 if (pci_dev_is_disconnected(adev->pdev)) 5094 amdgpu_amdkfd_device_fini_sw(adev); 5095 5096 amdgpu_device_ip_fini_early(adev); 5097 5098 amdgpu_irq_fini_hw(adev); 5099 5100 if (adev->mman.initialized) 5101 ttm_device_clear_dma_mappings(&adev->mman.bdev); 5102 5103 amdgpu_gart_dummy_page_fini(adev); 5104 5105 if (pci_dev_is_disconnected(adev->pdev)) 5106 amdgpu_device_unmap_mmio(adev); 5107 5108} 5109 5110void amdgpu_device_fini_sw(struct amdgpu_device *adev) 5111{ 5112 int i, idx; 5113 bool px; 5114 5115 amdgpu_device_ip_fini(adev); 5116 amdgpu_fence_driver_sw_fini(adev); 5117 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); 5118 adev->accel_working = false; 5119 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); 5120 for (i = 0; i < MAX_XCP; ++i) { 5121 dma_fence_put(adev->isolation[i].spearhead); 5122 amdgpu_sync_free(&adev->isolation[i].active); 5123 amdgpu_sync_free(&adev->isolation[i].prev); 5124 } 5125 5126 amdgpu_reset_fini(adev); 5127 5128 /* free i2c buses */ 5129 amdgpu_i2c_fini(adev); 5130 5131 if (adev->bios) { 5132 if (amdgpu_emu_mode != 1) 5133 amdgpu_atombios_fini(adev); 5134 amdgpu_bios_release(adev); 5135 } 5136 5137 kfree(adev->fru_info); 5138 adev->fru_info = NULL; 5139 5140 kfree(adev->xcp_mgr); 5141 adev->xcp_mgr = NULL; 5142 5143 px = amdgpu_device_supports_px(adev); 5144 5145 if (px || (!dev_is_removable(&adev->pdev->dev) && 5146 apple_gmux_detect(NULL, NULL))) 5147 vga_switcheroo_unregister_client(adev->pdev); 5148 5149 if (px) 5150 vga_switcheroo_fini_domain_pm_ops(adev->dev); 5151 5152 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 5153 vga_client_unregister(adev->pdev); 5154 5155 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 5156 5157 iounmap(adev->rmmio); 5158 adev->rmmio = NULL; 5159 drm_dev_exit(idx); 5160 } 5161 5162 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 5163 amdgpu_pmu_fini(adev); 5164 if (adev->discovery.bin) 5165 amdgpu_discovery_fini(adev); 5166 5167 amdgpu_reset_put_reset_domain(adev->reset_domain); 5168 adev->reset_domain = NULL; 5169 5170 kfree(adev->pci_state); 5171 kfree(adev->pcie_reset_ctx.swds_pcistate); 5172 kfree(adev->pcie_reset_ctx.swus_pcistate); 5173} 5174 5175/** 5176 * amdgpu_device_evict_resources - evict device resources 5177 * @adev: amdgpu device object 5178 * 5179 * Evicts all ttm device resources(vram BOs, gart table) from the lru list 5180 * of the vram memory type. Mainly used for evicting device resources 5181 * at suspend time. 5182 * 5183 */ 5184static int amdgpu_device_evict_resources(struct amdgpu_device *adev) 5185{ 5186 int ret; 5187 5188 /* No need to evict vram on APUs unless going to S4 */ 5189 if (!adev->in_s4 && (adev->flags & AMD_IS_APU)) 5190 return 0; 5191 5192 /* No need to evict when going to S5 through S4 callbacks */ 5193 if (system_state == SYSTEM_POWER_OFF) 5194 return 0; 5195 5196 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 5197 if (ret) { 5198 dev_warn(adev->dev, "evicting device resources failed\n"); 5199 return ret; 5200 } 5201 5202 if (adev->in_s4) { 5203 ret = ttm_device_prepare_hibernation(&adev->mman.bdev); 5204 if (ret) 5205 dev_err(adev->dev, "prepare hibernation failed, %d\n", ret); 5206 } 5207 return ret; 5208} 5209 5210/* 5211 * Suspend & resume. 5212 */ 5213/** 5214 * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events 5215 * @nb: notifier block 5216 * @mode: suspend mode 5217 * @data: data 5218 * 5219 * This function is called when the system is about to suspend or hibernate. 5220 * It is used to set the appropriate flags so that eviction can be optimized 5221 * in the pm prepare callback. 5222 */ 5223static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 5224 void *data) 5225{ 5226 struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb); 5227 5228 switch (mode) { 5229 case PM_HIBERNATION_PREPARE: 5230 adev->in_s4 = true; 5231 break; 5232 case PM_POST_HIBERNATION: 5233 adev->in_s4 = false; 5234 break; 5235 } 5236 5237 return NOTIFY_DONE; 5238} 5239 5240/** 5241 * amdgpu_device_prepare - prepare for device suspend 5242 * 5243 * @dev: drm dev pointer 5244 * 5245 * Prepare to put the hw in the suspend state (all asics). 5246 * Returns 0 for success or an error on failure. 5247 * Called at driver suspend. 5248 */ 5249int amdgpu_device_prepare(struct drm_device *dev) 5250{ 5251 struct amdgpu_device *adev = drm_to_adev(dev); 5252 int i, r; 5253 5254 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 5255 return 0; 5256 5257 /* Evict the majority of BOs before starting suspend sequence */ 5258 r = amdgpu_device_evict_resources(adev); 5259 if (r) 5260 return r; 5261 5262 flush_delayed_work(&adev->gfx.gfx_off_delay_work); 5263 5264 for (i = 0; i < adev->num_ip_blocks; i++) { 5265 if (!adev->ip_blocks[i].status.valid) 5266 continue; 5267 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) 5268 continue; 5269 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]); 5270 if (r) 5271 return r; 5272 } 5273 5274 return 0; 5275} 5276 5277/** 5278 * amdgpu_device_complete - complete power state transition 5279 * 5280 * @dev: drm dev pointer 5281 * 5282 * Undo the changes from amdgpu_device_prepare. This will be 5283 * called on all resume transitions, including those that failed. 5284 */ 5285void amdgpu_device_complete(struct drm_device *dev) 5286{ 5287 struct amdgpu_device *adev = drm_to_adev(dev); 5288 int i; 5289 5290 for (i = 0; i < adev->num_ip_blocks; i++) { 5291 if (!adev->ip_blocks[i].status.valid) 5292 continue; 5293 if (!adev->ip_blocks[i].version->funcs->complete) 5294 continue; 5295 adev->ip_blocks[i].version->funcs->complete(&adev->ip_blocks[i]); 5296 } 5297} 5298 5299/** 5300 * amdgpu_device_suspend - initiate device suspend 5301 * 5302 * @dev: drm dev pointer 5303 * @notify_clients: notify in-kernel DRM clients 5304 * 5305 * Puts the hw in the suspend state (all asics). 5306 * Returns 0 for success or an error on failure. 5307 * Called at driver suspend. 5308 */ 5309int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients) 5310{ 5311 struct amdgpu_device *adev = drm_to_adev(dev); 5312 int r, rec; 5313 5314 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 5315 return 0; 5316 5317 adev->in_suspend = true; 5318 5319 if (amdgpu_sriov_vf(adev)) { 5320 if (!adev->in_runpm) 5321 amdgpu_amdkfd_suspend_process(adev); 5322 amdgpu_virt_fini_data_exchange(adev); 5323 r = amdgpu_virt_request_full_gpu(adev, false); 5324 if (r) 5325 return r; 5326 } 5327 5328 r = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D3); 5329 if (r) 5330 goto unwind_sriov; 5331 5332 if (notify_clients) 5333 drm_client_dev_suspend(adev_to_drm(adev)); 5334 5335 cancel_delayed_work_sync(&adev->delayed_init_work); 5336 5337 amdgpu_ras_suspend(adev); 5338 5339 r = amdgpu_device_ip_suspend_phase1(adev); 5340 if (r) 5341 goto unwind_smartshift; 5342 5343 amdgpu_amdkfd_suspend(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); 5344 r = amdgpu_userq_suspend(adev); 5345 if (r) 5346 goto unwind_ip_phase1; 5347 5348 r = amdgpu_device_evict_resources(adev); 5349 if (r) 5350 goto unwind_userq; 5351 5352 amdgpu_ttm_set_buffer_funcs_status(adev, false); 5353 5354 amdgpu_fence_driver_hw_fini(adev); 5355 5356 r = amdgpu_device_ip_suspend_phase2(adev); 5357 if (r) 5358 goto unwind_evict; 5359 5360 if (amdgpu_sriov_vf(adev)) 5361 amdgpu_virt_release_full_gpu(adev, false); 5362 5363 return 0; 5364 5365unwind_evict: 5366 if (adev->mman.buffer_funcs_ring->sched.ready) 5367 amdgpu_ttm_set_buffer_funcs_status(adev, true); 5368 amdgpu_fence_driver_hw_init(adev); 5369 5370unwind_userq: 5371 rec = amdgpu_userq_resume(adev); 5372 if (rec) { 5373 dev_warn(adev->dev, "failed to re-initialize user queues: %d\n", rec); 5374 return r; 5375 } 5376 rec = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); 5377 if (rec) { 5378 dev_warn(adev->dev, "failed to re-initialize kfd: %d\n", rec); 5379 return r; 5380 } 5381 5382unwind_ip_phase1: 5383 /* suspend phase 1 = resume phase 3 */ 5384 rec = amdgpu_device_ip_resume_phase3(adev); 5385 if (rec) { 5386 dev_warn(adev->dev, "failed to re-initialize IPs phase1: %d\n", rec); 5387 return r; 5388 } 5389 5390unwind_smartshift: 5391 rec = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0); 5392 if (rec) { 5393 dev_warn(adev->dev, "failed to re-update smart shift: %d\n", rec); 5394 return r; 5395 } 5396 5397 if (notify_clients) 5398 drm_client_dev_resume(adev_to_drm(adev)); 5399 5400 amdgpu_ras_resume(adev); 5401 5402unwind_sriov: 5403 if (amdgpu_sriov_vf(adev)) { 5404 rec = amdgpu_virt_request_full_gpu(adev, true); 5405 if (rec) { 5406 dev_warn(adev->dev, "failed to reinitialize sriov: %d\n", rec); 5407 return r; 5408 } 5409 } 5410 5411 adev->in_suspend = adev->in_s0ix = adev->in_s3 = false; 5412 5413 return r; 5414} 5415 5416static inline int amdgpu_virt_resume(struct amdgpu_device *adev) 5417{ 5418 int r; 5419 unsigned int prev_physical_node_id = adev->gmc.xgmi.physical_node_id; 5420 5421 /* During VM resume, QEMU programming of VF MSIX table (register GFXMSIX_VECT0_ADDR_LO) 5422 * may not work. The access could be blocked by nBIF protection as VF isn't in 5423 * exclusive access mode. Exclusive access is enabled now, disable/enable MSIX 5424 * so that QEMU reprograms MSIX table. 5425 */ 5426 amdgpu_restore_msix(adev); 5427 5428 r = adev->gfxhub.funcs->get_xgmi_info(adev); 5429 if (r) 5430 return r; 5431 5432 dev_info(adev->dev, "xgmi node, old id %d, new id %d\n", 5433 prev_physical_node_id, adev->gmc.xgmi.physical_node_id); 5434 5435 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 5436 adev->vm_manager.vram_base_offset += 5437 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 5438 5439 return 0; 5440} 5441 5442/** 5443 * amdgpu_device_resume - initiate device resume 5444 * 5445 * @dev: drm dev pointer 5446 * @notify_clients: notify in-kernel DRM clients 5447 * 5448 * Bring the hw back to operating state (all asics). 5449 * Returns 0 for success or an error on failure. 5450 * Called at driver resume. 5451 */ 5452int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) 5453{ 5454 struct amdgpu_device *adev = drm_to_adev(dev); 5455 int r = 0; 5456 5457 if (amdgpu_sriov_vf(adev)) { 5458 r = amdgpu_virt_request_full_gpu(adev, true); 5459 if (r) 5460 return r; 5461 } 5462 5463 if (amdgpu_virt_xgmi_migrate_enabled(adev)) { 5464 r = amdgpu_virt_resume(adev); 5465 if (r) 5466 goto exit; 5467 } 5468 5469 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 5470 return 0; 5471 5472 if (adev->in_s0ix) 5473 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); 5474 5475 /* post card */ 5476 if (amdgpu_device_need_post(adev)) { 5477 r = amdgpu_device_asic_init(adev); 5478 if (r) 5479 dev_err(adev->dev, "amdgpu asic init failed\n"); 5480 } 5481 5482 r = amdgpu_device_ip_resume(adev); 5483 5484 if (r) { 5485 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 5486 goto exit; 5487 } 5488 5489 r = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); 5490 if (r) 5491 goto exit; 5492 5493 r = amdgpu_userq_resume(adev); 5494 if (r) 5495 goto exit; 5496 5497 r = amdgpu_device_ip_late_init(adev); 5498 if (r) 5499 goto exit; 5500 5501 queue_delayed_work(system_wq, &adev->delayed_init_work, 5502 msecs_to_jiffies(AMDGPU_RESUME_MS)); 5503exit: 5504 if (amdgpu_sriov_vf(adev)) { 5505 amdgpu_virt_init_data_exchange(adev); 5506 amdgpu_virt_release_full_gpu(adev, true); 5507 5508 if (!r && !adev->in_runpm) 5509 r = amdgpu_amdkfd_resume_process(adev); 5510 } 5511 5512 if (r) 5513 return r; 5514 5515 /* Make sure IB tests flushed */ 5516 flush_delayed_work(&adev->delayed_init_work); 5517 5518 if (notify_clients) 5519 drm_client_dev_resume(adev_to_drm(adev)); 5520 5521 amdgpu_ras_resume(adev); 5522 5523 if (adev->mode_info.num_crtc) { 5524 /* 5525 * Most of the connector probing functions try to acquire runtime pm 5526 * refs to ensure that the GPU is powered on when connector polling is 5527 * performed. Since we're calling this from a runtime PM callback, 5528 * trying to acquire rpm refs will cause us to deadlock. 5529 * 5530 * Since we're guaranteed to be holding the rpm lock, it's safe to 5531 * temporarily disable the rpm helpers so this doesn't deadlock us. 5532 */ 5533#ifdef CONFIG_PM 5534 dev->dev->power.disable_depth++; 5535#endif 5536 if (!adev->dc_enabled) 5537 drm_helper_hpd_irq_event(dev); 5538 else 5539 drm_kms_helper_hotplug_event(dev); 5540#ifdef CONFIG_PM 5541 dev->dev->power.disable_depth--; 5542#endif 5543 } 5544 5545 amdgpu_vram_mgr_clear_reset_blocks(adev); 5546 adev->in_suspend = false; 5547 5548 if (amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0)) 5549 dev_warn(adev->dev, "smart shift update failed\n"); 5550 5551 return 0; 5552} 5553 5554/** 5555 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 5556 * 5557 * @adev: amdgpu_device pointer 5558 * 5559 * The list of all the hardware IPs that make up the asic is walked and 5560 * the check_soft_reset callbacks are run. check_soft_reset determines 5561 * if the asic is still hung or not. 5562 * Returns true if any of the IPs are still in a hung state, false if not. 5563 */ 5564static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 5565{ 5566 int i; 5567 bool asic_hang = false; 5568 5569 if (amdgpu_sriov_vf(adev)) 5570 return true; 5571 5572 if (amdgpu_asic_need_full_reset(adev)) 5573 return true; 5574 5575 for (i = 0; i < adev->num_ip_blocks; i++) { 5576 if (!adev->ip_blocks[i].status.valid) 5577 continue; 5578 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 5579 adev->ip_blocks[i].status.hang = 5580 adev->ip_blocks[i].version->funcs->check_soft_reset( 5581 &adev->ip_blocks[i]); 5582 if (adev->ip_blocks[i].status.hang) { 5583 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 5584 asic_hang = true; 5585 } 5586 } 5587 return asic_hang; 5588} 5589 5590/** 5591 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 5592 * 5593 * @adev: amdgpu_device pointer 5594 * 5595 * The list of all the hardware IPs that make up the asic is walked and the 5596 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 5597 * handles any IP specific hardware or software state changes that are 5598 * necessary for a soft reset to succeed. 5599 * Returns 0 on success, negative error code on failure. 5600 */ 5601static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 5602{ 5603 int i, r = 0; 5604 5605 for (i = 0; i < adev->num_ip_blocks; i++) { 5606 if (!adev->ip_blocks[i].status.valid) 5607 continue; 5608 if (adev->ip_blocks[i].status.hang && 5609 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 5610 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]); 5611 if (r) 5612 return r; 5613 } 5614 } 5615 5616 return 0; 5617} 5618 5619/** 5620 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 5621 * 5622 * @adev: amdgpu_device pointer 5623 * 5624 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 5625 * reset is necessary to recover. 5626 * Returns true if a full asic reset is required, false if not. 5627 */ 5628static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 5629{ 5630 int i; 5631 5632 if (amdgpu_asic_need_full_reset(adev)) 5633 return true; 5634 5635 for (i = 0; i < adev->num_ip_blocks; i++) { 5636 if (!adev->ip_blocks[i].status.valid) 5637 continue; 5638 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 5639 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 5640 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 5641 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 5642 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 5643 if (adev->ip_blocks[i].status.hang) { 5644 dev_info(adev->dev, "Some block need full reset!\n"); 5645 return true; 5646 } 5647 } 5648 } 5649 return false; 5650} 5651 5652/** 5653 * amdgpu_device_ip_soft_reset - do a soft reset 5654 * 5655 * @adev: amdgpu_device pointer 5656 * 5657 * The list of all the hardware IPs that make up the asic is walked and the 5658 * soft_reset callbacks are run if the block is hung. soft_reset handles any 5659 * IP specific hardware or software state changes that are necessary to soft 5660 * reset the IP. 5661 * Returns 0 on success, negative error code on failure. 5662 */ 5663static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 5664{ 5665 int i, r = 0; 5666 5667 for (i = 0; i < adev->num_ip_blocks; i++) { 5668 if (!adev->ip_blocks[i].status.valid) 5669 continue; 5670 if (adev->ip_blocks[i].status.hang && 5671 adev->ip_blocks[i].version->funcs->soft_reset) { 5672 r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]); 5673 if (r) 5674 return r; 5675 } 5676 } 5677 5678 return 0; 5679} 5680 5681/** 5682 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 5683 * 5684 * @adev: amdgpu_device pointer 5685 * 5686 * The list of all the hardware IPs that make up the asic is walked and the 5687 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 5688 * handles any IP specific hardware or software state changes that are 5689 * necessary after the IP has been soft reset. 5690 * Returns 0 on success, negative error code on failure. 5691 */ 5692static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 5693{ 5694 int i, r = 0; 5695 5696 for (i = 0; i < adev->num_ip_blocks; i++) { 5697 if (!adev->ip_blocks[i].status.valid) 5698 continue; 5699 if (adev->ip_blocks[i].status.hang && 5700 adev->ip_blocks[i].version->funcs->post_soft_reset) 5701 r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]); 5702 if (r) 5703 return r; 5704 } 5705 5706 return 0; 5707} 5708 5709/** 5710 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 5711 * 5712 * @adev: amdgpu_device pointer 5713 * @reset_context: amdgpu reset context pointer 5714 * 5715 * do VF FLR and reinitialize Asic 5716 * return 0 means succeeded otherwise failed 5717 */ 5718static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 5719 struct amdgpu_reset_context *reset_context) 5720{ 5721 int r; 5722 struct amdgpu_hive_info *hive = NULL; 5723 5724 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) { 5725 if (!amdgpu_ras_get_fed_status(adev)) 5726 amdgpu_virt_ready_to_reset(adev); 5727 amdgpu_virt_wait_reset(adev); 5728 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags); 5729 r = amdgpu_virt_request_full_gpu(adev, true); 5730 } else { 5731 r = amdgpu_virt_reset_gpu(adev); 5732 } 5733 if (r) 5734 return r; 5735 5736 amdgpu_ras_clear_err_state(adev); 5737 amdgpu_irq_gpu_reset_resume_helper(adev); 5738 5739 /* some sw clean up VF needs to do before recover */ 5740 amdgpu_virt_post_reset(adev); 5741 5742 /* Resume IP prior to SMC */ 5743 r = amdgpu_device_ip_reinit_early_sriov(adev); 5744 if (r) 5745 return r; 5746 5747 amdgpu_virt_init_data_exchange(adev); 5748 5749 r = amdgpu_device_fw_loading(adev); 5750 if (r) 5751 return r; 5752 5753 /* now we are okay to resume SMC/CP/SDMA */ 5754 r = amdgpu_device_ip_reinit_late_sriov(adev); 5755 if (r) 5756 return r; 5757 5758 hive = amdgpu_get_xgmi_hive(adev); 5759 /* Update PSP FW topology after reset */ 5760 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) 5761 r = amdgpu_xgmi_update_topology(hive, adev); 5762 if (hive) 5763 amdgpu_put_xgmi_hive(hive); 5764 if (r) 5765 return r; 5766 5767 r = amdgpu_ib_ring_tests(adev); 5768 if (r) 5769 return r; 5770 5771 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) 5772 amdgpu_inc_vram_lost(adev); 5773 5774 /* need to be called during full access so we can't do it later like 5775 * bare-metal does. 5776 */ 5777 amdgpu_amdkfd_post_reset(adev); 5778 amdgpu_virt_release_full_gpu(adev, true); 5779 5780 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */ 5781 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 5782 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 5783 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 5784 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) || 5785 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) 5786 amdgpu_ras_resume(adev); 5787 5788 amdgpu_virt_ras_telemetry_post_reset(adev); 5789 5790 return 0; 5791} 5792 5793/** 5794 * amdgpu_device_has_job_running - check if there is any unfinished job 5795 * 5796 * @adev: amdgpu_device pointer 5797 * 5798 * check if there is any job running on the device when guest driver receives 5799 * FLR notification from host driver. If there are still jobs running, then 5800 * the guest driver will not respond the FLR reset. Instead, let the job hit 5801 * the timeout and guest driver then issue the reset request. 5802 */ 5803bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 5804{ 5805 int i; 5806 5807 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5808 struct amdgpu_ring *ring = adev->rings[i]; 5809 5810 if (!amdgpu_ring_sched_ready(ring)) 5811 continue; 5812 5813 if (amdgpu_fence_count_emitted(ring)) 5814 return true; 5815 } 5816 return false; 5817} 5818 5819/** 5820 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 5821 * 5822 * @adev: amdgpu_device pointer 5823 * 5824 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 5825 * a hung GPU. 5826 */ 5827bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 5828{ 5829 5830 if (amdgpu_gpu_recovery == 0) 5831 goto disabled; 5832 5833 /* Skip soft reset check in fatal error mode */ 5834 if (!amdgpu_ras_is_poison_mode_supported(adev)) 5835 return true; 5836 5837 if (amdgpu_sriov_vf(adev)) 5838 return true; 5839 5840 if (amdgpu_gpu_recovery == -1) { 5841 switch (adev->asic_type) { 5842#ifdef CONFIG_DRM_AMDGPU_SI 5843 case CHIP_VERDE: 5844 case CHIP_TAHITI: 5845 case CHIP_PITCAIRN: 5846 case CHIP_OLAND: 5847 case CHIP_HAINAN: 5848#endif 5849#ifdef CONFIG_DRM_AMDGPU_CIK 5850 case CHIP_KAVERI: 5851 case CHIP_KABINI: 5852 case CHIP_MULLINS: 5853#endif 5854 case CHIP_CARRIZO: 5855 case CHIP_STONEY: 5856 case CHIP_CYAN_SKILLFISH: 5857 goto disabled; 5858 default: 5859 break; 5860 } 5861 } 5862 5863 return true; 5864 5865disabled: 5866 dev_info(adev->dev, "GPU recovery disabled.\n"); 5867 return false; 5868} 5869 5870int amdgpu_device_mode1_reset(struct amdgpu_device *adev) 5871{ 5872 u32 i; 5873 int ret = 0; 5874 5875 if (adev->bios) 5876 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 5877 5878 dev_info(adev->dev, "GPU mode1 reset\n"); 5879 5880 /* Cache the state before bus master disable. The saved config space 5881 * values are used in other cases like restore after mode-2 reset. 5882 */ 5883 amdgpu_device_cache_pci_state(adev->pdev); 5884 5885 /* disable BM */ 5886 pci_clear_master(adev->pdev); 5887 5888 if (amdgpu_dpm_is_mode1_reset_supported(adev)) { 5889 dev_info(adev->dev, "GPU smu mode1 reset\n"); 5890 ret = amdgpu_dpm_mode1_reset(adev); 5891 } else { 5892 dev_info(adev->dev, "GPU psp mode1 reset\n"); 5893 ret = psp_gpu_reset(adev); 5894 } 5895 5896 if (ret) 5897 goto mode1_reset_failed; 5898 5899 /* enable mmio access after mode 1 reset completed */ 5900 adev->no_hw_access = false; 5901 5902 /* ensure no_hw_access is updated before we access hw */ 5903 smp_mb(); 5904 5905 amdgpu_device_load_pci_state(adev->pdev); 5906 ret = amdgpu_psp_wait_for_bootloader(adev); 5907 if (ret) 5908 goto mode1_reset_failed; 5909 5910 /* wait for asic to come out of reset */ 5911 for (i = 0; i < adev->usec_timeout; i++) { 5912 u32 memsize = adev->nbio.funcs->get_memsize(adev); 5913 5914 if (memsize != 0xffffffff) 5915 break; 5916 udelay(1); 5917 } 5918 5919 if (i >= adev->usec_timeout) { 5920 ret = -ETIMEDOUT; 5921 goto mode1_reset_failed; 5922 } 5923 5924 if (adev->bios) 5925 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 5926 5927 return 0; 5928 5929mode1_reset_failed: 5930 dev_err(adev->dev, "GPU mode1 reset failed\n"); 5931 return ret; 5932} 5933 5934int amdgpu_device_link_reset(struct amdgpu_device *adev) 5935{ 5936 int ret = 0; 5937 5938 dev_info(adev->dev, "GPU link reset\n"); 5939 5940 if (!amdgpu_reset_in_dpc(adev)) 5941 ret = amdgpu_dpm_link_reset(adev); 5942 5943 if (ret) 5944 goto link_reset_failed; 5945 5946 ret = amdgpu_psp_wait_for_bootloader(adev); 5947 if (ret) 5948 goto link_reset_failed; 5949 5950 return 0; 5951 5952link_reset_failed: 5953 dev_err(adev->dev, "GPU link reset failed\n"); 5954 return ret; 5955} 5956 5957int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 5958 struct amdgpu_reset_context *reset_context) 5959{ 5960 int i, r = 0; 5961 struct amdgpu_job *job = NULL; 5962 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; 5963 bool need_full_reset = 5964 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5965 5966 if (reset_context->reset_req_dev == adev) 5967 job = reset_context->job; 5968 5969 if (amdgpu_sriov_vf(adev)) 5970 amdgpu_virt_pre_reset(adev); 5971 5972 amdgpu_fence_driver_isr_toggle(adev, true); 5973 5974 /* block all schedulers and reset given job's ring */ 5975 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5976 struct amdgpu_ring *ring = adev->rings[i]; 5977 5978 if (!amdgpu_ring_sched_ready(ring)) 5979 continue; 5980 5981 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 5982 amdgpu_fence_driver_force_completion(ring); 5983 } 5984 5985 amdgpu_fence_driver_isr_toggle(adev, false); 5986 5987 if (job && job->vm) 5988 drm_sched_increase_karma(&job->base); 5989 5990 r = amdgpu_reset_prepare_hwcontext(adev, reset_context); 5991 /* If reset handler not implemented, continue; otherwise return */ 5992 if (r == -EOPNOTSUPP) 5993 r = 0; 5994 else 5995 return r; 5996 5997 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 5998 if (!amdgpu_sriov_vf(adev)) { 5999 6000 if (!need_full_reset) 6001 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 6002 6003 if (!need_full_reset && amdgpu_gpu_recovery && 6004 amdgpu_device_ip_check_soft_reset(adev)) { 6005 amdgpu_device_ip_pre_soft_reset(adev); 6006 r = amdgpu_device_ip_soft_reset(adev); 6007 amdgpu_device_ip_post_soft_reset(adev); 6008 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 6009 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); 6010 need_full_reset = true; 6011 } 6012 } 6013 6014 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { 6015 dev_info(tmp_adev->dev, "Dumping IP State\n"); 6016 /* Trigger ip dump before we reset the asic */ 6017 for (i = 0; i < tmp_adev->num_ip_blocks; i++) 6018 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state) 6019 tmp_adev->ip_blocks[i].version->funcs 6020 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]); 6021 dev_info(tmp_adev->dev, "Dumping IP State Completed\n"); 6022 } 6023 6024 if (need_full_reset) 6025 r = amdgpu_device_ip_suspend(adev); 6026 if (need_full_reset) 6027 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 6028 else 6029 clear_bit(AMDGPU_NEED_FULL_RESET, 6030 &reset_context->flags); 6031 } 6032 6033 return r; 6034} 6035 6036int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) 6037{ 6038 struct list_head *device_list_handle; 6039 bool full_reset, vram_lost = false; 6040 struct amdgpu_device *tmp_adev; 6041 int r, init_level; 6042 6043 device_list_handle = reset_context->reset_device_list; 6044 6045 if (!device_list_handle) 6046 return -EINVAL; 6047 6048 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 6049 6050 /** 6051 * If it's reset on init, it's default init level, otherwise keep level 6052 * as recovery level. 6053 */ 6054 if (reset_context->method == AMD_RESET_METHOD_ON_INIT) 6055 init_level = AMDGPU_INIT_LEVEL_DEFAULT; 6056 else 6057 init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY; 6058 6059 r = 0; 6060 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6061 amdgpu_set_init_level(tmp_adev, init_level); 6062 if (full_reset) { 6063 /* post card */ 6064 amdgpu_reset_set_dpc_status(tmp_adev, false); 6065 amdgpu_ras_clear_err_state(tmp_adev); 6066 r = amdgpu_device_asic_init(tmp_adev); 6067 if (r) { 6068 dev_warn(tmp_adev->dev, "asic atom init failed!"); 6069 } else { 6070 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 6071 6072 r = amdgpu_device_ip_resume_phase1(tmp_adev); 6073 if (r) 6074 goto out; 6075 6076 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 6077 6078 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) 6079 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job); 6080 6081 if (vram_lost) { 6082 dev_info( 6083 tmp_adev->dev, 6084 "VRAM is lost due to GPU reset!\n"); 6085 amdgpu_inc_vram_lost(tmp_adev); 6086 } 6087 6088 r = amdgpu_device_fw_loading(tmp_adev); 6089 if (r) 6090 return r; 6091 6092 r = amdgpu_xcp_restore_partition_mode( 6093 tmp_adev->xcp_mgr); 6094 if (r) 6095 goto out; 6096 6097 r = amdgpu_device_ip_resume_phase2(tmp_adev); 6098 if (r) 6099 goto out; 6100 6101 if (tmp_adev->mman.buffer_funcs_ring->sched.ready) 6102 amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true); 6103 6104 r = amdgpu_device_ip_resume_phase3(tmp_adev); 6105 if (r) 6106 goto out; 6107 6108 if (vram_lost) 6109 amdgpu_device_fill_reset_magic(tmp_adev); 6110 6111 /* 6112 * Add this ASIC as tracked as reset was already 6113 * complete successfully. 6114 */ 6115 amdgpu_register_gpu_instance(tmp_adev); 6116 6117 if (!reset_context->hive && 6118 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 6119 amdgpu_xgmi_add_device(tmp_adev); 6120 6121 r = amdgpu_device_ip_late_init(tmp_adev); 6122 if (r) 6123 goto out; 6124 6125 r = amdgpu_userq_post_reset(tmp_adev, vram_lost); 6126 if (r) 6127 goto out; 6128 6129 drm_client_dev_resume(adev_to_drm(tmp_adev)); 6130 6131 /* 6132 * The GPU enters bad state once faulty pages 6133 * by ECC has reached the threshold, and ras 6134 * recovery is scheduled next. So add one check 6135 * here to break recovery if it indeed exceeds 6136 * bad page threshold, and remind user to 6137 * retire this GPU or setting one bigger 6138 * bad_page_threshold value to fix this once 6139 * probing driver again. 6140 */ 6141 if (!amdgpu_ras_is_rma(tmp_adev)) { 6142 /* must succeed. */ 6143 amdgpu_ras_resume(tmp_adev); 6144 } else { 6145 r = -EINVAL; 6146 goto out; 6147 } 6148 6149 /* Update PSP FW topology after reset */ 6150 if (reset_context->hive && 6151 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 6152 r = amdgpu_xgmi_update_topology( 6153 reset_context->hive, tmp_adev); 6154 } 6155 } 6156 6157out: 6158 if (!r) { 6159 /* IP init is complete now, set level as default */ 6160 amdgpu_set_init_level(tmp_adev, 6161 AMDGPU_INIT_LEVEL_DEFAULT); 6162 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 6163 r = amdgpu_ib_ring_tests(tmp_adev); 6164 if (r) { 6165 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 6166 r = -EAGAIN; 6167 goto end; 6168 } 6169 } 6170 6171 if (r) 6172 tmp_adev->asic_reset_res = r; 6173 } 6174 6175end: 6176 return r; 6177} 6178 6179int amdgpu_do_asic_reset(struct list_head *device_list_handle, 6180 struct amdgpu_reset_context *reset_context) 6181{ 6182 struct amdgpu_device *tmp_adev = NULL; 6183 bool need_full_reset, skip_hw_reset; 6184 int r = 0; 6185 6186 /* Try reset handler method first */ 6187 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 6188 reset_list); 6189 6190 reset_context->reset_device_list = device_list_handle; 6191 r = amdgpu_reset_perform_reset(tmp_adev, reset_context); 6192 /* If reset handler not implemented, continue; otherwise return */ 6193 if (r == -EOPNOTSUPP) 6194 r = 0; 6195 else 6196 return r; 6197 6198 /* Reset handler not implemented, use the default method */ 6199 need_full_reset = 6200 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 6201 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); 6202 6203 /* 6204 * ASIC reset has to be done on all XGMI hive nodes ASAP 6205 * to allow proper links negotiation in FW (within 1 sec) 6206 */ 6207 if (!skip_hw_reset && need_full_reset) { 6208 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6209 /* For XGMI run all resets in parallel to speed up the process */ 6210 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 6211 if (!queue_work(system_unbound_wq, 6212 &tmp_adev->xgmi_reset_work)) 6213 r = -EALREADY; 6214 } else 6215 r = amdgpu_asic_reset(tmp_adev); 6216 6217 if (r) { 6218 dev_err(tmp_adev->dev, 6219 "ASIC reset failed with error, %d for drm dev, %s", 6220 r, adev_to_drm(tmp_adev)->unique); 6221 goto out; 6222 } 6223 } 6224 6225 /* For XGMI wait for all resets to complete before proceed */ 6226 if (!r) { 6227 list_for_each_entry(tmp_adev, device_list_handle, 6228 reset_list) { 6229 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 6230 flush_work(&tmp_adev->xgmi_reset_work); 6231 r = tmp_adev->asic_reset_res; 6232 if (r) 6233 break; 6234 } 6235 } 6236 } 6237 } 6238 6239 if (!r && amdgpu_ras_intr_triggered()) { 6240 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6241 amdgpu_ras_reset_error_count(tmp_adev, 6242 AMDGPU_RAS_BLOCK__MMHUB); 6243 } 6244 6245 amdgpu_ras_intr_cleared(); 6246 } 6247 6248 r = amdgpu_device_reinit_after_reset(reset_context); 6249 if (r == -EAGAIN) 6250 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 6251 else 6252 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 6253 6254out: 6255 return r; 6256} 6257 6258static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev) 6259{ 6260 6261 switch (amdgpu_asic_reset_method(adev)) { 6262 case AMD_RESET_METHOD_MODE1: 6263 case AMD_RESET_METHOD_LINK: 6264 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 6265 break; 6266 case AMD_RESET_METHOD_MODE2: 6267 adev->mp1_state = PP_MP1_STATE_RESET; 6268 break; 6269 default: 6270 adev->mp1_state = PP_MP1_STATE_NONE; 6271 break; 6272 } 6273} 6274 6275static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev) 6276{ 6277 amdgpu_vf_error_trans_all(adev); 6278 adev->mp1_state = PP_MP1_STATE_NONE; 6279} 6280 6281static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 6282{ 6283 struct pci_dev *p = NULL; 6284 6285 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 6286 adev->pdev->bus->number, 1); 6287 if (p) { 6288 pm_runtime_enable(&(p->dev)); 6289 pm_runtime_resume(&(p->dev)); 6290 } 6291 6292 pci_dev_put(p); 6293} 6294 6295static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 6296{ 6297 enum amd_reset_method reset_method; 6298 struct pci_dev *p = NULL; 6299 u64 expires; 6300 6301 /* 6302 * For now, only BACO and mode1 reset are confirmed 6303 * to suffer the audio issue without proper suspended. 6304 */ 6305 reset_method = amdgpu_asic_reset_method(adev); 6306 if ((reset_method != AMD_RESET_METHOD_BACO) && 6307 (reset_method != AMD_RESET_METHOD_MODE1)) 6308 return -EINVAL; 6309 6310 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 6311 adev->pdev->bus->number, 1); 6312 if (!p) 6313 return -ENODEV; 6314 6315 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 6316 if (!expires) 6317 /* 6318 * If we cannot get the audio device autosuspend delay, 6319 * a fixed 4S interval will be used. Considering 3S is 6320 * the audio controller default autosuspend delay setting. 6321 * 4S used here is guaranteed to cover that. 6322 */ 6323 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 6324 6325 while (!pm_runtime_status_suspended(&(p->dev))) { 6326 if (!pm_runtime_suspend(&(p->dev))) 6327 break; 6328 6329 if (expires < ktime_get_mono_fast_ns()) { 6330 dev_warn(adev->dev, "failed to suspend display audio\n"); 6331 pci_dev_put(p); 6332 /* TODO: abort the succeeding gpu reset? */ 6333 return -ETIMEDOUT; 6334 } 6335 } 6336 6337 pm_runtime_disable(&(p->dev)); 6338 6339 pci_dev_put(p); 6340 return 0; 6341} 6342 6343static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) 6344{ 6345 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 6346 6347#if defined(CONFIG_DEBUG_FS) 6348 if (!amdgpu_sriov_vf(adev)) 6349 cancel_work(&adev->reset_work); 6350#endif 6351 cancel_work(&adev->userq_reset_work); 6352 6353 if (adev->kfd.dev) 6354 cancel_work(&adev->kfd.reset_work); 6355 6356 if (amdgpu_sriov_vf(adev)) 6357 cancel_work(&adev->virt.flr_work); 6358 6359 if (con && adev->ras_enabled) 6360 cancel_work(&con->recovery_work); 6361 6362} 6363 6364static int amdgpu_device_health_check(struct list_head *device_list_handle) 6365{ 6366 struct amdgpu_device *tmp_adev; 6367 int ret = 0; 6368 6369 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6370 ret |= amdgpu_device_bus_status_check(tmp_adev); 6371 } 6372 6373 return ret; 6374} 6375 6376static void amdgpu_device_recovery_prepare(struct amdgpu_device *adev, 6377 struct list_head *device_list, 6378 struct amdgpu_hive_info *hive) 6379{ 6380 struct amdgpu_device *tmp_adev = NULL; 6381 6382 /* 6383 * Build list of devices to reset. 6384 * In case we are in XGMI hive mode, resort the device list 6385 * to put adev in the 1st position. 6386 */ 6387 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) { 6388 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 6389 list_add_tail(&tmp_adev->reset_list, device_list); 6390 if (adev->shutdown) 6391 tmp_adev->shutdown = true; 6392 if (amdgpu_reset_in_dpc(adev)) 6393 tmp_adev->pcie_reset_ctx.in_link_reset = true; 6394 } 6395 if (!list_is_first(&adev->reset_list, device_list)) 6396 list_rotate_to_front(&adev->reset_list, device_list); 6397 } else { 6398 list_add_tail(&adev->reset_list, device_list); 6399 } 6400} 6401 6402static void amdgpu_device_recovery_get_reset_lock(struct amdgpu_device *adev, 6403 struct list_head *device_list) 6404{ 6405 struct amdgpu_device *tmp_adev = NULL; 6406 6407 if (list_empty(device_list)) 6408 return; 6409 tmp_adev = 6410 list_first_entry(device_list, struct amdgpu_device, reset_list); 6411 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); 6412} 6413 6414static void amdgpu_device_recovery_put_reset_lock(struct amdgpu_device *adev, 6415 struct list_head *device_list) 6416{ 6417 struct amdgpu_device *tmp_adev = NULL; 6418 6419 if (list_empty(device_list)) 6420 return; 6421 tmp_adev = 6422 list_first_entry(device_list, struct amdgpu_device, reset_list); 6423 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); 6424} 6425 6426static void amdgpu_device_halt_activities(struct amdgpu_device *adev, 6427 struct amdgpu_job *job, 6428 struct amdgpu_reset_context *reset_context, 6429 struct list_head *device_list, 6430 struct amdgpu_hive_info *hive, 6431 bool need_emergency_restart) 6432{ 6433 struct amdgpu_device *tmp_adev = NULL; 6434 int i; 6435 6436 /* block all schedulers and reset given job's ring */ 6437 list_for_each_entry(tmp_adev, device_list, reset_list) { 6438 amdgpu_device_set_mp1_state(tmp_adev); 6439 6440 /* 6441 * Try to put the audio codec into suspend state 6442 * before gpu reset started. 6443 * 6444 * Due to the power domain of the graphics device 6445 * is shared with AZ power domain. Without this, 6446 * we may change the audio hardware from behind 6447 * the audio driver's back. That will trigger 6448 * some audio codec errors. 6449 */ 6450 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 6451 tmp_adev->pcie_reset_ctx.audio_suspended = true; 6452 6453 amdgpu_ras_set_error_query_ready(tmp_adev, false); 6454 6455 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 6456 6457 amdgpu_amdkfd_pre_reset(tmp_adev, reset_context); 6458 6459 /* 6460 * Mark these ASICs to be reset as untracked first 6461 * And add them back after reset completed 6462 */ 6463 amdgpu_unregister_gpu_instance(tmp_adev); 6464 6465 drm_client_dev_suspend(adev_to_drm(tmp_adev)); 6466 6467 /* disable ras on ALL IPs */ 6468 if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev) && 6469 amdgpu_device_ip_need_full_reset(tmp_adev)) 6470 amdgpu_ras_suspend(tmp_adev); 6471 6472 amdgpu_userq_pre_reset(tmp_adev); 6473 6474 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6475 struct amdgpu_ring *ring = tmp_adev->rings[i]; 6476 6477 if (!amdgpu_ring_sched_ready(ring)) 6478 continue; 6479 6480 drm_sched_stop(&ring->sched, job ? &job->base : NULL); 6481 6482 if (need_emergency_restart) 6483 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 6484 } 6485 atomic_inc(&tmp_adev->gpu_reset_counter); 6486 } 6487} 6488 6489static int amdgpu_device_asic_reset(struct amdgpu_device *adev, 6490 struct list_head *device_list, 6491 struct amdgpu_reset_context *reset_context) 6492{ 6493 struct amdgpu_device *tmp_adev = NULL; 6494 int retry_limit = AMDGPU_MAX_RETRY_LIMIT; 6495 int r = 0; 6496 6497retry: /* Rest of adevs pre asic reset from XGMI hive. */ 6498 list_for_each_entry(tmp_adev, device_list, reset_list) { 6499 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context); 6500 /*TODO Should we stop ?*/ 6501 if (r) { 6502 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 6503 r, adev_to_drm(tmp_adev)->unique); 6504 tmp_adev->asic_reset_res = r; 6505 } 6506 } 6507 6508 /* Actual ASIC resets if needed.*/ 6509 /* Host driver will handle XGMI hive reset for SRIOV */ 6510 if (amdgpu_sriov_vf(adev)) { 6511 6512 /* Bail out of reset early */ 6513 if (amdgpu_ras_is_rma(adev)) 6514 return -ENODEV; 6515 6516 if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) { 6517 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n"); 6518 amdgpu_ras_set_fed(adev, true); 6519 set_bit(AMDGPU_HOST_FLR, &reset_context->flags); 6520 } 6521 6522 r = amdgpu_device_reset_sriov(adev, reset_context); 6523 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) { 6524 amdgpu_virt_release_full_gpu(adev, true); 6525 goto retry; 6526 } 6527 if (r) 6528 adev->asic_reset_res = r; 6529 } else { 6530 r = amdgpu_do_asic_reset(device_list, reset_context); 6531 if (r && r == -EAGAIN) 6532 goto retry; 6533 } 6534 6535 list_for_each_entry(tmp_adev, device_list, reset_list) { 6536 /* 6537 * Drop any pending non scheduler resets queued before reset is done. 6538 * Any reset scheduled after this point would be valid. Scheduler resets 6539 * were already dropped during drm_sched_stop and no new ones can come 6540 * in before drm_sched_start. 6541 */ 6542 amdgpu_device_stop_pending_resets(tmp_adev); 6543 } 6544 6545 return r; 6546} 6547 6548static int amdgpu_device_sched_resume(struct list_head *device_list, 6549 struct amdgpu_reset_context *reset_context, 6550 bool job_signaled) 6551{ 6552 struct amdgpu_device *tmp_adev = NULL; 6553 int i, r = 0; 6554 6555 /* Post ASIC reset for all devs .*/ 6556 list_for_each_entry(tmp_adev, device_list, reset_list) { 6557 6558 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6559 struct amdgpu_ring *ring = tmp_adev->rings[i]; 6560 6561 if (!amdgpu_ring_sched_ready(ring)) 6562 continue; 6563 6564 drm_sched_start(&ring->sched, 0); 6565 } 6566 6567 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) 6568 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 6569 6570 if (tmp_adev->asic_reset_res) { 6571 /* bad news, how to tell it to userspace ? 6572 * for ras error, we should report GPU bad status instead of 6573 * reset failure 6574 */ 6575 if (reset_context->src != AMDGPU_RESET_SRC_RAS || 6576 !amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) 6577 dev_info( 6578 tmp_adev->dev, 6579 "GPU reset(%d) failed with error %d \n", 6580 atomic_read( 6581 &tmp_adev->gpu_reset_counter), 6582 tmp_adev->asic_reset_res); 6583 amdgpu_vf_error_put(tmp_adev, 6584 AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, 6585 tmp_adev->asic_reset_res); 6586 if (!r) 6587 r = tmp_adev->asic_reset_res; 6588 tmp_adev->asic_reset_res = 0; 6589 } else { 6590 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", 6591 atomic_read(&tmp_adev->gpu_reset_counter)); 6592 if (amdgpu_acpi_smart_shift_update(tmp_adev, 6593 AMDGPU_SS_DEV_D0)) 6594 dev_warn(tmp_adev->dev, 6595 "smart shift update failed\n"); 6596 } 6597 } 6598 6599 return r; 6600} 6601 6602static void amdgpu_device_gpu_resume(struct amdgpu_device *adev, 6603 struct list_head *device_list, 6604 bool need_emergency_restart) 6605{ 6606 struct amdgpu_device *tmp_adev = NULL; 6607 6608 list_for_each_entry(tmp_adev, device_list, reset_list) { 6609 /* unlock kfd: SRIOV would do it separately */ 6610 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 6611 amdgpu_amdkfd_post_reset(tmp_adev); 6612 6613 /* kfd_post_reset will do nothing if kfd device is not initialized, 6614 * need to bring up kfd here if it's not be initialized before 6615 */ 6616 if (!adev->kfd.init_complete) 6617 amdgpu_amdkfd_device_init(adev); 6618 6619 if (tmp_adev->pcie_reset_ctx.audio_suspended) 6620 amdgpu_device_resume_display_audio(tmp_adev); 6621 6622 amdgpu_device_unset_mp1_state(tmp_adev); 6623 6624 amdgpu_ras_set_error_query_ready(tmp_adev, true); 6625 6626 } 6627} 6628 6629 6630/** 6631 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 6632 * 6633 * @adev: amdgpu_device pointer 6634 * @job: which job trigger hang 6635 * @reset_context: amdgpu reset context pointer 6636 * 6637 * Attempt to reset the GPU if it has hung (all asics). 6638 * Attempt to do soft-reset or full-reset and reinitialize Asic 6639 * Returns 0 for success or an error on failure. 6640 */ 6641 6642int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 6643 struct amdgpu_job *job, 6644 struct amdgpu_reset_context *reset_context) 6645{ 6646 struct list_head device_list; 6647 bool job_signaled = false; 6648 struct amdgpu_hive_info *hive = NULL; 6649 int r = 0; 6650 bool need_emergency_restart = false; 6651 /* save the pasid here as the job may be freed before the end of the reset */ 6652 int pasid = job ? job->pasid : -EINVAL; 6653 6654 /* 6655 * If it reaches here because of hang/timeout and a RAS error is 6656 * detected at the same time, let RAS recovery take care of it. 6657 */ 6658 if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) && 6659 !amdgpu_sriov_vf(adev) && 6660 reset_context->src != AMDGPU_RESET_SRC_RAS) { 6661 dev_dbg(adev->dev, 6662 "Gpu recovery from source: %d yielding to RAS error recovery handling", 6663 reset_context->src); 6664 return 0; 6665 } 6666 6667 /* 6668 * Special case: RAS triggered and full reset isn't supported 6669 */ 6670 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 6671 6672 /* 6673 * Flush RAM to disk so that after reboot 6674 * the user can read log and see why the system rebooted. 6675 */ 6676 if (need_emergency_restart && amdgpu_ras_get_context(adev) && 6677 amdgpu_ras_get_context(adev)->reboot) { 6678 dev_warn(adev->dev, "Emergency reboot."); 6679 6680 ksys_sync_helper(); 6681 emergency_restart(); 6682 } 6683 6684 dev_info(adev->dev, "GPU %s begin!. Source: %d\n", 6685 need_emergency_restart ? "jobs stop" : "reset", 6686 reset_context->src); 6687 6688 if (!amdgpu_sriov_vf(adev)) 6689 hive = amdgpu_get_xgmi_hive(adev); 6690 if (hive) 6691 mutex_lock(&hive->hive_lock); 6692 6693 reset_context->job = job; 6694 reset_context->hive = hive; 6695 INIT_LIST_HEAD(&device_list); 6696 6697 amdgpu_device_recovery_prepare(adev, &device_list, hive); 6698 6699 if (!amdgpu_sriov_vf(adev)) { 6700 r = amdgpu_device_health_check(&device_list); 6701 if (r) 6702 goto end_reset; 6703 } 6704 6705 /* Cannot be called after locking reset domain */ 6706 amdgpu_ras_pre_reset(adev, &device_list); 6707 6708 /* We need to lock reset domain only once both for XGMI and single device */ 6709 amdgpu_device_recovery_get_reset_lock(adev, &device_list); 6710 6711 amdgpu_device_halt_activities(adev, job, reset_context, &device_list, 6712 hive, need_emergency_restart); 6713 if (need_emergency_restart) 6714 goto skip_sched_resume; 6715 /* 6716 * Must check guilty signal here since after this point all old 6717 * HW fences are force signaled. 6718 * 6719 * job->base holds a reference to parent fence 6720 */ 6721 if (job && dma_fence_is_signaled(&job->hw_fence->base)) { 6722 job_signaled = true; 6723 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 6724 goto skip_hw_reset; 6725 } 6726 6727 r = amdgpu_device_asic_reset(adev, &device_list, reset_context); 6728 if (r) 6729 goto reset_unlock; 6730skip_hw_reset: 6731 r = amdgpu_device_sched_resume(&device_list, reset_context, job_signaled); 6732 if (r) 6733 goto reset_unlock; 6734skip_sched_resume: 6735 amdgpu_device_gpu_resume(adev, &device_list, need_emergency_restart); 6736reset_unlock: 6737 amdgpu_device_recovery_put_reset_lock(adev, &device_list); 6738 amdgpu_ras_post_reset(adev, &device_list); 6739end_reset: 6740 if (hive) { 6741 mutex_unlock(&hive->hive_lock); 6742 amdgpu_put_xgmi_hive(hive); 6743 } 6744 6745 if (r) 6746 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 6747 6748 atomic_set(&adev->reset_domain->reset_res, r); 6749 6750 if (!r) { 6751 struct amdgpu_task_info *ti = NULL; 6752 6753 /* 6754 * The job may already be freed at this point via the sched tdr workqueue so 6755 * use the cached pasid. 6756 */ 6757 if (pasid >= 0) 6758 ti = amdgpu_vm_get_task_info_pasid(adev, pasid); 6759 6760 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, 6761 ti ? &ti->task : NULL); 6762 6763 amdgpu_vm_put_task_info(ti); 6764 } 6765 6766 return r; 6767} 6768 6769/** 6770 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner 6771 * 6772 * @adev: amdgpu_device pointer 6773 * @speed: pointer to the speed of the link 6774 * @width: pointer to the width of the link 6775 * 6776 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 6777 * first physical partner to an AMD dGPU. 6778 * This will exclude any virtual switches and links. 6779 */ 6780static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, 6781 enum pci_bus_speed *speed, 6782 enum pcie_link_width *width) 6783{ 6784 struct pci_dev *parent = adev->pdev; 6785 6786 if (!speed || !width) 6787 return; 6788 6789 *speed = PCI_SPEED_UNKNOWN; 6790 *width = PCIE_LNK_WIDTH_UNKNOWN; 6791 6792 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) { 6793 while ((parent = pci_upstream_bridge(parent))) { 6794 /* skip upstream/downstream switches internal to dGPU*/ 6795 if (parent->vendor == PCI_VENDOR_ID_ATI) 6796 continue; 6797 *speed = pcie_get_speed_cap(parent); 6798 *width = pcie_get_width_cap(parent); 6799 break; 6800 } 6801 } else { 6802 /* use the current speeds rather than max if switching is not supported */ 6803 pcie_bandwidth_available(adev->pdev, NULL, speed, width); 6804 } 6805} 6806 6807/** 6808 * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU 6809 * 6810 * @adev: amdgpu_device pointer 6811 * @speed: pointer to the speed of the link 6812 * @width: pointer to the width of the link 6813 * 6814 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 6815 * AMD dGPU which may be a virtual upstream bridge. 6816 */ 6817static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev, 6818 enum pci_bus_speed *speed, 6819 enum pcie_link_width *width) 6820{ 6821 struct pci_dev *parent = adev->pdev; 6822 6823 if (!speed || !width) 6824 return; 6825 6826 parent = pci_upstream_bridge(parent); 6827 if (parent && parent->vendor == PCI_VENDOR_ID_ATI) { 6828 /* use the upstream/downstream switches internal to dGPU */ 6829 *speed = pcie_get_speed_cap(parent); 6830 *width = pcie_get_width_cap(parent); 6831 while ((parent = pci_upstream_bridge(parent))) { 6832 if (parent->vendor == PCI_VENDOR_ID_ATI) { 6833 /* use the upstream/downstream switches internal to dGPU */ 6834 *speed = pcie_get_speed_cap(parent); 6835 *width = pcie_get_width_cap(parent); 6836 } 6837 } 6838 } else { 6839 /* use the device itself */ 6840 *speed = pcie_get_speed_cap(adev->pdev); 6841 *width = pcie_get_width_cap(adev->pdev); 6842 } 6843} 6844 6845/** 6846 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 6847 * 6848 * @adev: amdgpu_device pointer 6849 * 6850 * Fetches and stores in the driver the PCIE capabilities (gen speed 6851 * and lanes) of the slot the device is in. Handles APUs and 6852 * virtualized environments where PCIE config space may not be available. 6853 */ 6854static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 6855{ 6856 enum pci_bus_speed speed_cap, platform_speed_cap; 6857 enum pcie_link_width platform_link_width, link_width; 6858 6859 if (amdgpu_pcie_gen_cap) 6860 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 6861 6862 if (amdgpu_pcie_lane_cap) 6863 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 6864 6865 /* covers APUs as well */ 6866 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { 6867 if (adev->pm.pcie_gen_mask == 0) 6868 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 6869 if (adev->pm.pcie_mlw_mask == 0) 6870 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 6871 return; 6872 } 6873 6874 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 6875 return; 6876 6877 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap, 6878 &platform_link_width); 6879 amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width); 6880 6881 if (adev->pm.pcie_gen_mask == 0) { 6882 /* asic caps */ 6883 if (speed_cap == PCI_SPEED_UNKNOWN) { 6884 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6885 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6886 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 6887 } else { 6888 if (speed_cap == PCIE_SPEED_32_0GT) 6889 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6890 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6891 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6892 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | 6893 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); 6894 else if (speed_cap == PCIE_SPEED_16_0GT) 6895 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6896 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6897 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6898 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 6899 else if (speed_cap == PCIE_SPEED_8_0GT) 6900 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6901 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6902 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 6903 else if (speed_cap == PCIE_SPEED_5_0GT) 6904 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6905 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 6906 else 6907 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 6908 } 6909 /* platform caps */ 6910 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 6911 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6912 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6913 } else { 6914 if (platform_speed_cap == PCIE_SPEED_32_0GT) 6915 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6916 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6917 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6918 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | 6919 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); 6920 else if (platform_speed_cap == PCIE_SPEED_16_0GT) 6921 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6922 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6923 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6924 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 6925 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 6926 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6927 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6928 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 6929 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 6930 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6931 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6932 else 6933 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 6934 6935 } 6936 } 6937 if (adev->pm.pcie_mlw_mask == 0) { 6938 /* asic caps */ 6939 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6940 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK; 6941 } else { 6942 switch (link_width) { 6943 case PCIE_LNK_X32: 6944 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 | 6945 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6946 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6947 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6948 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6949 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6950 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6951 break; 6952 case PCIE_LNK_X16: 6953 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6954 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6955 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6956 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6957 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6958 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6959 break; 6960 case PCIE_LNK_X12: 6961 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6962 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6963 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6964 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6965 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6966 break; 6967 case PCIE_LNK_X8: 6968 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6969 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6970 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6971 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6972 break; 6973 case PCIE_LNK_X4: 6974 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6975 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6976 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6977 break; 6978 case PCIE_LNK_X2: 6979 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6980 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6981 break; 6982 case PCIE_LNK_X1: 6983 adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1; 6984 break; 6985 default: 6986 break; 6987 } 6988 } 6989 /* platform caps */ 6990 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6991 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 6992 } else { 6993 switch (platform_link_width) { 6994 case PCIE_LNK_X32: 6995 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 6996 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6997 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6998 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6999 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 7000 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 7001 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 7002 break; 7003 case PCIE_LNK_X16: 7004 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 7005 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 7006 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 7007 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 7008 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 7009 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 7010 break; 7011 case PCIE_LNK_X12: 7012 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 7013 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 7014 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 7015 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 7016 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 7017 break; 7018 case PCIE_LNK_X8: 7019 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 7020 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 7021 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 7022 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 7023 break; 7024 case PCIE_LNK_X4: 7025 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 7026 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 7027 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 7028 break; 7029 case PCIE_LNK_X2: 7030 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 7031 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 7032 break; 7033 case PCIE_LNK_X1: 7034 adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 7035 break; 7036 default: 7037 break; 7038 } 7039 } 7040 } 7041} 7042 7043/** 7044 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR 7045 * 7046 * @adev: amdgpu_device pointer 7047 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev 7048 * 7049 * Return true if @peer_adev can access (DMA) @adev through the PCIe 7050 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of 7051 * @peer_adev. 7052 */ 7053bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, 7054 struct amdgpu_device *peer_adev) 7055{ 7056#ifdef CONFIG_HSA_AMD_P2P 7057 bool p2p_access = 7058 !adev->gmc.xgmi.connected_to_cpu && 7059 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); 7060 if (!p2p_access) 7061 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n", 7062 pci_name(peer_adev->pdev)); 7063 7064 bool is_large_bar = adev->gmc.visible_vram_size && 7065 adev->gmc.real_vram_size == adev->gmc.visible_vram_size; 7066 bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev); 7067 7068 if (!p2p_addressable) { 7069 uint64_t address_mask = peer_adev->dev->dma_mask ? 7070 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); 7071 resource_size_t aper_limit = 7072 adev->gmc.aper_base + adev->gmc.aper_size - 1; 7073 7074 p2p_addressable = !(adev->gmc.aper_base & address_mask || 7075 aper_limit & address_mask); 7076 } 7077 return pcie_p2p && is_large_bar && p2p_access && p2p_addressable; 7078#else 7079 return false; 7080#endif 7081} 7082 7083int amdgpu_device_baco_enter(struct amdgpu_device *adev) 7084{ 7085 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 7086 7087 if (!amdgpu_device_supports_baco(adev)) 7088 return -ENOTSUPP; 7089 7090 if (ras && adev->ras_enabled && 7091 adev->nbio.funcs->enable_doorbell_interrupt) 7092 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 7093 7094 return amdgpu_dpm_baco_enter(adev); 7095} 7096 7097int amdgpu_device_baco_exit(struct amdgpu_device *adev) 7098{ 7099 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 7100 int ret = 0; 7101 7102 if (!amdgpu_device_supports_baco(adev)) 7103 return -ENOTSUPP; 7104 7105 ret = amdgpu_dpm_baco_exit(adev); 7106 if (ret) 7107 return ret; 7108 7109 if (ras && adev->ras_enabled && 7110 adev->nbio.funcs->enable_doorbell_interrupt) 7111 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 7112 7113 if (amdgpu_passthrough(adev) && adev->nbio.funcs && 7114 adev->nbio.funcs->clear_doorbell_interrupt) 7115 adev->nbio.funcs->clear_doorbell_interrupt(adev); 7116 7117 return 0; 7118} 7119 7120/** 7121 * amdgpu_pci_error_detected - Called when a PCI error is detected. 7122 * @pdev: PCI device struct 7123 * @state: PCI channel state 7124 * 7125 * Description: Called when a PCI error is detected. 7126 * 7127 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 7128 */ 7129pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 7130{ 7131 struct drm_device *dev = pci_get_drvdata(pdev); 7132 struct amdgpu_device *adev = drm_to_adev(dev); 7133 struct amdgpu_hive_info *hive __free(xgmi_put_hive) = 7134 amdgpu_get_xgmi_hive(adev); 7135 struct amdgpu_reset_context reset_context; 7136 struct list_head device_list; 7137 7138 dev_info(adev->dev, "PCI error: detected callback!!\n"); 7139 7140 adev->pci_channel_state = state; 7141 7142 switch (state) { 7143 case pci_channel_io_normal: 7144 dev_info(adev->dev, "pci_channel_io_normal: state(%d)!!\n", state); 7145 return PCI_ERS_RESULT_CAN_RECOVER; 7146 case pci_channel_io_frozen: 7147 /* Fatal error, prepare for slot reset */ 7148 dev_info(adev->dev, "pci_channel_io_frozen: state(%d)!!\n", state); 7149 if (hive) { 7150 /* Hive devices should be able to support FW based 7151 * link reset on other devices, if not return. 7152 */ 7153 if (!amdgpu_dpm_is_link_reset_supported(adev)) { 7154 dev_warn(adev->dev, 7155 "No support for XGMI hive yet...\n"); 7156 return PCI_ERS_RESULT_DISCONNECT; 7157 } 7158 /* Set dpc status only if device is part of hive 7159 * Non-hive devices should be able to recover after 7160 * link reset. 7161 */ 7162 amdgpu_reset_set_dpc_status(adev, true); 7163 7164 mutex_lock(&hive->hive_lock); 7165 } 7166 memset(&reset_context, 0, sizeof(reset_context)); 7167 INIT_LIST_HEAD(&device_list); 7168 7169 amdgpu_device_recovery_prepare(adev, &device_list, hive); 7170 amdgpu_device_recovery_get_reset_lock(adev, &device_list); 7171 amdgpu_device_halt_activities(adev, NULL, &reset_context, &device_list, 7172 hive, false); 7173 if (hive) 7174 mutex_unlock(&hive->hive_lock); 7175 return PCI_ERS_RESULT_NEED_RESET; 7176 case pci_channel_io_perm_failure: 7177 /* Permanent error, prepare for device removal */ 7178 dev_info(adev->dev, "pci_channel_io_perm_failure: state(%d)!!\n", state); 7179 return PCI_ERS_RESULT_DISCONNECT; 7180 } 7181 7182 return PCI_ERS_RESULT_NEED_RESET; 7183} 7184 7185/** 7186 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 7187 * @pdev: pointer to PCI device 7188 */ 7189pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 7190{ 7191 struct drm_device *dev = pci_get_drvdata(pdev); 7192 struct amdgpu_device *adev = drm_to_adev(dev); 7193 7194 dev_info(adev->dev, "PCI error: mmio enabled callback!!\n"); 7195 7196 /* TODO - dump whatever for debugging purposes */ 7197 7198 /* This called only if amdgpu_pci_error_detected returns 7199 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 7200 * works, no need to reset slot. 7201 */ 7202 7203 return PCI_ERS_RESULT_RECOVERED; 7204} 7205 7206/** 7207 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 7208 * @pdev: PCI device struct 7209 * 7210 * Description: This routine is called by the pci error recovery 7211 * code after the PCI slot has been reset, just before we 7212 * should resume normal operations. 7213 */ 7214pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 7215{ 7216 struct drm_device *dev = pci_get_drvdata(pdev); 7217 struct amdgpu_device *adev = drm_to_adev(dev); 7218 struct amdgpu_reset_context reset_context; 7219 struct amdgpu_device *tmp_adev; 7220 struct amdgpu_hive_info *hive; 7221 struct list_head device_list; 7222 struct pci_dev *link_dev; 7223 int r = 0, i, timeout; 7224 u32 memsize; 7225 u16 status; 7226 7227 dev_info(adev->dev, "PCI error: slot reset callback!!\n"); 7228 7229 memset(&reset_context, 0, sizeof(reset_context)); 7230 INIT_LIST_HEAD(&device_list); 7231 hive = amdgpu_get_xgmi_hive(adev); 7232 if (hive) { 7233 mutex_lock(&hive->hive_lock); 7234 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 7235 list_add_tail(&tmp_adev->reset_list, &device_list); 7236 } else { 7237 list_add_tail(&adev->reset_list, &device_list); 7238 } 7239 7240 if (adev->pcie_reset_ctx.swus) 7241 link_dev = adev->pcie_reset_ctx.swus; 7242 else 7243 link_dev = adev->pdev; 7244 /* wait for asic to come out of reset, timeout = 10s */ 7245 timeout = 10000; 7246 do { 7247 usleep_range(10000, 10500); 7248 r = pci_read_config_word(link_dev, PCI_VENDOR_ID, &status); 7249 timeout -= 10; 7250 } while (timeout > 0 && (status != PCI_VENDOR_ID_ATI) && 7251 (status != PCI_VENDOR_ID_AMD)); 7252 7253 if ((status != PCI_VENDOR_ID_ATI) && (status != PCI_VENDOR_ID_AMD)) { 7254 r = -ETIME; 7255 goto out; 7256 } 7257 7258 amdgpu_device_load_switch_state(adev); 7259 /* Restore PCI confspace */ 7260 amdgpu_device_load_pci_state(pdev); 7261 7262 /* confirm ASIC came out of reset */ 7263 for (i = 0; i < adev->usec_timeout; i++) { 7264 memsize = amdgpu_asic_get_config_memsize(adev); 7265 7266 if (memsize != 0xffffffff) 7267 break; 7268 udelay(1); 7269 } 7270 if (memsize == 0xffffffff) { 7271 r = -ETIME; 7272 goto out; 7273 } 7274 7275 reset_context.method = AMD_RESET_METHOD_NONE; 7276 reset_context.reset_req_dev = adev; 7277 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 7278 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 7279 7280 if (hive) { 7281 reset_context.hive = hive; 7282 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 7283 tmp_adev->pcie_reset_ctx.in_link_reset = true; 7284 } else { 7285 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); 7286 } 7287 7288 r = amdgpu_device_asic_reset(adev, &device_list, &reset_context); 7289out: 7290 if (!r) { 7291 if (amdgpu_device_cache_pci_state(adev->pdev)) 7292 pci_restore_state(adev->pdev); 7293 dev_info(adev->dev, "PCIe error recovery succeeded\n"); 7294 } else { 7295 dev_err(adev->dev, "PCIe error recovery failed, err:%d\n", r); 7296 if (hive) { 7297 list_for_each_entry(tmp_adev, &device_list, reset_list) 7298 amdgpu_device_unset_mp1_state(tmp_adev); 7299 } 7300 amdgpu_device_recovery_put_reset_lock(adev, &device_list); 7301 } 7302 7303 if (hive) { 7304 mutex_unlock(&hive->hive_lock); 7305 amdgpu_put_xgmi_hive(hive); 7306 } 7307 7308 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 7309} 7310 7311/** 7312 * amdgpu_pci_resume() - resume normal ops after PCI reset 7313 * @pdev: pointer to PCI device 7314 * 7315 * Called when the error recovery driver tells us that its 7316 * OK to resume normal operation. 7317 */ 7318void amdgpu_pci_resume(struct pci_dev *pdev) 7319{ 7320 struct drm_device *dev = pci_get_drvdata(pdev); 7321 struct amdgpu_device *adev = drm_to_adev(dev); 7322 struct list_head device_list; 7323 struct amdgpu_hive_info *hive = NULL; 7324 struct amdgpu_device *tmp_adev = NULL; 7325 7326 dev_info(adev->dev, "PCI error: resume callback!!\n"); 7327 7328 /* Only continue execution for the case of pci_channel_io_frozen */ 7329 if (adev->pci_channel_state != pci_channel_io_frozen) 7330 return; 7331 7332 INIT_LIST_HEAD(&device_list); 7333 7334 hive = amdgpu_get_xgmi_hive(adev); 7335 if (hive) { 7336 mutex_lock(&hive->hive_lock); 7337 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 7338 tmp_adev->pcie_reset_ctx.in_link_reset = false; 7339 list_add_tail(&tmp_adev->reset_list, &device_list); 7340 } 7341 } else 7342 list_add_tail(&adev->reset_list, &device_list); 7343 7344 amdgpu_device_sched_resume(&device_list, NULL, NULL); 7345 amdgpu_device_gpu_resume(adev, &device_list, false); 7346 amdgpu_device_recovery_put_reset_lock(adev, &device_list); 7347 7348 if (hive) { 7349 mutex_unlock(&hive->hive_lock); 7350 amdgpu_put_xgmi_hive(hive); 7351 } 7352} 7353 7354static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev) 7355{ 7356 struct pci_dev *swus, *swds; 7357 int r; 7358 7359 swds = pci_upstream_bridge(adev->pdev); 7360 if (!swds || swds->vendor != PCI_VENDOR_ID_ATI || 7361 pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM) 7362 return; 7363 swus = pci_upstream_bridge(swds); 7364 if (!swus || 7365 (swus->vendor != PCI_VENDOR_ID_ATI && 7366 swus->vendor != PCI_VENDOR_ID_AMD) || 7367 pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM) 7368 return; 7369 7370 /* If already saved, return */ 7371 if (adev->pcie_reset_ctx.swus) 7372 return; 7373 /* Upstream bridge is ATI, assume it's SWUS/DS architecture */ 7374 r = pci_save_state(swds); 7375 if (r) 7376 return; 7377 adev->pcie_reset_ctx.swds_pcistate = pci_store_saved_state(swds); 7378 7379 r = pci_save_state(swus); 7380 if (r) 7381 return; 7382 adev->pcie_reset_ctx.swus_pcistate = pci_store_saved_state(swus); 7383 7384 adev->pcie_reset_ctx.swus = swus; 7385} 7386 7387static void amdgpu_device_load_switch_state(struct amdgpu_device *adev) 7388{ 7389 struct pci_dev *pdev; 7390 int r; 7391 7392 if (!adev->pcie_reset_ctx.swds_pcistate || 7393 !adev->pcie_reset_ctx.swus_pcistate) 7394 return; 7395 7396 pdev = adev->pcie_reset_ctx.swus; 7397 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swus_pcistate); 7398 if (!r) { 7399 pci_restore_state(pdev); 7400 } else { 7401 dev_warn(adev->dev, "Failed to load SWUS state, err:%d\n", r); 7402 return; 7403 } 7404 7405 pdev = pci_upstream_bridge(adev->pdev); 7406 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swds_pcistate); 7407 if (!r) 7408 pci_restore_state(pdev); 7409 else 7410 dev_warn(adev->dev, "Failed to load SWDS state, err:%d\n", r); 7411} 7412 7413bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 7414{ 7415 struct drm_device *dev = pci_get_drvdata(pdev); 7416 struct amdgpu_device *adev = drm_to_adev(dev); 7417 int r; 7418 7419 if (amdgpu_sriov_vf(adev)) 7420 return false; 7421 7422 r = pci_save_state(pdev); 7423 if (!r) { 7424 kfree(adev->pci_state); 7425 7426 adev->pci_state = pci_store_saved_state(pdev); 7427 7428 if (!adev->pci_state) { 7429 dev_err(adev->dev, "Failed to store PCI saved state"); 7430 return false; 7431 } 7432 } else { 7433 dev_warn(adev->dev, "Failed to save PCI state, err:%d\n", r); 7434 return false; 7435 } 7436 7437 amdgpu_device_cache_switch_state(adev); 7438 7439 return true; 7440} 7441 7442bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 7443{ 7444 struct drm_device *dev = pci_get_drvdata(pdev); 7445 struct amdgpu_device *adev = drm_to_adev(dev); 7446 int r; 7447 7448 if (!adev->pci_state) 7449 return false; 7450 7451 r = pci_load_saved_state(pdev, adev->pci_state); 7452 7453 if (!r) { 7454 pci_restore_state(pdev); 7455 } else { 7456 dev_warn(adev->dev, "Failed to load PCI state, err:%d\n", r); 7457 return false; 7458 } 7459 7460 return true; 7461} 7462 7463void amdgpu_device_flush_hdp(struct amdgpu_device *adev, 7464 struct amdgpu_ring *ring) 7465{ 7466#ifdef CONFIG_X86_64 7467 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 7468 return; 7469#endif 7470 if (adev->gmc.xgmi.connected_to_cpu) 7471 return; 7472 7473 if (ring && ring->funcs->emit_hdp_flush) { 7474 amdgpu_ring_emit_hdp_flush(ring); 7475 return; 7476 } 7477 7478 if (!ring && amdgpu_sriov_runtime(adev)) { 7479 if (!amdgpu_kiq_hdp_flush(adev)) 7480 return; 7481 } 7482 7483 amdgpu_hdp_flush(adev, ring); 7484} 7485 7486void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev, 7487 struct amdgpu_ring *ring) 7488{ 7489#ifdef CONFIG_X86_64 7490 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 7491 return; 7492#endif 7493 if (adev->gmc.xgmi.connected_to_cpu) 7494 return; 7495 7496 amdgpu_hdp_invalidate(adev, ring); 7497} 7498 7499int amdgpu_in_reset(struct amdgpu_device *adev) 7500{ 7501 return atomic_read(&adev->reset_domain->in_gpu_reset); 7502} 7503 7504/** 7505 * amdgpu_device_halt() - bring hardware to some kind of halt state 7506 * 7507 * @adev: amdgpu_device pointer 7508 * 7509 * Bring hardware to some kind of halt state so that no one can touch it 7510 * any more. It will help to maintain error context when error occurred. 7511 * Compare to a simple hang, the system will keep stable at least for SSH 7512 * access. Then it should be trivial to inspect the hardware state and 7513 * see what's going on. Implemented as following: 7514 * 7515 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc), 7516 * clears all CPU mappings to device, disallows remappings through page faults 7517 * 2. amdgpu_irq_disable_all() disables all interrupts 7518 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences 7519 * 4. set adev->no_hw_access to avoid potential crashes after setp 5 7520 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings 7521 * 6. pci_disable_device() and pci_wait_for_pending_transaction() 7522 * flush any in flight DMA operations 7523 */ 7524void amdgpu_device_halt(struct amdgpu_device *adev) 7525{ 7526 struct pci_dev *pdev = adev->pdev; 7527 struct drm_device *ddev = adev_to_drm(adev); 7528 7529 amdgpu_xcp_dev_unplug(adev); 7530 drm_dev_unplug(ddev); 7531 7532 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 7533 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 7534 7535 amdgpu_irq_disable_all(adev); 7536 7537 amdgpu_fence_driver_hw_fini(adev); 7538 7539 adev->no_hw_access = true; 7540 7541 amdgpu_device_unmap_mmio(adev); 7542 7543 pci_disable_device(pdev); 7544 pci_wait_for_pending_transaction(pdev); 7545} 7546 7547u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, 7548 u32 reg) 7549{ 7550 unsigned long flags, address, data; 7551 u32 r; 7552 7553 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 7554 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 7555 7556 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 7557 WREG32(address, reg * 4); 7558 (void)RREG32(address); 7559 r = RREG32(data); 7560 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 7561 return r; 7562} 7563 7564void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, 7565 u32 reg, u32 v) 7566{ 7567 unsigned long flags, address, data; 7568 7569 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 7570 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 7571 7572 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 7573 WREG32(address, reg * 4); 7574 (void)RREG32(address); 7575 WREG32(data, v); 7576 (void)RREG32(data); 7577 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 7578} 7579 7580/** 7581 * amdgpu_device_get_gang - return a reference to the current gang 7582 * @adev: amdgpu_device pointer 7583 * 7584 * Returns: A new reference to the current gang leader. 7585 */ 7586struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev) 7587{ 7588 struct dma_fence *fence; 7589 7590 rcu_read_lock(); 7591 fence = dma_fence_get_rcu_safe(&adev->gang_submit); 7592 rcu_read_unlock(); 7593 return fence; 7594} 7595 7596/** 7597 * amdgpu_device_switch_gang - switch to a new gang 7598 * @adev: amdgpu_device pointer 7599 * @gang: the gang to switch to 7600 * 7601 * Try to switch to a new gang. 7602 * Returns: NULL if we switched to the new gang or a reference to the current 7603 * gang leader. 7604 */ 7605struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, 7606 struct dma_fence *gang) 7607{ 7608 struct dma_fence *old = NULL; 7609 7610 dma_fence_get(gang); 7611 do { 7612 dma_fence_put(old); 7613 old = amdgpu_device_get_gang(adev); 7614 if (old == gang) 7615 break; 7616 7617 if (!dma_fence_is_signaled(old)) { 7618 dma_fence_put(gang); 7619 return old; 7620 } 7621 7622 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, 7623 old, gang) != old); 7624 7625 /* 7626 * Drop it once for the exchanged reference in adev and once for the 7627 * thread local reference acquired in amdgpu_device_get_gang(). 7628 */ 7629 dma_fence_put(old); 7630 dma_fence_put(old); 7631 return NULL; 7632} 7633 7634/** 7635 * amdgpu_device_enforce_isolation - enforce HW isolation 7636 * @adev: the amdgpu device pointer 7637 * @ring: the HW ring the job is supposed to run on 7638 * @job: the job which is about to be pushed to the HW ring 7639 * 7640 * Makes sure that only one client at a time can use the GFX block. 7641 * Returns: The dependency to wait on before the job can be pushed to the HW. 7642 * The function is called multiple times until NULL is returned. 7643 */ 7644struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, 7645 struct amdgpu_ring *ring, 7646 struct amdgpu_job *job) 7647{ 7648 struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; 7649 struct drm_sched_fence *f = job->base.s_fence; 7650 struct dma_fence *dep; 7651 void *owner; 7652 int r; 7653 7654 /* 7655 * For now enforce isolation only for the GFX block since we only need 7656 * the cleaner shader on those rings. 7657 */ 7658 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX && 7659 ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) 7660 return NULL; 7661 7662 /* 7663 * All submissions where enforce isolation is false are handled as if 7664 * they come from a single client. Use ~0l as the owner to distinct it 7665 * from kernel submissions where the owner is NULL. 7666 */ 7667 owner = job->enforce_isolation ? f->owner : (void *)~0l; 7668 7669 mutex_lock(&adev->enforce_isolation_mutex); 7670 7671 /* 7672 * The "spearhead" submission is the first one which changes the 7673 * ownership to its client. We always need to wait for it to be 7674 * pushed to the HW before proceeding with anything. 7675 */ 7676 if (&f->scheduled != isolation->spearhead && 7677 !dma_fence_is_signaled(isolation->spearhead)) { 7678 dep = isolation->spearhead; 7679 goto out_grab_ref; 7680 } 7681 7682 if (isolation->owner != owner) { 7683 7684 /* 7685 * Wait for any gang to be assembled before switching to a 7686 * different owner or otherwise we could deadlock the 7687 * submissions. 7688 */ 7689 if (!job->gang_submit) { 7690 dep = amdgpu_device_get_gang(adev); 7691 if (!dma_fence_is_signaled(dep)) 7692 goto out_return_dep; 7693 dma_fence_put(dep); 7694 } 7695 7696 dma_fence_put(isolation->spearhead); 7697 isolation->spearhead = dma_fence_get(&f->scheduled); 7698 amdgpu_sync_move(&isolation->active, &isolation->prev); 7699 trace_amdgpu_isolation(isolation->owner, owner); 7700 isolation->owner = owner; 7701 } 7702 7703 /* 7704 * Specifying the ring here helps to pipeline submissions even when 7705 * isolation is enabled. If that is not desired for testing NULL can be 7706 * used instead of the ring to enforce a CPU round trip while switching 7707 * between clients. 7708 */ 7709 dep = amdgpu_sync_peek_fence(&isolation->prev, ring); 7710 r = amdgpu_sync_fence(&isolation->active, &f->finished, GFP_NOWAIT); 7711 if (r) 7712 dev_warn(adev->dev, "OOM tracking isolation\n"); 7713 7714out_grab_ref: 7715 dma_fence_get(dep); 7716out_return_dep: 7717 mutex_unlock(&adev->enforce_isolation_mutex); 7718 return dep; 7719} 7720 7721bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev) 7722{ 7723 switch (adev->asic_type) { 7724#ifdef CONFIG_DRM_AMDGPU_SI 7725 case CHIP_HAINAN: 7726#endif 7727 case CHIP_TOPAZ: 7728 /* chips with no display hardware */ 7729 return false; 7730#ifdef CONFIG_DRM_AMDGPU_SI 7731 case CHIP_TAHITI: 7732 case CHIP_PITCAIRN: 7733 case CHIP_VERDE: 7734 case CHIP_OLAND: 7735#endif 7736#ifdef CONFIG_DRM_AMDGPU_CIK 7737 case CHIP_BONAIRE: 7738 case CHIP_HAWAII: 7739 case CHIP_KAVERI: 7740 case CHIP_KABINI: 7741 case CHIP_MULLINS: 7742#endif 7743 case CHIP_TONGA: 7744 case CHIP_FIJI: 7745 case CHIP_POLARIS10: 7746 case CHIP_POLARIS11: 7747 case CHIP_POLARIS12: 7748 case CHIP_VEGAM: 7749 case CHIP_CARRIZO: 7750 case CHIP_STONEY: 7751 /* chips with display hardware */ 7752 return true; 7753 default: 7754 /* IP discovery */ 7755 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) || 7756 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 7757 return false; 7758 return true; 7759 } 7760} 7761 7762uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, 7763 uint32_t inst, uint32_t reg_addr, char reg_name[], 7764 uint32_t expected_value, uint32_t mask) 7765{ 7766 uint32_t ret = 0; 7767 uint32_t old_ = 0; 7768 uint32_t tmp_ = RREG32(reg_addr); 7769 uint32_t loop = adev->usec_timeout; 7770 7771 while ((tmp_ & (mask)) != (expected_value)) { 7772 if (old_ != tmp_) { 7773 loop = adev->usec_timeout; 7774 old_ = tmp_; 7775 } else 7776 udelay(1); 7777 tmp_ = RREG32(reg_addr); 7778 loop--; 7779 if (!loop) { 7780 dev_warn( 7781 adev->dev, 7782 "Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn", 7783 inst, reg_name, (uint32_t)expected_value, 7784 (uint32_t)(tmp_ & (mask))); 7785 ret = -ETIMEDOUT; 7786 break; 7787 } 7788 } 7789 return ret; 7790} 7791 7792ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring) 7793{ 7794 ssize_t size = 0; 7795 7796 if (!ring || !ring->adev) 7797 return size; 7798 7799 if (amdgpu_device_should_recover_gpu(ring->adev)) 7800 size |= AMDGPU_RESET_TYPE_FULL; 7801 7802 if (unlikely(!ring->adev->debug_disable_soft_recovery) && 7803 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery) 7804 size |= AMDGPU_RESET_TYPE_SOFT_RESET; 7805 7806 return size; 7807} 7808 7809ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset) 7810{ 7811 ssize_t size = 0; 7812 7813 if (supported_reset == 0) { 7814 size += sysfs_emit_at(buf, size, "unsupported"); 7815 size += sysfs_emit_at(buf, size, "\n"); 7816 return size; 7817 7818 } 7819 7820 if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET) 7821 size += sysfs_emit_at(buf, size, "soft "); 7822 7823 if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE) 7824 size += sysfs_emit_at(buf, size, "queue "); 7825 7826 if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE) 7827 size += sysfs_emit_at(buf, size, "pipe "); 7828 7829 if (supported_reset & AMDGPU_RESET_TYPE_FULL) 7830 size += sysfs_emit_at(buf, size, "full "); 7831 7832 size += sysfs_emit_at(buf, size, "\n"); 7833 return size; 7834} 7835 7836void amdgpu_device_set_uid(struct amdgpu_uid *uid_info, 7837 enum amdgpu_uid_type type, uint8_t inst, 7838 uint64_t uid) 7839{ 7840 if (!uid_info) 7841 return; 7842 7843 if (type >= AMDGPU_UID_TYPE_MAX) { 7844 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n", 7845 type); 7846 return; 7847 } 7848 7849 if (inst >= AMDGPU_UID_INST_MAX) { 7850 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n", 7851 inst); 7852 return; 7853 } 7854 7855 if (uid_info->uid[type][inst] != 0) { 7856 dev_warn_once( 7857 uid_info->adev->dev, 7858 "Overwriting existing UID %llu for type %d instance %d\n", 7859 uid_info->uid[type][inst], type, inst); 7860 } 7861 7862 uid_info->uid[type][inst] = uid; 7863} 7864 7865u64 amdgpu_device_get_uid(struct amdgpu_uid *uid_info, 7866 enum amdgpu_uid_type type, uint8_t inst) 7867{ 7868 if (!uid_info) 7869 return 0; 7870 7871 if (type >= AMDGPU_UID_TYPE_MAX) { 7872 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n", 7873 type); 7874 return 0; 7875 } 7876 7877 if (inst >= AMDGPU_UID_INST_MAX) { 7878 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n", 7879 inst); 7880 return 0; 7881 } 7882 7883 return uid_info->uid[type][inst]; 7884}
~/
Read-only file preview.