gmc_v6_0.c

   1/*
   2 * Copyright 2014 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/module.h>
  26#include <linux/pci.h>
  27
  28#include <drm/drm_cache.h>
  29#include "amdgpu.h"
  30#include "gmc_v6_0.h"
  31#include "amdgpu_ucode.h"
  32#include "amdgpu_gem.h"
  33
  34#include "bif/bif_3_0_d.h"
  35#include "bif/bif_3_0_sh_mask.h"
  36#include "oss/oss_1_0_d.h"
  37#include "oss/oss_1_0_sh_mask.h"
  38#include "gmc/gmc_6_0_d.h"
  39#include "gmc/gmc_6_0_sh_mask.h"
  40#include "dce/dce_6_0_d.h"
  41#include "dce/dce_6_0_sh_mask.h"
  42#include "si_enums.h"
  43
  44static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev);
  45static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
  46static int gmc_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block);
  47
  48MODULE_FIRMWARE("amdgpu/tahiti_mc.bin");
  49MODULE_FIRMWARE("amdgpu/pitcairn_mc.bin");
  50MODULE_FIRMWARE("amdgpu/verde_mc.bin");
  51MODULE_FIRMWARE("amdgpu/oland_mc.bin");
  52MODULE_FIRMWARE("amdgpu/hainan_mc.bin");
  53MODULE_FIRMWARE("amdgpu/si58_mc.bin");
  54
  55#define MC_SEQ_MISC0__MT__MASK   0xf0000000
  56#define MC_SEQ_MISC0__MT__GDDR1  0x10000000
  57#define MC_SEQ_MISC0__MT__DDR2   0x20000000
  58#define MC_SEQ_MISC0__MT__GDDR3  0x30000000
  59#define MC_SEQ_MISC0__MT__GDDR4  0x40000000
  60#define MC_SEQ_MISC0__MT__GDDR5  0x50000000
  61#define MC_SEQ_MISC0__MT__HBM    0x60000000
  62#define MC_SEQ_MISC0__MT__DDR3   0xB0000000
  63
  64static void gmc_v6_0_mc_stop(struct amdgpu_device *adev)
  65{
  66	u32 blackout;
  67	struct amdgpu_ip_block *ip_block;
  68
  69	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GMC);
  70	if (!ip_block)
  71		return;
  72
  73	gmc_v6_0_wait_for_idle(ip_block);
  74
  75	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
  76	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
  77		/* Block CPU access */
  78		WREG32(mmBIF_FB_EN, 0);
  79		/* blackout the MC */
  80		blackout = REG_SET_FIELD(blackout,
  81					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
  82		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
  83	}
  84	/* wait for the MC to settle */
  85	udelay(100);
  86
  87}
  88
  89static void gmc_v6_0_mc_resume(struct amdgpu_device *adev)
  90{
  91	u32 tmp;
  92
  93	/* unblackout the MC */
  94	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
  95	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
  96	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
  97	/* allow CPU access */
  98	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
  99	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
 100	WREG32(mmBIF_FB_EN, tmp);
 101}
 102
 103static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
 104{
 105	const char *chip_name;
 106	int err;
 107
 108	DRM_DEBUG("\n");
 109
 110	switch (adev->asic_type) {
 111	case CHIP_TAHITI:
 112		chip_name = "tahiti";
 113		break;
 114	case CHIP_PITCAIRN:
 115		chip_name = "pitcairn";
 116		break;
 117	case CHIP_VERDE:
 118		chip_name = "verde";
 119		break;
 120	case CHIP_OLAND:
 121		chip_name = "oland";
 122		break;
 123	case CHIP_HAINAN:
 124		chip_name = "hainan";
 125		break;
 126	default:
 127		BUG();
 128	}
 129
 130	/* this memory configuration requires special firmware */
 131	if (((RREG32(mmMC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
 132		chip_name = "si58";
 133
 134	err = amdgpu_ucode_request(adev, &adev->gmc.fw, AMDGPU_UCODE_REQUIRED,
 135				   "amdgpu/%s_mc.bin", chip_name);
 136	if (err) {
 137		dev_err(adev->dev,
 138		       "si_mc: Failed to load firmware \"%s_mc.bin\"\n",
 139		       chip_name);
 140		amdgpu_ucode_release(&adev->gmc.fw);
 141	}
 142	return err;
 143}
 144
 145static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
 146{
 147	const __le32 *new_fw_data = NULL;
 148	u32 running;
 149	const __le32 *new_io_mc_regs = NULL;
 150	int i, regs_size, ucode_size;
 151	const struct mc_firmware_header_v1_0 *hdr;
 152
 153	if (!adev->gmc.fw)
 154		return -EINVAL;
 155
 156	hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;
 157
 158	amdgpu_ucode_print_mc_hdr(&hdr->header);
 159
 160	adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);
 161	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
 162	new_io_mc_regs = (const __le32 *)
 163		(adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
 164	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
 165	new_fw_data = (const __le32 *)
 166		(adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 167
 168	running = RREG32(mmMC_SEQ_SUP_CNTL) & MC_SEQ_SUP_CNTL__RUN_MASK;
 169
 170	if (running == 0) {
 171
 172		/* reset the engine and set to writable */
 173		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
 174		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
 175
 176		/* load mc io regs */
 177		for (i = 0; i < regs_size; i++) {
 178			WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
 179			WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
 180		}
 181		/* load the MC ucode */
 182		for (i = 0; i < ucode_size; i++)
 183			WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
 184
 185		/* put the engine back into the active state */
 186		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
 187		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
 188		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
 189
 190		/* wait for training to complete */
 191		for (i = 0; i < adev->usec_timeout; i++) {
 192			if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0_MASK)
 193				break;
 194			udelay(1);
 195		}
 196		for (i = 0; i < adev->usec_timeout; i++) {
 197			if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1_MASK)
 198				break;
 199			udelay(1);
 200		}
 201
 202	}
 203
 204	return 0;
 205}
 206
 207static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
 208				       struct amdgpu_gmc *mc)
 209{
 210	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 211
 212	base <<= 24;
 213
 214	amdgpu_gmc_set_agp_default(adev, mc);
 215	amdgpu_gmc_vram_location(adev, mc, base);
 216	amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_LOW);
 217}
 218
 219static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 220{
 221	int i, j;
 222	struct amdgpu_ip_block *ip_block;
 223
 224
 225	/* Initialize HDP */
 226	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
 227		WREG32((0xb05 + j), 0x00000000);
 228		WREG32((0xb06 + j), 0x00000000);
 229		WREG32((0xb07 + j), 0x00000000);
 230		WREG32((0xb08 + j), 0x00000000);
 231		WREG32((0xb09 + j), 0x00000000);
 232	}
 233	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
 234
 235	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GMC);
 236	if (!ip_block)
 237		return;
 238
 239	if (gmc_v6_0_wait_for_idle(ip_block))
 240		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 241
 242	if (adev->mode_info.num_crtc) {
 243		u32 tmp;
 244
 245		/* Lockout access through VGA aperture*/
 246		tmp = RREG32(mmVGA_HDP_CONTROL);
 247		tmp |= VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK;
 248		WREG32(mmVGA_HDP_CONTROL, tmp);
 249
 250		/* disable VGA render */
 251		tmp = RREG32(mmVGA_RENDER_CONTROL);
 252		tmp &= VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK;
 253		WREG32(mmVGA_RENDER_CONTROL, tmp);
 254	}
 255	/* Update configuration */
 256	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
 257	       adev->gmc.vram_start >> 12);
 258	WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 259	       adev->gmc.vram_end >> 12);
 260	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
 261	       adev->mem_scratch.gpu_addr >> 12);
 262	WREG32(mmMC_VM_AGP_BASE, 0);
 263	WREG32(mmMC_VM_AGP_TOP, adev->gmc.agp_end >> 22);
 264	WREG32(mmMC_VM_AGP_BOT, adev->gmc.agp_start >> 22);
 265
 266	if (gmc_v6_0_wait_for_idle(ip_block))
 267		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 268}
 269
 270static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 271{
 272
 273	u32 tmp;
 274	int chansize, numchan;
 275	int r;
 276
 277	tmp = RREG32(mmMC_ARB_RAMCFG);
 278	if (tmp & (1 << 11))
 279		chansize = 16;
 280	else if (tmp & MC_ARB_RAMCFG__CHANSIZE_MASK)
 281		chansize = 64;
 282	else
 283		chansize = 32;
 284
 285	tmp = RREG32(mmMC_SHARED_CHMAP);
 286	switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
 287	case 0:
 288	default:
 289		numchan = 1;
 290		break;
 291	case 1:
 292		numchan = 2;
 293		break;
 294	case 2:
 295		numchan = 4;
 296		break;
 297	case 3:
 298		numchan = 8;
 299		break;
 300	case 4:
 301		numchan = 3;
 302		break;
 303	case 5:
 304		numchan = 6;
 305		break;
 306	case 6:
 307		numchan = 10;
 308		break;
 309	case 7:
 310		numchan = 12;
 311		break;
 312	case 8:
 313		numchan = 16;
 314		break;
 315	}
 316	adev->gmc.vram_width = numchan * chansize;
 317	/* size in MB on si */
 318	adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 319	adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 320
 321	if (!(adev->flags & AMD_IS_APU)) {
 322		r = amdgpu_device_resize_fb_bar(adev);
 323		if (r)
 324			return r;
 325	}
 326	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
 327	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
 328	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 329
 330	/* set the gart size */
 331	if (amdgpu_gart_size == -1) {
 332		switch (adev->asic_type) {
 333		case CHIP_HAINAN:    /* no MM engines */
 334		default:
 335			adev->gmc.gart_size = 256ULL << 20;
 336			break;
 337		case CHIP_VERDE:    /* UVD, VCE do not support GPUVM */
 338		case CHIP_TAHITI:   /* UVD, VCE do not support GPUVM */
 339		case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
 340		case CHIP_OLAND:    /* UVD, VCE do not support GPUVM */
 341			adev->gmc.gart_size = 1024ULL << 20;
 342			break;
 343		}
 344	} else {
 345		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
 346	}
 347
 348	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
 349	gmc_v6_0_vram_gtt_location(adev, &adev->gmc);
 350
 351	return 0;
 352}
 353
 354static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 355					uint32_t vmhub, uint32_t flush_type)
 356{
 357	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
 358}
 359
 360static uint64_t gmc_v6_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 361					    unsigned int vmid, uint64_t pd_addr)
 362{
 363	uint32_t reg;
 364
 365	/* write new base address */
 366	if (vmid < 8)
 367		reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid;
 368	else
 369		reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vmid - 8);
 370	amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
 371
 372	/* bits 0-15 are the VM contexts0-15 */
 373	amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
 374
 375	return pd_addr;
 376}
 377
 378static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level,
 379				uint64_t *addr, uint64_t *flags)
 380{
 381	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
 382}
 383
 384static void gmc_v6_0_get_vm_pte(struct amdgpu_device *adev,
 385				struct amdgpu_vm *vm,
 386				struct amdgpu_bo *bo,
 387				uint32_t vm_flags,
 388				uint64_t *flags)
 389{
 390	*flags &= ~AMDGPU_PTE_EXECUTABLE;
 391	*flags &= ~AMDGPU_PTE_PRT;
 392}
 393
 394static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
 395					      bool value)
 396{
 397	u32 tmp;
 398
 399	tmp = RREG32(mmVM_CONTEXT1_CNTL);
 400	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 401			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 402	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 403			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 404	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 405			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 406	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 407			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 408	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 409			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 410	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 411			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 412	WREG32(mmVM_CONTEXT1_CNTL, tmp);
 413}
 414
 415 /**
 416  * gmc_v8_0_set_prt() - set PRT VM fault
 417  *
 418  * @adev: amdgpu_device pointer
 419  * @enable: enable/disable VM fault handling for PRT
 420  */
 421static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
 422{
 423	u32 tmp;
 424
 425	if (enable && !adev->gmc.prt_warning) {
 426		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
 427		adev->gmc.prt_warning = true;
 428	}
 429
 430	tmp = RREG32(mmVM_PRT_CNTL);
 431	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
 432			    CB_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
 433			    enable);
 434	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
 435			    TC_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
 436			    enable);
 437	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
 438			    L2_CACHE_STORE_INVALID_ENTRIES,
 439			    enable);
 440	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
 441			    L1_TLB_STORE_INVALID_ENTRIES,
 442			    enable);
 443	WREG32(mmVM_PRT_CNTL, tmp);
 444
 445	if (enable) {
 446		uint32_t low = AMDGPU_VA_RESERVED_BOTTOM >>
 447			AMDGPU_GPU_PAGE_SHIFT;
 448		uint32_t high = adev->vm_manager.max_pfn -
 449			(AMDGPU_VA_RESERVED_TOP >> AMDGPU_GPU_PAGE_SHIFT);
 450
 451		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
 452		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
 453		WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
 454		WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
 455		WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
 456		WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
 457		WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
 458		WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
 459	} else {
 460		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
 461		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
 462		WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
 463		WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
 464		WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
 465		WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
 466		WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
 467		WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
 468	}
 469}
 470
 471static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 472{
 473	uint64_t table_addr;
 474	u32 field;
 475	int i;
 476
 477	if (adev->gart.bo == NULL) {
 478		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
 479		return -EINVAL;
 480	}
 481	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
 482
 483	table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
 484
 485	/* Setup TLB control */
 486	WREG32(mmMC_VM_MX_L1_TLB_CNTL,
 487	       (0xA << 7) |
 488	       MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK |
 489	       MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING_MASK |
 490	       MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
 491	       MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK |
 492	       (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
 493	/* Setup L2 cache */
 494	WREG32(mmVM_L2_CNTL,
 495	       VM_L2_CNTL__ENABLE_L2_CACHE_MASK |
 496	       VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK |
 497	       VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
 498	       VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
 499	       (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
 500	       (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
 501	WREG32(mmVM_L2_CNTL2,
 502	       VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
 503	       VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
 504
 505	field = adev->vm_manager.fragment_size;
 506	WREG32(mmVM_L2_CNTL3,
 507	       VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
 508	       (field << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
 509	       (field << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
 510	/* setup context0 */
 511	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);
 512	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);
 513	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12);
 514	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 515			(u32)(adev->dummy_page_addr >> 12));
 516	WREG32(mmVM_CONTEXT0_CNTL2, 0);
 517	WREG32(mmVM_CONTEXT0_CNTL,
 518	       VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK |
 519	       (0UL << VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
 520	       VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK);
 521
 522	WREG32(0x575, 0);
 523	WREG32(0x576, 0);
 524	WREG32(0x577, 0);
 525
 526	/* empty context1-15 */
 527	/* set vm size, must be a multiple of 4 */
 528	WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
 529	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
 530	/* Assign the pt base to something valid for now; the pts used for
 531	 * the VMs are determined by the application and setup and assigned
 532	 * on the fly in the vm part of radeon_gart.c
 533	 */
 534	for (i = 1; i < AMDGPU_NUM_VMID; i++) {
 535		if (i < 8)
 536			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
 537			       table_addr >> 12);
 538		else
 539			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
 540			       table_addr >> 12);
 541	}
 542
 543	/* enable context1-15 */
 544	WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
 545	       (u32)(adev->dummy_page_addr >> 12));
 546	WREG32(mmVM_CONTEXT1_CNTL2, 4);
 547	WREG32(mmVM_CONTEXT1_CNTL,
 548	       VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK |
 549	       (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
 550	       ((adev->vm_manager.block_size - 9)
 551	       << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT));
 552	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
 553		gmc_v6_0_set_fault_enable_default(adev, false);
 554	else
 555		gmc_v6_0_set_fault_enable_default(adev, true);
 556
 557	gmc_v6_0_flush_gpu_tlb(adev, 0, 0, 0);
 558	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
 559		 (unsigned int)(adev->gmc.gart_size >> 20),
 560		 (unsigned long long)table_addr);
 561	return 0;
 562}
 563
 564static int gmc_v6_0_gart_init(struct amdgpu_device *adev)
 565{
 566	int r;
 567
 568	if (adev->gart.bo) {
 569		dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");
 570		return 0;
 571	}
 572	r = amdgpu_gart_init(adev);
 573	if (r)
 574		return r;
 575	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
 576	adev->gart.gart_pte_flags = 0;
 577	return amdgpu_gart_table_vram_alloc(adev);
 578}
 579
 580static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)
 581{
 582	/*unsigned i;
 583
 584	for (i = 1; i < 16; ++i) {
 585		uint32_t reg;
 586		if (i < 8)
 587			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i ;
 588		else
 589			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (i - 8);
 590		adev->vm_manager.saved_table_addr[i] = RREG32(reg);
 591	}*/
 592
 593	/* Disable all tables */
 594	WREG32(mmVM_CONTEXT0_CNTL, 0);
 595	WREG32(mmVM_CONTEXT1_CNTL, 0);
 596	/* Setup TLB control */
 597	WREG32(mmMC_VM_MX_L1_TLB_CNTL,
 598	       MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
 599	       (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
 600	/* Setup L2 cache */
 601	WREG32(mmVM_L2_CNTL,
 602	       VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
 603	       VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
 604	       (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
 605	       (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
 606	WREG32(mmVM_L2_CNTL2, 0);
 607	WREG32(mmVM_L2_CNTL3,
 608	       VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
 609	       (0UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
 610}
 611
 612static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
 613				     u32 status, u32 addr)
 614{
 615	u32 mc_id;
 616	u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
 617	u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 618					PROTECTIONS);
 619
 620	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 621			      MEMORY_CLIENT_ID);
 622
 623	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from %d\n",
 624	       protections, vmid, addr,
 625	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 626			     MEMORY_CLIENT_RW) ?
 627	       "write" : "read", mc_id);
 628}
 629
 630static const u32 mc_cg_registers[] = {
 631	mmMC_HUB_MISC_HUB_CG,
 632	mmMC_HUB_MISC_SIP_CG,
 633	mmMC_HUB_MISC_VM_CG,
 634	mmMC_XPB_CLK_GAT,
 635	mmATC_MISC_CG,
 636	mmMC_CITF_MISC_WR_CG,
 637	mmMC_CITF_MISC_RD_CG,
 638	mmMC_CITF_MISC_VM_CG,
 639	mmVM_L2_CG,
 640};
 641
 642static const u32 mc_cg_ls_en[] = {
 643	MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
 644	MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
 645	MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
 646	MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
 647	ATC_MISC_CG__MEM_LS_ENABLE_MASK,
 648	MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
 649	MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
 650	MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
 651	VM_L2_CG__MEM_LS_ENABLE_MASK,
 652};
 653
 654static const u32 mc_cg_en[] = {
 655	MC_HUB_MISC_HUB_CG__ENABLE_MASK,
 656	MC_HUB_MISC_SIP_CG__ENABLE_MASK,
 657	MC_HUB_MISC_VM_CG__ENABLE_MASK,
 658	MC_XPB_CLK_GAT__ENABLE_MASK,
 659	ATC_MISC_CG__ENABLE_MASK,
 660	MC_CITF_MISC_WR_CG__ENABLE_MASK,
 661	MC_CITF_MISC_RD_CG__ENABLE_MASK,
 662	MC_CITF_MISC_VM_CG__ENABLE_MASK,
 663	VM_L2_CG__ENABLE_MASK,
 664};
 665
 666static void gmc_v6_0_enable_mc_ls(struct amdgpu_device *adev,
 667				  bool enable)
 668{
 669	int i;
 670	u32 orig, data;
 671
 672	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
 673		orig = data = RREG32(mc_cg_registers[i]);
 674		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
 675			data |= mc_cg_ls_en[i];
 676		else
 677			data &= ~mc_cg_ls_en[i];
 678		if (data != orig)
 679			WREG32(mc_cg_registers[i], data);
 680	}
 681}
 682
 683static void gmc_v6_0_enable_mc_mgcg(struct amdgpu_device *adev,
 684				    bool enable)
 685{
 686	int i;
 687	u32 orig, data;
 688
 689	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
 690		orig = data = RREG32(mc_cg_registers[i]);
 691		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
 692			data |= mc_cg_en[i];
 693		else
 694			data &= ~mc_cg_en[i];
 695		if (data != orig)
 696			WREG32(mc_cg_registers[i], data);
 697	}
 698}
 699
 700static void gmc_v6_0_enable_bif_mgls(struct amdgpu_device *adev,
 701				     bool enable)
 702{
 703	u32 orig, data;
 704
 705	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
 706
 707	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
 708		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
 709		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
 710		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
 711		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
 712	} else {
 713		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
 714		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
 715		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
 716		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
 717	}
 718
 719	if (orig != data)
 720		WREG32_PCIE(ixPCIE_CNTL2, data);
 721}
 722
 723static void gmc_v6_0_enable_hdp_mgcg(struct amdgpu_device *adev,
 724				     bool enable)
 725{
 726	u32 orig, data;
 727
 728	orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
 729
 730	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
 731		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
 732	else
 733		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
 734
 735	if (orig != data)
 736		WREG32(mmHDP_HOST_PATH_CNTL, data);
 737}
 738
 739static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
 740				   bool enable)
 741{
 742	u32 orig, data;
 743
 744	orig = data = RREG32(mmHDP_MEM_POWER_LS);
 745
 746	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
 747		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
 748	else
 749		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
 750
 751	if (orig != data)
 752		WREG32(mmHDP_MEM_POWER_LS, data);
 753}
 754
 755static int gmc_v6_0_convert_vram_type(int mc_seq_vram_type)
 756{
 757	switch (mc_seq_vram_type) {
 758	case MC_SEQ_MISC0__MT__GDDR1:
 759		return AMDGPU_VRAM_TYPE_GDDR1;
 760	case MC_SEQ_MISC0__MT__DDR2:
 761		return AMDGPU_VRAM_TYPE_DDR2;
 762	case MC_SEQ_MISC0__MT__GDDR3:
 763		return AMDGPU_VRAM_TYPE_GDDR3;
 764	case MC_SEQ_MISC0__MT__GDDR4:
 765		return AMDGPU_VRAM_TYPE_GDDR4;
 766	case MC_SEQ_MISC0__MT__GDDR5:
 767		return AMDGPU_VRAM_TYPE_GDDR5;
 768	case MC_SEQ_MISC0__MT__DDR3:
 769		return AMDGPU_VRAM_TYPE_DDR3;
 770	default:
 771		return AMDGPU_VRAM_TYPE_UNKNOWN;
 772	}
 773}
 774
 775static int gmc_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 776{
 777	struct amdgpu_device *adev = ip_block->adev;
 778
 779	gmc_v6_0_set_gmc_funcs(adev);
 780	gmc_v6_0_set_irq_funcs(adev);
 781
 782	return 0;
 783}
 784
 785static int gmc_v6_0_late_init(struct amdgpu_ip_block *ip_block)
 786{
 787	struct amdgpu_device *adev = ip_block->adev;
 788
 789	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
 790		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
 791	else
 792		return 0;
 793}
 794
 795static unsigned int gmc_v6_0_get_vbios_fb_size(struct amdgpu_device *adev)
 796{
 797	u32 d1vga_control = RREG32(mmD1VGA_CONTROL);
 798	unsigned int size;
 799
 800	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
 801		size = AMDGPU_VBIOS_VGA_ALLOCATION;
 802	} else {
 803		u32 viewport = RREG32(mmVIEWPORT_SIZE);
 804
 805		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
 806			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
 807			4);
 808	}
 809	return size;
 810}
 811
 812static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 813{
 814	int r;
 815	struct amdgpu_device *adev = ip_block->adev;
 816
 817	set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
 818
 819	if (adev->flags & AMD_IS_APU) {
 820		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
 821	} else {
 822		u32 tmp = RREG32(mmMC_SEQ_MISC0);
 823
 824		tmp &= MC_SEQ_MISC0__MT__MASK;
 825		adev->gmc.vram_type = gmc_v6_0_convert_vram_type(tmp);
 826	}
 827
 828	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault);
 829	if (r)
 830		return r;
 831
 832	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault);
 833	if (r)
 834		return r;
 835
 836	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
 837
 838	adev->gmc.mc_mask = 0xffffffffffULL;
 839
 840	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));
 841	if (r) {
 842		dev_warn(adev->dev, "No suitable DMA available.\n");
 843		return r;
 844	}
 845	adev->need_swiotlb = drm_need_swiotlb(40);
 846
 847	r = gmc_v6_0_init_microcode(adev);
 848	if (r) {
 849		dev_err(adev->dev, "Failed to load mc firmware!\n");
 850		return r;
 851	}
 852
 853	r = gmc_v6_0_mc_init(adev);
 854	if (r)
 855		return r;
 856
 857	amdgpu_gmc_get_vbios_allocations(adev);
 858
 859	r = amdgpu_bo_init(adev);
 860	if (r)
 861		return r;
 862
 863	r = gmc_v6_0_gart_init(adev);
 864	if (r)
 865		return r;
 866
 867	/*
 868	 * number of VMs
 869	 * VMID 0 is reserved for System
 870	 * amdgpu graphics/compute will use VMIDs 1-7
 871	 * amdkfd will use VMIDs 8-15
 872	 */
 873	adev->vm_manager.first_kfd_vmid = 8;
 874	amdgpu_vm_manager_init(adev);
 875
 876	/* base offset of vram pages */
 877	if (adev->flags & AMD_IS_APU) {
 878		u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
 879
 880		tmp <<= 22;
 881		adev->vm_manager.vram_base_offset = tmp;
 882	} else {
 883		adev->vm_manager.vram_base_offset = 0;
 884	}
 885
 886	return 0;
 887}
 888
 889static int gmc_v6_0_sw_fini(struct amdgpu_ip_block *ip_block)
 890{
 891	struct amdgpu_device *adev = ip_block->adev;
 892
 893	amdgpu_gem_force_release(adev);
 894	amdgpu_vm_manager_fini(adev);
 895	amdgpu_gart_table_vram_free(adev);
 896	amdgpu_bo_fini(adev);
 897	amdgpu_ucode_release(&adev->gmc.fw);
 898
 899	return 0;
 900}
 901
 902static int gmc_v6_0_hw_init(struct amdgpu_ip_block *ip_block)
 903{
 904	int r;
 905	struct amdgpu_device *adev = ip_block->adev;
 906
 907	gmc_v6_0_mc_program(adev);
 908
 909	if (!(adev->flags & AMD_IS_APU)) {
 910		r = gmc_v6_0_mc_load_microcode(adev);
 911		if (r) {
 912			dev_err(adev->dev, "Failed to load MC firmware!\n");
 913			return r;
 914		}
 915	}
 916
 917	r = gmc_v6_0_gart_enable(adev);
 918	if (r)
 919		return r;
 920
 921	if (amdgpu_emu_mode == 1)
 922		return amdgpu_gmc_vram_checking(adev);
 923
 924	return 0;
 925}
 926
 927static int gmc_v6_0_hw_fini(struct amdgpu_ip_block *ip_block)
 928{
 929	struct amdgpu_device *adev = ip_block->adev;
 930
 931	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
 932	gmc_v6_0_gart_disable(adev);
 933
 934	return 0;
 935}
 936
 937static int gmc_v6_0_suspend(struct amdgpu_ip_block *ip_block)
 938{
 939	gmc_v6_0_hw_fini(ip_block);
 940
 941	return 0;
 942}
 943
 944static int gmc_v6_0_resume(struct amdgpu_ip_block *ip_block)
 945{
 946	int r;
 947	struct amdgpu_device *adev = ip_block->adev;
 948
 949	r = gmc_v6_0_hw_init(ip_block);
 950	if (r)
 951		return r;
 952
 953	amdgpu_vmid_reset_all(adev);
 954
 955	return 0;
 956}
 957
 958static bool gmc_v6_0_is_idle(struct amdgpu_ip_block *ip_block)
 959{
 960	struct amdgpu_device *adev = ip_block->adev;
 961
 962	u32 tmp = RREG32(mmSRBM_STATUS);
 963
 964	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
 965		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
 966		return false;
 967
 968	return true;
 969}
 970
 971static int gmc_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 972{
 973	unsigned int i;
 974	struct amdgpu_device *adev = ip_block->adev;
 975
 976	for (i = 0; i < adev->usec_timeout; i++) {
 977		if (gmc_v6_0_is_idle(ip_block))
 978			return 0;
 979		udelay(1);
 980	}
 981	return -ETIMEDOUT;
 982
 983}
 984
 985static int gmc_v6_0_soft_reset(struct amdgpu_ip_block *ip_block)
 986{
 987	struct amdgpu_device *adev = ip_block->adev;
 988
 989	u32 srbm_soft_reset = 0;
 990	u32 tmp = RREG32(mmSRBM_STATUS);
 991
 992	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
 993		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
 994						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
 995
 996	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
 997		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
 998		if (!(adev->flags & AMD_IS_APU))
 999			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1000							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1001	}
1002
1003	if (srbm_soft_reset) {
1004		gmc_v6_0_mc_stop(adev);
1005
1006		if (gmc_v6_0_wait_for_idle(ip_block))
1007			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1008
1009		tmp = RREG32(mmSRBM_SOFT_RESET);
1010		tmp |= srbm_soft_reset;
1011		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1012		WREG32(mmSRBM_SOFT_RESET, tmp);
1013		tmp = RREG32(mmSRBM_SOFT_RESET);
1014
1015		udelay(50);
1016
1017		tmp &= ~srbm_soft_reset;
1018		WREG32(mmSRBM_SOFT_RESET, tmp);
1019		tmp = RREG32(mmSRBM_SOFT_RESET);
1020
1021		udelay(50);
1022
1023		gmc_v6_0_mc_resume(adev);
1024		udelay(50);
1025	}
1026
1027	return 0;
1028}
1029
1030static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1031					     struct amdgpu_irq_src *src,
1032					     unsigned int type,
1033					     enum amdgpu_interrupt_state state)
1034{
1035	u32 tmp;
1036	u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1037		    VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1038		    VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1039		    VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1040		    VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1041		    VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1042
1043	switch (state) {
1044	case AMDGPU_IRQ_STATE_DISABLE:
1045		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1046		tmp &= ~bits;
1047		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1048		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1049		tmp &= ~bits;
1050		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1051		break;
1052	case AMDGPU_IRQ_STATE_ENABLE:
1053		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1054		tmp |= bits;
1055		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1056		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1057		tmp |= bits;
1058		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1059		break;
1060	default:
1061		break;
1062	}
1063
1064	return 0;
1065}
1066
1067static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
1068				      struct amdgpu_irq_src *source,
1069				      struct amdgpu_iv_entry *entry)
1070{
1071	u32 addr, status;
1072
1073	/* Delegate to the soft IRQ handler ring */
1074	if (adev->irq.ih_soft.enabled && entry->ih != &adev->irq.ih_soft) {
1075		amdgpu_irq_delegate(adev, entry, 4);
1076		return 1;
1077	}
1078
1079	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1080	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1081	WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1082
1083	if (!addr && !status)
1084		return 0;
1085
1086	amdgpu_vm_update_fault_cache(adev, entry->pasid,
1087				     ((u64)addr) << AMDGPU_GPU_PAGE_SHIFT,
1088				     status, AMDGPU_GFXHUB(0));
1089
1090	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1091		gmc_v6_0_set_fault_enable_default(adev, false);
1092
1093	if (printk_ratelimit()) {
1094		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1095			entry->src_id, entry->src_data[0]);
1096		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1097			addr);
1098		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1099			status);
1100		gmc_v6_0_vm_decode_fault(adev, status, addr);
1101	}
1102
1103	return 0;
1104}
1105
1106static int gmc_v6_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1107					  enum amd_clockgating_state state)
1108{
1109	struct amdgpu_device *adev = ip_block->adev;
1110	bool gate = false;
1111
1112	if (state == AMD_CG_STATE_GATE)
1113		gate = true;
1114
1115	if (!(adev->flags & AMD_IS_APU)) {
1116		gmc_v6_0_enable_mc_mgcg(adev, gate);
1117		gmc_v6_0_enable_mc_ls(adev, gate);
1118	}
1119	gmc_v6_0_enable_bif_mgls(adev, gate);
1120	gmc_v6_0_enable_hdp_mgcg(adev, gate);
1121	gmc_v6_0_enable_hdp_ls(adev, gate);
1122
1123	return 0;
1124}
1125
1126static int gmc_v6_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
1127					  enum amd_powergating_state state)
1128{
1129	return 0;
1130}
1131
1132static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
1133	.name = "gmc_v6_0",
1134	.early_init = gmc_v6_0_early_init,
1135	.late_init = gmc_v6_0_late_init,
1136	.sw_init = gmc_v6_0_sw_init,
1137	.sw_fini = gmc_v6_0_sw_fini,
1138	.hw_init = gmc_v6_0_hw_init,
1139	.hw_fini = gmc_v6_0_hw_fini,
1140	.suspend = gmc_v6_0_suspend,
1141	.resume = gmc_v6_0_resume,
1142	.is_idle = gmc_v6_0_is_idle,
1143	.wait_for_idle = gmc_v6_0_wait_for_idle,
1144	.soft_reset = gmc_v6_0_soft_reset,
1145	.set_clockgating_state = gmc_v6_0_set_clockgating_state,
1146	.set_powergating_state = gmc_v6_0_set_powergating_state,
1147};
1148
1149static const struct amdgpu_gmc_funcs gmc_v6_0_gmc_funcs = {
1150	.flush_gpu_tlb = gmc_v6_0_flush_gpu_tlb,
1151	.emit_flush_gpu_tlb = gmc_v6_0_emit_flush_gpu_tlb,
1152	.set_prt = gmc_v6_0_set_prt,
1153	.get_vm_pde = gmc_v6_0_get_vm_pde,
1154	.get_vm_pte = gmc_v6_0_get_vm_pte,
1155	.get_vbios_fb_size = gmc_v6_0_get_vbios_fb_size,
1156};
1157
1158static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = {
1159	.set = gmc_v6_0_vm_fault_interrupt_state,
1160	.process = gmc_v6_0_process_interrupt,
1161};
1162
1163static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev)
1164{
1165	adev->gmc.gmc_funcs = &gmc_v6_0_gmc_funcs;
1166}
1167
1168static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1169{
1170	adev->gmc.vm_fault.num_types = 1;
1171	adev->gmc.vm_fault.funcs = &gmc_v6_0_irq_funcs;
1172}
1173
1174const struct amdgpu_ip_block_version gmc_v6_0_ip_block = {
1175	.type = AMD_IP_BLOCK_TYPE_GMC,
1176	.major = 6,
1177	.minor = 0,
1178	.rev = 0,
1179	.funcs = &gmc_v6_0_ip_funcs,
1180};