gmc_v11_0.c

   1/*
   2 * Copyright 2021 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23#include <linux/firmware.h>
  24#include <linux/pci.h>
  25
  26#include <drm/drm_cache.h>
  27
  28#include "amdgpu.h"
  29#include "amdgpu_atomfirmware.h"
  30#include "gmc_v11_0.h"
  31#include "umc_v8_10.h"
  32#include "athub/athub_3_0_0_sh_mask.h"
  33#include "athub/athub_3_0_0_offset.h"
  34#include "dcn/dcn_3_2_0_offset.h"
  35#include "dcn/dcn_3_2_0_sh_mask.h"
  36#include "oss/osssys_6_0_0_offset.h"
  37#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
  38#include "navi10_enum.h"
  39#include "soc15.h"
  40#include "soc15d.h"
  41#include "soc15_common.h"
  42#include "nbio_v4_3.h"
  43#include "gfxhub_v3_0.h"
  44#include "gfxhub_v3_0_3.h"
  45#include "gfxhub_v11_5_0.h"
  46#include "mmhub_v3_0.h"
  47#include "mmhub_v3_0_1.h"
  48#include "mmhub_v3_0_2.h"
  49#include "mmhub_v3_3.h"
  50#include "athub_v3_0.h"
  51
  52
  53static int gmc_v11_0_ecc_interrupt_state(struct amdgpu_device *adev,
  54					 struct amdgpu_irq_src *src,
  55					 unsigned int type,
  56					 enum amdgpu_interrupt_state state)
  57{
  58	return 0;
  59}
  60
  61static int
  62gmc_v11_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
  63				   struct amdgpu_irq_src *src, unsigned int type,
  64				   enum amdgpu_interrupt_state state)
  65{
  66	switch (state) {
  67	case AMDGPU_IRQ_STATE_DISABLE:
  68		/* MM HUB */
  69		amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false);
  70		/* GFX HUB */
  71		/* This works because this interrupt is only
  72		 * enabled at init/resume and disabled in
  73		 * fini/suspend, so the overall state doesn't
  74		 * change over the course of suspend/resume.
  75		 */
  76		if (!adev->in_s0ix && (adev->in_runpm || adev->in_suspend ||
  77							   amdgpu_in_reset(adev)))
  78			amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false);
  79		break;
  80	case AMDGPU_IRQ_STATE_ENABLE:
  81		/* MM HUB */
  82		amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true);
  83		/* GFX HUB */
  84		/* This works because this interrupt is only
  85		 * enabled at init/resume and disabled in
  86		 * fini/suspend, so the overall state doesn't
  87		 * change over the course of suspend/resume.
  88		 */
  89		if (!adev->in_s0ix)
  90			amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true);
  91		break;
  92	default:
  93		break;
  94	}
  95
  96	return 0;
  97}
  98
  99static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev,
 100				       struct amdgpu_irq_src *source,
 101				       struct amdgpu_iv_entry *entry)
 102{
 103	uint32_t vmhub_index = entry->client_id == SOC21_IH_CLIENTID_VMC ?
 104			       AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0);
 105	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index];
 106	bool retry_fault = !!(entry->src_data[1] &
 107			      AMDGPU_GMC9_FAULT_SOURCE_DATA_RETRY);
 108	bool write_fault = !!(entry->src_data[1] &
 109			      AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE);
 110	uint32_t status = 0;
 111	u64 addr;
 112
 113	addr = (u64)entry->src_data[0] << 12;
 114	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 115
 116	if (retry_fault) {
 117		/* Returning 1 here also prevents sending the IV to the KFD */
 118
 119		/* Process it only if it's the first fault for this address */
 120		if (entry->ih != &adev->irq.ih_soft &&
 121		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
 122					     entry->timestamp))
 123			return 1;
 124
 125		/* Delegate it to a different ring if the hardware hasn't
 126		 * already done it.
 127		 */
 128		if (entry->ih == &adev->irq.ih) {
 129			amdgpu_irq_delegate(adev, entry, 8);
 130			return 1;
 131		}
 132
 133		/* Try to handle the recoverable page faults by filling page
 134		 * tables
 135		 */
 136		if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr,
 137					   entry->timestamp, write_fault))
 138			return 1;
 139	}
 140
 141	if (!amdgpu_sriov_vf(adev)) {
 142		/*
 143		 * Issue a dummy read to wait for the status register to
 144		 * be updated to avoid reading an incorrect value due to
 145		 * the new fast GRBM interface.
 146		 */
 147		if (entry->vmid_src == AMDGPU_GFXHUB(0))
 148			RREG32(hub->vm_l2_pro_fault_status);
 149
 150		status = RREG32(hub->vm_l2_pro_fault_status);
 151		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
 152
 153		amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status,
 154					     entry->vmid_src ? AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0));
 155	}
 156
 157	if (printk_ratelimit()) {
 158		struct amdgpu_task_info *task_info;
 159
 160		dev_err(adev->dev,
 161			"[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",
 162			entry->vmid_src ? "mmhub" : "gfxhub",
 163			entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
 164		task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
 165		if (task_info) {
 166			amdgpu_vm_print_task_info(adev, task_info);
 167			amdgpu_vm_put_task_info(task_info);
 168		}
 169
 170		dev_err(adev->dev, "  in page starting at address 0x%016llx from client %d\n",
 171				addr, entry->client_id);
 172
 173		/* Only print L2 fault status if the status register could be read and
 174		 * contains useful information
 175		 */
 176		if (status != 0)
 177			hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
 178	}
 179
 180	return 0;
 181}
 182
 183static const struct amdgpu_irq_src_funcs gmc_v11_0_irq_funcs = {
 184	.set = gmc_v11_0_vm_fault_interrupt_state,
 185	.process = gmc_v11_0_process_interrupt,
 186};
 187
 188static const struct amdgpu_irq_src_funcs gmc_v11_0_ecc_funcs = {
 189	.set = gmc_v11_0_ecc_interrupt_state,
 190	.process = amdgpu_umc_process_ecc_irq,
 191};
 192
 193static void gmc_v11_0_set_irq_funcs(struct amdgpu_device *adev)
 194{
 195	adev->gmc.vm_fault.num_types = 1;
 196	adev->gmc.vm_fault.funcs = &gmc_v11_0_irq_funcs;
 197
 198	if (!amdgpu_sriov_vf(adev)) {
 199		adev->gmc.ecc_irq.num_types = 1;
 200		adev->gmc.ecc_irq.funcs = &gmc_v11_0_ecc_funcs;
 201	}
 202}
 203
 204/**
 205 * gmc_v11_0_use_invalidate_semaphore - judge whether to use semaphore
 206 *
 207 * @adev: amdgpu_device pointer
 208 * @vmhub: vmhub type
 209 *
 210 */
 211static bool gmc_v11_0_use_invalidate_semaphore(struct amdgpu_device *adev,
 212				       uint32_t vmhub)
 213{
 214	return ((vmhub == AMDGPU_MMHUB0(0)) &&
 215		(!amdgpu_sriov_vf(adev)));
 216}
 217
 218static bool gmc_v11_0_get_vmid_pasid_mapping_info(
 219					struct amdgpu_device *adev,
 220					uint8_t vmid, uint16_t *p_pasid)
 221{
 222	*p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff;
 223
 224	return !!(*p_pasid);
 225}
 226
 227/**
 228 * gmc_v11_0_flush_gpu_tlb - gart tlb flush callback
 229 *
 230 * @adev: amdgpu_device pointer
 231 * @vmid: vm instance to flush
 232 * @vmhub: which hub to flush
 233 * @flush_type: the flush type
 234 *
 235 * Flush the TLB for the requested page table.
 236 */
 237static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 238					uint32_t vmhub, uint32_t flush_type)
 239{
 240	bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(adev, vmhub);
 241	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
 242	u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
 243	/* Use register 17 for GART */
 244	const unsigned int eng = 17;
 245	unsigned char hub_ip;
 246	u32 sem, req, ack;
 247	unsigned int i;
 248	u32 tmp;
 249
 250	if ((vmhub == AMDGPU_GFXHUB(0)) && !adev->gfx.is_poweron)
 251		return;
 252
 253	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;
 254	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
 255	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
 256
 257	/* flush hdp cache */
 258	amdgpu_device_flush_hdp(adev, NULL);
 259
 260	/* This is necessary for SRIOV as well as for GFXOFF to function
 261	 * properly under bare metal
 262	 */
 263	if ((adev->gfx.kiq[0].ring.sched.ready || adev->mes.ring[0].sched.ready) &&
 264	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
 265		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
 266						 1 << vmid, GET_INST(GC, 0));
 267		return;
 268	}
 269
 270	/* This path is needed before KIQ/MES/GFXOFF are set up */
 271	hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ? GC_HWIP : MMHUB_HWIP;
 272
 273	spin_lock(&adev->gmc.invalidate_lock);
 274	/*
 275	 * It may lose gpuvm invalidate acknowldege state across power-gating
 276	 * off cycle, add semaphore acquire before invalidation and semaphore
 277	 * release after invalidation to avoid entering power gated state
 278	 * to WA the Issue
 279	 */
 280
 281	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 282	if (use_semaphore) {
 283		for (i = 0; i < adev->usec_timeout; i++) {
 284			/* a read return value of 1 means semaphore acuqire */
 285			tmp = RREG32_RLC_NO_KIQ(sem, hub_ip);
 286			if (tmp & 0x1)
 287				break;
 288			udelay(1);
 289		}
 290
 291		if (i >= adev->usec_timeout)
 292			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
 293	}
 294
 295	WREG32_RLC_NO_KIQ(req, inv_req, hub_ip);
 296
 297	/* Wait for ACK with a delay.*/
 298	for (i = 0; i < adev->usec_timeout; i++) {
 299		tmp = RREG32_RLC_NO_KIQ(ack, hub_ip);
 300		tmp &= 1 << vmid;
 301		if (tmp)
 302			break;
 303
 304		udelay(1);
 305	}
 306
 307	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 308	if (use_semaphore)
 309		WREG32_RLC_NO_KIQ(sem, 0, hub_ip);
 310
 311	/* Issue additional private vm invalidation to MMHUB */
 312	if ((vmhub != AMDGPU_GFXHUB(0)) &&
 313	    (hub->vm_l2_bank_select_reserved_cid2) &&
 314		!amdgpu_sriov_vf(adev)) {
 315		inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
 316		/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
 317		inv_req |= (1 << 25);
 318		/* Issue private invalidation */
 319		WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
 320		/* Read back to ensure invalidation is done*/
 321		RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
 322	}
 323
 324	spin_unlock(&adev->gmc.invalidate_lock);
 325
 326	if (i >= adev->usec_timeout)
 327		dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
 328}
 329
 330/**
 331 * gmc_v11_0_flush_gpu_tlb_pasid - tlb flush via pasid
 332 *
 333 * @adev: amdgpu_device pointer
 334 * @pasid: pasid to be flush
 335 * @flush_type: the flush type
 336 * @all_hub: flush all hubs
 337 * @inst: is used to select which instance of KIQ to use for the invalidation
 338 *
 339 * Flush the TLB for the requested pasid.
 340 */
 341static void gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 342					  uint16_t pasid, uint32_t flush_type,
 343					  bool all_hub, uint32_t inst)
 344{
 345	uint16_t queried;
 346	int vmid, i;
 347
 348	for (vmid = 1; vmid < 16; vmid++) {
 349		bool valid;
 350
 351		valid = gmc_v11_0_get_vmid_pasid_mapping_info(adev, vmid,
 352							      &queried);
 353		if (!valid || queried != pasid)
 354			continue;
 355
 356		if (all_hub) {
 357			for_each_set_bit(i, adev->vmhubs_mask,
 358					 AMDGPU_MAX_VMHUBS)
 359				gmc_v11_0_flush_gpu_tlb(adev, vmid, i,
 360							flush_type);
 361		} else {
 362			gmc_v11_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0),
 363						flush_type);
 364		}
 365	}
 366}
 367
 368static uint64_t gmc_v11_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 369					     unsigned int vmid, uint64_t pd_addr)
 370{
 371	bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
 372	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 373	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
 374	unsigned int eng = ring->vm_inv_eng;
 375
 376	/*
 377	 * It may lose gpuvm invalidate acknowldege state across power-gating
 378	 * off cycle, add semaphore acquire before invalidation and semaphore
 379	 * release after invalidation to avoid entering power gated state
 380	 * to WA the Issue
 381	 */
 382
 383	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 384	if (use_semaphore)
 385		/* a read return value of 1 means semaphore acuqire */
 386		amdgpu_ring_emit_reg_wait(ring,
 387					  hub->vm_inv_eng0_sem +
 388					  hub->eng_distance * eng, 0x1, 0x1);
 389
 390	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
 391			      (hub->ctx_addr_distance * vmid),
 392			      lower_32_bits(pd_addr));
 393
 394	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
 395			      (hub->ctx_addr_distance * vmid),
 396			      upper_32_bits(pd_addr));
 397
 398	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
 399					    hub->eng_distance * eng,
 400					    hub->vm_inv_eng0_ack +
 401					    hub->eng_distance * eng,
 402					    req, 1 << vmid);
 403
 404	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
 405	if (use_semaphore)
 406		/*
 407		 * add semaphore release after invalidation,
 408		 * write with 0 means semaphore release
 409		 */
 410		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
 411				      hub->eng_distance * eng, 0);
 412
 413	return pd_addr;
 414}
 415
 416static void gmc_v11_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
 417					 unsigned int pasid)
 418{
 419	struct amdgpu_device *adev = ring->adev;
 420	uint32_t reg;
 421
 422	if (ring->vm_hub == AMDGPU_GFXHUB(0))
 423		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
 424	else
 425		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;
 426
 427	amdgpu_ring_emit_wreg(ring, reg, pasid);
 428}
 429
 430/*
 431 * PTE format:
 432 * 63:59 reserved
 433 * 58:57 reserved
 434 * 56 F
 435 * 55 L
 436 * 54 reserved
 437 * 53:52 SW
 438 * 51 T
 439 * 50:48 mtype
 440 * 47:12 4k physical page base address
 441 * 11:7 fragment
 442 * 6 write
 443 * 5 read
 444 * 4 exe
 445 * 3 Z
 446 * 2 snooped
 447 * 1 system
 448 * 0 valid
 449 *
 450 * PDE format:
 451 * 63:59 block fragment size
 452 * 58:55 reserved
 453 * 54 P
 454 * 53:48 reserved
 455 * 47:6 physical base address of PD or PTE
 456 * 5:3 reserved
 457 * 2 C
 458 * 1 system
 459 * 0 valid
 460 */
 461
 462static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level,
 463				 uint64_t *addr, uint64_t *flags)
 464{
 465	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
 466		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
 467	BUG_ON(*addr & 0xFFFF00000000003FULL);
 468
 469	if (!adev->gmc.translate_further)
 470		return;
 471
 472	if (level == AMDGPU_VM_PDB1) {
 473		/* Set the block fragment size */
 474		if (!(*flags & AMDGPU_PDE_PTE))
 475			*flags |= AMDGPU_PDE_BFS(0x9);
 476
 477	} else if (level == AMDGPU_VM_PDB0) {
 478		if (*flags & AMDGPU_PDE_PTE)
 479			*flags &= ~AMDGPU_PDE_PTE;
 480		else
 481			*flags |= AMDGPU_PTE_TF;
 482	}
 483}
 484
 485static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev,
 486				 struct amdgpu_vm *vm,
 487				 struct amdgpu_bo *bo,
 488				 uint32_t vm_flags,
 489				 uint64_t *flags)
 490{
 491	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
 492		*flags |= AMDGPU_PTE_EXECUTABLE;
 493	else
 494		*flags &= ~AMDGPU_PTE_EXECUTABLE;
 495
 496	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
 497	case AMDGPU_VM_MTYPE_DEFAULT:
 498	case AMDGPU_VM_MTYPE_NC:
 499	default:
 500		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_NC);
 501		break;
 502	case AMDGPU_VM_MTYPE_WC:
 503		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_WC);
 504		break;
 505	case AMDGPU_VM_MTYPE_CC:
 506		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_CC);
 507		break;
 508	case AMDGPU_VM_MTYPE_UC:
 509		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
 510		break;
 511	}
 512
 513	if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
 514		*flags |= AMDGPU_PTE_NOALLOC;
 515	else
 516		*flags &= ~AMDGPU_PTE_NOALLOC;
 517
 518	if (vm_flags & AMDGPU_VM_PAGE_PRT) {
 519		*flags |= AMDGPU_PTE_PRT;
 520		*flags |= AMDGPU_PTE_SNOOPED;
 521		*flags |= AMDGPU_PTE_LOG;
 522		*flags |= AMDGPU_PTE_SYSTEM;
 523		*flags &= ~AMDGPU_PTE_VALID;
 524	}
 525
 526	if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
 527			       AMDGPU_GEM_CREATE_EXT_COHERENT |
 528			       AMDGPU_GEM_CREATE_UNCACHED))
 529		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
 530}
 531
 532static unsigned int gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev)
 533{
 534	u32 d1vga_control = RREG32_SOC15(DCE, 0, regD1VGA_CONTROL);
 535	unsigned int size;
 536
 537	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
 538		size = AMDGPU_VBIOS_VGA_ALLOCATION;
 539	} else {
 540		u32 viewport;
 541		u32 pitch;
 542
 543		viewport = RREG32_SOC15(DCE, 0, regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
 544		pitch = RREG32_SOC15(DCE, 0, regHUBPREQ0_DCSURF_SURFACE_PITCH);
 545		size = (REG_GET_FIELD(viewport,
 546					HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
 547				REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) *
 548				4);
 549	}
 550
 551	return size;
 552}
 553
 554static const struct amdgpu_gmc_funcs gmc_v11_0_gmc_funcs = {
 555	.flush_gpu_tlb = gmc_v11_0_flush_gpu_tlb,
 556	.flush_gpu_tlb_pasid = gmc_v11_0_flush_gpu_tlb_pasid,
 557	.emit_flush_gpu_tlb = gmc_v11_0_emit_flush_gpu_tlb,
 558	.emit_pasid_mapping = gmc_v11_0_emit_pasid_mapping,
 559	.get_vm_pde = gmc_v11_0_get_vm_pde,
 560	.get_vm_pte = gmc_v11_0_get_vm_pte,
 561	.get_vbios_fb_size = gmc_v11_0_get_vbios_fb_size,
 562};
 563
 564static void gmc_v11_0_set_gmc_funcs(struct amdgpu_device *adev)
 565{
 566	adev->gmc.gmc_funcs = &gmc_v11_0_gmc_funcs;
 567}
 568
 569static void gmc_v11_0_set_umc_funcs(struct amdgpu_device *adev)
 570{
 571	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
 572	case IP_VERSION(8, 10, 0):
 573		adev->umc.channel_inst_num = UMC_V8_10_CHANNEL_INSTANCE_NUM;
 574		adev->umc.umc_inst_num = UMC_V8_10_UMC_INSTANCE_NUM;
 575		adev->umc.max_ras_err_cnt_per_query = UMC_V8_10_TOTAL_CHANNEL_NUM(adev);
 576		adev->umc.channel_offs = UMC_V8_10_PER_CHANNEL_OFFSET;
 577		adev->umc.retire_unit = UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM;
 578		if (adev->umc.node_inst_num == 4)
 579			adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl_ext0[0][0][0];
 580		else
 581			adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl[0][0][0];
 582		adev->umc.ras = &umc_v8_10_ras;
 583		break;
 584	case IP_VERSION(8, 11, 0):
 585		break;
 586	default:
 587		break;
 588	}
 589}
 590
 591
 592static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev)
 593{
 594	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
 595	case IP_VERSION(3, 0, 1):
 596		adev->mmhub.funcs = &mmhub_v3_0_1_funcs;
 597		break;
 598	case IP_VERSION(3, 0, 2):
 599		adev->mmhub.funcs = &mmhub_v3_0_2_funcs;
 600		break;
 601	case IP_VERSION(3, 3, 0):
 602	case IP_VERSION(3, 3, 1):
 603	case IP_VERSION(3, 3, 2):
 604		adev->mmhub.funcs = &mmhub_v3_3_funcs;
 605		break;
 606	default:
 607		adev->mmhub.funcs = &mmhub_v3_0_funcs;
 608		break;
 609	}
 610}
 611
 612static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev)
 613{
 614	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 615	case IP_VERSION(11, 0, 3):
 616		adev->gfxhub.funcs = &gfxhub_v3_0_3_funcs;
 617		break;
 618	case IP_VERSION(11, 5, 0):
 619	case IP_VERSION(11, 5, 1):
 620	case IP_VERSION(11, 5, 2):
 621	case IP_VERSION(11, 5, 3):
 622		adev->gfxhub.funcs = &gfxhub_v11_5_0_funcs;
 623		break;
 624	default:
 625		adev->gfxhub.funcs = &gfxhub_v3_0_funcs;
 626		break;
 627	}
 628}
 629
 630static int gmc_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 631{
 632	struct amdgpu_device *adev = ip_block->adev;
 633
 634	gmc_v11_0_set_gfxhub_funcs(adev);
 635	gmc_v11_0_set_mmhub_funcs(adev);
 636	gmc_v11_0_set_gmc_funcs(adev);
 637	gmc_v11_0_set_irq_funcs(adev);
 638	gmc_v11_0_set_umc_funcs(adev);
 639
 640	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
 641	adev->gmc.shared_aperture_end =
 642		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
 643	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
 644	adev->gmc.private_aperture_end =
 645		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
 646	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
 647
 648	return 0;
 649}
 650
 651static int gmc_v11_0_late_init(struct amdgpu_ip_block *ip_block)
 652{
 653	struct amdgpu_device *adev = ip_block->adev;
 654	int r;
 655
 656	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
 657	if (r)
 658		return r;
 659
 660	r = amdgpu_gmc_ras_late_init(adev);
 661	if (r)
 662		return r;
 663
 664	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
 665}
 666
 667static void gmc_v11_0_vram_gtt_location(struct amdgpu_device *adev,
 668					struct amdgpu_gmc *mc)
 669{
 670	u64 base = 0;
 671
 672	base = adev->mmhub.funcs->get_fb_location(adev);
 673
 674	amdgpu_gmc_set_agp_default(adev, mc);
 675	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
 676	amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_HIGH);
 677	if (!amdgpu_sriov_vf(adev) &&
 678	    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 5, 0)) &&
 679	    (amdgpu_agp == 1))
 680		amdgpu_gmc_agp_location(adev, mc);
 681
 682	/* base offset of vram pages */
 683	if (amdgpu_sriov_vf(adev))
 684		adev->vm_manager.vram_base_offset = 0;
 685	else
 686		adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev);
 687}
 688
 689/**
 690 * gmc_v11_0_mc_init - initialize the memory controller driver params
 691 *
 692 * @adev: amdgpu_device pointer
 693 *
 694 * Look up the amount of vram, vram width, and decide how to place
 695 * vram and gart within the GPU's physical address space.
 696 * Returns 0 for success.
 697 */
 698static int gmc_v11_0_mc_init(struct amdgpu_device *adev)
 699{
 700	int r;
 701
 702	/* size in MB on si */
 703	adev->gmc.mc_vram_size =
 704		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
 705	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
 706
 707	if (!(adev->flags & AMD_IS_APU)) {
 708		r = amdgpu_device_resize_fb_bar(adev);
 709		if (r)
 710			return r;
 711	}
 712	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
 713	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
 714
 715#ifdef CONFIG_X86_64
 716	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
 717		adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev);
 718		adev->gmc.aper_size = adev->gmc.real_vram_size;
 719	}
 720#endif
 721	/* In case the PCI BAR is larger than the actual amount of vram */
 722	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 723	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
 724		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
 725
 726	/* set the gart size */
 727	if (amdgpu_gart_size == -1)
 728		adev->gmc.gart_size = 512ULL << 20;
 729	else
 730		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
 731
 732	gmc_v11_0_vram_gtt_location(adev, &adev->gmc);
 733
 734	return 0;
 735}
 736
 737static int gmc_v11_0_gart_init(struct amdgpu_device *adev)
 738{
 739	int r;
 740
 741	if (adev->gart.bo) {
 742		WARN(1, "PCIE GART already initialized\n");
 743		return 0;
 744	}
 745
 746	/* Initialize common gart structure */
 747	r = amdgpu_gart_init(adev);
 748	if (r)
 749		return r;
 750
 751	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
 752	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC) |
 753				 AMDGPU_PTE_EXECUTABLE;
 754
 755	return amdgpu_gart_table_vram_alloc(adev);
 756}
 757
 758static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 759{
 760	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
 761	struct amdgpu_device *adev = ip_block->adev;
 762
 763	adev->mmhub.funcs->init(adev);
 764
 765	adev->gfxhub.funcs->init(adev);
 766
 767	spin_lock_init(&adev->gmc.invalidate_lock);
 768
 769	r = amdgpu_atomfirmware_get_vram_info(adev,
 770					      &vram_width, &vram_type, &vram_vendor);
 771	adev->gmc.vram_width = vram_width;
 772
 773	adev->gmc.vram_type = vram_type;
 774	adev->gmc.vram_vendor = vram_vendor;
 775
 776	/* The mall_size is already calculated as mall_size_per_umc * num_umc.
 777	 * However, for gfx1151, which features a 2-to-1 UMC mapping,
 778	 * the result must be multiplied by 2 to determine the actual mall size.
 779	 */
 780	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 781	case IP_VERSION(11, 5, 1):
 782		adev->gmc.mall_size *= 2;
 783		break;
 784	default:
 785		break;
 786	}
 787
 788	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 789	case IP_VERSION(11, 0, 0):
 790	case IP_VERSION(11, 0, 1):
 791	case IP_VERSION(11, 0, 2):
 792	case IP_VERSION(11, 0, 3):
 793	case IP_VERSION(11, 0, 4):
 794	case IP_VERSION(11, 5, 0):
 795	case IP_VERSION(11, 5, 1):
 796	case IP_VERSION(11, 5, 2):
 797	case IP_VERSION(11, 5, 3):
 798		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
 799		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
 800		/*
 801		 * To fulfill 4-level page support,
 802		 * vm size is 256TB (48bit), maximum size,
 803		 * block size 512 (9bit)
 804		 */
 805		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
 806		break;
 807	default:
 808		break;
 809	}
 810
 811	/* This interrupt is VMC page fault.*/
 812	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VMC,
 813			      VMC_1_0__SRCID__VM_FAULT,
 814			      &adev->gmc.vm_fault);
 815
 816	if (r)
 817		return r;
 818
 819	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
 820			      UTCL2_1_0__SRCID__FAULT,
 821			      &adev->gmc.vm_fault);
 822	if (r)
 823		return r;
 824
 825	if (!amdgpu_sriov_vf(adev)) {
 826		/* interrupt sent to DF. */
 827		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0,
 828				      &adev->gmc.ecc_irq);
 829		if (r)
 830			return r;
 831	}
 832
 833	/*
 834	 * Set the internal MC address mask This is the max address of the GPU's
 835	 * internal address space.
 836	 */
 837	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
 838
 839	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
 840	if (r) {
 841		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
 842		return r;
 843	}
 844
 845	adev->need_swiotlb = drm_need_swiotlb(44);
 846
 847	r = gmc_v11_0_mc_init(adev);
 848	if (r)
 849		return r;
 850
 851	amdgpu_gmc_get_vbios_allocations(adev);
 852
 853	/* Memory manager */
 854	r = amdgpu_bo_init(adev);
 855	if (r)
 856		return r;
 857
 858	r = gmc_v11_0_gart_init(adev);
 859	if (r)
 860		return r;
 861
 862	/*
 863	 * number of VMs
 864	 * VMID 0 is reserved for System
 865	 * amdgpu graphics/compute will use VMIDs 1-7
 866	 * amdkfd will use VMIDs 8-15
 867	 */
 868	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
 869
 870	amdgpu_vm_manager_init(adev);
 871
 872	r = amdgpu_gmc_ras_sw_init(adev);
 873	if (r)
 874		return r;
 875
 876	return 0;
 877}
 878
 879/**
 880 * gmc_v11_0_gart_fini - vm fini callback
 881 *
 882 * @adev: amdgpu_device pointer
 883 *
 884 * Tears down the driver GART/VM setup (CIK).
 885 */
 886static void gmc_v11_0_gart_fini(struct amdgpu_device *adev)
 887{
 888	amdgpu_gart_table_vram_free(adev);
 889}
 890
 891static int gmc_v11_0_sw_fini(struct amdgpu_ip_block *ip_block)
 892{
 893	struct amdgpu_device *adev = ip_block->adev;
 894
 895	amdgpu_vm_manager_fini(adev);
 896	gmc_v11_0_gart_fini(adev);
 897	amdgpu_gem_force_release(adev);
 898	amdgpu_bo_fini(adev);
 899
 900	return 0;
 901}
 902
 903static void gmc_v11_0_init_golden_registers(struct amdgpu_device *adev)
 904{
 905	if (amdgpu_sriov_vf(adev)) {
 906		struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
 907
 908		WREG32(hub->vm_contexts_disable, 0);
 909		return;
 910	}
 911}
 912
 913/**
 914 * gmc_v11_0_gart_enable - gart enable
 915 *
 916 * @adev: amdgpu_device pointer
 917 */
 918static int gmc_v11_0_gart_enable(struct amdgpu_device *adev)
 919{
 920	int r;
 921	bool value;
 922
 923	if (adev->gart.bo == NULL) {
 924		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
 925		return -EINVAL;
 926	}
 927
 928	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
 929
 930	r = adev->mmhub.funcs->gart_enable(adev);
 931	if (r)
 932		return r;
 933
 934	/* Flush HDP after it is initialized */
 935	amdgpu_device_flush_hdp(adev, NULL);
 936
 937	value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS;
 938
 939	adev->mmhub.funcs->set_fault_enable_default(adev, value);
 940	gmc_v11_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0);
 941
 942	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 943		 (unsigned int)(adev->gmc.gart_size >> 20),
 944		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
 945
 946	return 0;
 947}
 948
 949static int gmc_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 950{
 951	struct amdgpu_device *adev = ip_block->adev;
 952	int r;
 953
 954	adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
 955
 956	/* The sequence of these two function calls matters.*/
 957	gmc_v11_0_init_golden_registers(adev);
 958
 959	r = gmc_v11_0_gart_enable(adev);
 960	if (r)
 961		return r;
 962
 963	if (adev->umc.funcs && adev->umc.funcs->init_registers)
 964		adev->umc.funcs->init_registers(adev);
 965
 966	return 0;
 967}
 968
 969/**
 970 * gmc_v11_0_gart_disable - gart disable
 971 *
 972 * @adev: amdgpu_device pointer
 973 *
 974 * This disables all VM page table.
 975 */
 976static void gmc_v11_0_gart_disable(struct amdgpu_device *adev)
 977{
 978	adev->mmhub.funcs->gart_disable(adev);
 979}
 980
 981static int gmc_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
 982{
 983	struct amdgpu_device *adev = ip_block->adev;
 984
 985	if (amdgpu_sriov_vf(adev)) {
 986		/* full access mode, so don't touch any GMC register */
 987		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
 988		return 0;
 989	}
 990
 991	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
 992
 993	if (adev->gmc.ecc_irq.funcs &&
 994		amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))
 995		amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
 996
 997	gmc_v11_0_gart_disable(adev);
 998
 999	return 0;
1000}
1001
1002static int gmc_v11_0_suspend(struct amdgpu_ip_block *ip_block)
1003{
1004	gmc_v11_0_hw_fini(ip_block);
1005
1006	return 0;
1007}
1008
1009static int gmc_v11_0_resume(struct amdgpu_ip_block *ip_block)
1010{
1011	int r;
1012
1013	r = gmc_v11_0_hw_init(ip_block);
1014	if (r)
1015		return r;
1016
1017	amdgpu_vmid_reset_all(ip_block->adev);
1018
1019	return 0;
1020}
1021
1022static bool gmc_v11_0_is_idle(struct amdgpu_ip_block *ip_block)
1023{
1024	/* MC is always ready in GMC v11.*/
1025	return true;
1026}
1027
1028static int gmc_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
1029{
1030	/* There is no need to wait for MC idle in GMC v11.*/
1031	return 0;
1032}
1033
1034static int gmc_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1035					   enum amd_clockgating_state state)
1036{
1037	int r;
1038	struct amdgpu_device *adev = ip_block->adev;
1039
1040	r = adev->mmhub.funcs->set_clockgating(adev, state);
1041	if (r)
1042		return r;
1043
1044	return athub_v3_0_set_clockgating(adev, state);
1045}
1046
1047static void gmc_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
1048{
1049	struct amdgpu_device *adev = ip_block->adev;
1050
1051	adev->mmhub.funcs->get_clockgating(adev, flags);
1052
1053	athub_v3_0_get_clockgating(adev, flags);
1054}
1055
1056static int gmc_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
1057					   enum amd_powergating_state state)
1058{
1059	return 0;
1060}
1061
1062const struct amd_ip_funcs gmc_v11_0_ip_funcs = {
1063	.name = "gmc_v11_0",
1064	.early_init = gmc_v11_0_early_init,
1065	.sw_init = gmc_v11_0_sw_init,
1066	.hw_init = gmc_v11_0_hw_init,
1067	.late_init = gmc_v11_0_late_init,
1068	.sw_fini = gmc_v11_0_sw_fini,
1069	.hw_fini = gmc_v11_0_hw_fini,
1070	.suspend = gmc_v11_0_suspend,
1071	.resume = gmc_v11_0_resume,
1072	.is_idle = gmc_v11_0_is_idle,
1073	.wait_for_idle = gmc_v11_0_wait_for_idle,
1074	.set_clockgating_state = gmc_v11_0_set_clockgating_state,
1075	.set_powergating_state = gmc_v11_0_set_powergating_state,
1076	.get_clockgating_state = gmc_v11_0_get_clockgating_state,
1077};
1078
1079const struct amdgpu_ip_block_version gmc_v11_0_ip_block = {
1080	.type = AMD_IP_BLOCK_TYPE_GMC,
1081	.major = 11,
1082	.minor = 0,
1083	.rev = 0,
1084	.funcs = &gmc_v11_0_ip_funcs,
1085};