amdgpu_gmc.c

   1/*
   2 * Copyright 2018 Advanced Micro Devices, Inc.
   3 * All Rights Reserved.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the
   7 * "Software"), to deal in the Software without restriction, including
   8 * without limitation the rights to use, copy, modify, merge, publish,
   9 * distribute, sub license, and/or sell copies of the Software, and to
  10 * permit persons to whom the Software is furnished to do so, subject to
  11 * the following conditions:
  12 *
  13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20 *
  21 * The above copyright notice and this permission notice (including the
  22 * next paragraph) shall be included in all copies or substantial portions
  23 * of the Software.
  24 *
  25 */
  26
  27#include <linux/io-64-nonatomic-lo-hi.h>
  28#ifdef CONFIG_X86
  29#include <asm/hypervisor.h>
  30#endif
  31
  32#include "amdgpu.h"
  33#include "amdgpu_gmc.h"
  34#include "amdgpu_ras.h"
  35#include "amdgpu_reset.h"
  36#include "amdgpu_xgmi.h"
  37
  38#include <drm/drm_drv.h>
  39#include <drm/ttm/ttm_tt.h>
  40
  41static const u64 four_gb = 0x100000000ULL;
  42
  43bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev)
  44{
  45	return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev);
  46}
  47
  48/**
  49 * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0
  50 *
  51 * @adev: amdgpu_device pointer
  52 *
  53 * Allocate video memory for pdb0 and map it for CPU access
  54 * Returns 0 for success, error for failure.
  55 */
  56int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev)
  57{
  58	int r;
  59	struct amdgpu_bo_param bp;
  60	u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
  61	uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21;
  62	uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift;
  63
  64	memset(&bp, 0, sizeof(bp));
  65	bp.size = PAGE_ALIGN((npdes + 1) * 8);
  66	bp.byte_align = PAGE_SIZE;
  67	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
  68	bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
  69		AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
  70	bp.type = ttm_bo_type_kernel;
  71	bp.resv = NULL;
  72	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
  73
  74	r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo);
  75	if (r)
  76		return r;
  77
  78	r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false);
  79	if (unlikely(r != 0))
  80		goto bo_reserve_failure;
  81
  82	r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM);
  83	if (r)
  84		goto bo_pin_failure;
  85	r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0);
  86	if (r)
  87		goto bo_kmap_failure;
  88
  89	amdgpu_bo_unreserve(adev->gmc.pdb0_bo);
  90	return 0;
  91
  92bo_kmap_failure:
  93	amdgpu_bo_unpin(adev->gmc.pdb0_bo);
  94bo_pin_failure:
  95	amdgpu_bo_unreserve(adev->gmc.pdb0_bo);
  96bo_reserve_failure:
  97	amdgpu_bo_unref(&adev->gmc.pdb0_bo);
  98	return r;
  99}
 100
 101/**
 102 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO
 103 *
 104 * @bo: the BO to get the PDE for
 105 * @level: the level in the PD hirarchy
 106 * @addr: resulting addr
 107 * @flags: resulting flags
 108 *
 109 * Get the address and flags to be used for a PDE (Page Directory Entry).
 110 */
 111void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
 112			       uint64_t *addr, uint64_t *flags)
 113{
 114	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 115
 116	switch (bo->tbo.resource->mem_type) {
 117	case TTM_PL_TT:
 118		*addr = bo->tbo.ttm->dma_address[0];
 119		break;
 120	case TTM_PL_VRAM:
 121		*addr = amdgpu_bo_gpu_offset(bo);
 122		break;
 123	default:
 124		*addr = 0;
 125		break;
 126	}
 127	*flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, bo->tbo.resource);
 128	amdgpu_gmc_get_vm_pde(adev, level, addr, flags);
 129}
 130
 131/*
 132 * amdgpu_gmc_pd_addr - return the address of the root directory
 133 */
 134uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo)
 135{
 136	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 137	uint64_t pd_addr;
 138
 139	/* TODO: move that into ASIC specific code */
 140	if (adev->asic_type >= CHIP_VEGA10) {
 141		uint64_t flags = AMDGPU_PTE_VALID;
 142
 143		amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags);
 144		pd_addr |= flags;
 145	} else {
 146		pd_addr = amdgpu_bo_gpu_offset(bo);
 147	}
 148	return pd_addr;
 149}
 150
 151/**
 152 * amdgpu_gmc_set_pte_pde - update the page tables using CPU
 153 *
 154 * @adev: amdgpu_device pointer
 155 * @cpu_pt_addr: cpu address of the page table
 156 * @gpu_page_idx: entry in the page table to update
 157 * @addr: dst addr to write into pte/pde
 158 * @flags: access flags
 159 *
 160 * Update the page tables using CPU.
 161 */
 162int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
 163				uint32_t gpu_page_idx, uint64_t addr,
 164				uint64_t flags)
 165{
 166	void __iomem *ptr = (void *)cpu_pt_addr;
 167	uint64_t value;
 168
 169	/*
 170	 * The following is for PTE only. GART does not have PDEs.
 171	*/
 172	value = addr & 0x0000FFFFFFFFF000ULL;
 173	value |= flags;
 174	writeq(value, ptr + (gpu_page_idx * 8));
 175
 176	return 0;
 177}
 178
 179/**
 180 * amdgpu_gmc_agp_addr - return the address in the AGP address space
 181 *
 182 * @bo: TTM BO which needs the address, must be in GTT domain
 183 *
 184 * Tries to figure out how to access the BO through the AGP aperture. Returns
 185 * AMDGPU_BO_INVALID_OFFSET if that is not possible.
 186 */
 187uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo)
 188{
 189	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 190
 191	if (!bo->ttm)
 192		return AMDGPU_BO_INVALID_OFFSET;
 193
 194	if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached)
 195		return AMDGPU_BO_INVALID_OFFSET;
 196
 197	if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size)
 198		return AMDGPU_BO_INVALID_OFFSET;
 199
 200	return adev->gmc.agp_start + bo->ttm->dma_address[0];
 201}
 202
 203/**
 204 * amdgpu_gmc_vram_location - try to find VRAM location
 205 *
 206 * @adev: amdgpu device structure holding all necessary information
 207 * @mc: memory controller structure holding memory information
 208 * @base: base address at which to put VRAM
 209 *
 210 * Function will try to place VRAM at base address provided
 211 * as parameter.
 212 */
 213void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
 214			      u64 base)
 215{
 216	uint64_t vis_limit = (uint64_t)amdgpu_vis_vram_limit << 20;
 217	uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 218
 219	mc->vram_start = base;
 220	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 221	if (limit < mc->real_vram_size)
 222		mc->real_vram_size = limit;
 223
 224	if (vis_limit && vis_limit < mc->visible_vram_size)
 225		mc->visible_vram_size = vis_limit;
 226
 227	if (mc->real_vram_size < mc->visible_vram_size)
 228		mc->visible_vram_size = mc->real_vram_size;
 229
 230	if (mc->xgmi.num_physical_nodes == 0) {
 231		mc->fb_start = mc->vram_start;
 232		mc->fb_end = mc->vram_end;
 233	}
 234	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 235			mc->mc_vram_size >> 20, mc->vram_start,
 236			mc->vram_end, mc->real_vram_size >> 20);
 237}
 238
 239/** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture
 240 *
 241 * @adev: amdgpu device structure holding all necessary information
 242 * @mc: memory controller structure holding memory information
 243 *
 244 * This function is only used if use GART for FB translation. In such
 245 * case, we use sysvm aperture (vmid0 page tables) for both vram
 246 * and gart (aka system memory) access.
 247 *
 248 * GPUVM (and our organization of vmid0 page tables) require sysvm
 249 * aperture to be placed at a location aligned with 8 times of native
 250 * page size. For example, if vm_context0_cntl.page_table_block_size
 251 * is 12, then native page size is 8G (2M*2^12), sysvm should start
 252 * with a 64G aligned address. For simplicity, we just put sysvm at
 253 * address 0. So vram start at address 0 and gart is right after vram.
 254 */
 255void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
 256{
 257	u64 hive_vram_start = 0;
 258	u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1;
 259	mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id;
 260	mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1;
 261	/* node_segment_size may not 4GB aligned on SRIOV, align up is needed. */
 262	mc->gart_start = ALIGN(hive_vram_end + 1, four_gb);
 263	mc->gart_end = mc->gart_start + mc->gart_size - 1;
 264	if (amdgpu_virt_xgmi_migrate_enabled(adev)) {
 265		/* set mc->vram_start to 0 to switch the returned GPU address of
 266		 * amdgpu_bo_create_reserved() from FB aperture to GART aperture.
 267		 */
 268		mc->vram_start = 0;
 269		mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 270		mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size);
 271	} else {
 272		mc->fb_start = hive_vram_start;
 273		mc->fb_end = hive_vram_end;
 274	}
 275	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 276			mc->mc_vram_size >> 20, mc->vram_start,
 277			mc->vram_end, mc->real_vram_size >> 20);
 278	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
 279			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 280}
 281
 282/**
 283 * amdgpu_gmc_gart_location - try to find GART location
 284 *
 285 * @adev: amdgpu device structure holding all necessary information
 286 * @mc: memory controller structure holding memory information
 287 * @gart_placement: GART placement policy with respect to VRAM
 288 *
 289 * Function will try to place GART before or after VRAM.
 290 * If GART size is bigger than space left then we ajust GART size.
 291 * Thus function will never fails.
 292 */
 293void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
 294			      enum amdgpu_gart_placement gart_placement)
 295{
 296	u64 size_af, size_bf;
 297	/*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/
 298	u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1);
 299
 300	/* VCE doesn't like it when BOs cross a 4GB segment, so align
 301	 * the GART base on a 4GB boundary as well.
 302	 */
 303	size_bf = mc->fb_start;
 304	size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb);
 305
 306	if (mc->gart_size > max(size_bf, size_af)) {
 307		dev_warn(adev->dev, "limiting GART\n");
 308		mc->gart_size = max(size_bf, size_af);
 309	}
 310
 311	switch (gart_placement) {
 312	case AMDGPU_GART_PLACEMENT_HIGH:
 313		mc->gart_start = max_mc_address - mc->gart_size + 1;
 314		break;
 315	case AMDGPU_GART_PLACEMENT_LOW:
 316		mc->gart_start = 0;
 317		break;
 318	case AMDGPU_GART_PLACEMENT_BEST_FIT:
 319	default:
 320		if ((size_bf >= mc->gart_size && size_bf < size_af) ||
 321		    (size_af < mc->gart_size))
 322			mc->gart_start = 0;
 323		else
 324			mc->gart_start = max_mc_address - mc->gart_size + 1;
 325		break;
 326	}
 327
 328	mc->gart_start &= ~(four_gb - 1);
 329	mc->gart_end = mc->gart_start + mc->gart_size - 1;
 330	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
 331			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 332}
 333
 334/**
 335 * amdgpu_gmc_agp_location - try to find AGP location
 336 * @adev: amdgpu device structure holding all necessary information
 337 * @mc: memory controller structure holding memory information
 338 *
 339 * Function will place try to find a place for the AGP BAR in the MC address
 340 * space.
 341 *
 342 * AGP BAR will be assigned the largest available hole in the address space.
 343 * Should be called after VRAM and GART locations are setup.
 344 */
 345void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
 346{
 347	const uint64_t sixteen_gb = 1ULL << 34;
 348	const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1);
 349	u64 size_af, size_bf;
 350
 351	if (mc->fb_start > mc->gart_start) {
 352		size_bf = (mc->fb_start & sixteen_gb_mask) -
 353			ALIGN(mc->gart_end + 1, sixteen_gb);
 354		size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb);
 355	} else {
 356		size_bf = mc->fb_start & sixteen_gb_mask;
 357		size_af = (mc->gart_start & sixteen_gb_mask) -
 358			ALIGN(mc->fb_end + 1, sixteen_gb);
 359	}
 360
 361	if (size_bf > size_af) {
 362		mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask;
 363		mc->agp_size = size_bf;
 364	} else {
 365		mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb);
 366		mc->agp_size = size_af;
 367	}
 368
 369	mc->agp_end = mc->agp_start + mc->agp_size - 1;
 370	dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n",
 371			mc->agp_size >> 20, mc->agp_start, mc->agp_end);
 372}
 373
 374/**
 375 * amdgpu_gmc_set_agp_default - Set the default AGP aperture value.
 376 * @adev: amdgpu device structure holding all necessary information
 377 * @mc: memory controller structure holding memory information
 378 *
 379 * To disable the AGP aperture, you need to set the start to a larger
 380 * value than the end.  This function sets the default value which
 381 * can then be overridden using amdgpu_gmc_agp_location() if you want
 382 * to enable the AGP aperture on a specific chip.
 383 *
 384 */
 385void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev,
 386				struct amdgpu_gmc *mc)
 387{
 388	mc->agp_start = 0xffffffffffff;
 389	mc->agp_end = 0;
 390	mc->agp_size = 0;
 391}
 392
 393/**
 394 * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid
 395 *
 396 * @addr: 48 bit physical address, page aligned (36 significant bits)
 397 * @pasid: 16 bit process address space identifier
 398 */
 399static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid)
 400{
 401	return addr << 4 | pasid;
 402}
 403
 404/**
 405 * amdgpu_gmc_filter_faults - filter VM faults
 406 *
 407 * @adev: amdgpu device structure
 408 * @ih: interrupt ring that the fault received from
 409 * @addr: address of the VM fault
 410 * @pasid: PASID of the process causing the fault
 411 * @timestamp: timestamp of the fault
 412 *
 413 * Returns:
 414 * True if the fault was filtered and should not be processed further.
 415 * False if the fault is a new one and needs to be handled.
 416 */
 417bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev,
 418			      struct amdgpu_ih_ring *ih, uint64_t addr,
 419			      uint16_t pasid, uint64_t timestamp)
 420{
 421	struct amdgpu_gmc *gmc = &adev->gmc;
 422	uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid);
 423	struct amdgpu_gmc_fault *fault;
 424	uint32_t hash;
 425
 426	/* Stale retry fault if timestamp goes backward */
 427	if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp))
 428		return true;
 429
 430	/* If we don't have space left in the ring buffer return immediately */
 431	stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) -
 432		AMDGPU_GMC_FAULT_TIMEOUT;
 433	if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp)
 434		return true;
 435
 436	/* Try to find the fault in the hash */
 437	hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
 438	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
 439	while (fault->timestamp >= stamp) {
 440		uint64_t tmp;
 441
 442		if (atomic64_read(&fault->key) == key) {
 443			/*
 444			 * if we get a fault which is already present in
 445			 * the fault_ring and the timestamp of
 446			 * the fault is after the expired timestamp,
 447			 * then this is a new fault that needs to be added
 448			 * into the fault ring.
 449			 */
 450			if (fault->timestamp_expiry != 0 &&
 451			    amdgpu_ih_ts_after(fault->timestamp_expiry,
 452					       timestamp))
 453				break;
 454			else
 455				return true;
 456		}
 457
 458		tmp = fault->timestamp;
 459		fault = &gmc->fault_ring[fault->next];
 460
 461		/* Check if the entry was reused */
 462		if (fault->timestamp >= tmp)
 463			break;
 464	}
 465
 466	/* Add the fault to the ring */
 467	fault = &gmc->fault_ring[gmc->last_fault];
 468	atomic64_set(&fault->key, key);
 469	fault->timestamp = timestamp;
 470
 471	/* And update the hash */
 472	fault->next = gmc->fault_hash[hash].idx;
 473	gmc->fault_hash[hash].idx = gmc->last_fault++;
 474	return false;
 475}
 476
 477/**
 478 * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter
 479 *
 480 * @adev: amdgpu device structure
 481 * @addr: address of the VM fault
 482 * @pasid: PASID of the process causing the fault
 483 *
 484 * Remove the address from fault filter, then future vm fault on this address
 485 * will pass to retry fault handler to recover.
 486 */
 487void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
 488				     uint16_t pasid)
 489{
 490	struct amdgpu_gmc *gmc = &adev->gmc;
 491	uint64_t key = amdgpu_gmc_fault_key(addr, pasid);
 492	struct amdgpu_ih_ring *ih;
 493	struct amdgpu_gmc_fault *fault;
 494	uint32_t last_wptr;
 495	uint64_t last_ts;
 496	uint32_t hash;
 497	uint64_t tmp;
 498
 499	if (adev->irq.retry_cam_enabled)
 500		return;
 501	else if (adev->irq.ih1.ring_size)
 502		ih = &adev->irq.ih1;
 503	else if (adev->irq.ih_soft.enabled)
 504		ih = &adev->irq.ih_soft;
 505	else
 506		return;
 507
 508	/* Get the WPTR of the last entry in IH ring */
 509	last_wptr = amdgpu_ih_get_wptr(adev, ih);
 510	/* Order wptr with ring data. */
 511	rmb();
 512	/* Get the timetamp of the last entry in IH ring */
 513	last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1);
 514
 515	hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
 516	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
 517	do {
 518		if (atomic64_read(&fault->key) == key) {
 519			/*
 520			 * Update the timestamp when this fault
 521			 * expired.
 522			 */
 523			fault->timestamp_expiry = last_ts;
 524			break;
 525		}
 526
 527		tmp = fault->timestamp;
 528		fault = &gmc->fault_ring[fault->next];
 529	} while (fault->timestamp < tmp);
 530}
 531
 532int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev)
 533{
 534	int r;
 535
 536	/* umc ras block */
 537	r = amdgpu_umc_ras_sw_init(adev);
 538	if (r)
 539		return r;
 540
 541	/* mmhub ras block */
 542	r = amdgpu_mmhub_ras_sw_init(adev);
 543	if (r)
 544		return r;
 545
 546	/* hdp ras block */
 547	r = amdgpu_hdp_ras_sw_init(adev);
 548	if (r)
 549		return r;
 550
 551	/* mca.x ras block */
 552	r = amdgpu_mca_mp0_ras_sw_init(adev);
 553	if (r)
 554		return r;
 555
 556	r = amdgpu_mca_mp1_ras_sw_init(adev);
 557	if (r)
 558		return r;
 559
 560	r = amdgpu_mca_mpio_ras_sw_init(adev);
 561	if (r)
 562		return r;
 563
 564	/* xgmi ras block */
 565	r = amdgpu_xgmi_ras_sw_init(adev);
 566	if (r)
 567		return r;
 568
 569	return 0;
 570}
 571
 572int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev)
 573{
 574	return 0;
 575}
 576
 577void amdgpu_gmc_ras_fini(struct amdgpu_device *adev)
 578{
 579
 580}
 581
 582	/*
 583	 * The latest engine allocation on gfx9/10 is:
 584	 * Engine 2, 3: firmware
 585	 * Engine 0, 1, 4~16: amdgpu ring,
 586	 *                    subject to change when ring number changes
 587	 * Engine 17: Gart flushes
 588	 */
 589#define AMDGPU_VMHUB_INV_ENG_BITMAP		0x1FFF3
 590
 591int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 592{
 593	struct amdgpu_ring *ring;
 594	unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = {0};
 595	unsigned i;
 596	unsigned vmhub, inv_eng;
 597	struct amdgpu_ring *shared_ring;
 598
 599	/* init the vm inv eng for all vmhubs */
 600	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
 601		vm_inv_engs[i] = AMDGPU_VMHUB_INV_ENG_BITMAP;
 602		/* reserve engine 5 for firmware */
 603		if (adev->enable_mes)
 604			vm_inv_engs[i] &= ~(1 << 5);
 605		/* reserve engine 6 for uni mes */
 606		if (adev->enable_uni_mes)
 607			vm_inv_engs[i] &= ~(1 << 6);
 608		/* reserve mmhub engine 3 for firmware */
 609		if (adev->enable_umsch_mm)
 610			vm_inv_engs[i] &= ~(1 << 3);
 611	}
 612
 613	for (i = 0; i < adev->num_rings; ++i) {
 614		ring = adev->rings[i];
 615		vmhub = ring->vm_hub;
 616
 617		if (ring == &adev->mes.ring[0] ||
 618		    ring == &adev->mes.ring[1] ||
 619		    ring == &adev->umsch_mm.ring ||
 620		    ring == &adev->cper.ring_buf)
 621			continue;
 622
 623		/* Skip if the ring is a shared ring */
 624		if (amdgpu_sdma_is_shared_inv_eng(adev, ring))
 625			continue;
 626
 627		inv_eng = ffs(vm_inv_engs[vmhub]);
 628		if (!inv_eng) {
 629			dev_err(adev->dev, "no VM inv eng for ring %s\n",
 630				ring->name);
 631			return -EINVAL;
 632		}
 633
 634		ring->vm_inv_eng = inv_eng - 1;
 635		vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng);
 636
 637		dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n",
 638			 ring->name, ring->vm_inv_eng, ring->vm_hub);
 639		/* SDMA has a special packet which allows it to use the same
 640		 * invalidation engine for all the rings in one instance.
 641		 * Therefore, we do not allocate a separate VM invalidation engine
 642		 * for SDMA page rings. Instead, they share the VM invalidation
 643		 * engine with the SDMA gfx ring. This change ensures efficient
 644		 * resource management and avoids the issue of insufficient VM
 645		 * invalidation engines.
 646		 */
 647		shared_ring = amdgpu_sdma_get_shared_ring(adev, ring);
 648		if (shared_ring) {
 649			shared_ring->vm_inv_eng = ring->vm_inv_eng;
 650			dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n",
 651					ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub);
 652			continue;
 653		}
 654	}
 655
 656	return 0;
 657}
 658
 659void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 660			      uint32_t vmhub, uint32_t flush_type)
 661{
 662	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 663	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
 664	struct dma_fence *fence;
 665	struct amdgpu_job *job;
 666	int r;
 667
 668	if (!hub->sdma_invalidation_workaround || vmid ||
 669	    !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready ||
 670	    !ring->sched.ready) {
 671		/*
 672		 * A GPU reset should flush all TLBs anyway, so no need to do
 673		 * this while one is ongoing.
 674		 */
 675		if (!down_read_trylock(&adev->reset_domain->sem))
 676			return;
 677
 678		if (adev->gmc.flush_tlb_needs_extra_type_2)
 679			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,
 680							   vmhub, 2);
 681
 682		if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)
 683			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,
 684							   vmhub, 0);
 685
 686		adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub,
 687						   flush_type);
 688		up_read(&adev->reset_domain->sem);
 689		return;
 690	}
 691
 692	/* The SDMA on Navi 1x has a bug which can theoretically result in memory
 693	 * corruption if an invalidation happens at the same time as an VA
 694	 * translation. Avoid this by doing the invalidation from the SDMA
 695	 * itself at least for GART.
 696	 */
 697	mutex_lock(&adev->mman.gtt_window_lock);
 698	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.high_pr,
 699				     AMDGPU_FENCE_OWNER_UNDEFINED,
 700				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
 701				     &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB);
 702	if (r)
 703		goto error_alloc;
 704
 705	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
 706	job->vm_needs_flush = true;
 707	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
 708	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 709	fence = amdgpu_job_submit(job);
 710	mutex_unlock(&adev->mman.gtt_window_lock);
 711
 712	dma_fence_wait(fence, false);
 713	dma_fence_put(fence);
 714
 715	return;
 716
 717error_alloc:
 718	mutex_unlock(&adev->mman.gtt_window_lock);
 719	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
 720}
 721
 722int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
 723				   uint32_t flush_type, bool all_hub,
 724				   uint32_t inst)
 725{
 726	struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
 727	struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
 728	unsigned int ndw;
 729	int r, cnt = 0;
 730	uint32_t seq;
 731
 732	/*
 733	 * A GPU reset should flush all TLBs anyway, so no need to do
 734	 * this while one is ongoing.
 735	 */
 736	if (!down_read_trylock(&adev->reset_domain->sem))
 737		return 0;
 738
 739	if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
 740
 741		if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) {
 742			r = 0;
 743			goto error_unlock_reset;
 744		}
 745
 746		if (adev->gmc.flush_tlb_needs_extra_type_2)
 747			adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
 748								 2, all_hub,
 749								 inst);
 750
 751		if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)
 752			adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
 753								 0, all_hub,
 754								 inst);
 755
 756		adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
 757							 flush_type, all_hub,
 758							 inst);
 759		r = 0;
 760	} else {
 761		/* 2 dwords flush + 8 dwords fence */
 762		ndw = kiq->pmf->invalidate_tlbs_size + 8;
 763
 764		if (adev->gmc.flush_tlb_needs_extra_type_2)
 765			ndw += kiq->pmf->invalidate_tlbs_size;
 766
 767		if (adev->gmc.flush_tlb_needs_extra_type_0)
 768			ndw += kiq->pmf->invalidate_tlbs_size;
 769
 770		spin_lock(&adev->gfx.kiq[inst].ring_lock);
 771		r = amdgpu_ring_alloc(ring, ndw);
 772		if (r) {
 773			spin_unlock(&adev->gfx.kiq[inst].ring_lock);
 774			goto error_unlock_reset;
 775		}
 776		if (adev->gmc.flush_tlb_needs_extra_type_2)
 777			kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
 778
 779		if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
 780			kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
 781
 782		kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
 783		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
 784		if (r) {
 785			amdgpu_ring_undo(ring);
 786			spin_unlock(&adev->gfx.kiq[inst].ring_lock);
 787			goto error_unlock_reset;
 788		}
 789
 790		amdgpu_ring_commit(ring);
 791		spin_unlock(&adev->gfx.kiq[inst].ring_lock);
 792
 793		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 794
 795		might_sleep();
 796		while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
 797		       !amdgpu_reset_pending(adev->reset_domain)) {
 798			msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
 799			r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 800		}
 801
 802		if (cnt > MAX_KIQ_REG_TRY) {
 803			dev_err(adev->dev, "timeout waiting for kiq fence\n");
 804			r = -ETIME;
 805		} else
 806			r = 0;
 807	}
 808
 809error_unlock_reset:
 810	up_read(&adev->reset_domain->sem);
 811	return r;
 812}
 813
 814void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
 815				      uint32_t reg0, uint32_t reg1,
 816				      uint32_t ref, uint32_t mask,
 817				      uint32_t xcc_inst)
 818{
 819	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst];
 820	struct amdgpu_ring *ring = &kiq->ring;
 821	signed long r, cnt = 0;
 822	unsigned long flags;
 823	uint32_t seq;
 824
 825	if (adev->mes.ring[0].sched.ready) {
 826		amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1,
 827					      ref, mask);
 828		return;
 829	}
 830
 831	spin_lock_irqsave(&kiq->ring_lock, flags);
 832	amdgpu_ring_alloc(ring, 32);
 833	amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1,
 834					    ref, mask);
 835	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
 836	if (r)
 837		goto failed_undo;
 838
 839	amdgpu_ring_commit(ring);
 840	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 841
 842	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 843
 844	/* don't wait anymore for IRQ context */
 845	if (r < 1 && in_interrupt())
 846		goto failed_kiq;
 847
 848	might_sleep();
 849	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
 850	       !amdgpu_reset_pending(adev->reset_domain)) {
 851
 852		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
 853		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
 854	}
 855
 856	if (cnt > MAX_KIQ_REG_TRY)
 857		goto failed_kiq;
 858
 859	return;
 860
 861failed_undo:
 862	amdgpu_ring_undo(ring);
 863	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 864failed_kiq:
 865	dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1);
 866}
 867
 868/**
 869 * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ
 870 * @adev: amdgpu_device pointer
 871 *
 872 * Check and set if an the device @adev supports Trusted Memory
 873 * Zones (TMZ).
 874 */
 875void amdgpu_gmc_tmz_set(struct amdgpu_device *adev)
 876{
 877	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 878	/* RAVEN */
 879	case IP_VERSION(9, 2, 2):
 880	case IP_VERSION(9, 1, 0):
 881	/* RENOIR looks like RAVEN */
 882	case IP_VERSION(9, 3, 0):
 883	/* GC 10.3.7 */
 884	case IP_VERSION(10, 3, 7):
 885	/* GC 11.0.1 */
 886	case IP_VERSION(11, 0, 1):
 887		if (amdgpu_tmz == 0) {
 888			adev->gmc.tmz_enabled = false;
 889			dev_info(adev->dev,
 890				 "Trusted Memory Zone (TMZ) feature disabled (cmd line)\n");
 891		} else {
 892			adev->gmc.tmz_enabled = true;
 893			dev_info(adev->dev,
 894				 "Trusted Memory Zone (TMZ) feature enabled\n");
 895		}
 896		break;
 897	case IP_VERSION(10, 1, 10):
 898	case IP_VERSION(10, 1, 1):
 899	case IP_VERSION(10, 1, 2):
 900	case IP_VERSION(10, 1, 3):
 901	case IP_VERSION(10, 3, 0):
 902	case IP_VERSION(10, 3, 2):
 903	case IP_VERSION(10, 3, 4):
 904	case IP_VERSION(10, 3, 5):
 905	case IP_VERSION(10, 3, 6):
 906	/* VANGOGH */
 907	case IP_VERSION(10, 3, 1):
 908	/* YELLOW_CARP*/
 909	case IP_VERSION(10, 3, 3):
 910	case IP_VERSION(11, 0, 4):
 911	case IP_VERSION(11, 5, 0):
 912	case IP_VERSION(11, 5, 1):
 913	case IP_VERSION(11, 5, 2):
 914	case IP_VERSION(11, 5, 3):
 915		/* Don't enable it by default yet.
 916		 */
 917		if (amdgpu_tmz < 1) {
 918			adev->gmc.tmz_enabled = false;
 919			dev_info(adev->dev,
 920				 "Trusted Memory Zone (TMZ) feature disabled as experimental (default)\n");
 921		} else {
 922			adev->gmc.tmz_enabled = true;
 923			dev_info(adev->dev,
 924				 "Trusted Memory Zone (TMZ) feature enabled as experimental (cmd line)\n");
 925		}
 926		break;
 927	default:
 928		adev->gmc.tmz_enabled = false;
 929		dev_info(adev->dev,
 930			 "Trusted Memory Zone (TMZ) feature not supported\n");
 931		break;
 932	}
 933}
 934
 935/**
 936 * amdgpu_gmc_noretry_set -- set per asic noretry defaults
 937 * @adev: amdgpu_device pointer
 938 *
 939 * Set a per asic default for the no-retry parameter.
 940 *
 941 */
 942void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
 943{
 944	struct amdgpu_gmc *gmc = &adev->gmc;
 945	uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0);
 946	bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) ||
 947				gc_ver == IP_VERSION(9, 4, 0) ||
 948				gc_ver == IP_VERSION(9, 4, 1) ||
 949				gc_ver == IP_VERSION(9, 4, 2) ||
 950				gc_ver == IP_VERSION(9, 4, 3) ||
 951				gc_ver == IP_VERSION(9, 4, 4) ||
 952				gc_ver == IP_VERSION(9, 5, 0) ||
 953				gc_ver >= IP_VERSION(10, 3, 0));
 954
 955	if (!amdgpu_sriov_xnack_support(adev))
 956		gmc->noretry = 1;
 957	else
 958		gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry;
 959}
 960
 961void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,
 962				   bool enable)
 963{
 964	struct amdgpu_vmhub *hub;
 965	u32 tmp, reg, i;
 966
 967	hub = &adev->vmhub[hub_type];
 968	for (i = 0; i < 16; i++) {
 969		reg = hub->vm_context0_cntl + hub->ctx_distance * i;
 970
 971		tmp = (hub_type == AMDGPU_GFXHUB(0)) ?
 972			RREG32_SOC15_IP(GC, reg) :
 973			RREG32_SOC15_IP(MMHUB, reg);
 974
 975		if (enable)
 976			tmp |= hub->vm_cntx_cntl_vm_fault;
 977		else
 978			tmp &= ~hub->vm_cntx_cntl_vm_fault;
 979
 980		(hub_type == AMDGPU_GFXHUB(0)) ?
 981			WREG32_SOC15_IP(GC, reg, tmp) :
 982			WREG32_SOC15_IP(MMHUB, reg, tmp);
 983	}
 984}
 985
 986void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev)
 987{
 988	unsigned size;
 989
 990	/*
 991	 * Some ASICs need to reserve a region of video memory to avoid access
 992	 * from driver
 993	 */
 994	adev->mman.stolen_reserved_offset = 0;
 995	adev->mman.stolen_reserved_size = 0;
 996
 997	/*
 998	 * TODO:
 999	 * Currently there is a bug where some memory client outside
1000	 * of the driver writes to first 8M of VRAM on S3 resume,
1001	 * this overrides GART which by default gets placed in first 8M and
1002	 * causes VM_FAULTS once GTT is accessed.
1003	 * Keep the stolen memory reservation until the while this is not solved.
1004	 */
1005	switch (adev->asic_type) {
1006	case CHIP_VEGA10:
1007		adev->mman.keep_stolen_vga_memory = true;
1008		/*
1009		 * VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area.
1010		 */
1011#ifdef CONFIG_X86
1012		if (amdgpu_sriov_vf(adev) && hypervisor_is_type(X86_HYPER_MS_HYPERV)) {
1013			adev->mman.stolen_reserved_offset = 0x500000;
1014			adev->mman.stolen_reserved_size = 0x200000;
1015		}
1016#endif
1017		break;
1018	case CHIP_RAVEN:
1019	case CHIP_RENOIR:
1020		adev->mman.keep_stolen_vga_memory = true;
1021		break;
1022	case CHIP_POLARIS10:
1023	case CHIP_POLARIS11:
1024	case CHIP_POLARIS12:
1025		/* MacBookPros with switchable graphics put VRAM at 0 when
1026		 * the iGPU is enabled which results in cursor issues if
1027		 * the cursor ends up at 0.  Reserve vram at 0 in that case.
1028		 */
1029		if (adev->gmc.vram_start == 0)
1030			adev->mman.keep_stolen_vga_memory = true;
1031		break;
1032	default:
1033		adev->mman.keep_stolen_vga_memory = false;
1034		break;
1035	}
1036
1037	if (amdgpu_sriov_vf(adev) ||
1038	    !amdgpu_device_has_display_hardware(adev)) {
1039		size = 0;
1040	} else {
1041		size = amdgpu_gmc_get_vbios_fb_size(adev);
1042
1043		if (adev->mman.keep_stolen_vga_memory)
1044			size = max(size, (unsigned)AMDGPU_VBIOS_VGA_ALLOCATION);
1045	}
1046
1047	/* set to 0 if the pre-OS buffer uses up most of vram */
1048	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
1049		size = 0;
1050
1051	if (size > AMDGPU_VBIOS_VGA_ALLOCATION) {
1052		adev->mman.stolen_vga_size = AMDGPU_VBIOS_VGA_ALLOCATION;
1053		adev->mman.stolen_extended_size = size - adev->mman.stolen_vga_size;
1054	} else {
1055		adev->mman.stolen_vga_size = size;
1056		adev->mman.stolen_extended_size = 0;
1057	}
1058}
1059
1060/**
1061 * amdgpu_gmc_init_pdb0 - initialize PDB0
1062 *
1063 * @adev: amdgpu_device pointer
1064 *
1065 * This function is only used when GART page table is used
1066 * for FB address translatioin. In such a case, we construct
1067 * a 2-level system VM page table: PDB0->PTB, to cover both
1068 * VRAM of the hive and system memory.
1069 *
1070 * PDB0 is static, initialized once on driver initialization.
1071 * The first n entries of PDB0 are used as PTE by setting
1072 * P bit to 1, pointing to VRAM. The n+1'th entry points
1073 * to a big PTB covering system memory.
1074 *
1075 */
1076void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
1077{
1078	int i;
1079	uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW?
1080	/* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M
1081	 */
1082	u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
1083	u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21;
1084	u64 vram_addr, vram_end;
1085	u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo);
1086	int idx;
1087
1088	if (!drm_dev_enter(adev_to_drm(adev), &idx))
1089		return;
1090
1091	flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE;
1092	flags |= AMDGPU_PTE_WRITEABLE;
1093	flags |= AMDGPU_PTE_SNOOPED;
1094	flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1));
1095	flags |= AMDGPU_PDE_PTE_FLAG(adev);
1096
1097	vram_addr = adev->vm_manager.vram_base_offset;
1098	if (!amdgpu_virt_xgmi_migrate_enabled(adev))
1099		vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1100	vram_end = vram_addr + vram_size;
1101
1102	/* The first n PDE0 entries are used as PTE,
1103	 * pointing to vram
1104	 */
1105	for (i = 0; vram_addr < vram_end; i++, vram_addr += pde0_page_size)
1106		amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags);
1107
1108	/* The n+1'th PDE0 entry points to a huge
1109	 * PTB who has more than 512 entries each
1110	 * pointing to a 4K system page
1111	 */
1112	flags = AMDGPU_PTE_VALID;
1113	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0);
1114	/* Requires gart_ptb_gpu_pa to be 4K aligned */
1115	amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags);
1116	drm_dev_exit(idx);
1117}
1118
1119/**
1120 * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC
1121 * address
1122 *
1123 * @adev: amdgpu_device pointer
1124 * @mc_addr: MC address of buffer
1125 */
1126uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr)
1127{
1128	return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset;
1129}
1130
1131/**
1132 * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from
1133 * GPU's view
1134 *
1135 * @adev: amdgpu_device pointer
1136 * @bo: amdgpu buffer object
1137 */
1138uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo)
1139{
1140	return amdgpu_gmc_vram_mc2pa(adev, amdgpu_bo_gpu_offset(bo));
1141}
1142
1143int amdgpu_gmc_vram_checking(struct amdgpu_device *adev)
1144{
1145	struct amdgpu_bo *vram_bo = NULL;
1146	uint64_t vram_gpu = 0;
1147	void *vram_ptr = NULL;
1148
1149	int ret, size = 0x100000;
1150	uint8_t cptr[10];
1151
1152	ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
1153				AMDGPU_GEM_DOMAIN_VRAM,
1154				&vram_bo,
1155				&vram_gpu,
1156				&vram_ptr);
1157	if (ret)
1158		return ret;
1159
1160	memset(vram_ptr, 0x86, size);
1161	memset(cptr, 0x86, 10);
1162
1163	/**
1164	 * Check the start, the mid, and the end of the memory if the content of
1165	 * each byte is the pattern "0x86". If yes, we suppose the vram bo is
1166	 * workable.
1167	 *
1168	 * Note: If check the each byte of whole 1M bo, it will cost too many
1169	 * seconds, so here, we just pick up three parts for emulation.
1170	 */
1171	ret = memcmp(vram_ptr, cptr, 10);
1172	if (ret) {
1173		ret = -EIO;
1174		goto release_buffer;
1175	}
1176
1177	ret = memcmp(vram_ptr + (size / 2), cptr, 10);
1178	if (ret) {
1179		ret = -EIO;
1180		goto release_buffer;
1181	}
1182
1183	ret = memcmp(vram_ptr + size - 10, cptr, 10);
1184	if (ret) {
1185		ret = -EIO;
1186		goto release_buffer;
1187	}
1188
1189release_buffer:
1190	amdgpu_bo_free_kernel(&vram_bo, &vram_gpu,
1191			&vram_ptr);
1192
1193	return ret;
1194}
1195
1196static const char *nps_desc[] = {
1197	[AMDGPU_NPS1_PARTITION_MODE] = "NPS1",
1198	[AMDGPU_NPS2_PARTITION_MODE] = "NPS2",
1199	[AMDGPU_NPS3_PARTITION_MODE] = "NPS3",
1200	[AMDGPU_NPS4_PARTITION_MODE] = "NPS4",
1201	[AMDGPU_NPS6_PARTITION_MODE] = "NPS6",
1202	[AMDGPU_NPS8_PARTITION_MODE] = "NPS8",
1203};
1204
1205static ssize_t available_memory_partition_show(struct device *dev,
1206					       struct device_attribute *addr,
1207					       char *buf)
1208{
1209	struct drm_device *ddev = dev_get_drvdata(dev);
1210	struct amdgpu_device *adev = drm_to_adev(ddev);
1211	int size = 0, mode;
1212	char *sep = "";
1213
1214	for_each_inst(mode, adev->gmc.supported_nps_modes) {
1215		size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]);
1216		sep = ", ";
1217	}
1218	size += sysfs_emit_at(buf, size, "\n");
1219
1220	return size;
1221}
1222
1223static ssize_t current_memory_partition_store(struct device *dev,
1224					      struct device_attribute *attr,
1225					      const char *buf, size_t count)
1226{
1227	struct drm_device *ddev = dev_get_drvdata(dev);
1228	struct amdgpu_device *adev = drm_to_adev(ddev);
1229	enum amdgpu_memory_partition mode;
1230	struct amdgpu_hive_info *hive;
1231	int i;
1232
1233	mode = UNKNOWN_MEMORY_PARTITION_MODE;
1234	for_each_inst(i, adev->gmc.supported_nps_modes) {
1235		if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) {
1236			mode = i;
1237			break;
1238		}
1239	}
1240
1241	if (mode == UNKNOWN_MEMORY_PARTITION_MODE)
1242		return -EINVAL;
1243
1244	if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) {
1245		dev_info(
1246			adev->dev,
1247			"requested NPS mode is same as current NPS mode, skipping\n");
1248		return count;
1249	}
1250
1251	/* If device is part of hive, all devices in the hive should request the
1252	 * same mode. Hence store the requested mode in hive.
1253	 */
1254	hive = amdgpu_get_xgmi_hive(adev);
1255	if (hive) {
1256		atomic_set(&hive->requested_nps_mode, mode);
1257		amdgpu_put_xgmi_hive(hive);
1258	} else {
1259		adev->gmc.requested_nps_mode = mode;
1260	}
1261
1262	dev_info(
1263		adev->dev,
1264		"NPS mode change requested, please remove and reload the driver\n");
1265
1266	return count;
1267}
1268
1269static ssize_t current_memory_partition_show(
1270	struct device *dev, struct device_attribute *addr, char *buf)
1271{
1272	struct drm_device *ddev = dev_get_drvdata(dev);
1273	struct amdgpu_device *adev = drm_to_adev(ddev);
1274	enum amdgpu_memory_partition mode;
1275
1276	/* Only minimal precaution taken to reject requests while in reset */
1277	if (amdgpu_in_reset(adev))
1278		return -EPERM;
1279
1280	mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
1281	if ((mode >= ARRAY_SIZE(nps_desc)) ||
1282	    (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode))
1283		return sysfs_emit(buf, "UNKNOWN\n");
1284
1285	return sysfs_emit(buf, "%s\n", nps_desc[mode]);
1286}
1287
1288static DEVICE_ATTR_RW(current_memory_partition);
1289static DEVICE_ATTR_RO(available_memory_partition);
1290
1291int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev)
1292{
1293	bool nps_switch_support;
1294	int r = 0;
1295
1296	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
1297		return 0;
1298
1299	nps_switch_support = (hweight32(adev->gmc.supported_nps_modes &
1300					AMDGPU_ALL_NPS_MASK) > 1);
1301	if (!nps_switch_support)
1302		dev_attr_current_memory_partition.attr.mode &=
1303			~(S_IWUSR | S_IWGRP | S_IWOTH);
1304	else
1305		r = device_create_file(adev->dev,
1306				       &dev_attr_available_memory_partition);
1307
1308	if (r)
1309		return r;
1310
1311	return device_create_file(adev->dev,
1312				  &dev_attr_current_memory_partition);
1313}
1314
1315void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev)
1316{
1317	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
1318		return;
1319
1320	device_remove_file(adev->dev, &dev_attr_current_memory_partition);
1321	device_remove_file(adev->dev, &dev_attr_available_memory_partition);
1322}
1323
1324int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
1325				 struct amdgpu_mem_partition_info *mem_ranges,
1326				 uint8_t *exp_ranges)
1327{
1328	struct amdgpu_gmc_memrange *ranges;
1329	int range_cnt, ret, i, j;
1330	uint32_t nps_type;
1331	bool refresh;
1332
1333	if (!mem_ranges || !exp_ranges)
1334		return -EINVAL;
1335
1336	refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
1337		  (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
1338	ret = amdgpu_discovery_get_nps_info(adev, &nps_type, &ranges,
1339					    &range_cnt, refresh);
1340
1341	if (ret)
1342		return ret;
1343
1344	/* TODO: For now, expect ranges and partition count to be the same.
1345	 * Adjust if there are holes expected in any NPS domain.
1346	 */
1347	if (*exp_ranges && (range_cnt != *exp_ranges)) {
1348		dev_warn(
1349			adev->dev,
1350			"NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d",
1351			*exp_ranges, nps_type, range_cnt);
1352		ret = -EINVAL;
1353		goto err;
1354	}
1355
1356	for (i = 0; i < range_cnt; ++i) {
1357		if (ranges[i].base_address >= ranges[i].limit_address) {
1358			dev_warn(
1359				adev->dev,
1360				"Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx",
1361				nps_type, i, ranges[i].base_address,
1362				ranges[i].limit_address);
1363			ret = -EINVAL;
1364			goto err;
1365		}
1366
1367		/* Check for overlaps, not expecting any now */
1368		for (j = i - 1; j >= 0; j--) {
1369			if (max(ranges[j].base_address,
1370				ranges[i].base_address) <=
1371			    min(ranges[j].limit_address,
1372				ranges[i].limit_address)) {
1373				dev_warn(
1374					adev->dev,
1375					"overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]",
1376					ranges[j].base_address,
1377					ranges[j].limit_address,
1378					ranges[i].base_address,
1379					ranges[i].limit_address);
1380				ret = -EINVAL;
1381				goto err;
1382			}
1383		}
1384
1385		mem_ranges[i].range.fpfn =
1386			(ranges[i].base_address -
1387			 adev->vm_manager.vram_base_offset) >>
1388			AMDGPU_GPU_PAGE_SHIFT;
1389		mem_ranges[i].range.lpfn =
1390			(ranges[i].limit_address -
1391			 adev->vm_manager.vram_base_offset) >>
1392			AMDGPU_GPU_PAGE_SHIFT;
1393		mem_ranges[i].size =
1394			ranges[i].limit_address - ranges[i].base_address + 1;
1395	}
1396
1397	if (!*exp_ranges)
1398		*exp_ranges = range_cnt;
1399err:
1400	kvfree(ranges);
1401
1402	return ret;
1403}
1404
1405int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
1406					int nps_mode)
1407{
1408	/* Not supported on VF devices and APUs */
1409	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
1410		return -EOPNOTSUPP;
1411
1412	if (!adev->psp.funcs) {
1413		dev_err(adev->dev,
1414			"PSP interface not available for nps mode change request");
1415		return -EINVAL;
1416	}
1417
1418	return psp_memory_partition(&adev->psp, nps_mode);
1419}
1420
1421static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev,
1422						  int req_nps_mode,
1423						  int cur_nps_mode)
1424{
1425	return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) ==
1426			BIT(req_nps_mode)) &&
1427		req_nps_mode != cur_nps_mode);
1428}
1429
1430void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
1431{
1432	int req_nps_mode, cur_nps_mode, r;
1433	struct amdgpu_hive_info *hive;
1434
1435	if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes ||
1436	    !adev->gmc.gmc_funcs->request_mem_partition_mode)
1437		return;
1438
1439	cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
1440	hive = amdgpu_get_xgmi_hive(adev);
1441	if (hive) {
1442		req_nps_mode = atomic_read(&hive->requested_nps_mode);
1443		if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode,
1444						    cur_nps_mode)) {
1445			amdgpu_put_xgmi_hive(hive);
1446			return;
1447		}
1448		r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode);
1449		amdgpu_put_xgmi_hive(hive);
1450		goto out;
1451	}
1452
1453	req_nps_mode = adev->gmc.requested_nps_mode;
1454	if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode))
1455		return;
1456
1457	/* even if this fails, we should let driver unload w/o blocking */
1458	r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode);
1459out:
1460	if (r)
1461		dev_err(adev->dev, "NPS mode change request failed\n");
1462	else
1463		dev_info(
1464			adev->dev,
1465			"NPS mode change request done, reload driver to complete the change\n");
1466}
1467
1468bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev)
1469{
1470	if (adev->gmc.gmc_funcs->need_reset_on_init)
1471		return adev->gmc.gmc_funcs->need_reset_on_init(adev);
1472
1473	return false;
1474}
1475
1476enum amdgpu_memory_partition
1477amdgpu_gmc_get_vf_memory_partition(struct amdgpu_device *adev)
1478{
1479	switch (adev->gmc.num_mem_partitions) {
1480	case 0:
1481		return UNKNOWN_MEMORY_PARTITION_MODE;
1482	case 1:
1483		return AMDGPU_NPS1_PARTITION_MODE;
1484	case 2:
1485		return AMDGPU_NPS2_PARTITION_MODE;
1486	case 4:
1487		return AMDGPU_NPS4_PARTITION_MODE;
1488	case 8:
1489		return AMDGPU_NPS8_PARTITION_MODE;
1490	default:
1491		return AMDGPU_NPS1_PARTITION_MODE;
1492	}
1493}
1494
1495enum amdgpu_memory_partition
1496amdgpu_gmc_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes)
1497{
1498	enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE;
1499
1500	if (adev->nbio.funcs &&
1501	    adev->nbio.funcs->get_memory_partition_mode)
1502		mode = adev->nbio.funcs->get_memory_partition_mode(adev,
1503								   supp_modes);
1504	else
1505		dev_warn(adev->dev, "memory partition mode query is not supported\n");
1506
1507	return mode;
1508}
1509
1510enum amdgpu_memory_partition
1511amdgpu_gmc_query_memory_partition(struct amdgpu_device *adev)
1512{
1513	if (amdgpu_sriov_vf(adev))
1514		return amdgpu_gmc_get_vf_memory_partition(adev);
1515	else
1516		return amdgpu_gmc_get_memory_partition(adev, NULL);
1517}
1518
1519static bool amdgpu_gmc_validate_partition_info(struct amdgpu_device *adev)
1520{
1521	enum amdgpu_memory_partition mode;
1522	u32 supp_modes;
1523	bool valid;
1524
1525	mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes);
1526
1527	/* Mode detected by hardware not present in supported modes */
1528	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) &&
1529	    !(BIT(mode - 1) & supp_modes))
1530		return false;
1531
1532	switch (mode) {
1533	case UNKNOWN_MEMORY_PARTITION_MODE:
1534	case AMDGPU_NPS1_PARTITION_MODE:
1535		valid = (adev->gmc.num_mem_partitions == 1);
1536		break;
1537	case AMDGPU_NPS2_PARTITION_MODE:
1538		valid = (adev->gmc.num_mem_partitions == 2);
1539		break;
1540	case AMDGPU_NPS4_PARTITION_MODE:
1541		valid = (adev->gmc.num_mem_partitions == 3 ||
1542			 adev->gmc.num_mem_partitions == 4);
1543		break;
1544	case AMDGPU_NPS8_PARTITION_MODE:
1545		valid = (adev->gmc.num_mem_partitions == 8);
1546		break;
1547	default:
1548		valid = false;
1549	}
1550
1551	return valid;
1552}
1553
1554static bool amdgpu_gmc_is_node_present(int *node_ids, int num_ids, int nid)
1555{
1556	int i;
1557
1558	/* Check if node with id 'nid' is present in 'node_ids' array */
1559	for (i = 0; i < num_ids; ++i)
1560		if (node_ids[i] == nid)
1561			return true;
1562
1563	return false;
1564}
1565
1566static void
1567amdgpu_gmc_init_acpi_mem_ranges(struct amdgpu_device *adev,
1568				struct amdgpu_mem_partition_info *mem_ranges)
1569{
1570	struct amdgpu_numa_info numa_info;
1571	int node_ids[AMDGPU_MAX_MEM_RANGES];
1572	int num_ranges = 0, ret;
1573	int num_xcc, xcc_id;
1574	uint32_t xcc_mask;
1575
1576	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1577	xcc_mask = (1U << num_xcc) - 1;
1578
1579	for_each_inst(xcc_id, xcc_mask)	{
1580		ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info);
1581		if (ret)
1582			continue;
1583
1584		if (numa_info.nid == NUMA_NO_NODE) {
1585			mem_ranges[0].size = numa_info.size;
1586			mem_ranges[0].numa.node = numa_info.nid;
1587			num_ranges = 1;
1588			break;
1589		}
1590
1591		if (amdgpu_gmc_is_node_present(node_ids, num_ranges,
1592					     numa_info.nid))
1593			continue;
1594
1595		node_ids[num_ranges] = numa_info.nid;
1596		mem_ranges[num_ranges].numa.node = numa_info.nid;
1597		mem_ranges[num_ranges].size = numa_info.size;
1598		++num_ranges;
1599	}
1600
1601	adev->gmc.num_mem_partitions = num_ranges;
1602}
1603
1604void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev,
1605				   struct amdgpu_mem_partition_info *mem_ranges)
1606{
1607	enum amdgpu_memory_partition mode;
1608	u32 start_addr = 0, size;
1609	int i, r, l;
1610
1611	mode = amdgpu_gmc_query_memory_partition(adev);
1612
1613	switch (mode) {
1614	case UNKNOWN_MEMORY_PARTITION_MODE:
1615		adev->gmc.num_mem_partitions = 0;
1616		break;
1617	case AMDGPU_NPS1_PARTITION_MODE:
1618		adev->gmc.num_mem_partitions = 1;
1619		break;
1620	case AMDGPU_NPS2_PARTITION_MODE:
1621		adev->gmc.num_mem_partitions = 2;
1622		break;
1623	case AMDGPU_NPS4_PARTITION_MODE:
1624		if (adev->flags & AMD_IS_APU)
1625			adev->gmc.num_mem_partitions = 3;
1626		else
1627			adev->gmc.num_mem_partitions = 4;
1628		break;
1629	case AMDGPU_NPS8_PARTITION_MODE:
1630		adev->gmc.num_mem_partitions = 8;
1631		break;
1632	default:
1633		adev->gmc.num_mem_partitions = 1;
1634		break;
1635	}
1636
1637	/* Use NPS range info, if populated */
1638	r = amdgpu_gmc_get_nps_memranges(adev, mem_ranges,
1639					 &adev->gmc.num_mem_partitions);
1640	if (!r) {
1641		l = 0;
1642		for (i = 1; i < adev->gmc.num_mem_partitions; ++i) {
1643			if (mem_ranges[i].range.lpfn >
1644			    mem_ranges[i - 1].range.lpfn)
1645				l = i;
1646		}
1647
1648	} else {
1649		if (!adev->gmc.num_mem_partitions) {
1650			dev_warn(adev->dev,
1651				 "Not able to detect NPS mode, fall back to NPS1\n");
1652			adev->gmc.num_mem_partitions = 1;
1653		}
1654		/* Fallback to sw based calculation */
1655		size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT;
1656		size /= adev->gmc.num_mem_partitions;
1657
1658		for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
1659			mem_ranges[i].range.fpfn = start_addr;
1660			mem_ranges[i].size =
1661				((u64)size << AMDGPU_GPU_PAGE_SHIFT);
1662			mem_ranges[i].range.lpfn = start_addr + size - 1;
1663			start_addr += size;
1664		}
1665
1666		l = adev->gmc.num_mem_partitions - 1;
1667	}
1668
1669	/* Adjust the last one */
1670	mem_ranges[l].range.lpfn =
1671		(adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1;
1672	mem_ranges[l].size =
1673		adev->gmc.real_vram_size -
1674		((u64)mem_ranges[l].range.fpfn << AMDGPU_GPU_PAGE_SHIFT);
1675}
1676
1677int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev)
1678{
1679	bool valid;
1680
1681	adev->gmc.mem_partitions = kcalloc(AMDGPU_MAX_MEM_RANGES,
1682					   sizeof(struct amdgpu_mem_partition_info),
1683					   GFP_KERNEL);
1684	if (!adev->gmc.mem_partitions)
1685		return -ENOMEM;
1686
1687	if (adev->gmc.is_app_apu)
1688		amdgpu_gmc_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions);
1689	else
1690		amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
1691
1692	if (amdgpu_sriov_vf(adev))
1693		valid = true;
1694	else
1695		valid = amdgpu_gmc_validate_partition_info(adev);
1696	if (!valid) {
1697		/* TODO: handle invalid case */
1698		dev_warn(adev->dev,
1699			 "Mem ranges not matching with hardware config\n");
1700	}
1701
1702	return 0;
1703}