amdgpu_vm.c

   1/*
   2 * Copyright 2008 Advanced Micro Devices, Inc.
   3 * Copyright 2008 Red Hat Inc.
   4 * Copyright 2009 Jerome Glisse.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice shall be included in
  14 * all copies or substantial portions of the Software.
  15 *
  16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 * OTHER DEALINGS IN THE SOFTWARE.
  23 *
  24 * Authors: Dave Airlie
  25 *          Alex Deucher
  26 *          Jerome Glisse
  27 */
  28
  29#include <linux/dma-fence-array.h>
  30#include <linux/interval_tree_generic.h>
  31#include <linux/idr.h>
  32#include <linux/dma-buf.h>
  33
  34#include <drm/amdgpu_drm.h>
  35#include <drm/drm_drv.h>
  36#include <drm/ttm/ttm_tt.h>
  37#include <drm/drm_exec.h>
  38#include "amdgpu.h"
  39#include "amdgpu_vm.h"
  40#include "amdgpu_trace.h"
  41#include "amdgpu_amdkfd.h"
  42#include "amdgpu_gmc.h"
  43#include "amdgpu_xgmi.h"
  44#include "amdgpu_dma_buf.h"
  45#include "amdgpu_res_cursor.h"
  46#include "kfd_svm.h"
  47
  48/**
  49 * DOC: GPUVM
  50 *
  51 * GPUVM is the MMU functionality provided on the GPU.
  52 * GPUVM is similar to the legacy GART on older asics, however
  53 * rather than there being a single global GART table
  54 * for the entire GPU, there can be multiple GPUVM page tables active
  55 * at any given time.  The GPUVM page tables can contain a mix
  56 * VRAM pages and system pages (both memory and MMIO) and system pages
  57 * can be mapped as snooped (cached system pages) or unsnooped
  58 * (uncached system pages).
  59 *
  60 * Each active GPUVM has an ID associated with it and there is a page table
  61 * linked with each VMID.  When executing a command buffer,
  62 * the kernel tells the engine what VMID to use for that command
  63 * buffer.  VMIDs are allocated dynamically as commands are submitted.
  64 * The userspace drivers maintain their own address space and the kernel
  65 * sets up their pages tables accordingly when they submit their
  66 * command buffers and a VMID is assigned.
  67 * The hardware supports up to 16 active GPUVMs at any given time.
  68 *
  69 * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending
  70 * on the ASIC family.  GPUVM supports RWX attributes on each page as well
  71 * as other features such as encryption and caching attributes.
  72 *
  73 * VMID 0 is special.  It is the GPUVM used for the kernel driver.  In
  74 * addition to an aperture managed by a page table, VMID 0 also has
  75 * several other apertures.  There is an aperture for direct access to VRAM
  76 * and there is a legacy AGP aperture which just forwards accesses directly
  77 * to the matching system physical addresses (or IOVAs when an IOMMU is
  78 * present).  These apertures provide direct access to these memories without
  79 * incurring the overhead of a page table.  VMID 0 is used by the kernel
  80 * driver for tasks like memory management.
  81 *
  82 * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.
  83 * For user applications, each application can have their own unique GPUVM
  84 * address space.  The application manages the address space and the kernel
  85 * driver manages the GPUVM page tables for each process.  If an GPU client
  86 * accesses an invalid page, it will generate a GPU page fault, similar to
  87 * accessing an invalid page on a CPU.
  88 */
  89
  90#define START(node) ((node)->start)
  91#define LAST(node) ((node)->last)
  92
  93INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  94		     START, LAST, static, amdgpu_vm_it)
  95
  96#undef START
  97#undef LAST
  98
  99/**
 100 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
 101 */
 102struct amdgpu_prt_cb {
 103
 104	/**
 105	 * @adev: amdgpu device
 106	 */
 107	struct amdgpu_device *adev;
 108
 109	/**
 110	 * @cb: callback
 111	 */
 112	struct dma_fence_cb cb;
 113};
 114
 115/**
 116 * struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence
 117 */
 118struct amdgpu_vm_tlb_seq_struct {
 119	/**
 120	 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on
 121	 */
 122	struct amdgpu_vm *vm;
 123
 124	/**
 125	 * @cb: callback
 126	 */
 127	struct dma_fence_cb cb;
 128};
 129
 130/**
 131 * amdgpu_vm_assert_locked - check if VM is correctly locked
 132 * @vm: the VM which schould be tested
 133 *
 134 * Asserts that the VM root PD is locked.
 135 */
 136static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
 137{
 138	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 139}
 140
 141/**
 142 * amdgpu_vm_bo_evicted - vm_bo is evicted
 143 *
 144 * @vm_bo: vm_bo which is evicted
 145 *
 146 * State for PDs/PTs and per VM BOs which are not at the location they should
 147 * be.
 148 */
 149static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 150{
 151	struct amdgpu_vm *vm = vm_bo->vm;
 152	struct amdgpu_bo *bo = vm_bo->bo;
 153
 154	vm_bo->moved = true;
 155	amdgpu_vm_assert_locked(vm);
 156	spin_lock(&vm_bo->vm->status_lock);
 157	if (bo->tbo.type == ttm_bo_type_kernel)
 158		list_move(&vm_bo->vm_status, &vm->evicted);
 159	else
 160		list_move_tail(&vm_bo->vm_status, &vm->evicted);
 161	spin_unlock(&vm_bo->vm->status_lock);
 162}
 163/**
 164 * amdgpu_vm_bo_moved - vm_bo is moved
 165 *
 166 * @vm_bo: vm_bo which is moved
 167 *
 168 * State for per VM BOs which are moved, but that change is not yet reflected
 169 * in the page tables.
 170 */
 171static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 172{
 173	amdgpu_vm_assert_locked(vm_bo->vm);
 174	spin_lock(&vm_bo->vm->status_lock);
 175	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
 176	spin_unlock(&vm_bo->vm->status_lock);
 177}
 178
 179/**
 180 * amdgpu_vm_bo_idle - vm_bo is idle
 181 *
 182 * @vm_bo: vm_bo which is now idle
 183 *
 184 * State for PDs/PTs and per VM BOs which have gone through the state machine
 185 * and are now idle.
 186 */
 187static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 188{
 189	amdgpu_vm_assert_locked(vm_bo->vm);
 190	spin_lock(&vm_bo->vm->status_lock);
 191	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
 192	spin_unlock(&vm_bo->vm->status_lock);
 193	vm_bo->moved = false;
 194}
 195
 196/**
 197 * amdgpu_vm_bo_invalidated - vm_bo is invalidated
 198 *
 199 * @vm_bo: vm_bo which is now invalidated
 200 *
 201 * State for normal BOs which are invalidated and that change not yet reflected
 202 * in the PTs.
 203 */
 204static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 205{
 206	spin_lock(&vm_bo->vm->status_lock);
 207	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
 208	spin_unlock(&vm_bo->vm->status_lock);
 209}
 210
 211/**
 212 * amdgpu_vm_bo_evicted_user - vm_bo is evicted
 213 *
 214 * @vm_bo: vm_bo which is evicted
 215 *
 216 * State for BOs used by user mode queues which are not at the location they
 217 * should be.
 218 */
 219static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
 220{
 221	vm_bo->moved = true;
 222	spin_lock(&vm_bo->vm->status_lock);
 223	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
 224	spin_unlock(&vm_bo->vm->status_lock);
 225}
 226
 227/**
 228 * amdgpu_vm_bo_relocated - vm_bo is reloacted
 229 *
 230 * @vm_bo: vm_bo which is relocated
 231 *
 232 * State for PDs/PTs which needs to update their parent PD.
 233 * For the root PD, just move to idle state.
 234 */
 235static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 236{
 237	amdgpu_vm_assert_locked(vm_bo->vm);
 238	if (vm_bo->bo->parent) {
 239		spin_lock(&vm_bo->vm->status_lock);
 240		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 241		spin_unlock(&vm_bo->vm->status_lock);
 242	} else {
 243		amdgpu_vm_bo_idle(vm_bo);
 244	}
 245}
 246
 247/**
 248 * amdgpu_vm_bo_done - vm_bo is done
 249 *
 250 * @vm_bo: vm_bo which is now done
 251 *
 252 * State for normal BOs which are invalidated and that change has been updated
 253 * in the PTs.
 254 */
 255static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 256{
 257	amdgpu_vm_assert_locked(vm_bo->vm);
 258	spin_lock(&vm_bo->vm->status_lock);
 259	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
 260	spin_unlock(&vm_bo->vm->status_lock);
 261}
 262
 263/**
 264 * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
 265 * @vm: the VM which state machine to reset
 266 *
 267 * Move all vm_bo object in the VM into a state where they will be updated
 268 * again during validation.
 269 */
 270static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 271{
 272	struct amdgpu_vm_bo_base *vm_bo, *tmp;
 273
 274	amdgpu_vm_assert_locked(vm);
 275
 276	spin_lock(&vm->status_lock);
 277	list_splice_init(&vm->done, &vm->invalidated);
 278	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
 279		vm_bo->moved = true;
 280
 281	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
 282		struct amdgpu_bo *bo = vm_bo->bo;
 283
 284		vm_bo->moved = true;
 285		if (!bo || bo->tbo.type != ttm_bo_type_kernel)
 286			list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
 287		else if (bo->parent)
 288			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 289	}
 290	spin_unlock(&vm->status_lock);
 291}
 292
 293/**
 294 * amdgpu_vm_update_shared - helper to update shared memory stat
 295 * @base: base structure for tracking BO usage in a VM
 296 *
 297 * Takes the vm status_lock and updates the shared memory stat. If the basic
 298 * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
 299 * as well.
 300 */
 301static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 302{
 303	struct amdgpu_vm *vm = base->vm;
 304	struct amdgpu_bo *bo = base->bo;
 305	uint64_t size = amdgpu_bo_size(bo);
 306	uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo);
 307	bool shared;
 308
 309	dma_resv_assert_held(bo->tbo.base.resv);
 310	spin_lock(&vm->status_lock);
 311	shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 312	if (base->shared != shared) {
 313		base->shared = shared;
 314		if (shared) {
 315			vm->stats[bo_memtype].drm.shared += size;
 316			vm->stats[bo_memtype].drm.private -= size;
 317		} else {
 318			vm->stats[bo_memtype].drm.shared -= size;
 319			vm->stats[bo_memtype].drm.private += size;
 320		}
 321	}
 322	spin_unlock(&vm->status_lock);
 323}
 324
 325/**
 326 * amdgpu_vm_bo_update_shared - callback when bo gets shared/unshared
 327 * @bo: amdgpu buffer object
 328 *
 329 * Update the per VM stats for all the vm if needed from private to shared or
 330 * vice versa.
 331 */
 332void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
 333{
 334	struct amdgpu_vm_bo_base *base;
 335
 336	for (base = bo->vm_bo; base; base = base->next)
 337		amdgpu_vm_update_shared(base);
 338}
 339
 340/**
 341 * amdgpu_vm_update_stats_locked - helper to update normal memory stat
 342 * @base: base structure for tracking BO usage in a VM
 343 * @res:  the ttm_resource to use for the purpose of accounting, may or may not
 344 *        be bo->tbo.resource
 345 * @sign: if we should add (+1) or subtract (-1) from the stat
 346 *
 347 * Caller need to have the vm status_lock held. Useful for when multiple update
 348 * need to happen at the same time.
 349 */
 350static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
 351			    struct ttm_resource *res, int sign)
 352{
 353	struct amdgpu_vm *vm = base->vm;
 354	struct amdgpu_bo *bo = base->bo;
 355	int64_t size = sign * amdgpu_bo_size(bo);
 356	uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo);
 357
 358	/* For drm-total- and drm-shared-, BO are accounted by their preferred
 359	 * placement, see also amdgpu_bo_mem_stats_placement.
 360	 */
 361	if (base->shared)
 362		vm->stats[bo_memtype].drm.shared += size;
 363	else
 364		vm->stats[bo_memtype].drm.private += size;
 365
 366	if (res && res->mem_type < __AMDGPU_PL_NUM) {
 367		uint32_t res_memtype = res->mem_type;
 368
 369		vm->stats[res_memtype].drm.resident += size;
 370		/* BO only count as purgeable if it is resident,
 371		 * since otherwise there's nothing to purge.
 372		 */
 373		if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
 374			vm->stats[res_memtype].drm.purgeable += size;
 375		if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
 376			vm->stats[bo_memtype].evicted += size;
 377	}
 378}
 379
 380/**
 381 * amdgpu_vm_update_stats - helper to update normal memory stat
 382 * @base: base structure for tracking BO usage in a VM
 383 * @res:  the ttm_resource to use for the purpose of accounting, may or may not
 384 *        be bo->tbo.resource
 385 * @sign: if we should add (+1) or subtract (-1) from the stat
 386 *
 387 * Updates the basic memory stat when bo is added/deleted/moved.
 388 */
 389void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
 390			    struct ttm_resource *res, int sign)
 391{
 392	struct amdgpu_vm *vm = base->vm;
 393
 394	spin_lock(&vm->status_lock);
 395	amdgpu_vm_update_stats_locked(base, res, sign);
 396	spin_unlock(&vm->status_lock);
 397}
 398
 399/**
 400 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
 401 *
 402 * @base: base structure for tracking BO usage in a VM
 403 * @vm: vm to which bo is to be added
 404 * @bo: amdgpu buffer object
 405 *
 406 * Initialize a bo_va_base structure and add it to the appropriate lists
 407 *
 408 */
 409void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 410			    struct amdgpu_vm *vm, struct amdgpu_bo *bo)
 411{
 412	base->vm = vm;
 413	base->bo = bo;
 414	base->next = NULL;
 415	INIT_LIST_HEAD(&base->vm_status);
 416
 417	if (!bo)
 418		return;
 419	base->next = bo->vm_bo;
 420	bo->vm_bo = base;
 421
 422	spin_lock(&vm->status_lock);
 423	base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 424	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
 425	spin_unlock(&vm->status_lock);
 426
 427	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
 428		return;
 429
 430	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 431
 432	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
 433	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
 434		amdgpu_vm_bo_relocated(base);
 435	else
 436		amdgpu_vm_bo_idle(base);
 437
 438	if (bo->preferred_domains &
 439	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
 440		return;
 441
 442	/*
 443	 * we checked all the prerequisites, but it looks like this per vm bo
 444	 * is currently evicted. add the bo to the evicted list to make sure it
 445	 * is validated on next vm use to avoid fault.
 446	 * */
 447	amdgpu_vm_bo_evicted(base);
 448}
 449
 450/**
 451 * amdgpu_vm_lock_pd - lock PD in drm_exec
 452 *
 453 * @vm: vm providing the BOs
 454 * @exec: drm execution context
 455 * @num_fences: number of extra fences to reserve
 456 *
 457 * Lock the VM root PD in the DRM execution context.
 458 */
 459int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
 460		      unsigned int num_fences)
 461{
 462	/* We need at least two fences for the VM PD/PT updates */
 463	return drm_exec_prepare_obj(exec, &vm->root.bo->tbo.base,
 464				    2 + num_fences);
 465}
 466
 467/**
 468 * amdgpu_vm_lock_done_list - lock all BOs on the done list
 469 * @vm: vm providing the BOs
 470 * @exec: drm execution context
 471 * @num_fences: number of extra fences to reserve
 472 *
 473 * Lock the BOs on the done list in the DRM execution context.
 474 */
 475int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
 476			     unsigned int num_fences)
 477{
 478	struct list_head *prev = &vm->done;
 479	struct amdgpu_bo_va *bo_va;
 480	struct amdgpu_bo *bo;
 481	int ret;
 482
 483	/* We can only trust prev->next while holding the lock */
 484	spin_lock(&vm->status_lock);
 485	while (!list_is_head(prev->next, &vm->done)) {
 486		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
 487
 488		bo = bo_va->base.bo;
 489		if (bo) {
 490			amdgpu_bo_ref(bo);
 491			spin_unlock(&vm->status_lock);
 492
 493			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
 494			amdgpu_bo_unref(&bo);
 495			if (unlikely(ret))
 496				return ret;
 497
 498			spin_lock(&vm->status_lock);
 499		}
 500		prev = prev->next;
 501	}
 502	spin_unlock(&vm->status_lock);
 503
 504	return 0;
 505}
 506
 507/**
 508 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
 509 *
 510 * @adev: amdgpu device pointer
 511 * @vm: vm providing the BOs
 512 *
 513 * Move all BOs to the end of LRU and remember their positions to put them
 514 * together.
 515 */
 516void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 517				struct amdgpu_vm *vm)
 518{
 519	spin_lock(&adev->mman.bdev.lru_lock);
 520	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
 521	spin_unlock(&adev->mman.bdev.lru_lock);
 522}
 523
 524/* Create scheduler entities for page table updates */
 525static int amdgpu_vm_init_entities(struct amdgpu_device *adev,
 526				   struct amdgpu_vm *vm)
 527{
 528	int r;
 529
 530	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
 531				  adev->vm_manager.vm_pte_scheds,
 532				  adev->vm_manager.vm_pte_num_scheds, NULL);
 533	if (r)
 534		goto error;
 535
 536	return drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
 537				     adev->vm_manager.vm_pte_scheds,
 538				     adev->vm_manager.vm_pte_num_scheds, NULL);
 539
 540error:
 541	drm_sched_entity_destroy(&vm->immediate);
 542	return r;
 543}
 544
 545/* Destroy the entities for page table updates again */
 546static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm)
 547{
 548	drm_sched_entity_destroy(&vm->immediate);
 549	drm_sched_entity_destroy(&vm->delayed);
 550}
 551
 552/**
 553 * amdgpu_vm_generation - return the page table re-generation counter
 554 * @adev: the amdgpu_device
 555 * @vm: optional VM to check, might be NULL
 556 *
 557 * Returns a page table re-generation token to allow checking if submissions
 558 * are still valid to use this VM. The VM parameter might be NULL in which case
 559 * just the VRAM lost counter will be used.
 560 */
 561uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 562{
 563	uint64_t result = (u64)atomic_read(&adev->vram_lost_counter) << 32;
 564
 565	if (!vm)
 566		return result;
 567
 568	result += lower_32_bits(vm->generation);
 569	/* Add one if the page tables will be re-generated on next CS */
 570	if (drm_sched_entity_error(&vm->delayed))
 571		++result;
 572
 573	return result;
 574}
 575
 576/**
 577 * amdgpu_vm_validate - validate evicted BOs tracked in the VM
 578 *
 579 * @adev: amdgpu device pointer
 580 * @vm: vm providing the BOs
 581 * @ticket: optional reservation ticket used to reserve the VM
 582 * @validate: callback to do the validation
 583 * @param: parameter for the validation callback
 584 *
 585 * Validate the page table BOs and per-VM BOs on command submission if
 586 * necessary. If a ticket is given, also try to validate evicted user queue
 587 * BOs. They must already be reserved with the given ticket.
 588 *
 589 * Returns:
 590 * Validation result.
 591 */
 592int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 593		       struct ww_acquire_ctx *ticket,
 594		       int (*validate)(void *p, struct amdgpu_bo *bo),
 595		       void *param)
 596{
 597	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
 598	struct amdgpu_vm_bo_base *bo_base;
 599	struct amdgpu_bo *bo;
 600	int r;
 601
 602	if (vm->generation != new_vm_generation) {
 603		vm->generation = new_vm_generation;
 604		amdgpu_vm_bo_reset_state_machine(vm);
 605		amdgpu_vm_fini_entities(vm);
 606		r = amdgpu_vm_init_entities(adev, vm);
 607		if (r)
 608			return r;
 609	}
 610
 611	spin_lock(&vm->status_lock);
 612	while (!list_empty(&vm->evicted)) {
 613		bo_base = list_first_entry(&vm->evicted,
 614					   struct amdgpu_vm_bo_base,
 615					   vm_status);
 616		spin_unlock(&vm->status_lock);
 617
 618		bo = bo_base->bo;
 619
 620		r = validate(param, bo);
 621		if (r)
 622			return r;
 623
 624		if (bo->tbo.type != ttm_bo_type_kernel) {
 625			amdgpu_vm_bo_moved(bo_base);
 626		} else {
 627			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
 628			amdgpu_vm_bo_relocated(bo_base);
 629		}
 630		spin_lock(&vm->status_lock);
 631	}
 632	while (ticket && !list_empty(&vm->evicted_user)) {
 633		bo_base = list_first_entry(&vm->evicted_user,
 634					   struct amdgpu_vm_bo_base,
 635					   vm_status);
 636		spin_unlock(&vm->status_lock);
 637
 638		bo = bo_base->bo;
 639		dma_resv_assert_held(bo->tbo.base.resv);
 640
 641		r = validate(param, bo);
 642		if (r)
 643			return r;
 644
 645		amdgpu_vm_bo_invalidated(bo_base);
 646
 647		spin_lock(&vm->status_lock);
 648	}
 649	spin_unlock(&vm->status_lock);
 650
 651	amdgpu_vm_eviction_lock(vm);
 652	vm->evicting = false;
 653	amdgpu_vm_eviction_unlock(vm);
 654
 655	return 0;
 656}
 657
 658/**
 659 * amdgpu_vm_ready - check VM is ready for updates
 660 *
 661 * @vm: VM to check
 662 *
 663 * Check if all VM PDs/PTs are ready for updates
 664 *
 665 * Returns:
 666 * True if VM is not evicting and all VM entities are not stopped
 667 */
 668bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 669{
 670	bool ret;
 671
 672	amdgpu_vm_assert_locked(vm);
 673
 674	amdgpu_vm_eviction_lock(vm);
 675	ret = !vm->evicting;
 676	amdgpu_vm_eviction_unlock(vm);
 677
 678	spin_lock(&vm->status_lock);
 679	ret &= list_empty(&vm->evicted);
 680	spin_unlock(&vm->status_lock);
 681
 682	spin_lock(&vm->immediate.lock);
 683	ret &= !vm->immediate.stopped;
 684	spin_unlock(&vm->immediate.lock);
 685
 686	spin_lock(&vm->delayed.lock);
 687	ret &= !vm->delayed.stopped;
 688	spin_unlock(&vm->delayed.lock);
 689
 690	return ret;
 691}
 692
 693/**
 694 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 695 *
 696 * @adev: amdgpu_device pointer
 697 */
 698void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 699{
 700	const struct amdgpu_ip_block *ip_block;
 701	bool has_compute_vm_bug;
 702	struct amdgpu_ring *ring;
 703	int i;
 704
 705	has_compute_vm_bug = false;
 706
 707	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 708	if (ip_block) {
 709		/* Compute has a VM bug for GFX version < 7.
 710		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
 711		if (ip_block->version->major <= 7)
 712			has_compute_vm_bug = true;
 713		else if (ip_block->version->major == 8)
 714			if (adev->gfx.mec_fw_version < 673)
 715				has_compute_vm_bug = true;
 716	}
 717
 718	for (i = 0; i < adev->num_rings; i++) {
 719		ring = adev->rings[i];
 720		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
 721			/* only compute rings */
 722			ring->has_compute_vm_bug = has_compute_vm_bug;
 723		else
 724			ring->has_compute_vm_bug = false;
 725	}
 726}
 727
 728/**
 729 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
 730 *
 731 * @ring: ring on which the job will be submitted
 732 * @job: job to submit
 733 *
 734 * Returns:
 735 * True if sync is needed.
 736 */
 737bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 738				  struct amdgpu_job *job)
 739{
 740	struct amdgpu_device *adev = ring->adev;
 741	unsigned vmhub = ring->vm_hub;
 742	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 743
 744	if (job->vmid == 0)
 745		return false;
 746
 747	if (job->vm_needs_flush || ring->has_compute_vm_bug)
 748		return true;
 749
 750	if (ring->funcs->emit_gds_switch && job->gds_switch_needed)
 751		return true;
 752
 753	if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid]))
 754		return true;
 755
 756	return false;
 757}
 758
 759/**
 760 * amdgpu_vm_flush - hardware flush the vm
 761 *
 762 * @ring: ring to use for flush
 763 * @job:  related job
 764 * @need_pipe_sync: is pipe sync needed
 765 *
 766 * Emit a VM flush when it is necessary.
 767 *
 768 * Returns:
 769 * 0 on success, errno otherwise.
 770 */
 771int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 772		    bool need_pipe_sync)
 773{
 774	struct amdgpu_device *adev = ring->adev;
 775	struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];
 776	unsigned vmhub = ring->vm_hub;
 777	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 778	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
 779	bool spm_update_needed = job->spm_update_needed;
 780	bool gds_switch_needed = ring->funcs->emit_gds_switch &&
 781		job->gds_switch_needed;
 782	bool vm_flush_needed = job->vm_needs_flush;
 783	bool cleaner_shader_needed = false;
 784	bool pasid_mapping_needed = false;
 785	struct dma_fence *fence = NULL;
 786	unsigned int patch;
 787	int r;
 788
 789	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
 790		gds_switch_needed = true;
 791		vm_flush_needed = true;
 792		pasid_mapping_needed = true;
 793		spm_update_needed = true;
 794	}
 795
 796	mutex_lock(&id_mgr->lock);
 797	if (id->pasid != job->pasid || !id->pasid_mapping ||
 798	    !dma_fence_is_signaled(id->pasid_mapping))
 799		pasid_mapping_needed = true;
 800	mutex_unlock(&id_mgr->lock);
 801
 802	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
 803	vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
 804			job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
 805	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
 806		ring->funcs->emit_wreg;
 807
 808	cleaner_shader_needed = job->run_cleaner_shader &&
 809		adev->gfx.enable_cleaner_shader &&
 810		ring->funcs->emit_cleaner_shader && job->base.s_fence &&
 811		&job->base.s_fence->scheduled == isolation->spearhead;
 812
 813	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync &&
 814	    !cleaner_shader_needed)
 815		return 0;
 816
 817	amdgpu_ring_ib_begin(ring);
 818	if (ring->funcs->init_cond_exec)
 819		patch = amdgpu_ring_init_cond_exec(ring,
 820						   ring->cond_exe_gpu_addr);
 821
 822	if (need_pipe_sync)
 823		amdgpu_ring_emit_pipeline_sync(ring);
 824
 825	if (cleaner_shader_needed)
 826		ring->funcs->emit_cleaner_shader(ring);
 827
 828	if (vm_flush_needed) {
 829		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
 830		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
 831	}
 832
 833	if (pasid_mapping_needed)
 834		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
 835
 836	if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)
 837		adev->gfx.rlc.funcs->update_spm_vmid(adev, ring, job->vmid);
 838
 839	if (ring->funcs->emit_gds_switch &&
 840	    gds_switch_needed) {
 841		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 842					    job->gds_size, job->gws_base,
 843					    job->gws_size, job->oa_base,
 844					    job->oa_size);
 845	}
 846
 847	if (vm_flush_needed || pasid_mapping_needed || cleaner_shader_needed) {
 848		r = amdgpu_fence_emit(ring, job->hw_vm_fence, 0);
 849		if (r)
 850			return r;
 851		fence = &job->hw_vm_fence->base;
 852		/* get a ref for the job */
 853		dma_fence_get(fence);
 854	}
 855
 856	if (vm_flush_needed) {
 857		mutex_lock(&id_mgr->lock);
 858		dma_fence_put(id->last_flush);
 859		id->last_flush = dma_fence_get(fence);
 860		id->current_gpu_reset_count =
 861			atomic_read(&adev->gpu_reset_counter);
 862		mutex_unlock(&id_mgr->lock);
 863	}
 864
 865	if (pasid_mapping_needed) {
 866		mutex_lock(&id_mgr->lock);
 867		id->pasid = job->pasid;
 868		dma_fence_put(id->pasid_mapping);
 869		id->pasid_mapping = dma_fence_get(fence);
 870		mutex_unlock(&id_mgr->lock);
 871	}
 872
 873	/*
 874	 * Make sure that all other submissions wait for the cleaner shader to
 875	 * finish before we push them to the HW.
 876	 */
 877	if (cleaner_shader_needed) {
 878		trace_amdgpu_cleaner_shader(ring, fence);
 879		mutex_lock(&adev->enforce_isolation_mutex);
 880		dma_fence_put(isolation->spearhead);
 881		isolation->spearhead = dma_fence_get(fence);
 882		mutex_unlock(&adev->enforce_isolation_mutex);
 883	}
 884	dma_fence_put(fence);
 885
 886	amdgpu_ring_patch_cond_exec(ring, patch);
 887
 888	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
 889	if (ring->funcs->emit_switch_buffer) {
 890		amdgpu_ring_emit_switch_buffer(ring);
 891		amdgpu_ring_emit_switch_buffer(ring);
 892	}
 893
 894	amdgpu_ring_ib_end(ring);
 895	return 0;
 896}
 897
 898/**
 899 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 900 *
 901 * @vm: requested vm
 902 * @bo: requested buffer object
 903 *
 904 * Find @bo inside the requested vm.
 905 * Search inside the @bos vm list for the requested vm
 906 * Returns the found bo_va or NULL if none is found
 907 *
 908 * Object has to be reserved!
 909 *
 910 * Returns:
 911 * Found bo_va or NULL.
 912 */
 913struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 914				       struct amdgpu_bo *bo)
 915{
 916	struct amdgpu_vm_bo_base *base;
 917
 918	for (base = bo->vm_bo; base; base = base->next) {
 919		if (base->vm != vm)
 920			continue;
 921
 922		return container_of(base, struct amdgpu_bo_va, base);
 923	}
 924	return NULL;
 925}
 926
 927/**
 928 * amdgpu_vm_map_gart - Resolve gart mapping of addr
 929 *
 930 * @pages_addr: optional DMA address to use for lookup
 931 * @addr: the unmapped addr
 932 *
 933 * Look up the physical address of the page that the pte resolves
 934 * to.
 935 *
 936 * Returns:
 937 * The pointer for the page table entry.
 938 */
 939uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 940{
 941	uint64_t result;
 942
 943	/* page table offset */
 944	result = pages_addr[addr >> PAGE_SHIFT];
 945
 946	/* in case cpu page size != gpu page size*/
 947	result |= addr & (~PAGE_MASK);
 948
 949	result &= 0xFFFFFFFFFFFFF000ULL;
 950
 951	return result;
 952}
 953
 954/**
 955 * amdgpu_vm_update_pdes - make sure that all directories are valid
 956 *
 957 * @adev: amdgpu_device pointer
 958 * @vm: requested vm
 959 * @immediate: submit immediately to the paging queue
 960 *
 961 * Makes sure all directories are up to date.
 962 *
 963 * Returns:
 964 * 0 for success, error for failure.
 965 */
 966int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 967			  struct amdgpu_vm *vm, bool immediate)
 968{
 969	struct amdgpu_vm_update_params params;
 970	struct amdgpu_vm_bo_base *entry;
 971	bool flush_tlb_needed = false;
 972	LIST_HEAD(relocated);
 973	int r, idx;
 974
 975	amdgpu_vm_assert_locked(vm);
 976
 977	spin_lock(&vm->status_lock);
 978	list_splice_init(&vm->relocated, &relocated);
 979	spin_unlock(&vm->status_lock);
 980
 981	if (list_empty(&relocated))
 982		return 0;
 983
 984	if (!drm_dev_enter(adev_to_drm(adev), &idx))
 985		return -ENODEV;
 986
 987	memset(&params, 0, sizeof(params));
 988	params.adev = adev;
 989	params.vm = vm;
 990	params.immediate = immediate;
 991
 992	r = vm->update_funcs->prepare(&params, NULL,
 993				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES);
 994	if (r)
 995		goto error;
 996
 997	list_for_each_entry(entry, &relocated, vm_status) {
 998		/* vm_flush_needed after updating moved PDEs */
 999		flush_tlb_needed |= entry->moved;
1000
1001		r = amdgpu_vm_pde_update(&params, entry);
1002		if (r)
1003			goto error;
1004	}
1005
1006	r = vm->update_funcs->commit(&params, &vm->last_update);
1007	if (r)
1008		goto error;
1009
1010	if (flush_tlb_needed)
1011		atomic64_inc(&vm->tlb_seq);
1012
1013	while (!list_empty(&relocated)) {
1014		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
1015					 vm_status);
1016		amdgpu_vm_bo_idle(entry);
1017	}
1018
1019error:
1020	drm_dev_exit(idx);
1021	return r;
1022}
1023
1024/**
1025 * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
1026 * @fence: unused
1027 * @cb: the callback structure
1028 *
1029 * Increments the tlb sequence to make sure that future CS execute a VM flush.
1030 */
1031static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
1032				 struct dma_fence_cb *cb)
1033{
1034	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
1035
1036	tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
1037	atomic64_inc(&tlb_cb->vm->tlb_seq);
1038	kfree(tlb_cb);
1039}
1040
1041/**
1042 * amdgpu_vm_tlb_flush - prepare TLB flush
1043 *
1044 * @params: parameters for update
1045 * @fence: input fence to sync TLB flush with
1046 * @tlb_cb: the callback structure
1047 *
1048 * Increments the tlb sequence to make sure that future CS execute a VM flush.
1049 */
1050static void
1051amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
1052		    struct dma_fence **fence,
1053		    struct amdgpu_vm_tlb_seq_struct *tlb_cb)
1054{
1055	struct amdgpu_vm *vm = params->vm;
1056
1057	tlb_cb->vm = vm;
1058	if (!fence || !*fence) {
1059		amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
1060		return;
1061	}
1062
1063	if (!dma_fence_add_callback(*fence, &tlb_cb->cb,
1064				    amdgpu_vm_tlb_seq_cb)) {
1065		dma_fence_put(vm->last_tlb_flush);
1066		vm->last_tlb_flush = dma_fence_get(*fence);
1067	} else {
1068		amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
1069	}
1070
1071	/* Prepare a TLB flush fence to be attached to PTs */
1072	/* The check for need_tlb_fence should be dropped once we
1073	 * sort out the issues with KIQ/MES TLB invalidation timeouts.
1074	 */
1075	if (!params->unlocked && vm->need_tlb_fence) {
1076		amdgpu_vm_tlb_fence_create(params->adev, vm, fence);
1077
1078		/* Makes sure no PD/PT is freed before the flush */
1079		dma_resv_add_fence(vm->root.bo->tbo.base.resv, *fence,
1080				   DMA_RESV_USAGE_BOOKKEEP);
1081	}
1082}
1083
1084/**
1085 * amdgpu_vm_update_range - update a range in the vm page table
1086 *
1087 * @adev: amdgpu_device pointer to use for commands
1088 * @vm: the VM to update the range
1089 * @immediate: immediate submission in a page fault
1090 * @unlocked: unlocked invalidation during MM callback
1091 * @flush_tlb: trigger tlb invalidation after update completed
1092 * @allow_override: change MTYPE for local NUMA nodes
1093 * @sync: fences we need to sync to
1094 * @start: start of mapped range
1095 * @last: last mapped entry
1096 * @flags: flags for the entries
1097 * @offset: offset into nodes and pages_addr
1098 * @vram_base: base for vram mappings
1099 * @res: ttm_resource to map
1100 * @pages_addr: DMA addresses to use for mapping
1101 * @fence: optional resulting fence
1102 *
1103 * Fill in the page table entries between @start and @last.
1104 *
1105 * Returns:
1106 * 0 for success, negative erro code for failure.
1107 */
1108int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1109			   bool immediate, bool unlocked, bool flush_tlb,
1110			   bool allow_override, struct amdgpu_sync *sync,
1111			   uint64_t start, uint64_t last, uint64_t flags,
1112			   uint64_t offset, uint64_t vram_base,
1113			   struct ttm_resource *res, dma_addr_t *pages_addr,
1114			   struct dma_fence **fence)
1115{
1116	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
1117	struct amdgpu_vm_update_params params;
1118	struct amdgpu_res_cursor cursor;
1119	int r, idx;
1120
1121	if (!drm_dev_enter(adev_to_drm(adev), &idx))
1122		return -ENODEV;
1123
1124	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
1125	if (!tlb_cb) {
1126		drm_dev_exit(idx);
1127		return -ENOMEM;
1128	}
1129
1130	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
1131	 * heavy-weight flush TLB unconditionally.
1132	 */
1133	flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
1134		     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0);
1135
1136	/*
1137	 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
1138	 */
1139	flush_tlb |= amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 0, 0);
1140
1141	memset(&params, 0, sizeof(params));
1142	params.adev = adev;
1143	params.vm = vm;
1144	params.immediate = immediate;
1145	params.pages_addr = pages_addr;
1146	params.unlocked = unlocked;
1147	params.needs_flush = flush_tlb;
1148	params.allow_override = allow_override;
1149	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
1150
1151	amdgpu_vm_eviction_lock(vm);
1152	if (vm->evicting) {
1153		r = -EBUSY;
1154		goto error_free;
1155	}
1156
1157	if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
1158		struct dma_fence *tmp = dma_fence_get_stub();
1159
1160		amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
1161		swap(vm->last_unlocked, tmp);
1162		dma_fence_put(tmp);
1163	}
1164
1165	r = vm->update_funcs->prepare(&params, sync,
1166				      AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
1167	if (r)
1168		goto error_free;
1169
1170	amdgpu_res_first(pages_addr ? NULL : res, offset,
1171			 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
1172	while (cursor.remaining) {
1173		uint64_t tmp, num_entries, addr;
1174
1175		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
1176		if (pages_addr) {
1177			bool contiguous = true;
1178
1179			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
1180				uint64_t pfn = cursor.start >> PAGE_SHIFT;
1181				uint64_t count;
1182
1183				contiguous = pages_addr[pfn + 1] ==
1184					pages_addr[pfn] + PAGE_SIZE;
1185
1186				tmp = num_entries /
1187					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1188				for (count = 2; count < tmp; ++count) {
1189					uint64_t idx = pfn + count;
1190
1191					if (contiguous != (pages_addr[idx] ==
1192					    pages_addr[idx - 1] + PAGE_SIZE))
1193						break;
1194				}
1195				if (!contiguous)
1196					count--;
1197				num_entries = count *
1198					AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1199			}
1200
1201			if (!contiguous) {
1202				addr = cursor.start;
1203				params.pages_addr = pages_addr;
1204			} else {
1205				addr = pages_addr[cursor.start >> PAGE_SHIFT];
1206				params.pages_addr = NULL;
1207			}
1208
1209		} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT_FLAG(adev))) {
1210			addr = vram_base + cursor.start;
1211		} else {
1212			addr = 0;
1213		}
1214
1215		tmp = start + num_entries;
1216		r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags);
1217		if (r)
1218			goto error_free;
1219
1220		amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
1221		start = tmp;
1222	}
1223
1224	r = vm->update_funcs->commit(&params, fence);
1225	if (r)
1226		goto error_free;
1227
1228	if (params.needs_flush) {
1229		amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
1230		tlb_cb = NULL;
1231	}
1232
1233	amdgpu_vm_pt_free_list(adev, &params);
1234
1235error_free:
1236	kfree(tlb_cb);
1237	amdgpu_vm_eviction_unlock(vm);
1238	drm_dev_exit(idx);
1239	return r;
1240}
1241
1242void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
1243			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
1244{
1245	spin_lock(&vm->status_lock);
1246	memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
1247	spin_unlock(&vm->status_lock);
1248}
1249
1250/**
1251 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1252 *
1253 * @adev: amdgpu_device pointer
1254 * @bo_va: requested BO and VM object
1255 * @clear: if true clear the entries
1256 *
1257 * Fill in the page table entries for @bo_va.
1258 *
1259 * Returns:
1260 * 0 for success, -EINVAL for failure.
1261 */
1262int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
1263			bool clear)
1264{
1265	struct amdgpu_bo *bo = bo_va->base.bo;
1266	struct amdgpu_vm *vm = bo_va->base.vm;
1267	struct amdgpu_bo_va_mapping *mapping;
1268	struct dma_fence **last_update;
1269	dma_addr_t *pages_addr = NULL;
1270	struct ttm_resource *mem;
1271	struct amdgpu_sync sync;
1272	bool flush_tlb = clear;
1273	uint64_t vram_base;
1274	uint64_t flags;
1275	bool uncached;
1276	int r;
1277
1278	amdgpu_sync_create(&sync);
1279	if (clear) {
1280		mem = NULL;
1281
1282		/* Implicitly sync to command submissions in the same VM before
1283		 * unmapping.
1284		 */
1285		r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv,
1286				     AMDGPU_SYNC_EQ_OWNER, vm);
1287		if (r)
1288			goto error_free;
1289		if (bo) {
1290			r = amdgpu_sync_kfd(&sync, bo->tbo.base.resv);
1291			if (r)
1292				goto error_free;
1293		}
1294	} else if (!bo) {
1295		mem = NULL;
1296
1297		/* PRT map operations don't need to sync to anything. */
1298
1299	} else {
1300		struct drm_gem_object *obj = &bo->tbo.base;
1301
1302		if (drm_gem_is_imported(obj) && bo_va->is_xgmi) {
1303			struct dma_buf *dma_buf = obj->import_attach->dmabuf;
1304			struct drm_gem_object *gobj = dma_buf->priv;
1305			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
1306
1307			if (abo->tbo.resource &&
1308			    abo->tbo.resource->mem_type == TTM_PL_VRAM)
1309				bo = gem_to_amdgpu_bo(gobj);
1310		}
1311		mem = bo->tbo.resource;
1312		if (mem && (mem->mem_type == TTM_PL_TT ||
1313			    mem->mem_type == AMDGPU_PL_PREEMPT))
1314			pages_addr = bo->tbo.ttm->dma_address;
1315
1316		/* Implicitly sync to moving fences before mapping anything */
1317		r = amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv,
1318				     AMDGPU_SYNC_EXPLICIT, vm);
1319		if (r)
1320			goto error_free;
1321	}
1322
1323	if (bo) {
1324		struct amdgpu_device *bo_adev;
1325
1326		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1327
1328		if (amdgpu_bo_encrypted(bo))
1329			flags |= AMDGPU_PTE_TMZ;
1330
1331		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1332		vram_base = bo_adev->vm_manager.vram_base_offset;
1333		uncached = (bo->flags & AMDGPU_GEM_CREATE_UNCACHED) != 0;
1334	} else {
1335		flags = 0x0;
1336		vram_base = 0;
1337		uncached = false;
1338	}
1339
1340	if (clear || amdgpu_vm_is_bo_always_valid(vm, bo))
1341		last_update = &vm->last_update;
1342	else
1343		last_update = &bo_va->last_pt_update;
1344
1345	if (!clear && bo_va->base.moved) {
1346		flush_tlb = true;
1347		list_splice_init(&bo_va->valids, &bo_va->invalids);
1348
1349	} else if (bo_va->cleared != clear) {
1350		list_splice_init(&bo_va->valids, &bo_va->invalids);
1351	}
1352
1353	list_for_each_entry(mapping, &bo_va->invalids, list) {
1354		uint64_t update_flags = flags;
1355
1356		/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1357		 * but in case of something, we filter the flags in first place
1358		 */
1359		if (!(mapping->flags & AMDGPU_VM_PAGE_READABLE))
1360			update_flags &= ~AMDGPU_PTE_READABLE;
1361		if (!(mapping->flags & AMDGPU_VM_PAGE_WRITEABLE))
1362			update_flags &= ~AMDGPU_PTE_WRITEABLE;
1363
1364		/* Apply ASIC specific mapping flags */
1365		amdgpu_gmc_get_vm_pte(adev, vm, bo, mapping->flags,
1366				      &update_flags);
1367
1368		trace_amdgpu_vm_bo_update(mapping);
1369
1370		r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
1371					   !uncached, &sync, mapping->start,
1372					   mapping->last, update_flags,
1373					   mapping->offset, vram_base, mem,
1374					   pages_addr, last_update);
1375		if (r)
1376			goto error_free;
1377	}
1378
1379	/* If the BO is not in its preferred location add it back to
1380	 * the evicted list so that it gets validated again on the
1381	 * next command submission.
1382	 */
1383	if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
1384		if (bo->tbo.resource &&
1385		    !(bo->preferred_domains &
1386		      amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)))
1387			amdgpu_vm_bo_evicted(&bo_va->base);
1388		else
1389			amdgpu_vm_bo_idle(&bo_va->base);
1390	} else {
1391		amdgpu_vm_bo_done(&bo_va->base);
1392	}
1393
1394	list_splice_init(&bo_va->invalids, &bo_va->valids);
1395	bo_va->cleared = clear;
1396	bo_va->base.moved = false;
1397
1398	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1399		list_for_each_entry(mapping, &bo_va->valids, list)
1400			trace_amdgpu_vm_bo_mapping(mapping);
1401	}
1402
1403error_free:
1404	amdgpu_sync_free(&sync);
1405	return r;
1406}
1407
1408/**
1409 * amdgpu_vm_update_prt_state - update the global PRT state
1410 *
1411 * @adev: amdgpu_device pointer
1412 */
1413static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1414{
1415	unsigned long flags;
1416	bool enable;
1417
1418	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1419	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1420	adev->gmc.gmc_funcs->set_prt(adev, enable);
1421	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1422}
1423
1424/**
1425 * amdgpu_vm_prt_get - add a PRT user
1426 *
1427 * @adev: amdgpu_device pointer
1428 */
1429static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1430{
1431	if (!adev->gmc.gmc_funcs->set_prt)
1432		return;
1433
1434	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1435		amdgpu_vm_update_prt_state(adev);
1436}
1437
1438/**
1439 * amdgpu_vm_prt_put - drop a PRT user
1440 *
1441 * @adev: amdgpu_device pointer
1442 */
1443static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1444{
1445	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1446		amdgpu_vm_update_prt_state(adev);
1447}
1448
1449/**
1450 * amdgpu_vm_prt_cb - callback for updating the PRT status
1451 *
1452 * @fence: fence for the callback
1453 * @_cb: the callback function
1454 */
1455static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1456{
1457	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1458
1459	amdgpu_vm_prt_put(cb->adev);
1460	kfree(cb);
1461}
1462
1463/**
1464 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1465 *
1466 * @adev: amdgpu_device pointer
1467 * @fence: fence for the callback
1468 */
1469static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1470				 struct dma_fence *fence)
1471{
1472	struct amdgpu_prt_cb *cb;
1473
1474	if (!adev->gmc.gmc_funcs->set_prt)
1475		return;
1476
1477	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1478	if (!cb) {
1479		/* Last resort when we are OOM */
1480		if (fence)
1481			dma_fence_wait(fence, false);
1482
1483		amdgpu_vm_prt_put(adev);
1484	} else {
1485		cb->adev = adev;
1486		if (!fence || dma_fence_add_callback(fence, &cb->cb,
1487						     amdgpu_vm_prt_cb))
1488			amdgpu_vm_prt_cb(fence, &cb->cb);
1489	}
1490}
1491
1492/**
1493 * amdgpu_vm_free_mapping - free a mapping
1494 *
1495 * @adev: amdgpu_device pointer
1496 * @vm: requested vm
1497 * @mapping: mapping to be freed
1498 * @fence: fence of the unmap operation
1499 *
1500 * Free a mapping and make sure we decrease the PRT usage count if applicable.
1501 */
1502static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1503				   struct amdgpu_vm *vm,
1504				   struct amdgpu_bo_va_mapping *mapping,
1505				   struct dma_fence *fence)
1506{
1507	if (mapping->flags & AMDGPU_VM_PAGE_PRT)
1508		amdgpu_vm_add_prt_cb(adev, fence);
1509	kfree(mapping);
1510}
1511
1512/**
1513 * amdgpu_vm_prt_fini - finish all prt mappings
1514 *
1515 * @adev: amdgpu_device pointer
1516 * @vm: requested vm
1517 *
1518 * Register a cleanup callback to disable PRT support after VM dies.
1519 */
1520static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1521{
1522	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1523	struct dma_resv_iter cursor;
1524	struct dma_fence *fence;
1525
1526	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1527		/* Add a callback for each fence in the reservation object */
1528		amdgpu_vm_prt_get(adev);
1529		amdgpu_vm_add_prt_cb(adev, fence);
1530	}
1531}
1532
1533/**
1534 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1535 *
1536 * @adev: amdgpu_device pointer
1537 * @vm: requested vm
1538 * @fence: optional resulting fence (unchanged if no work needed to be done
1539 * or if an error occurred)
1540 *
1541 * Make sure all freed BOs are cleared in the PT.
1542 * PTs have to be reserved and mutex must be locked!
1543 *
1544 * Returns:
1545 * 0 for success.
1546 *
1547 */
1548int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1549			  struct amdgpu_vm *vm,
1550			  struct dma_fence **fence)
1551{
1552	struct amdgpu_bo_va_mapping *mapping;
1553	struct dma_fence *f = NULL;
1554	struct amdgpu_sync sync;
1555	int r;
1556
1557
1558	/*
1559	 * Implicitly sync to command submissions in the same VM before
1560	 * unmapping.
1561	 */
1562	amdgpu_sync_create(&sync);
1563	r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv,
1564			     AMDGPU_SYNC_EQ_OWNER, vm);
1565	if (r)
1566		goto error_free;
1567
1568	while (!list_empty(&vm->freed)) {
1569		mapping = list_first_entry(&vm->freed,
1570			struct amdgpu_bo_va_mapping, list);
1571		list_del(&mapping->list);
1572
1573		r = amdgpu_vm_update_range(adev, vm, false, false, true, false,
1574					   &sync, mapping->start, mapping->last,
1575					   0, 0, 0, NULL, NULL, &f);
1576		amdgpu_vm_free_mapping(adev, vm, mapping, f);
1577		if (r) {
1578			dma_fence_put(f);
1579			goto error_free;
1580		}
1581	}
1582
1583	if (fence && f) {
1584		dma_fence_put(*fence);
1585		*fence = f;
1586	} else {
1587		dma_fence_put(f);
1588	}
1589
1590error_free:
1591	amdgpu_sync_free(&sync);
1592	return r;
1593
1594}
1595
1596/**
1597 * amdgpu_vm_handle_moved - handle moved BOs in the PT
1598 *
1599 * @adev: amdgpu_device pointer
1600 * @vm: requested vm
1601 * @ticket: optional reservation ticket used to reserve the VM
1602 *
1603 * Make sure all BOs which are moved are updated in the PTs.
1604 *
1605 * Returns:
1606 * 0 for success.
1607 *
1608 * PTs have to be reserved!
1609 */
1610int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1611			   struct amdgpu_vm *vm,
1612			   struct ww_acquire_ctx *ticket)
1613{
1614	struct amdgpu_bo_va *bo_va;
1615	struct dma_resv *resv;
1616	bool clear, unlock;
1617	int r;
1618
1619	spin_lock(&vm->status_lock);
1620	while (!list_empty(&vm->moved)) {
1621		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
1622					 base.vm_status);
1623		spin_unlock(&vm->status_lock);
1624
1625		/* Per VM BOs never need to bo cleared in the page tables */
1626		r = amdgpu_vm_bo_update(adev, bo_va, false);
1627		if (r)
1628			return r;
1629		spin_lock(&vm->status_lock);
1630	}
1631
1632	while (!list_empty(&vm->invalidated)) {
1633		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1634					 base.vm_status);
1635		resv = bo_va->base.bo->tbo.base.resv;
1636		spin_unlock(&vm->status_lock);
1637
1638		/* Try to reserve the BO to avoid clearing its ptes */
1639		if (!adev->debug_vm && dma_resv_trylock(resv)) {
1640			clear = false;
1641			unlock = true;
1642		/* The caller is already holding the reservation lock */
1643		} else if (ticket && dma_resv_locking_ctx(resv) == ticket) {
1644			clear = false;
1645			unlock = false;
1646		/* Somebody else is using the BO right now */
1647		} else {
1648			clear = true;
1649			unlock = false;
1650		}
1651
1652		r = amdgpu_vm_bo_update(adev, bo_va, clear);
1653
1654		if (unlock)
1655			dma_resv_unlock(resv);
1656		if (r)
1657			return r;
1658
1659		/* Remember evicted DMABuf imports in compute VMs for later
1660		 * validation
1661		 */
1662		if (vm->is_compute_context &&
1663		    drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
1664		    (!bo_va->base.bo->tbo.resource ||
1665		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
1666			amdgpu_vm_bo_evicted_user(&bo_va->base);
1667
1668		spin_lock(&vm->status_lock);
1669	}
1670	spin_unlock(&vm->status_lock);
1671
1672	return 0;
1673}
1674
1675/**
1676 * amdgpu_vm_flush_compute_tlb - Flush TLB on compute VM
1677 *
1678 * @adev: amdgpu_device pointer
1679 * @vm: requested vm
1680 * @flush_type: flush type
1681 * @xcc_mask: mask of XCCs that belong to the compute partition in need of a TLB flush.
1682 *
1683 * Flush TLB if needed for a compute VM.
1684 *
1685 * Returns:
1686 * 0 for success.
1687 */
1688int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
1689				struct amdgpu_vm *vm,
1690				uint32_t flush_type,
1691				uint32_t xcc_mask)
1692{
1693	uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
1694	bool all_hub = false;
1695	int xcc = 0, r = 0;
1696
1697	WARN_ON_ONCE(!vm->is_compute_context);
1698
1699	/*
1700	 * It can be that we race and lose here, but that is extremely unlikely
1701	 * and the worst thing which could happen is that we flush the changes
1702	 * into the TLB once more which is harmless.
1703	 */
1704	if (atomic64_xchg(&vm->kfd_last_flushed_seq, tlb_seq) == tlb_seq)
1705		return 0;
1706
1707	if (adev->family == AMDGPU_FAMILY_AI ||
1708	    adev->family == AMDGPU_FAMILY_RV)
1709		all_hub = true;
1710
1711	for_each_inst(xcc, xcc_mask) {
1712		r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, flush_type,
1713						   all_hub, xcc);
1714		if (r)
1715			break;
1716	}
1717	return r;
1718}
1719
1720/**
1721 * amdgpu_vm_bo_add - add a bo to a specific vm
1722 *
1723 * @adev: amdgpu_device pointer
1724 * @vm: requested vm
1725 * @bo: amdgpu buffer object
1726 *
1727 * Add @bo into the requested vm.
1728 * Add @bo to the list of bos associated with the vm
1729 *
1730 * Returns:
1731 * Newly added bo_va or NULL for failure
1732 *
1733 * Object has to be reserved!
1734 */
1735struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1736				      struct amdgpu_vm *vm,
1737				      struct amdgpu_bo *bo)
1738{
1739	struct amdgpu_bo_va *bo_va;
1740
1741	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1742	if (bo_va == NULL) {
1743		return NULL;
1744	}
1745	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
1746
1747	bo_va->ref_count = 1;
1748	bo_va->last_pt_update = dma_fence_get_stub();
1749	INIT_LIST_HEAD(&bo_va->valids);
1750	INIT_LIST_HEAD(&bo_va->invalids);
1751
1752	if (!bo)
1753		return bo_va;
1754
1755	dma_resv_assert_held(bo->tbo.base.resv);
1756	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1757		bo_va->is_xgmi = true;
1758		/* Power up XGMI if it can be potentially used */
1759		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1760	}
1761
1762	return bo_va;
1763}
1764
1765
1766/**
1767 * amdgpu_vm_bo_insert_map - insert a new mapping
1768 *
1769 * @adev: amdgpu_device pointer
1770 * @bo_va: bo_va to store the address
1771 * @mapping: the mapping to insert
1772 *
1773 * Insert a new mapping into all structures.
1774 */
1775static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1776				    struct amdgpu_bo_va *bo_va,
1777				    struct amdgpu_bo_va_mapping *mapping)
1778{
1779	struct amdgpu_vm *vm = bo_va->base.vm;
1780	struct amdgpu_bo *bo = bo_va->base.bo;
1781
1782	mapping->bo_va = bo_va;
1783	list_add(&mapping->list, &bo_va->invalids);
1784	amdgpu_vm_it_insert(mapping, &vm->va);
1785
1786	if (mapping->flags & AMDGPU_VM_PAGE_PRT)
1787		amdgpu_vm_prt_get(adev);
1788
1789	if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved)
1790		amdgpu_vm_bo_moved(&bo_va->base);
1791
1792	trace_amdgpu_vm_bo_map(bo_va, mapping);
1793}
1794
1795/* Validate operation parameters to prevent potential abuse */
1796static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,
1797					  struct amdgpu_bo *bo,
1798					  uint64_t saddr,
1799					  uint64_t offset,
1800					  uint64_t size)
1801{
1802	uint64_t tmp, lpfn;
1803
1804	if (saddr & AMDGPU_GPU_PAGE_MASK
1805	    || offset & AMDGPU_GPU_PAGE_MASK
1806	    || size & AMDGPU_GPU_PAGE_MASK)
1807		return -EINVAL;
1808
1809	if (check_add_overflow(saddr, size, &tmp)
1810	    || check_add_overflow(offset, size, &tmp)
1811	    || size == 0 /* which also leads to end < begin */)
1812		return -EINVAL;
1813
1814	/* make sure object fit at this offset */
1815	if (bo && offset + size > amdgpu_bo_size(bo))
1816		return -EINVAL;
1817
1818	/* Ensure last pfn not exceed max_pfn */
1819	lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT;
1820	if (lpfn >= adev->vm_manager.max_pfn)
1821		return -EINVAL;
1822
1823	return 0;
1824}
1825
1826/**
1827 * amdgpu_vm_bo_map - map bo inside a vm
1828 *
1829 * @adev: amdgpu_device pointer
1830 * @bo_va: bo_va to store the address
1831 * @saddr: where to map the BO
1832 * @offset: requested offset in the BO
1833 * @size: BO size in bytes
1834 * @flags: attributes of pages (read/write/valid/etc.)
1835 *
1836 * Add a mapping of the BO at the specefied addr into the VM.
1837 *
1838 * Returns:
1839 * 0 for success, error for failure.
1840 *
1841 * Object has to be reserved and unreserved outside!
1842 */
1843int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1844		     struct amdgpu_bo_va *bo_va,
1845		     uint64_t saddr, uint64_t offset,
1846		     uint64_t size, uint32_t flags)
1847{
1848	struct amdgpu_bo_va_mapping *mapping, *tmp;
1849	struct amdgpu_bo *bo = bo_va->base.bo;
1850	struct amdgpu_vm *vm = bo_va->base.vm;
1851	uint64_t eaddr;
1852	int r;
1853
1854	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1855	if (r)
1856		return r;
1857
1858	saddr /= AMDGPU_GPU_PAGE_SIZE;
1859	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1860
1861	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1862	if (tmp) {
1863		/* bo and tmp overlap, invalid addr */
1864		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1865			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1866			tmp->start, tmp->last + 1);
1867		return -EINVAL;
1868	}
1869
1870	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1871	if (!mapping)
1872		return -ENOMEM;
1873
1874	mapping->start = saddr;
1875	mapping->last = eaddr;
1876	mapping->offset = offset;
1877	mapping->flags = flags;
1878
1879	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1880
1881	return 0;
1882}
1883
1884/**
1885 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1886 *
1887 * @adev: amdgpu_device pointer
1888 * @bo_va: bo_va to store the address
1889 * @saddr: where to map the BO
1890 * @offset: requested offset in the BO
1891 * @size: BO size in bytes
1892 * @flags: attributes of pages (read/write/valid/etc.)
1893 *
1894 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1895 * mappings as we do so.
1896 *
1897 * Returns:
1898 * 0 for success, error for failure.
1899 *
1900 * Object has to be reserved and unreserved outside!
1901 */
1902int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1903			     struct amdgpu_bo_va *bo_va,
1904			     uint64_t saddr, uint64_t offset,
1905			     uint64_t size, uint32_t flags)
1906{
1907	struct amdgpu_bo_va_mapping *mapping;
1908	struct amdgpu_bo *bo = bo_va->base.bo;
1909	uint64_t eaddr;
1910	int r;
1911
1912	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1913	if (r)
1914		return r;
1915
1916	/* Allocate all the needed memory */
1917	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1918	if (!mapping)
1919		return -ENOMEM;
1920
1921	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1922	if (r) {
1923		kfree(mapping);
1924		return r;
1925	}
1926
1927	saddr /= AMDGPU_GPU_PAGE_SIZE;
1928	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1929
1930	mapping->start = saddr;
1931	mapping->last = eaddr;
1932	mapping->offset = offset;
1933	mapping->flags = flags;
1934
1935	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1936
1937	return 0;
1938}
1939
1940/**
1941 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1942 *
1943 * @adev: amdgpu_device pointer
1944 * @bo_va: bo_va to remove the address from
1945 * @saddr: where to the BO is mapped
1946 *
1947 * Remove a mapping of the BO at the specefied addr from the VM.
1948 *
1949 * Returns:
1950 * 0 for success, error for failure.
1951 *
1952 * Object has to be reserved and unreserved outside!
1953 */
1954int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1955		       struct amdgpu_bo_va *bo_va,
1956		       uint64_t saddr)
1957{
1958	struct amdgpu_bo_va_mapping *mapping;
1959	struct amdgpu_vm *vm = bo_va->base.vm;
1960	bool valid = true;
1961	int r;
1962
1963	saddr /= AMDGPU_GPU_PAGE_SIZE;
1964
1965	list_for_each_entry(mapping, &bo_va->valids, list) {
1966		if (mapping->start == saddr)
1967			break;
1968	}
1969
1970	if (&mapping->list == &bo_va->valids) {
1971		valid = false;
1972
1973		list_for_each_entry(mapping, &bo_va->invalids, list) {
1974			if (mapping->start == saddr)
1975				break;
1976		}
1977
1978		if (&mapping->list == &bo_va->invalids)
1979			return -ENOENT;
1980	}
1981
1982	/* It's unlikely to happen that the mapping userq hasn't been idled
1983	 * during user requests GEM unmap IOCTL except for forcing the unmap
1984	 * from user space.
1985	 */
1986	if (unlikely(atomic_read(&bo_va->userq_va_mapped) > 0)) {
1987		r = amdgpu_userq_gem_va_unmap_validate(adev, mapping, saddr);
1988		if (unlikely(r == -EBUSY))
1989			dev_warn_once(adev->dev,
1990				      "Attempt to unmap an active userq buffer\n");
1991	}
1992
1993	list_del(&mapping->list);
1994	amdgpu_vm_it_remove(mapping, &vm->va);
1995	mapping->bo_va = NULL;
1996	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1997
1998	if (valid)
1999		list_add(&mapping->list, &vm->freed);
2000	else
2001		amdgpu_vm_free_mapping(adev, vm, mapping,
2002				       bo_va->last_pt_update);
2003
2004	return 0;
2005}
2006
2007/**
2008 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2009 *
2010 * @adev: amdgpu_device pointer
2011 * @vm: VM structure to use
2012 * @saddr: start of the range
2013 * @size: size of the range
2014 *
2015 * Remove all mappings in a range, split them as appropriate.
2016 *
2017 * Returns:
2018 * 0 for success, error for failure.
2019 */
2020int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2021				struct amdgpu_vm *vm,
2022				uint64_t saddr, uint64_t size)
2023{
2024	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2025	LIST_HEAD(removed);
2026	uint64_t eaddr;
2027	int r;
2028
2029	r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size);
2030	if (r)
2031		return r;
2032
2033	saddr /= AMDGPU_GPU_PAGE_SIZE;
2034	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
2035
2036	/* Allocate all the needed memory */
2037	before = kzalloc(sizeof(*before), GFP_KERNEL);
2038	if (!before)
2039		return -ENOMEM;
2040	INIT_LIST_HEAD(&before->list);
2041
2042	after = kzalloc(sizeof(*after), GFP_KERNEL);
2043	if (!after) {
2044		kfree(before);
2045		return -ENOMEM;
2046	}
2047	INIT_LIST_HEAD(&after->list);
2048
2049	/* Now gather all removed mappings */
2050	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2051	while (tmp) {
2052		/* Remember mapping split at the start */
2053		if (tmp->start < saddr) {
2054			before->start = tmp->start;
2055			before->last = saddr - 1;
2056			before->offset = tmp->offset;
2057			before->flags = tmp->flags;
2058			before->bo_va = tmp->bo_va;
2059			list_add(&before->list, &tmp->bo_va->invalids);
2060		}
2061
2062		/* Remember mapping split at the end */
2063		if (tmp->last > eaddr) {
2064			after->start = eaddr + 1;
2065			after->last = tmp->last;
2066			after->offset = tmp->offset;
2067			after->offset += (after->start - tmp->start) << PAGE_SHIFT;
2068			after->flags = tmp->flags;
2069			after->bo_va = tmp->bo_va;
2070			list_add(&after->list, &tmp->bo_va->invalids);
2071		}
2072
2073		list_del(&tmp->list);
2074		list_add(&tmp->list, &removed);
2075
2076		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2077	}
2078
2079	/* And free them up */
2080	list_for_each_entry_safe(tmp, next, &removed, list) {
2081		amdgpu_vm_it_remove(tmp, &vm->va);
2082		list_del(&tmp->list);
2083
2084		if (tmp->start < saddr)
2085		    tmp->start = saddr;
2086		if (tmp->last > eaddr)
2087		    tmp->last = eaddr;
2088
2089		tmp->bo_va = NULL;
2090		list_add(&tmp->list, &vm->freed);
2091		trace_amdgpu_vm_bo_unmap(NULL, tmp);
2092	}
2093
2094	/* Insert partial mapping before the range */
2095	if (!list_empty(&before->list)) {
2096		struct amdgpu_bo *bo = before->bo_va->base.bo;
2097
2098		amdgpu_vm_it_insert(before, &vm->va);
2099		if (before->flags & AMDGPU_VM_PAGE_PRT)
2100			amdgpu_vm_prt_get(adev);
2101
2102		if (amdgpu_vm_is_bo_always_valid(vm, bo) &&
2103		    !before->bo_va->base.moved)
2104			amdgpu_vm_bo_moved(&before->bo_va->base);
2105	} else {
2106		kfree(before);
2107	}
2108
2109	/* Insert partial mapping after the range */
2110	if (!list_empty(&after->list)) {
2111		struct amdgpu_bo *bo = after->bo_va->base.bo;
2112
2113		amdgpu_vm_it_insert(after, &vm->va);
2114		if (after->flags & AMDGPU_VM_PAGE_PRT)
2115			amdgpu_vm_prt_get(adev);
2116
2117		if (amdgpu_vm_is_bo_always_valid(vm, bo) &&
2118		    !after->bo_va->base.moved)
2119			amdgpu_vm_bo_moved(&after->bo_va->base);
2120	} else {
2121		kfree(after);
2122	}
2123
2124	return 0;
2125}
2126
2127/**
2128 * amdgpu_vm_bo_lookup_mapping - find mapping by address
2129 *
2130 * @vm: the requested VM
2131 * @addr: the address
2132 *
2133 * Find a mapping by it's address.
2134 *
2135 * Returns:
2136 * The amdgpu_bo_va_mapping matching for addr or NULL
2137 *
2138 */
2139struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2140							 uint64_t addr)
2141{
2142	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2143}
2144
2145/**
2146 * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2147 *
2148 * @vm: the requested vm
2149 * @ticket: CS ticket
2150 *
2151 * Trace all mappings of BOs reserved during a command submission.
2152 */
2153void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2154{
2155	struct amdgpu_bo_va_mapping *mapping;
2156
2157	if (!trace_amdgpu_vm_bo_cs_enabled())
2158		return;
2159
2160	for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2161	     mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2162		if (mapping->bo_va && mapping->bo_va->base.bo) {
2163			struct amdgpu_bo *bo;
2164
2165			bo = mapping->bo_va->base.bo;
2166			if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
2167			    ticket)
2168				continue;
2169		}
2170
2171		trace_amdgpu_vm_bo_cs(mapping);
2172	}
2173}
2174
2175/**
2176 * amdgpu_vm_bo_del - remove a bo from a specific vm
2177 *
2178 * @adev: amdgpu_device pointer
2179 * @bo_va: requested bo_va
2180 *
2181 * Remove @bo_va->bo from the requested vm.
2182 *
2183 * Object have to be reserved!
2184 */
2185void amdgpu_vm_bo_del(struct amdgpu_device *adev,
2186		      struct amdgpu_bo_va *bo_va)
2187{
2188	struct amdgpu_bo_va_mapping *mapping, *next;
2189	struct amdgpu_bo *bo = bo_va->base.bo;
2190	struct amdgpu_vm *vm = bo_va->base.vm;
2191	struct amdgpu_vm_bo_base **base;
2192
2193	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
2194
2195	if (bo) {
2196		dma_resv_assert_held(bo->tbo.base.resv);
2197		if (amdgpu_vm_is_bo_always_valid(vm, bo))
2198			ttm_bo_set_bulk_move(&bo->tbo, NULL);
2199
2200		for (base = &bo_va->base.bo->vm_bo; *base;
2201		     base = &(*base)->next) {
2202			if (*base != &bo_va->base)
2203				continue;
2204
2205			amdgpu_vm_update_stats(*base, bo->tbo.resource, -1);
2206			*base = bo_va->base.next;
2207			break;
2208		}
2209	}
2210
2211	spin_lock(&vm->status_lock);
2212	list_del(&bo_va->base.vm_status);
2213	spin_unlock(&vm->status_lock);
2214
2215	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2216		list_del(&mapping->list);
2217		amdgpu_vm_it_remove(mapping, &vm->va);
2218		mapping->bo_va = NULL;
2219		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2220		list_add(&mapping->list, &vm->freed);
2221	}
2222	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2223		list_del(&mapping->list);
2224		amdgpu_vm_it_remove(mapping, &vm->va);
2225		amdgpu_vm_free_mapping(adev, vm, mapping,
2226				       bo_va->last_pt_update);
2227	}
2228
2229	dma_fence_put(bo_va->last_pt_update);
2230
2231	if (bo && bo_va->is_xgmi)
2232		amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
2233
2234	kfree(bo_va);
2235}
2236
2237/**
2238 * amdgpu_vm_evictable - check if we can evict a VM
2239 *
2240 * @bo: A page table of the VM.
2241 *
2242 * Check if it is possible to evict a VM.
2243 */
2244bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
2245{
2246	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
2247
2248	/* Page tables of a destroyed VM can go away immediately */
2249	if (!bo_base || !bo_base->vm)
2250		return true;
2251
2252	/* Don't evict VM page tables while they are busy */
2253	if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))
2254		return false;
2255
2256	/* Try to block ongoing updates */
2257	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
2258		return false;
2259
2260	/* Don't evict VM page tables while they are updated */
2261	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
2262		amdgpu_vm_eviction_unlock(bo_base->vm);
2263		return false;
2264	}
2265
2266	bo_base->vm->evicting = true;
2267	amdgpu_vm_eviction_unlock(bo_base->vm);
2268	return true;
2269}
2270
2271/**
2272 * amdgpu_vm_bo_invalidate - mark the bo as invalid
2273 *
2274 * @bo: amdgpu buffer object
2275 * @evicted: is the BO evicted
2276 *
2277 * Mark @bo as invalid.
2278 */
2279void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
2280{
2281	struct amdgpu_vm_bo_base *bo_base;
2282
2283	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2284		struct amdgpu_vm *vm = bo_base->vm;
2285
2286		if (evicted && amdgpu_vm_is_bo_always_valid(vm, bo)) {
2287			amdgpu_vm_bo_evicted(bo_base);
2288			continue;
2289		}
2290
2291		if (bo_base->moved)
2292			continue;
2293		bo_base->moved = true;
2294
2295		if (bo->tbo.type == ttm_bo_type_kernel)
2296			amdgpu_vm_bo_relocated(bo_base);
2297		else if (amdgpu_vm_is_bo_always_valid(vm, bo))
2298			amdgpu_vm_bo_moved(bo_base);
2299		else
2300			amdgpu_vm_bo_invalidated(bo_base);
2301	}
2302}
2303
2304/**
2305 * amdgpu_vm_bo_move - handle BO move
2306 *
2307 * @bo: amdgpu buffer object
2308 * @new_mem: the new placement of the BO move
2309 * @evicted: is the BO evicted
2310 *
2311 * Update the memory stats for the new placement and mark @bo as invalid.
2312 */
2313void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
2314		       bool evicted)
2315{
2316	struct amdgpu_vm_bo_base *bo_base;
2317
2318	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2319		struct amdgpu_vm *vm = bo_base->vm;
2320
2321		spin_lock(&vm->status_lock);
2322		amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
2323		amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
2324		spin_unlock(&vm->status_lock);
2325	}
2326
2327	amdgpu_vm_bo_invalidate(bo, evicted);
2328}
2329
2330/**
2331 * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2332 *
2333 * @vm_size: VM size
2334 *
2335 * Returns:
2336 * VM page table as power of two
2337 */
2338static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2339{
2340	/* Total bits covered by PD + PTs */
2341	unsigned bits = ilog2(vm_size) + 18;
2342
2343	/* Make sure the PD is 4K in size up to 8GB address space.
2344	   Above that split equal between PD and PTs */
2345	if (vm_size <= 8)
2346		return (bits - 9);
2347	else
2348		return ((bits + 3) / 2);
2349}
2350
2351/**
2352 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2353 *
2354 * @adev: amdgpu_device pointer
2355 * @min_vm_size: the minimum vm size in GB if it's set auto
2356 * @fragment_size_default: Default PTE fragment size
2357 * @max_level: max VMPT level
2358 * @max_bits: max address space size in bits
2359 *
2360 */
2361void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2362			   uint32_t fragment_size_default, unsigned max_level,
2363			   unsigned max_bits)
2364{
2365	unsigned int max_size = 1 << (max_bits - 30);
2366	unsigned int vm_size;
2367	uint64_t tmp;
2368
2369	/* adjust vm size first */
2370	if (amdgpu_vm_size != -1) {
2371		vm_size = amdgpu_vm_size;
2372		if (vm_size > max_size) {
2373			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2374				 amdgpu_vm_size, max_size);
2375			vm_size = max_size;
2376		}
2377	} else {
2378		struct sysinfo si;
2379		unsigned int phys_ram_gb;
2380
2381		/* Optimal VM size depends on the amount of physical
2382		 * RAM available. Underlying requirements and
2383		 * assumptions:
2384		 *
2385		 *  - Need to map system memory and VRAM from all GPUs
2386		 *     - VRAM from other GPUs not known here
2387		 *     - Assume VRAM <= system memory
2388		 *  - On GFX8 and older, VM space can be segmented for
2389		 *    different MTYPEs
2390		 *  - Need to allow room for fragmentation, guard pages etc.
2391		 *
2392		 * This adds up to a rough guess of system memory x3.
2393		 * Round up to power of two to maximize the available
2394		 * VM size with the given page table size.
2395		 */
2396		si_meminfo(&si);
2397		phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2398			       (1 << 30) - 1) >> 30;
2399		vm_size = roundup_pow_of_two(
2400			clamp(phys_ram_gb * 3, min_vm_size, max_size));
2401	}
2402
2403	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2404
2405	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2406	if (amdgpu_vm_block_size != -1)
2407		tmp >>= amdgpu_vm_block_size - 9;
2408	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2409	adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp);
2410	switch (adev->vm_manager.num_level) {
2411	case 4:
2412		adev->vm_manager.root_level = AMDGPU_VM_PDB3;
2413		break;
2414	case 3:
2415		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2416		break;
2417	case 2:
2418		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2419		break;
2420	case 1:
2421		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2422		break;
2423	default:
2424		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2425	}
2426	/* block size depends on vm size and hw setup*/
2427	if (amdgpu_vm_block_size != -1)
2428		adev->vm_manager.block_size =
2429			min((unsigned)amdgpu_vm_block_size, max_bits
2430			    - AMDGPU_GPU_PAGE_SHIFT
2431			    - 9 * adev->vm_manager.num_level);
2432	else if (adev->vm_manager.num_level > 1)
2433		adev->vm_manager.block_size = 9;
2434	else
2435		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2436
2437	if (amdgpu_vm_fragment_size == -1)
2438		adev->vm_manager.fragment_size = fragment_size_default;
2439	else
2440		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2441
2442	dev_info(
2443		adev->dev,
2444		"vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2445		vm_size, adev->vm_manager.num_level + 1,
2446		adev->vm_manager.block_size, adev->vm_manager.fragment_size);
2447}
2448
2449/**
2450 * amdgpu_vm_wait_idle - wait for the VM to become idle
2451 *
2452 * @vm: VM object to wait for
2453 * @timeout: timeout to wait for VM to become idle
2454 */
2455long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2456{
2457	timeout = drm_sched_entity_flush(&vm->immediate, timeout);
2458	if (timeout <= 0)
2459		return timeout;
2460
2461	return drm_sched_entity_flush(&vm->delayed, timeout);
2462}
2463
2464static void amdgpu_vm_destroy_task_info(struct kref *kref)
2465{
2466	struct amdgpu_task_info *ti = container_of(kref, struct amdgpu_task_info, refcount);
2467
2468	kfree(ti);
2469}
2470
2471static inline struct amdgpu_vm *
2472amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid)
2473{
2474	struct amdgpu_vm *vm;
2475	unsigned long flags;
2476
2477	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
2478	vm = xa_load(&adev->vm_manager.pasids, pasid);
2479	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
2480
2481	return vm;
2482}
2483
2484/**
2485 * amdgpu_vm_put_task_info - reference down the vm task_info ptr
2486 *
2487 * @task_info: task_info struct under discussion.
2488 *
2489 * frees the vm task_info ptr at the last put
2490 */
2491void amdgpu_vm_put_task_info(struct amdgpu_task_info *task_info)
2492{
2493	if (task_info)
2494		kref_put(&task_info->refcount, amdgpu_vm_destroy_task_info);
2495}
2496
2497/**
2498 * amdgpu_vm_get_task_info_vm - Extracts task info for a vm.
2499 *
2500 * @vm: VM to get info from
2501 *
2502 * Returns the reference counted task_info structure, which must be
2503 * referenced down with amdgpu_vm_put_task_info.
2504 */
2505struct amdgpu_task_info *
2506amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm)
2507{
2508	struct amdgpu_task_info *ti = NULL;
2509
2510	if (vm) {
2511		ti = vm->task_info;
2512		kref_get(&vm->task_info->refcount);
2513	}
2514
2515	return ti;
2516}
2517
2518/**
2519 * amdgpu_vm_get_task_info_pasid - Extracts task info for a PASID.
2520 *
2521 * @adev: drm device pointer
2522 * @pasid: PASID identifier for VM
2523 *
2524 * Returns the reference counted task_info structure, which must be
2525 * referenced down with amdgpu_vm_put_task_info.
2526 */
2527struct amdgpu_task_info *
2528amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid)
2529{
2530	return amdgpu_vm_get_task_info_vm(
2531			amdgpu_vm_get_vm_from_pasid(adev, pasid));
2532}
2533
2534static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm)
2535{
2536	vm->task_info = kzalloc(sizeof(struct amdgpu_task_info), GFP_KERNEL);
2537	if (!vm->task_info)
2538		return -ENOMEM;
2539
2540	kref_init(&vm->task_info->refcount);
2541	return 0;
2542}
2543
2544/**
2545 * amdgpu_vm_set_task_info - Sets VMs task info.
2546 *
2547 * @vm: vm for which to set the info
2548 */
2549void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
2550{
2551	if (!vm->task_info)
2552		return;
2553
2554	if (vm->task_info->task.pid == current->pid)
2555		return;
2556
2557	vm->task_info->task.pid = current->pid;
2558	get_task_comm(vm->task_info->task.comm, current);
2559
2560	if (current->group_leader->mm != current->mm)
2561		return;
2562
2563	vm->task_info->tgid = current->group_leader->pid;
2564	get_task_comm(vm->task_info->process_name, current->group_leader);
2565}
2566
2567/**
2568 * amdgpu_vm_init - initialize a vm instance
2569 *
2570 * @adev: amdgpu_device pointer
2571 * @vm: requested vm
2572 * @xcp_id: GPU partition selection id
2573 * @pasid: the pasid the VM is using on this GPU
2574 *
2575 * Init @vm fields.
2576 *
2577 * Returns:
2578 * 0 for success, error for failure.
2579 */
2580int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2581		   int32_t xcp_id, uint32_t pasid)
2582{
2583	struct amdgpu_bo *root_bo;
2584	struct amdgpu_bo_vm *root;
2585	int r, i;
2586
2587	vm->va = RB_ROOT_CACHED;
2588	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2589		vm->reserved_vmid[i] = NULL;
2590	INIT_LIST_HEAD(&vm->evicted);
2591	INIT_LIST_HEAD(&vm->evicted_user);
2592	INIT_LIST_HEAD(&vm->relocated);
2593	INIT_LIST_HEAD(&vm->moved);
2594	INIT_LIST_HEAD(&vm->idle);
2595	INIT_LIST_HEAD(&vm->invalidated);
2596	spin_lock_init(&vm->status_lock);
2597	INIT_LIST_HEAD(&vm->freed);
2598	INIT_LIST_HEAD(&vm->done);
2599	INIT_KFIFO(vm->faults);
2600
2601	r = amdgpu_vm_init_entities(adev, vm);
2602	if (r)
2603		return r;
2604
2605	ttm_lru_bulk_move_init(&vm->lru_bulk_move);
2606
2607	vm->is_compute_context = false;
2608	vm->need_tlb_fence = amdgpu_userq_enabled(&adev->ddev);
2609
2610	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2611				    AMDGPU_VM_USE_CPU_FOR_GFX);
2612
2613	dev_dbg(adev->dev, "VM update mode is %s\n",
2614		vm->use_cpu_for_update ? "CPU" : "SDMA");
2615	WARN_ONCE((vm->use_cpu_for_update &&
2616		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2617		  "CPU update of VM recommended only for large BAR system\n");
2618
2619	if (vm->use_cpu_for_update)
2620		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2621	else
2622		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2623
2624	vm->last_update = dma_fence_get_stub();
2625	vm->last_unlocked = dma_fence_get_stub();
2626	vm->last_tlb_flush = dma_fence_get_stub();
2627	vm->generation = amdgpu_vm_generation(adev, NULL);
2628
2629	mutex_init(&vm->eviction_lock);
2630	vm->evicting = false;
2631	vm->tlb_fence_context = dma_fence_context_alloc(1);
2632
2633	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2634				false, &root, xcp_id);
2635	if (r)
2636		goto error_free_delayed;
2637
2638	root_bo = amdgpu_bo_ref(&root->bo);
2639	r = amdgpu_bo_reserve(root_bo, true);
2640	if (r) {
2641		amdgpu_bo_unref(&root_bo);
2642		goto error_free_delayed;
2643	}
2644
2645	amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
2646	r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);
2647	if (r)
2648		goto error_free_root;
2649
2650	r = amdgpu_vm_pt_clear(adev, vm, root, false);
2651	if (r)
2652		goto error_free_root;
2653
2654	r = amdgpu_vm_create_task_info(vm);
2655	if (r)
2656		dev_dbg(adev->dev, "Failed to create task info for VM\n");
2657
2658	/* Store new PASID in XArray (if non-zero) */
2659	if (pasid != 0) {
2660		r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm, GFP_KERNEL));
2661		if (r < 0)
2662			goto error_free_root;
2663
2664		vm->pasid = pasid;
2665	}
2666
2667	amdgpu_bo_unreserve(vm->root.bo);
2668	amdgpu_bo_unref(&root_bo);
2669
2670	return 0;
2671
2672error_free_root:
2673	/* If PASID was partially set, erase it from XArray before failing */
2674	if (vm->pasid != 0) {
2675		xa_erase_irq(&adev->vm_manager.pasids, vm->pasid);
2676		vm->pasid = 0;
2677	}
2678	amdgpu_vm_pt_free_root(adev, vm);
2679	amdgpu_bo_unreserve(vm->root.bo);
2680	amdgpu_bo_unref(&root_bo);
2681
2682error_free_delayed:
2683	dma_fence_put(vm->last_tlb_flush);
2684	dma_fence_put(vm->last_unlocked);
2685	ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
2686	amdgpu_vm_fini_entities(vm);
2687
2688	return r;
2689}
2690
2691/**
2692 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2693 *
2694 * @adev: amdgpu_device pointer
2695 * @vm: requested vm
2696 *
2697 * This only works on GFX VMs that don't have any BOs added and no
2698 * page tables allocated yet.
2699 *
2700 * Changes the following VM parameters:
2701 * - use_cpu_for_update
2702 * - pte_supports_ats
2703 *
2704 * Reinitializes the page directory to reflect the changed ATS
2705 * setting.
2706 *
2707 * Returns:
2708 * 0 for success, -errno for errors.
2709 */
2710int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2711{
2712	int r;
2713
2714	r = amdgpu_bo_reserve(vm->root.bo, true);
2715	if (r)
2716		return r;
2717
2718	/* Update VM state */
2719	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2720				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2721	dev_dbg(adev->dev, "VM update mode is %s\n",
2722		vm->use_cpu_for_update ? "CPU" : "SDMA");
2723	WARN_ONCE((vm->use_cpu_for_update &&
2724		   !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2725		  "CPU update of VM recommended only for large BAR system\n");
2726
2727	if (vm->use_cpu_for_update) {
2728		/* Sync with last SDMA update/clear before switching to CPU */
2729		r = amdgpu_bo_sync_wait(vm->root.bo,
2730					AMDGPU_FENCE_OWNER_UNDEFINED, true);
2731		if (r)
2732			goto unreserve_bo;
2733
2734		vm->update_funcs = &amdgpu_vm_cpu_funcs;
2735		r = amdgpu_vm_pt_map_tables(adev, vm);
2736		if (r)
2737			goto unreserve_bo;
2738
2739	} else {
2740		vm->update_funcs = &amdgpu_vm_sdma_funcs;
2741	}
2742
2743	dma_fence_put(vm->last_update);
2744	vm->last_update = dma_fence_get_stub();
2745	vm->is_compute_context = true;
2746	vm->need_tlb_fence = true;
2747
2748unreserve_bo:
2749	amdgpu_bo_unreserve(vm->root.bo);
2750	return r;
2751}
2752
2753static int amdgpu_vm_stats_is_zero(struct amdgpu_vm *vm)
2754{
2755	for (int i = 0; i < __AMDGPU_PL_NUM; ++i) {
2756		if (!(drm_memory_stats_is_zero(&vm->stats[i].drm) &&
2757		      vm->stats[i].evicted == 0))
2758			return false;
2759	}
2760	return true;
2761}
2762
2763/**
2764 * amdgpu_vm_fini - tear down a vm instance
2765 *
2766 * @adev: amdgpu_device pointer
2767 * @vm: requested vm
2768 *
2769 * Tear down @vm.
2770 * Unbind the VM and remove all bos from the vm bo list
2771 */
2772void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2773{
2774	struct amdgpu_bo_va_mapping *mapping, *tmp;
2775	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2776	struct amdgpu_bo *root;
2777	unsigned long flags;
2778	int i;
2779
2780	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2781
2782	root = amdgpu_bo_ref(vm->root.bo);
2783	amdgpu_bo_reserve(root, true);
2784	/* Remove PASID mapping before destroying VM */
2785	if (vm->pasid != 0) {
2786		xa_erase_irq(&adev->vm_manager.pasids, vm->pasid);
2787		vm->pasid = 0;
2788	}
2789	dma_fence_wait(vm->last_unlocked, false);
2790	dma_fence_put(vm->last_unlocked);
2791	dma_fence_wait(vm->last_tlb_flush, false);
2792	/* Make sure that all fence callbacks have completed */
2793	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
2794	spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags);
2795	dma_fence_put(vm->last_tlb_flush);
2796
2797	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2798		if (mapping->flags & AMDGPU_VM_PAGE_PRT && prt_fini_needed) {
2799			amdgpu_vm_prt_fini(adev, vm);
2800			prt_fini_needed = false;
2801		}
2802
2803		list_del(&mapping->list);
2804		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2805	}
2806
2807	amdgpu_vm_pt_free_root(adev, vm);
2808	amdgpu_bo_unreserve(root);
2809	amdgpu_bo_unref(&root);
2810	WARN_ON(vm->root.bo);
2811
2812	amdgpu_vm_fini_entities(vm);
2813
2814	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2815		dev_err(adev->dev, "still active bo inside vm\n");
2816	}
2817	rbtree_postorder_for_each_entry_safe(mapping, tmp,
2818					     &vm->va.rb_root, rb) {
2819		/* Don't remove the mapping here, we don't want to trigger a
2820		 * rebalance and the tree is about to be destroyed anyway.
2821		 */
2822		list_del(&mapping->list);
2823		kfree(mapping);
2824	}
2825
2826	dma_fence_put(vm->last_update);
2827
2828	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
2829		amdgpu_vmid_free_reserved(adev, vm, i);
2830	}
2831
2832	ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
2833
2834	if (!amdgpu_vm_stats_is_zero(vm)) {
2835		struct amdgpu_task_info *ti = vm->task_info;
2836
2837		dev_warn(adev->dev,
2838			 "VM memory stats for proc %s(%d) task %s(%d) is non-zero when fini\n",
2839			 ti->process_name, ti->task.pid, ti->task.comm, ti->tgid);
2840	}
2841
2842	amdgpu_vm_put_task_info(vm->task_info);
2843}
2844
2845/**
2846 * amdgpu_vm_manager_init - init the VM manager
2847 *
2848 * @adev: amdgpu_device pointer
2849 *
2850 * Initialize the VM manager structures
2851 */
2852void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2853{
2854	/* Concurrent flushes are only possible starting with Vega10 and
2855	 * are broken on Navi10 and Navi14.
2856	 */
2857	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
2858					      adev->asic_type == CHIP_NAVI10 ||
2859					      adev->asic_type == CHIP_NAVI14);
2860	amdgpu_vmid_mgr_init(adev);
2861
2862	spin_lock_init(&adev->vm_manager.prt_lock);
2863	atomic_set(&adev->vm_manager.num_prt_users, 0);
2864
2865	/* If not overridden by the user, by default, only in large BAR systems
2866	 * Compute VM tables will be updated by CPU
2867	 */
2868#ifdef CONFIG_X86_64
2869	if (amdgpu_vm_update_mode == -1) {
2870		/* For asic with VF MMIO access protection
2871		 * avoid using CPU for VM table updates
2872		 */
2873		if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&
2874		    !amdgpu_sriov_vf_mmio_access_protection(adev))
2875			adev->vm_manager.vm_update_mode =
2876				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2877		else
2878			adev->vm_manager.vm_update_mode = 0;
2879	} else
2880		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2881#else
2882	adev->vm_manager.vm_update_mode = 0;
2883#endif
2884
2885	xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
2886}
2887
2888/**
2889 * amdgpu_vm_manager_fini - cleanup VM manager
2890 *
2891 * @adev: amdgpu_device pointer
2892 *
2893 * Cleanup the VM manager and free resources.
2894 */
2895void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2896{
2897	WARN_ON(!xa_empty(&adev->vm_manager.pasids));
2898	xa_destroy(&adev->vm_manager.pasids);
2899
2900	amdgpu_vmid_mgr_fini(adev);
2901	amdgpu_pasid_mgr_cleanup();
2902}
2903
2904/**
2905 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
2906 *
2907 * @dev: drm device pointer
2908 * @data: drm_amdgpu_vm
2909 * @filp: drm file pointer
2910 *
2911 * Returns:
2912 * 0 for success, -errno for errors.
2913 */
2914int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2915{
2916	union drm_amdgpu_vm *args = data;
2917	struct amdgpu_device *adev = drm_to_adev(dev);
2918	struct amdgpu_fpriv *fpriv = filp->driver_priv;
2919	struct amdgpu_vm *vm = &fpriv->vm;
2920
2921	/* No valid flags defined yet */
2922	if (args->in.flags)
2923		return -EINVAL;
2924
2925	switch (args->in.op) {
2926	case AMDGPU_VM_OP_RESERVE_VMID:
2927		/* We only have requirement to reserve vmid from gfxhub */
2928		return amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(0));
2929	case AMDGPU_VM_OP_UNRESERVE_VMID:
2930		amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0));
2931		break;
2932	default:
2933		return -EINVAL;
2934	}
2935
2936	return 0;
2937}
2938
2939/**
2940 * amdgpu_vm_handle_fault - graceful handling of VM faults.
2941 * @adev: amdgpu device pointer
2942 * @pasid: PASID of the VM
2943 * @ts: Timestamp of the fault
2944 * @vmid: VMID, only used for GFX 9.4.3.
2945 * @node_id: Node_id received in IH cookie. Only applicable for
2946 *           GFX 9.4.3.
2947 * @addr: Address of the fault
2948 * @write_fault: true is write fault, false is read fault
2949 *
2950 * Try to gracefully handle a VM fault. Return true if the fault was handled and
2951 * shouldn't be reported any more.
2952 */
2953bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
2954			    u32 vmid, u32 node_id, uint64_t addr, uint64_t ts,
2955			    bool write_fault)
2956{
2957	bool is_compute_context = false;
2958	struct amdgpu_bo *root;
2959	unsigned long irqflags;
2960	uint64_t value, flags;
2961	struct amdgpu_vm *vm;
2962	int r;
2963
2964	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2965	vm = xa_load(&adev->vm_manager.pasids, pasid);
2966	if (vm) {
2967		root = amdgpu_bo_ref(vm->root.bo);
2968		is_compute_context = vm->is_compute_context;
2969	} else {
2970		root = NULL;
2971	}
2972	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2973
2974	if (!root)
2975		return false;
2976
2977	if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid,
2978	    node_id, addr >> PAGE_SHIFT, ts, write_fault)) {
2979		amdgpu_bo_unref(&root);
2980		return true;
2981	}
2982
2983	addr /= AMDGPU_GPU_PAGE_SIZE;
2984
2985	r = amdgpu_bo_reserve(root, true);
2986	if (r)
2987		goto error_unref;
2988
2989	/* Double check that the VM still exists */
2990	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2991	vm = xa_load(&adev->vm_manager.pasids, pasid);
2992	if (vm && vm->root.bo != root)
2993		vm = NULL;
2994	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2995	if (!vm)
2996		goto error_unlock;
2997
2998	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
2999		AMDGPU_PTE_SYSTEM;
3000
3001	if (is_compute_context) {
3002		/* Intentionally setting invalid PTE flag
3003		 * combination to force a no-retry-fault
3004		 */
3005		flags = AMDGPU_VM_NORETRY_FLAGS;
3006		value = 0;
3007	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
3008		/* Redirect the access to the dummy page */
3009		value = adev->dummy_page_addr;
3010		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
3011			AMDGPU_PTE_WRITEABLE;
3012
3013	} else {
3014		/* Let the hw retry silently on the PTE */
3015		value = 0;
3016	}
3017
3018	r = dma_resv_reserve_fences(root->tbo.base.resv, 1);
3019	if (r) {
3020		pr_debug("failed %d to reserve fence slot\n", r);
3021		goto error_unlock;
3022	}
3023
3024	r = amdgpu_vm_update_range(adev, vm, true, false, false, false,
3025				   NULL, addr, addr, flags, value, 0, NULL, NULL, NULL);
3026	if (r)
3027		goto error_unlock;
3028
3029	r = amdgpu_vm_update_pdes(adev, vm, true);
3030
3031error_unlock:
3032	amdgpu_bo_unreserve(root);
3033	if (r < 0)
3034		dev_err(adev->dev, "Can't handle page fault (%d)\n", r);
3035
3036error_unref:
3037	amdgpu_bo_unref(&root);
3038
3039	return false;
3040}
3041
3042#if defined(CONFIG_DEBUG_FS)
3043/**
3044 * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
3045 *
3046 * @vm: Requested VM for printing BO info
3047 * @m: debugfs file
3048 *
3049 * Print BO information in debugfs file for the VM
3050 */
3051void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
3052{
3053	struct amdgpu_bo_va *bo_va, *tmp;
3054	u64 total_idle = 0;
3055	u64 total_evicted = 0;
3056	u64 total_relocated = 0;
3057	u64 total_moved = 0;
3058	u64 total_invalidated = 0;
3059	u64 total_done = 0;
3060	unsigned int total_idle_objs = 0;
3061	unsigned int total_evicted_objs = 0;
3062	unsigned int total_relocated_objs = 0;
3063	unsigned int total_moved_objs = 0;
3064	unsigned int total_invalidated_objs = 0;
3065	unsigned int total_done_objs = 0;
3066	unsigned int id = 0;
3067
3068	amdgpu_vm_assert_locked(vm);
3069
3070	spin_lock(&vm->status_lock);
3071	seq_puts(m, "\tIdle BOs:\n");
3072	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
3073		if (!bo_va->base.bo)
3074			continue;
3075		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3076	}
3077	total_idle_objs = id;
3078	id = 0;
3079
3080	seq_puts(m, "\tEvicted BOs:\n");
3081	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
3082		if (!bo_va->base.bo)
3083			continue;
3084		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3085	}
3086	total_evicted_objs = id;
3087	id = 0;
3088
3089	seq_puts(m, "\tRelocated BOs:\n");
3090	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
3091		if (!bo_va->base.bo)
3092			continue;
3093		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3094	}
3095	total_relocated_objs = id;
3096	id = 0;
3097
3098	seq_puts(m, "\tMoved BOs:\n");
3099	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
3100		if (!bo_va->base.bo)
3101			continue;
3102		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3103	}
3104	total_moved_objs = id;
3105	id = 0;
3106
3107	seq_puts(m, "\tInvalidated BOs:\n");
3108	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
3109		if (!bo_va->base.bo)
3110			continue;
3111		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
3112	}
3113	total_invalidated_objs = id;
3114	id = 0;
3115
3116	seq_puts(m, "\tDone BOs:\n");
3117	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
3118		if (!bo_va->base.bo)
3119			continue;
3120		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
3121	}
3122	spin_unlock(&vm->status_lock);
3123	total_done_objs = id;
3124
3125	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
3126		   total_idle_objs);
3127	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
3128		   total_evicted_objs);
3129	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
3130		   total_relocated_objs);
3131	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
3132		   total_moved_objs);
3133	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
3134		   total_invalidated_objs);
3135	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
3136		   total_done_objs);
3137}
3138#endif
3139
3140/**
3141 * amdgpu_vm_update_fault_cache - update cached fault into.
3142 * @adev: amdgpu device pointer
3143 * @pasid: PASID of the VM
3144 * @addr: Address of the fault
3145 * @status: GPUVM fault status register
3146 * @vmhub: which vmhub got the fault
3147 *
3148 * Cache the fault info for later use by userspace in debugging.
3149 */
3150void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
3151				  unsigned int pasid,
3152				  uint64_t addr,
3153				  uint32_t status,
3154				  unsigned int vmhub)
3155{
3156	struct amdgpu_vm *vm;
3157	unsigned long flags;
3158
3159	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
3160
3161	vm = xa_load(&adev->vm_manager.pasids, pasid);
3162	/* Don't update the fault cache if status is 0.  In the multiple
3163	 * fault case, subsequent faults will return a 0 status which is
3164	 * useless for userspace and replaces the useful fault status, so
3165	 * only update if status is non-0.
3166	 */
3167	if (vm && status) {
3168		vm->fault_info.addr = addr;
3169		vm->fault_info.status = status;
3170		/*
3171		 * Update the fault information globally for later usage
3172		 * when vm could be stale or freed.
3173		 */
3174		adev->vm_manager.fault_info.addr = addr;
3175		adev->vm_manager.fault_info.vmhub = vmhub;
3176		adev->vm_manager.fault_info.status = status;
3177
3178		if (AMDGPU_IS_GFXHUB(vmhub)) {
3179			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_GFX;
3180			vm->fault_info.vmhub |=
3181				(vmhub - AMDGPU_GFXHUB_START) << AMDGPU_VMHUB_IDX_SHIFT;
3182		} else if (AMDGPU_IS_MMHUB0(vmhub)) {
3183			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM0;
3184			vm->fault_info.vmhub |=
3185				(vmhub - AMDGPU_MMHUB0_START) << AMDGPU_VMHUB_IDX_SHIFT;
3186		} else if (AMDGPU_IS_MMHUB1(vmhub)) {
3187			vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM1;
3188			vm->fault_info.vmhub |=
3189				(vmhub - AMDGPU_MMHUB1_START) << AMDGPU_VMHUB_IDX_SHIFT;
3190		} else {
3191			WARN_ONCE(1, "Invalid vmhub %u\n", vmhub);
3192		}
3193	}
3194	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
3195}
3196
3197/**
3198 * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
3199 *
3200 * @vm: VM to test against.
3201 * @bo: BO to be tested.
3202 *
3203 * Returns true if the BO shares the dma_resv object with the root PD and is
3204 * always guaranteed to be valid inside the VM.
3205 */
3206bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
3207{
3208	return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv;
3209}
3210
3211void amdgpu_vm_print_task_info(struct amdgpu_device *adev,
3212			       struct amdgpu_task_info *task_info)
3213{
3214	dev_err(adev->dev,
3215		" Process %s pid %d thread %s pid %d\n",
3216		task_info->process_name, task_info->tgid,
3217		task_info->task.comm, task_info->task.pid);
3218}