xe_pt.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2022 Intel Corporation
   4 */
   5
   6#include "xe_pt.h"
   7
   8#include "regs/xe_gtt_defs.h"
   9#include "xe_bo.h"
  10#include "xe_device.h"
  11#include "xe_drm_client.h"
  12#include "xe_exec_queue.h"
  13#include "xe_gt.h"
  14#include "xe_migrate.h"
  15#include "xe_pt_types.h"
  16#include "xe_pt_walk.h"
  17#include "xe_res_cursor.h"
  18#include "xe_sched_job.h"
  19#include "xe_svm.h"
  20#include "xe_sync.h"
  21#include "xe_tlb_inval_job.h"
  22#include "xe_trace.h"
  23#include "xe_ttm_stolen_mgr.h"
  24#include "xe_userptr.h"
  25#include "xe_vm.h"
  26
  27struct xe_pt_dir {
  28	struct xe_pt pt;
  29	/** @children: Array of page-table child nodes */
  30	struct xe_ptw *children[XE_PDES];
  31	/** @staging: Array of page-table staging nodes */
  32	struct xe_ptw *staging[XE_PDES];
  33};
  34
  35#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
  36#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
  37#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
  38#else
  39#define xe_pt_set_addr(__xe_pt, __addr)
  40#define xe_pt_addr(__xe_pt) 0ull
  41#endif
  42
  43static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48};
  44static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48};
  45
  46#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
  47
  48static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
  49{
  50	return container_of(pt, struct xe_pt_dir, pt);
  51}
  52
  53static struct xe_pt *
  54xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)
  55{
  56	return container_of(pt_dir->staging[index], struct xe_pt, base);
  57}
  58
  59static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
  60			     unsigned int level)
  61{
  62	struct xe_device *xe = tile_to_xe(tile);
  63	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
  64	u8 id = tile->id;
  65
  66	if (!xe_vm_has_scratch(vm))
  67		return 0;
  68
  69	if (level > MAX_HUGEPTE_LEVEL)
  70		return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo,
  71						 0);
  72
  73	return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) |
  74		XE_PTE_NULL;
  75}
  76
  77static void xe_pt_free(struct xe_pt *pt)
  78{
  79	if (pt->level)
  80		kfree(as_xe_pt_dir(pt));
  81	else
  82		kfree(pt);
  83}
  84
  85/**
  86 * xe_pt_create() - Create a page-table.
  87 * @vm: The vm to create for.
  88 * @tile: The tile to create for.
  89 * @level: The page-table level.
  90 * @exec: The drm_exec object used to lock the vm.
  91 *
  92 * Allocate and initialize a single struct xe_pt metadata structure. Also
  93 * create the corresponding page-table bo, but don't initialize it. If the
  94 * level is grater than zero, then it's assumed to be a directory page-
  95 * table and the directory structure is also allocated and initialized to
  96 * NULL pointers.
  97 *
  98 * Return: A valid struct xe_pt pointer on success, Pointer error code on
  99 * error.
 100 */
 101struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
 102			   unsigned int level, struct drm_exec *exec)
 103{
 104	struct xe_pt *pt;
 105	struct xe_bo *bo;
 106	u32 bo_flags;
 107	int err;
 108
 109	if (level) {
 110		struct xe_pt_dir *dir = kzalloc(sizeof(*dir), GFP_KERNEL);
 111
 112		pt = (dir) ? &dir->pt : NULL;
 113	} else {
 114		pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 115	}
 116	if (!pt)
 117		return ERR_PTR(-ENOMEM);
 118
 119	bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) |
 120		   XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
 121		   XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE;
 122	if (vm->xef) /* userspace */
 123		bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
 124
 125	pt->level = level;
 126
 127	drm_WARN_ON(&vm->xe->drm, IS_ERR_OR_NULL(exec));
 128	bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
 129				  ttm_bo_type_kernel,
 130				  bo_flags, exec);
 131	if (IS_ERR(bo)) {
 132		err = PTR_ERR(bo);
 133		goto err_kfree;
 134	}
 135	pt->bo = bo;
 136	pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;
 137	pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;
 138
 139	if (vm->xef)
 140		xe_drm_client_add_bo(vm->xef->client, pt->bo);
 141	xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);
 142
 143	return pt;
 144
 145err_kfree:
 146	xe_pt_free(pt);
 147	return ERR_PTR(err);
 148}
 149ALLOW_ERROR_INJECTION(xe_pt_create, ERRNO);
 150
 151/**
 152 * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
 153 * entries.
 154 * @tile: The tile the scratch pagetable of which to use.
 155 * @vm: The vm we populate for.
 156 * @pt: The pagetable the bo of which to initialize.
 157 *
 158 * Populate the page-table bo of @pt with entries pointing into the tile's
 159 * scratch page-table tree if any. Otherwise populate with zeros.
 160 */
 161void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
 162			  struct xe_pt *pt)
 163{
 164	struct iosys_map *map = &pt->bo->vmap;
 165	u64 empty;
 166	int i;
 167
 168	if (!xe_vm_has_scratch(vm)) {
 169		/*
 170		 * FIXME: Some memory is allocated already allocated to zero?
 171		 * Find out which memory that is and avoid this memset...
 172		 */
 173		xe_map_memset(vm->xe, map, 0, 0, SZ_4K);
 174	} else {
 175		empty = __xe_pt_empty_pte(tile, vm, pt->level);
 176		for (i = 0; i < XE_PDES; i++)
 177			xe_pt_write(vm->xe, map, i, empty);
 178	}
 179}
 180
 181/**
 182 * xe_pt_shift() - Return the ilog2 value of the size of the address range of
 183 * a page-table at a certain level.
 184 * @level: The level.
 185 *
 186 * Return: The ilog2 value of the size of the address range of a page-table
 187 * at level @level.
 188 */
 189unsigned int xe_pt_shift(unsigned int level)
 190{
 191	return XE_PTE_SHIFT + XE_PDE_SHIFT * level;
 192}
 193
 194/**
 195 * xe_pt_destroy() - Destroy a page-table tree.
 196 * @pt: The root of the page-table tree to destroy.
 197 * @flags: vm flags. Currently unused.
 198 * @deferred: List head of lockless list for deferred putting. NULL for
 199 *            immediate putting.
 200 *
 201 * Puts the page-table bo, recursively calls xe_pt_destroy on all children
 202 * and finally frees @pt. TODO: Can we remove the @flags argument?
 203 */
 204void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
 205{
 206	int i;
 207
 208	if (!pt)
 209		return;
 210
 211	XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
 212	xe_bo_unpin(pt->bo);
 213	xe_bo_put_deferred(pt->bo, deferred);
 214
 215	if (pt->level > 0 && pt->num_live) {
 216		struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
 217
 218		for (i = 0; i < XE_PDES; i++) {
 219			if (xe_pt_entry_staging(pt_dir, i))
 220				xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags,
 221					      deferred);
 222		}
 223	}
 224	xe_pt_free(pt);
 225}
 226
 227/**
 228 * xe_pt_clear() - Clear a page-table.
 229 * @xe: xe device.
 230 * @pt: The page-table.
 231 *
 232 * Clears page-table by setting to zero.
 233 */
 234void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt)
 235{
 236	struct iosys_map *map = &pt->bo->vmap;
 237
 238	xe_map_memset(xe, map, 0, 0, SZ_4K);
 239}
 240
 241/**
 242 * DOC: Pagetable building
 243 *
 244 * Below we use the term "page-table" for both page-directories, containing
 245 * pointers to lower level page-directories or page-tables, and level 0
 246 * page-tables that contain only page-table-entries pointing to memory pages.
 247 *
 248 * When inserting an address range in an already existing page-table tree
 249 * there will typically be a set of page-tables that are shared with other
 250 * address ranges, and a set that are private to this address range.
 251 * The set of shared page-tables can be at most two per level,
 252 * and those can't be updated immediately because the entries of those
 253 * page-tables may still be in use by the gpu for other mappings. Therefore
 254 * when inserting entries into those, we instead stage those insertions by
 255 * adding insertion data into struct xe_vm_pgtable_update structures. This
 256 * data, (subtrees for the cpu and page-table-entries for the gpu) is then
 257 * added in a separate commit step. CPU-data is committed while still under the
 258 * vm lock, the object lock and for userptr, the notifier lock in read mode.
 259 * The GPU async data is committed either by the GPU or CPU after fulfilling
 260 * relevant dependencies.
 261 * For non-shared page-tables (and, in fact, for shared ones that aren't
 262 * existing at the time of staging), we add the data in-place without the
 263 * special update structures. This private part of the page-table tree will
 264 * remain disconnected from the vm page-table tree until data is committed to
 265 * the shared page tables of the vm tree in the commit phase.
 266 */
 267
 268struct xe_pt_update {
 269	/** @update: The update structure we're building for this parent. */
 270	struct xe_vm_pgtable_update *update;
 271	/** @parent: The parent. Used to detect a parent change. */
 272	struct xe_pt *parent;
 273	/** @preexisting: Whether the parent was pre-existing or allocated */
 274	bool preexisting;
 275};
 276
 277/**
 278 * struct xe_pt_stage_bind_walk - Walk state for the stage_bind walk.
 279 */
 280struct xe_pt_stage_bind_walk {
 281	/** @base: The base class. */
 282	struct xe_pt_walk base;
 283
 284	/* Input parameters for the walk */
 285	/** @vm: The vm we're building for. */
 286	struct xe_vm *vm;
 287	/** @tile: The tile we're building for. */
 288	struct xe_tile *tile;
 289	/** @default_vram_pte: PTE flag only template for VRAM. No address is associated */
 290	u64 default_vram_pte;
 291	/** @default_system_pte: PTE flag only template for System. No address is associated */
 292	u64 default_system_pte;
 293	/** @dma_offset: DMA offset to add to the PTE. */
 294	u64 dma_offset;
 295	/**
 296	 * @needs_64K: This address range enforces 64K alignment and
 297	 * granularity on VRAM.
 298	 */
 299	bool needs_64K;
 300	/** @clear_pt: clear page table entries during the bind walk */
 301	bool clear_pt;
 302	/**
 303	 * @vma: VMA being mapped
 304	 */
 305	struct xe_vma *vma;
 306
 307	/* Also input, but is updated during the walk*/
 308	/** @curs: The DMA address cursor. */
 309	struct xe_res_cursor *curs;
 310	/** @va_curs_start: The Virtual address corresponding to @curs->start */
 311	u64 va_curs_start;
 312
 313	/* Output */
 314	/** @wupd: Walk output data for page-table updates. */
 315	struct xe_walk_update {
 316		/** @wupd.entries: Caller provided storage. */
 317		struct xe_vm_pgtable_update *entries;
 318		/** @wupd.num_used_entries: Number of update @entries used. */
 319		unsigned int num_used_entries;
 320		/** @wupd.updates: Tracks the update entry at a given level */
 321		struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1];
 322	} wupd;
 323
 324	/* Walk state */
 325	/**
 326	 * @l0_end_addr: The end address of the current l0 leaf. Used for
 327	 * 64K granularity detection.
 328	 */
 329	u64 l0_end_addr;
 330	/** @addr_64K: The start address of the current 64K chunk. */
 331	u64 addr_64K;
 332	/** @found_64K: Whether @add_64K actually points to a 64K chunk. */
 333	bool found_64K;
 334};
 335
 336static int
 337xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
 338		 pgoff_t offset, bool alloc_entries)
 339{
 340	struct xe_pt_update *upd = &wupd->updates[parent->level];
 341	struct xe_vm_pgtable_update *entry;
 342
 343	/*
 344	 * For *each level*, we could only have one active
 345	 * struct xt_pt_update at any one time. Once we move on to a
 346	 * new parent and page-directory, the old one is complete, and
 347	 * updates are either already stored in the build tree or in
 348	 * @wupd->entries
 349	 */
 350	if (likely(upd->parent == parent))
 351		return 0;
 352
 353	upd->parent = parent;
 354	upd->preexisting = true;
 355
 356	if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1)
 357		return -EINVAL;
 358
 359	entry = wupd->entries + wupd->num_used_entries++;
 360	upd->update = entry;
 361	entry->ofs = offset;
 362	entry->pt_bo = parent->bo;
 363	entry->pt = parent;
 364	entry->flags = 0;
 365	entry->qwords = 0;
 366	entry->pt_bo->update_index = -1;
 367
 368	if (alloc_entries) {
 369		entry->pt_entries = kmalloc_array(XE_PDES,
 370						  sizeof(*entry->pt_entries),
 371						  GFP_KERNEL);
 372		if (!entry->pt_entries)
 373			return -ENOMEM;
 374	}
 375
 376	return 0;
 377}
 378
 379/*
 380 * NOTE: This is a very frequently called function so we allow ourselves
 381 * to annotate (using branch prediction hints) the fastpath of updating a
 382 * non-pre-existing pagetable with leaf ptes.
 383 */
 384static int
 385xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
 386		   pgoff_t offset, struct xe_pt *xe_child, u64 pte)
 387{
 388	struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level];
 389	struct xe_pt_update *child_upd = xe_child ?
 390		&xe_walk->wupd.updates[xe_child->level] : NULL;
 391	int ret;
 392
 393	ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true);
 394	if (unlikely(ret))
 395		return ret;
 396
 397	/*
 398	 * Register this new pagetable so that it won't be recognized as
 399	 * a shared pagetable by a subsequent insertion.
 400	 */
 401	if (unlikely(child_upd)) {
 402		child_upd->update = NULL;
 403		child_upd->parent = xe_child;
 404		child_upd->preexisting = false;
 405	}
 406
 407	if (likely(!upd->preexisting)) {
 408		/* Continue building a non-connected subtree. */
 409		struct iosys_map *map = &parent->bo->vmap;
 410
 411		if (unlikely(xe_child)) {
 412			parent->base.children[offset] = &xe_child->base;
 413			parent->base.staging[offset] = &xe_child->base;
 414		}
 415
 416		xe_pt_write(xe_walk->vm->xe, map, offset, pte);
 417		parent->num_live++;
 418	} else {
 419		/* Shared pt. Stage update. */
 420		unsigned int idx;
 421		struct xe_vm_pgtable_update *entry = upd->update;
 422
 423		idx = offset - entry->ofs;
 424		entry->pt_entries[idx].pt = xe_child;
 425		entry->pt_entries[idx].pte = pte;
 426		entry->qwords++;
 427	}
 428
 429	return 0;
 430}
 431
 432static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level,
 433				   struct xe_pt_stage_bind_walk *xe_walk)
 434{
 435	u64 size, dma;
 436
 437	if (level > MAX_HUGEPTE_LEVEL)
 438		return false;
 439
 440	/* Does the virtual range requested cover a huge pte? */
 441	if (!xe_pt_covers(addr, next, level, &xe_walk->base))
 442		return false;
 443
 444	/* Does the DMA segment cover the whole pte? */
 445	if (next - xe_walk->va_curs_start > xe_walk->curs->size)
 446		return false;
 447
 448	/* null VMA's do not have dma addresses */
 449	if (xe_vma_is_null(xe_walk->vma))
 450		return true;
 451
 452	/* if we are clearing page table, no dma addresses*/
 453	if (xe_walk->clear_pt)
 454		return true;
 455
 456	/* Is the DMA address huge PTE size aligned? */
 457	size = next - addr;
 458	dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs);
 459
 460	return IS_ALIGNED(dma, size);
 461}
 462
 463/*
 464 * Scan the requested mapping to check whether it can be done entirely
 465 * with 64K PTEs.
 466 */
 467static bool
 468xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
 469{
 470	struct xe_res_cursor curs = *xe_walk->curs;
 471
 472	if (!IS_ALIGNED(addr, SZ_64K))
 473		return false;
 474
 475	if (next > xe_walk->l0_end_addr)
 476		return false;
 477
 478	/* null VMA's do not have dma addresses */
 479	if (xe_vma_is_null(xe_walk->vma))
 480		return true;
 481
 482	xe_res_next(&curs, addr - xe_walk->va_curs_start);
 483	for (; addr < next; addr += SZ_64K) {
 484		if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K)
 485			return false;
 486
 487		xe_res_next(&curs, SZ_64K);
 488	}
 489
 490	return addr == next;
 491}
 492
 493/*
 494 * For non-compact "normal" 4K level-0 pagetables, we want to try to group
 495 * addresses together in 64K-contigous regions to add a 64K TLB hint for the
 496 * device to the PTE.
 497 * This function determines whether the address is part of such a
 498 * segment. For VRAM in normal pagetables, this is strictly necessary on
 499 * some devices.
 500 */
 501static bool
 502xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
 503{
 504	/* Address is within an already found 64k region */
 505	if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K)
 506		return true;
 507
 508	xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk);
 509	xe_walk->addr_64K = addr;
 510
 511	return xe_walk->found_64K;
 512}
 513
 514static int
 515xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset,
 516		       unsigned int level, u64 addr, u64 next,
 517		       struct xe_ptw **child,
 518		       enum page_walk_action *action,
 519		       struct xe_pt_walk *walk)
 520{
 521	struct xe_pt_stage_bind_walk *xe_walk =
 522		container_of(walk, typeof(*xe_walk), base);
 523	u16 pat_index = xe_walk->vma->attr.pat_index;
 524	struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base);
 525	struct xe_vm *vm = xe_walk->vm;
 526	struct xe_pt *xe_child;
 527	bool covers;
 528	int ret = 0;
 529	u64 pte;
 530
 531	/* Is this a leaf entry ?*/
 532	if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) {
 533		struct xe_res_cursor *curs = xe_walk->curs;
 534		bool is_null = xe_vma_is_null(xe_walk->vma);
 535		bool is_vram = is_null ? false : xe_res_is_vram(curs);
 536
 537		XE_WARN_ON(xe_walk->va_curs_start != addr);
 538
 539		if (xe_walk->clear_pt) {
 540			pte = 0;
 541		} else {
 542			pte = vm->pt_ops->pte_encode_vma(is_null ? 0 :
 543							 xe_res_dma(curs) +
 544							 xe_walk->dma_offset,
 545							 xe_walk->vma,
 546							 pat_index, level);
 547			if (!is_null)
 548				pte |= is_vram ? xe_walk->default_vram_pte :
 549					xe_walk->default_system_pte;
 550
 551			/*
 552			 * Set the XE_PTE_PS64 hint if possible, otherwise if
 553			 * this device *requires* 64K PTE size for VRAM, fail.
 554			 */
 555			if (level == 0 && !xe_parent->is_compact) {
 556				if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) {
 557					xe_walk->vma->gpuva.flags |=
 558							XE_VMA_PTE_64K;
 559					pte |= XE_PTE_PS64;
 560				} else if (XE_WARN_ON(xe_walk->needs_64K &&
 561					   is_vram)) {
 562					return -EINVAL;
 563				}
 564			}
 565		}
 566
 567		ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte);
 568		if (unlikely(ret))
 569			return ret;
 570
 571		if (!is_null && !xe_walk->clear_pt)
 572			xe_res_next(curs, next - addr);
 573		xe_walk->va_curs_start = next;
 574		xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level);
 575		*action = ACTION_CONTINUE;
 576
 577		return ret;
 578	}
 579
 580	/*
 581	 * Descending to lower level. Determine if we need to allocate a
 582	 * new page table or -directory, which we do if there is no
 583	 * previous one or there is one we can completely replace.
 584	 */
 585	if (level == 1) {
 586		walk->shifts = xe_normal_pt_shifts;
 587		xe_walk->l0_end_addr = next;
 588	}
 589
 590	covers = xe_pt_covers(addr, next, level, &xe_walk->base);
 591	if (covers || !*child) {
 592		u64 flags = 0;
 593
 594		xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1,
 595					xe_vm_validation_exec(vm));
 596		if (IS_ERR(xe_child))
 597			return PTR_ERR(xe_child);
 598
 599		xe_pt_set_addr(xe_child,
 600			       round_down(addr, 1ull << walk->shifts[level]));
 601
 602		if (!covers)
 603			xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);
 604
 605		*child = &xe_child->base;
 606
 607		/*
 608		 * Prefer the compact pagetable layout for L0 if possible. Only
 609		 * possible if VMA covers entire 2MB region as compact 64k and
 610		 * 4k pages cannot be mixed within a 2MB region.
 611		 * TODO: Suballocate the pt bo to avoid wasting a lot of
 612		 * memory.
 613		 */
 614		if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&
 615		    covers && xe_pt_scan_64K(addr, next, xe_walk)) {
 616			walk->shifts = xe_compact_pt_shifts;
 617			xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT;
 618			flags |= XE_PDE_64K;
 619			xe_child->is_compact = true;
 620		}
 621
 622		pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0) | flags;
 623		ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child,
 624					 pte);
 625	}
 626
 627	*action = ACTION_SUBTREE;
 628	return ret;
 629}
 630
 631static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
 632	.pt_entry = xe_pt_stage_bind_entry,
 633};
 634
 635/*
 636 * Default atomic expectations for different allocation scenarios are as follows:
 637 *
 638 * 1. Traditional API: When the VM is not in LR mode:
 639 *    - Device atomics are expected to function with all allocations.
 640 *
 641 * 2. Compute/SVM API: When the VM is in LR mode:
 642 *    - Device atomics are the default behavior when the bo is placed in a single region.
 643 *    - In all other cases device atomics will be disabled with AE=0 until an application
 644 *      request differently using a ioctl like madvise.
 645 */
 646static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma)
 647{
 648	if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
 649		return false;
 650
 651	return true;
 652}
 653
 654static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma)
 655{
 656	struct xe_device *xe = vm->xe;
 657	struct xe_bo *bo = xe_vma_bo(vma);
 658
 659	if (!xe->info.has_device_atomics_on_smem ||
 660	    vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
 661		return false;
 662
 663	if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE)
 664		return true;
 665
 666	/*
 667	 * If a SMEM+LMEM allocation is backed by SMEM, a device
 668	 * atomics will cause a gpu page fault and which then
 669	 * gets migrated to LMEM, bind such allocations with
 670	 * device atomics enabled.
 671	 */
 672	return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) ||
 673				 (bo && xe_bo_has_single_placement(bo))));
 674}
 675
 676/**
 677 * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
 678 * range.
 679 * @tile: The tile we're building for.
 680 * @vma: The vma indicating the address range.
 681 * @range: The range indicating the address range.
 682 * @entries: Storage for the update entries used for connecting the tree to
 683 * the main tree at commit time.
 684 * @num_entries: On output contains the number of @entries used.
 685 * @clear_pt: Clear the page table entries.
 686 *
 687 * This function builds a disconnected page-table tree for a given address
 688 * range. The tree is connected to the main vm tree for the gpu using
 689 * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind().
 690 * The function builds xe_vm_pgtable_update structures for already existing
 691 * shared page-tables, and non-existing shared and non-shared page-tables
 692 * are built and populated directly.
 693 *
 694 * Return 0 on success, negative error code on error.
 695 */
 696static int
 697xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 698		 struct xe_svm_range *range,
 699		 struct xe_vm_pgtable_update *entries,
 700		 u32 *num_entries, bool clear_pt)
 701{
 702	struct xe_device *xe = tile_to_xe(tile);
 703	struct xe_bo *bo = xe_vma_bo(vma);
 704	struct xe_res_cursor curs;
 705	struct xe_vm *vm = xe_vma_vm(vma);
 706	struct xe_pt_stage_bind_walk xe_walk = {
 707		.base = {
 708			.ops = &xe_pt_stage_bind_ops,
 709			.shifts = xe_normal_pt_shifts,
 710			.max_level = XE_PT_HIGHEST_LEVEL,
 711			.staging = true,
 712		},
 713		.vm = vm,
 714		.tile = tile,
 715		.curs = &curs,
 716		.va_curs_start = range ? xe_svm_range_start(range) :
 717			xe_vma_start(vma),
 718		.vma = vma,
 719		.wupd.entries = entries,
 720		.clear_pt = clear_pt,
 721	};
 722	struct xe_pt *pt = vm->pt_root[tile->id];
 723	int ret;
 724
 725	if (range) {
 726		/* Move this entire thing to xe_svm.c? */
 727		xe_svm_notifier_lock(vm);
 728		if (!xe_svm_range_pages_valid(range)) {
 729			xe_svm_range_debug(range, "BIND PREPARE - RETRY");
 730			xe_svm_notifier_unlock(vm);
 731			return -EAGAIN;
 732		}
 733		if (xe_svm_range_has_dma_mapping(range)) {
 734			xe_res_first_dma(range->base.pages.dma_addr, 0,
 735					 xe_svm_range_size(range),
 736					 &curs);
 737			xe_svm_range_debug(range, "BIND PREPARE - MIXED");
 738		} else {
 739			xe_assert(xe, false);
 740		}
 741		/*
 742		 * Note, when unlocking the resource cursor dma addresses may become
 743		 * stale, but the bind will be aborted anyway at commit time.
 744		 */
 745		xe_svm_notifier_unlock(vm);
 746	}
 747
 748	xe_walk.needs_64K = (vm->flags & XE_VM_FLAG_64K);
 749	if (clear_pt)
 750		goto walk_pt;
 751
 752	if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) {
 753		xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0;
 754		xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ?
 755			XE_USM_PPGTT_PTE_AE : 0;
 756	}
 757
 758	xe_walk.default_vram_pte |= XE_PPGTT_PTE_DM;
 759	xe_walk.dma_offset = bo ? vram_region_gpu_offset(bo->ttm.resource) : 0;
 760	if (!range)
 761		xe_bo_assert_held(bo);
 762
 763	if (!xe_vma_is_null(vma) && !range) {
 764		if (xe_vma_is_userptr(vma))
 765			xe_res_first_dma(to_userptr_vma(vma)->userptr.pages.dma_addr, 0,
 766					 xe_vma_size(vma), &curs);
 767		else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
 768			xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
 769				     xe_vma_size(vma), &curs);
 770		else
 771			xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
 772					xe_vma_size(vma), &curs);
 773	} else if (!range) {
 774		curs.size = xe_vma_size(vma);
 775	}
 776
 777walk_pt:
 778	ret = xe_pt_walk_range(&pt->base, pt->level,
 779			       range ? xe_svm_range_start(range) : xe_vma_start(vma),
 780			       range ? xe_svm_range_end(range) : xe_vma_end(vma),
 781			       &xe_walk.base);
 782
 783	*num_entries = xe_walk.wupd.num_used_entries;
 784	return ret;
 785}
 786
 787/**
 788 * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
 789 * shared pagetable.
 790 * @addr: The start address within the non-shared pagetable.
 791 * @end: The end address within the non-shared pagetable.
 792 * @level: The level of the non-shared pagetable.
 793 * @walk: Walk info. The function adjusts the walk action.
 794 * @action: next action to perform (see enum page_walk_action)
 795 * @offset: Ignored on input, First non-shared entry on output.
 796 * @end_offset: Ignored on input, Last non-shared entry + 1 on output.
 797 *
 798 * A non-shared page-table has some entries that belong to the address range
 799 * and others that don't. This function determines the entries that belong
 800 * fully to the address range. Depending on level, some entries may
 801 * partially belong to the address range (that can't happen at level 0).
 802 * The function detects that and adjust those offsets to not include those
 803 * partial entries. Iff it does detect partial entries, we know that there must
 804 * be shared page tables also at lower levels, so it adjusts the walk action
 805 * accordingly.
 806 *
 807 * Return: true if there were non-shared entries, false otherwise.
 808 */
 809static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level,
 810				    struct xe_pt_walk *walk,
 811				    enum page_walk_action *action,
 812				    pgoff_t *offset, pgoff_t *end_offset)
 813{
 814	u64 size = 1ull << walk->shifts[level];
 815
 816	*offset = xe_pt_offset(addr, level, walk);
 817	*end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset;
 818
 819	if (!level)
 820		return true;
 821
 822	/*
 823	 * If addr or next are not size aligned, there are shared pts at lower
 824	 * level, so in that case traverse down the subtree
 825	 */
 826	*action = ACTION_CONTINUE;
 827	if (!IS_ALIGNED(addr, size)) {
 828		*action = ACTION_SUBTREE;
 829		(*offset)++;
 830	}
 831
 832	if (!IS_ALIGNED(end, size)) {
 833		*action = ACTION_SUBTREE;
 834		(*end_offset)--;
 835	}
 836
 837	return *end_offset > *offset;
 838}
 839
 840struct xe_pt_zap_ptes_walk {
 841	/** @base: The walk base-class */
 842	struct xe_pt_walk base;
 843
 844	/* Input parameters for the walk */
 845	/** @tile: The tile we're building for */
 846	struct xe_tile *tile;
 847
 848	/* Output */
 849	/** @needs_invalidate: Whether we need to invalidate TLB*/
 850	bool needs_invalidate;
 851};
 852
 853static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
 854				unsigned int level, u64 addr, u64 next,
 855				struct xe_ptw **child,
 856				enum page_walk_action *action,
 857				struct xe_pt_walk *walk)
 858{
 859	struct xe_pt_zap_ptes_walk *xe_walk =
 860		container_of(walk, typeof(*xe_walk), base);
 861	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
 862	pgoff_t end_offset;
 863
 864	XE_WARN_ON(!*child);
 865	XE_WARN_ON(!level);
 866
 867	/*
 868	 * Note that we're called from an entry callback, and we're dealing
 869	 * with the child of that entry rather than the parent, so need to
 870	 * adjust level down.
 871	 */
 872	if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset,
 873				    &end_offset)) {
 874		xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,
 875			      offset * sizeof(u64), 0,
 876			      (end_offset - offset) * sizeof(u64));
 877		xe_walk->needs_invalidate = true;
 878	}
 879
 880	return 0;
 881}
 882
 883static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
 884	.pt_entry = xe_pt_zap_ptes_entry,
 885};
 886
 887/**
 888 * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
 889 * @tile: The tile we're zapping for.
 890 * @vma: GPU VMA detailing address range.
 891 *
 892 * Eviction and Userptr invalidation needs to be able to zap the
 893 * gpu ptes of a given address range in pagefaulting mode.
 894 * In order to be able to do that, that function needs access to the shared
 895 * page-table entrieaso it can either clear the leaf PTEs or
 896 * clear the pointers to lower-level page-tables. The caller is required
 897 * to hold the necessary locks to ensure neither the page-table connectivity
 898 * nor the page-table entries of the range is updated from under us.
 899 *
 900 * Return: Whether ptes were actually updated and a TLB invalidation is
 901 * required.
 902 */
 903bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
 904{
 905	struct xe_pt_zap_ptes_walk xe_walk = {
 906		.base = {
 907			.ops = &xe_pt_zap_ptes_ops,
 908			.shifts = xe_normal_pt_shifts,
 909			.max_level = XE_PT_HIGHEST_LEVEL,
 910		},
 911		.tile = tile,
 912	};
 913	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
 914	u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated);
 915
 916	if (xe_vma_bo(vma))
 917		xe_bo_assert_held(xe_vma_bo(vma));
 918	else if (xe_vma_is_userptr(vma))
 919		lockdep_assert_held(&xe_vma_vm(vma)->svm.gpusvm.notifier_lock);
 920
 921	if (!(pt_mask & BIT(tile->id)))
 922		return false;
 923
 924	(void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
 925				xe_vma_end(vma), &xe_walk.base);
 926
 927	return xe_walk.needs_invalidate;
 928}
 929
 930/**
 931 * xe_pt_zap_ptes_range() - Zap (zero) gpu ptes of a SVM range
 932 * @tile: The tile we're zapping for.
 933 * @vm: The VM we're zapping for.
 934 * @range: The SVM range we're zapping for.
 935 *
 936 * SVM invalidation needs to be able to zap the gpu ptes of a given address
 937 * range. In order to be able to do that, that function needs access to the
 938 * shared page-table entries so it can either clear the leaf PTEs or
 939 * clear the pointers to lower-level page-tables. The caller is required
 940 * to hold the SVM notifier lock.
 941 *
 942 * Return: Whether ptes were actually updated and a TLB invalidation is
 943 * required.
 944 */
 945bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
 946			  struct xe_svm_range *range)
 947{
 948	struct xe_pt_zap_ptes_walk xe_walk = {
 949		.base = {
 950			.ops = &xe_pt_zap_ptes_ops,
 951			.shifts = xe_normal_pt_shifts,
 952			.max_level = XE_PT_HIGHEST_LEVEL,
 953		},
 954		.tile = tile,
 955	};
 956	struct xe_pt *pt = vm->pt_root[tile->id];
 957	u8 pt_mask = (range->tile_present & ~range->tile_invalidated);
 958
 959	/*
 960	 * Locking rules:
 961	 *
 962	 * - notifier_lock (write): full protection against page table changes
 963	 *   and MMU notifier invalidations.
 964	 *
 965	 * - notifier_lock (read) + vm_lock (write): combined protection against
 966	 *   invalidations and concurrent page table modifications. (e.g., madvise)
 967	 *
 968	 */
 969	lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
 970		       (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
 971		       lockdep_is_held_type(&vm->lock, 0)));
 972
 973	if (!(pt_mask & BIT(tile->id)))
 974		return false;
 975
 976	(void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range),
 977				xe_svm_range_end(range), &xe_walk.base);
 978
 979	return xe_walk.needs_invalidate;
 980}
 981
 982static void
 983xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
 984		       struct iosys_map *map, void *data,
 985		       u32 qword_ofs, u32 num_qwords,
 986		       const struct xe_vm_pgtable_update *update)
 987{
 988	struct xe_pt_entry *ptes = update->pt_entries;
 989	u64 *ptr = data;
 990	u32 i;
 991
 992	for (i = 0; i < num_qwords; i++) {
 993		if (map)
 994			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
 995				  sizeof(u64), u64, ptes[i].pte);
 996		else
 997			ptr[i] = ptes[i].pte;
 998	}
 999}
1000
1001static void xe_pt_cancel_bind(struct xe_vma *vma,
1002			      struct xe_vm_pgtable_update *entries,
1003			      u32 num_entries)
1004{
1005	u32 i, j;
1006
1007	for (i = 0; i < num_entries; i++) {
1008		struct xe_pt *pt = entries[i].pt;
1009
1010		if (!pt)
1011			continue;
1012
1013		if (pt->level) {
1014			for (j = 0; j < entries[i].qwords; j++)
1015				xe_pt_destroy(entries[i].pt_entries[j].pt,
1016					      xe_vma_vm(vma)->flags, NULL);
1017		}
1018
1019		kfree(entries[i].pt_entries);
1020		entries[i].pt_entries = NULL;
1021		entries[i].qwords = 0;
1022	}
1023}
1024
1025#define XE_INVALID_VMA	((struct xe_vma *)(0xdeaddeadull))
1026
1027static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)
1028{
1029	struct xe_vm *vm;
1030
1031	if (vma == XE_INVALID_VMA)
1032		return;
1033
1034	vm = xe_vma_vm(vma);
1035	lockdep_assert_held(&vm->lock);
1036
1037	if (!xe_vma_has_no_bo(vma))
1038		dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
1039
1040	xe_vm_assert_held(vm);
1041}
1042
1043static void xe_pt_commit_locks_assert(struct xe_vma *vma)
1044{
1045	struct xe_vm *vm;
1046
1047	if (vma == XE_INVALID_VMA)
1048		return;
1049
1050	vm = xe_vma_vm(vma);
1051	xe_pt_commit_prepare_locks_assert(vma);
1052
1053	if (xe_vma_is_userptr(vma))
1054		xe_svm_assert_held_read(vm);
1055}
1056
1057static void xe_pt_commit(struct xe_vma *vma,
1058			 struct xe_vm_pgtable_update *entries,
1059			 u32 num_entries, struct llist_head *deferred)
1060{
1061	u32 i, j;
1062
1063	xe_pt_commit_locks_assert(vma);
1064
1065	for (i = 0; i < num_entries; i++) {
1066		struct xe_pt *pt = entries[i].pt;
1067		struct xe_pt_dir *pt_dir;
1068
1069		if (!pt->level)
1070			continue;
1071
1072		pt_dir = as_xe_pt_dir(pt);
1073		for (j = 0; j < entries[i].qwords; j++) {
1074			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
1075			int j_ = j + entries[i].ofs;
1076
1077			pt_dir->children[j_] = pt_dir->staging[j_];
1078			xe_pt_destroy(oldpte, (vma == XE_INVALID_VMA) ? 0 :
1079				      xe_vma_vm(vma)->flags, deferred);
1080		}
1081	}
1082}
1083
1084static void xe_pt_abort_bind(struct xe_vma *vma,
1085			     struct xe_vm_pgtable_update *entries,
1086			     u32 num_entries, bool rebind)
1087{
1088	int i, j;
1089
1090	xe_pt_commit_prepare_locks_assert(vma);
1091
1092	for (i = num_entries - 1; i >= 0; --i) {
1093		struct xe_pt *pt = entries[i].pt;
1094		struct xe_pt_dir *pt_dir;
1095
1096		if (!rebind)
1097			pt->num_live -= entries[i].qwords;
1098
1099		if (!pt->level)
1100			continue;
1101
1102		pt_dir = as_xe_pt_dir(pt);
1103		for (j = 0; j < entries[i].qwords; j++) {
1104			u32 j_ = j + entries[i].ofs;
1105			struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_);
1106			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
1107
1108			pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;
1109			xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
1110		}
1111	}
1112}
1113
1114static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
1115				      struct xe_vm_pgtable_update *entries,
1116				      u32 num_entries, bool rebind)
1117{
1118	u32 i, j;
1119
1120	xe_pt_commit_prepare_locks_assert(vma);
1121
1122	for (i = 0; i < num_entries; i++) {
1123		struct xe_pt *pt = entries[i].pt;
1124		struct xe_pt_dir *pt_dir;
1125
1126		if (!rebind)
1127			pt->num_live += entries[i].qwords;
1128
1129		if (!pt->level)
1130			continue;
1131
1132		pt_dir = as_xe_pt_dir(pt);
1133		for (j = 0; j < entries[i].qwords; j++) {
1134			u32 j_ = j + entries[i].ofs;
1135			struct xe_pt *newpte = entries[i].pt_entries[j].pt;
1136			struct xe_pt *oldpte = NULL;
1137
1138			if (xe_pt_entry_staging(pt_dir, j_))
1139				oldpte = xe_pt_entry_staging(pt_dir, j_);
1140
1141			pt_dir->staging[j_] = &newpte->base;
1142			entries[i].pt_entries[j].pt = oldpte;
1143		}
1144	}
1145}
1146
1147static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
1148			    u32 num_entries)
1149{
1150	u32 i;
1151
1152	for (i = 0; i < num_entries; i++)
1153		kfree(entries[i].pt_entries);
1154}
1155
1156static int
1157xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
1158		   struct xe_svm_range *range,
1159		   struct xe_vm_pgtable_update *entries,
1160		   u32 *num_entries, bool invalidate_on_bind)
1161{
1162	int err;
1163
1164	*num_entries = 0;
1165	err = xe_pt_stage_bind(tile, vma, range, entries, num_entries,
1166			       invalidate_on_bind);
1167	if (!err)
1168		xe_tile_assert(tile, *num_entries);
1169
1170	return err;
1171}
1172
1173static void xe_vm_dbg_print_entries(struct xe_device *xe,
1174				    const struct xe_vm_pgtable_update *entries,
1175				    unsigned int num_entries, bool bind)
1176#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
1177{
1178	unsigned int i;
1179
1180	vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind",
1181	       num_entries);
1182	for (i = 0; i < num_entries; i++) {
1183		const struct xe_vm_pgtable_update *entry = &entries[i];
1184		struct xe_pt *xe_pt = entry->pt;
1185		u64 page_size = 1ull << xe_pt_shift(xe_pt->level);
1186		u64 end;
1187		u64 start;
1188
1189		xe_assert(xe, !entry->pt->is_compact);
1190		start = entry->ofs * page_size;
1191		end = start + page_size * entry->qwords;
1192		vm_dbg(&xe->drm,
1193		       "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n",
1194		       i, xe_pt->level, entry->ofs, entry->qwords,
1195		       xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0);
1196	}
1197}
1198#else
1199{}
1200#endif
1201
1202static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
1203{
1204	int i;
1205
1206	for (i = 0; i < num_syncs; i++) {
1207		struct dma_fence *fence = syncs[i].fence;
1208
1209		if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1210				       &fence->flags))
1211			return false;
1212	}
1213
1214	return true;
1215}
1216
1217static int job_test_add_deps(struct xe_sched_job *job,
1218			     struct dma_resv *resv,
1219			     enum dma_resv_usage usage)
1220{
1221	if (!job) {
1222		if (!dma_resv_test_signaled(resv, usage))
1223			return -ETIME;
1224
1225		return 0;
1226	}
1227
1228	return xe_sched_job_add_deps(job, resv, usage);
1229}
1230
1231static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job)
1232{
1233	struct xe_bo *bo = xe_vma_bo(vma);
1234
1235	xe_bo_assert_held(bo);
1236
1237	if (bo && !bo->vm)
1238		return job_test_add_deps(job, bo->ttm.base.resv,
1239					 DMA_RESV_USAGE_KERNEL);
1240
1241	return 0;
1242}
1243
1244static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
1245		       struct xe_sched_job *job)
1246{
1247	int err = 0;
1248
1249	/*
1250	 * No need to check for is_cpu_addr_mirror here as vma_add_deps is a
1251	 * NOP if VMA is_cpu_addr_mirror
1252	 */
1253
1254	switch (op->base.op) {
1255	case DRM_GPUVA_OP_MAP:
1256		if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1257			break;
1258
1259		err = vma_add_deps(op->map.vma, job);
1260		break;
1261	case DRM_GPUVA_OP_REMAP:
1262		if (op->remap.prev)
1263			err = vma_add_deps(op->remap.prev, job);
1264		if (!err && op->remap.next)
1265			err = vma_add_deps(op->remap.next, job);
1266		break;
1267	case DRM_GPUVA_OP_UNMAP:
1268		break;
1269	case DRM_GPUVA_OP_PREFETCH:
1270		err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job);
1271		break;
1272	case DRM_GPUVA_OP_DRIVER:
1273		break;
1274	default:
1275		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1276	}
1277
1278	return err;
1279}
1280
1281static int xe_pt_vm_dependencies(struct xe_sched_job *job,
1282				 struct xe_tlb_inval_job *ijob,
1283				 struct xe_tlb_inval_job *mjob,
1284				 struct xe_vm *vm,
1285				 struct xe_vma_ops *vops,
1286				 struct xe_vm_pgtable_update_ops *pt_update_ops,
1287				 struct xe_range_fence_tree *rftree)
1288{
1289	struct xe_range_fence *rtfence;
1290	struct dma_fence *fence;
1291	struct xe_vma_op *op;
1292	int err = 0, i;
1293
1294	xe_vm_assert_held(vm);
1295
1296	if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
1297		return -ETIME;
1298
1299	if (!job && !xe_exec_queue_is_idle(pt_update_ops->q))
1300		return -ETIME;
1301
1302	if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) {
1303		err = job_test_add_deps(job, xe_vm_resv(vm),
1304					pt_update_ops->wait_vm_bookkeep ?
1305					DMA_RESV_USAGE_BOOKKEEP :
1306					DMA_RESV_USAGE_KERNEL);
1307		if (err)
1308			return err;
1309	}
1310
1311	rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start,
1312					    pt_update_ops->last);
1313	while (rtfence) {
1314		fence = rtfence->fence;
1315
1316		if (!dma_fence_is_signaled(fence)) {
1317			/*
1318			 * Is this a CPU update? GPU is busy updating, so return
1319			 * an error
1320			 */
1321			if (!job)
1322				return -ETIME;
1323
1324			dma_fence_get(fence);
1325			err = drm_sched_job_add_dependency(&job->drm, fence);
1326			if (err)
1327				return err;
1328		}
1329
1330		rtfence = xe_range_fence_tree_next(rtfence,
1331						   pt_update_ops->start,
1332						   pt_update_ops->last);
1333	}
1334
1335	list_for_each_entry(op, &vops->list, link) {
1336		err = op_add_deps(vm, op, job);
1337		if (err)
1338			return err;
1339	}
1340
1341	for (i = 0; job && !err && i < vops->num_syncs; i++)
1342		err = xe_sync_entry_add_deps(&vops->syncs[i], job);
1343
1344	if (job) {
1345		if (ijob) {
1346			err = xe_tlb_inval_job_alloc_dep(ijob);
1347			if (err)
1348				return err;
1349		}
1350
1351		if (mjob) {
1352			err = xe_tlb_inval_job_alloc_dep(mjob);
1353			if (err)
1354				return err;
1355		}
1356	}
1357
1358	return err;
1359}
1360
1361static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
1362{
1363	struct xe_vma_ops *vops = pt_update->vops;
1364	struct xe_vm *vm = vops->vm;
1365	struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
1366	struct xe_vm_pgtable_update_ops *pt_update_ops =
1367		&vops->pt_update_ops[pt_update->tile_id];
1368
1369	return xe_pt_vm_dependencies(pt_update->job, pt_update->ijob,
1370				     pt_update->mjob, vm, pt_update->vops,
1371				     pt_update_ops, rftree);
1372}
1373
1374#if IS_ENABLED(CONFIG_DRM_GPUSVM)
1375#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
1376
1377static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1378{
1379	u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;
1380	static u32 count;
1381
1382	if (count++ % divisor == divisor - 1) {
1383		uvma->userptr.divisor = divisor << 1;
1384		return true;
1385	}
1386
1387	return false;
1388}
1389
1390#else
1391
1392static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1393{
1394	return false;
1395}
1396
1397#endif
1398
1399static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
1400			     struct xe_vm_pgtable_update_ops *pt_update)
1401{
1402	struct xe_userptr_vma *uvma;
1403	unsigned long notifier_seq;
1404
1405	xe_svm_assert_held_read(vm);
1406
1407	if (!xe_vma_is_userptr(vma))
1408		return 0;
1409
1410	uvma = to_userptr_vma(vma);
1411	if (xe_pt_userptr_inject_eagain(uvma))
1412		xe_vma_userptr_force_invalidate(uvma);
1413
1414	notifier_seq = uvma->userptr.pages.notifier_seq;
1415
1416	if (!mmu_interval_read_retry(&uvma->userptr.notifier,
1417				     notifier_seq))
1418		return 0;
1419
1420	if (xe_vm_in_fault_mode(vm))
1421		return -EAGAIN;
1422
1423	/*
1424	 * Just continue the operation since exec or rebind worker
1425	 * will take care of rebinding.
1426	 */
1427	return 0;
1428}
1429
1430static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
1431				struct xe_vm_pgtable_update_ops *pt_update)
1432{
1433	int err = 0;
1434
1435	xe_svm_assert_held_read(vm);
1436
1437	switch (op->base.op) {
1438	case DRM_GPUVA_OP_MAP:
1439		if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1440			break;
1441
1442		err = vma_check_userptr(vm, op->map.vma, pt_update);
1443		break;
1444	case DRM_GPUVA_OP_REMAP:
1445		if (op->remap.prev && !op->remap.skip_prev)
1446			err = vma_check_userptr(vm, op->remap.prev, pt_update);
1447		if (!err && op->remap.next && !op->remap.skip_next)
1448			err = vma_check_userptr(vm, op->remap.next, pt_update);
1449		break;
1450	case DRM_GPUVA_OP_UNMAP:
1451		break;
1452	case DRM_GPUVA_OP_PREFETCH:
1453		if (xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va))) {
1454			struct xe_svm_range *range = op->map_range.range;
1455			unsigned long i;
1456
1457			xe_assert(vm->xe,
1458				  xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va)));
1459			xa_for_each(&op->prefetch_range.range, i, range) {
1460				xe_svm_range_debug(range, "PRE-COMMIT");
1461
1462				if (!xe_svm_range_pages_valid(range)) {
1463					xe_svm_range_debug(range, "PRE-COMMIT - RETRY");
1464					return -ENODATA;
1465				}
1466			}
1467		} else {
1468			err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), pt_update);
1469		}
1470		break;
1471#if IS_ENABLED(CONFIG_DRM_XE_GPUSVM)
1472	case DRM_GPUVA_OP_DRIVER:
1473		if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
1474			struct xe_svm_range *range = op->map_range.range;
1475
1476			xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
1477
1478			xe_svm_range_debug(range, "PRE-COMMIT");
1479
1480			if (!xe_svm_range_pages_valid(range)) {
1481				xe_svm_range_debug(range, "PRE-COMMIT - RETRY");
1482				return -EAGAIN;
1483			}
1484		}
1485		break;
1486#endif
1487	default:
1488		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1489	}
1490
1491	return err;
1492}
1493
1494static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
1495{
1496	struct xe_vm *vm = pt_update->vops->vm;
1497	struct xe_vma_ops *vops = pt_update->vops;
1498	struct xe_vm_pgtable_update_ops *pt_update_ops =
1499		&vops->pt_update_ops[pt_update->tile_id];
1500	struct xe_vma_op *op;
1501	int err;
1502
1503	err = xe_pt_pre_commit(pt_update);
1504	if (err)
1505		return err;
1506
1507	xe_svm_notifier_lock(vm);
1508
1509	list_for_each_entry(op, &vops->list, link) {
1510		err = op_check_svm_userptr(vm, op, pt_update_ops);
1511		if (err) {
1512			xe_svm_notifier_unlock(vm);
1513			break;
1514		}
1515	}
1516
1517	return err;
1518}
1519#endif
1520
1521struct xe_pt_stage_unbind_walk {
1522	/** @base: The pagewalk base-class. */
1523	struct xe_pt_walk base;
1524
1525	/* Input parameters for the walk */
1526	/** @tile: The tile we're unbinding from. */
1527	struct xe_tile *tile;
1528
1529	/**
1530	 * @modified_start: Walk range start, modified to include any
1531	 * shared pagetables that we're the only user of and can thus
1532	 * treat as private.
1533	 */
1534	u64 modified_start;
1535	/** @modified_end: Walk range start, modified like @modified_start. */
1536	u64 modified_end;
1537
1538	/* Output */
1539	/* @wupd: Structure to track the page-table updates we're building */
1540	struct xe_walk_update wupd;
1541};
1542
1543/*
1544 * Check whether this range is the only one populating this pagetable,
1545 * and in that case, update the walk range checks so that higher levels don't
1546 * view us as a shared pagetable.
1547 */
1548static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
1549			     const struct xe_pt *child,
1550			     enum page_walk_action *action,
1551			     struct xe_pt_walk *walk)
1552{
1553	struct xe_pt_stage_unbind_walk *xe_walk =
1554		container_of(walk, typeof(*xe_walk), base);
1555	unsigned int shift = walk->shifts[level];
1556	u64 size = 1ull << shift;
1557
1558	if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) &&
1559	    ((next - addr) >> shift) == child->num_live) {
1560		u64 size = 1ull << walk->shifts[level + 1];
1561
1562		*action = ACTION_CONTINUE;
1563
1564		if (xe_walk->modified_start >= addr)
1565			xe_walk->modified_start = round_down(addr, size);
1566		if (xe_walk->modified_end <= next)
1567			xe_walk->modified_end = round_up(next, size);
1568
1569		return true;
1570	}
1571
1572	return false;
1573}
1574
1575static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
1576				    unsigned int level, u64 addr, u64 next,
1577				    struct xe_ptw **child,
1578				    enum page_walk_action *action,
1579				    struct xe_pt_walk *walk)
1580{
1581	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1582
1583	XE_WARN_ON(!*child);
1584	XE_WARN_ON(!level);
1585
1586	xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
1587
1588	return 0;
1589}
1590
1591static int
1592xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
1593				unsigned int level, u64 addr, u64 next,
1594				struct xe_ptw **child,
1595				enum page_walk_action *action,
1596				struct xe_pt_walk *walk)
1597{
1598	struct xe_pt_stage_unbind_walk *xe_walk =
1599		container_of(walk, typeof(*xe_walk), base);
1600	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1601	pgoff_t end_offset;
1602	u64 size = 1ull << walk->shifts[--level];
1603	int err;
1604
1605	if (!IS_ALIGNED(addr, size))
1606		addr = xe_walk->modified_start;
1607	if (!IS_ALIGNED(next, size))
1608		next = xe_walk->modified_end;
1609
1610	/* Parent == *child is the root pt. Don't kill it. */
1611	if (parent != *child &&
1612	    xe_pt_check_kill(addr, next, level, xe_child, action, walk))
1613		return 0;
1614
1615	if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset,
1616				     &end_offset))
1617		return 0;
1618
1619	err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
1620	if (err)
1621		return err;
1622
1623	xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
1624
1625	return 0;
1626}
1627
1628static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = {
1629	.pt_entry = xe_pt_stage_unbind_entry,
1630	.pt_post_descend = xe_pt_stage_unbind_post_descend,
1631};
1632
1633/**
1634 * xe_pt_stage_unbind() - Build page-table update structures for an unbind
1635 * operation
1636 * @tile: The tile we're unbinding for.
1637 * @vm: The vm
1638 * @vma: The vma we're unbinding.
1639 * @range: The range we're unbinding.
1640 * @entries: Caller-provided storage for the update structures.
1641 *
1642 * Builds page-table update structures for an unbind operation. The function
1643 * will attempt to remove all page-tables that we're the only user
1644 * of, and for that to work, the unbind operation must be committed in the
1645 * same critical section that blocks racing binds to the same page-table tree.
1646 *
1647 * Return: The number of entries used.
1648 */
1649static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
1650				       struct xe_vm *vm,
1651				       struct xe_vma *vma,
1652				       struct xe_svm_range *range,
1653				       struct xe_vm_pgtable_update *entries)
1654{
1655	u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
1656	u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
1657	struct xe_pt_stage_unbind_walk xe_walk = {
1658		.base = {
1659			.ops = &xe_pt_stage_unbind_ops,
1660			.shifts = xe_normal_pt_shifts,
1661			.max_level = XE_PT_HIGHEST_LEVEL,
1662			.staging = true,
1663		},
1664		.tile = tile,
1665		.modified_start = start,
1666		.modified_end = end,
1667		.wupd.entries = entries,
1668	};
1669	struct xe_pt *pt = vm->pt_root[tile->id];
1670
1671	(void)xe_pt_walk_shared(&pt->base, pt->level, start, end,
1672				&xe_walk.base);
1673
1674	return xe_walk.wupd.num_used_entries;
1675}
1676
1677static void
1678xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
1679				  struct xe_tile *tile, struct iosys_map *map,
1680				  void *ptr, u32 qword_ofs, u32 num_qwords,
1681				  const struct xe_vm_pgtable_update *update)
1682{
1683	struct xe_vm *vm = pt_update->vops->vm;
1684	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
1685	int i;
1686
1687	if (map && map->is_iomem)
1688		for (i = 0; i < num_qwords; ++i)
1689			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
1690				  sizeof(u64), u64, empty);
1691	else if (map)
1692		memset64(map->vaddr + qword_ofs * sizeof(u64), empty,
1693			 num_qwords);
1694	else
1695		memset64(ptr, empty, num_qwords);
1696}
1697
1698static void xe_pt_abort_unbind(struct xe_vma *vma,
1699			       struct xe_vm_pgtable_update *entries,
1700			       u32 num_entries)
1701{
1702	int i, j;
1703
1704	xe_pt_commit_prepare_locks_assert(vma);
1705
1706	for (i = num_entries - 1; i >= 0; --i) {
1707		struct xe_vm_pgtable_update *entry = &entries[i];
1708		struct xe_pt *pt = entry->pt;
1709		struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
1710
1711		pt->num_live += entry->qwords;
1712
1713		if (!pt->level)
1714			continue;
1715
1716		for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
1717			pt_dir->staging[j] =
1718				entries[i].pt_entries[j - entry->ofs].pt ?
1719				&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
1720	}
1721}
1722
1723static void
1724xe_pt_commit_prepare_unbind(struct xe_vma *vma,
1725			    struct xe_vm_pgtable_update *entries,
1726			    u32 num_entries)
1727{
1728	int i, j;
1729
1730	xe_pt_commit_prepare_locks_assert(vma);
1731
1732	for (i = 0; i < num_entries; ++i) {
1733		struct xe_vm_pgtable_update *entry = &entries[i];
1734		struct xe_pt *pt = entry->pt;
1735		struct xe_pt_dir *pt_dir;
1736
1737		pt->num_live -= entry->qwords;
1738		if (!pt->level)
1739			continue;
1740
1741		pt_dir = as_xe_pt_dir(pt);
1742		for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
1743			entry->pt_entries[j - entry->ofs].pt =
1744				xe_pt_entry_staging(pt_dir, j);
1745			pt_dir->staging[j] = NULL;
1746		}
1747	}
1748}
1749
1750static void
1751xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
1752				 u64 start, u64 end)
1753{
1754	u64 last;
1755	u32 current_op = pt_update_ops->current_op;
1756	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1757	int i, level = 0;
1758
1759	for (i = 0; i < pt_op->num_entries; i++) {
1760		const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
1761
1762		if (entry->pt->level > level)
1763			level = entry->pt->level;
1764	}
1765
1766	/* Greedy (non-optimal) calculation but simple */
1767	start = ALIGN_DOWN(start, 0x1ull << xe_pt_shift(level));
1768	last = ALIGN(end, 0x1ull << xe_pt_shift(level)) - 1;
1769
1770	if (start < pt_update_ops->start)
1771		pt_update_ops->start = start;
1772	if (last > pt_update_ops->last)
1773		pt_update_ops->last = last;
1774}
1775
1776static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma)
1777{
1778	int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0;
1779
1780	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1781		return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv,
1782					       xe->info.tile_count << shift);
1783
1784	return 0;
1785}
1786
1787static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
1788			   struct xe_vm_pgtable_update_ops *pt_update_ops,
1789			   struct xe_vma *vma, bool invalidate_on_bind)
1790{
1791	u32 current_op = pt_update_ops->current_op;
1792	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1793	int err;
1794
1795	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
1796	xe_bo_assert_held(xe_vma_bo(vma));
1797
1798	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1799	       "Preparing bind, with range [%llx...%llx)\n",
1800	       xe_vma_start(vma), xe_vma_end(vma) - 1);
1801
1802	pt_op->vma = NULL;
1803	pt_op->bind = true;
1804	pt_op->rebind = BIT(tile->id) & vma->tile_present;
1805
1806	err = vma_reserve_fences(tile_to_xe(tile), vma);
1807	if (err)
1808		return err;
1809
1810	err = xe_pt_prepare_bind(tile, vma, NULL, pt_op->entries,
1811				 &pt_op->num_entries, invalidate_on_bind);
1812	if (!err) {
1813		xe_tile_assert(tile, pt_op->num_entries <=
1814			       ARRAY_SIZE(pt_op->entries));
1815		xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
1816					pt_op->num_entries, true);
1817
1818		xe_pt_update_ops_rfence_interval(pt_update_ops,
1819						 xe_vma_start(vma),
1820						 xe_vma_end(vma));
1821		++pt_update_ops->current_op;
1822		pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
1823
1824		/*
1825		 * If rebind, we have to invalidate TLB on !LR vms to invalidate
1826		 * cached PTEs point to freed memory. On LR vms this is done
1827		 * automatically when the context is re-enabled by the rebind worker,
1828		 * or in fault mode it was invalidated on PTE zapping.
1829		 *
1830		 * If !rebind, and scratch enabled VMs, there is a chance the scratch
1831		 * PTE is already cached in the TLB so it needs to be invalidated.
1832		 * On !LR VMs this is done in the ring ops preceding a batch, but on
1833		 * LR, in particular on user-space batch buffer chaining, it needs to
1834		 * be done here.
1835		 */
1836		if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
1837		     xe_vm_in_lr_mode(vm)))
1838			pt_update_ops->needs_invalidation = true;
1839		else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
1840			/* We bump also if batch_invalidate_tlb is true */
1841			vm->tlb_flush_seqno++;
1842
1843		vma->tile_staged |= BIT(tile->id);
1844		pt_op->vma = vma;
1845		xe_pt_commit_prepare_bind(vma, pt_op->entries,
1846					  pt_op->num_entries, pt_op->rebind);
1847	} else {
1848		xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
1849	}
1850
1851	return err;
1852}
1853
1854static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,
1855			      struct xe_vm_pgtable_update_ops *pt_update_ops,
1856			      struct xe_vma *vma, struct xe_svm_range *range)
1857{
1858	u32 current_op = pt_update_ops->current_op;
1859	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1860	int err;
1861
1862	xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma));
1863
1864	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1865	       "Preparing bind, with range [%lx...%lx)\n",
1866	       xe_svm_range_start(range), xe_svm_range_end(range) - 1);
1867
1868	pt_op->vma = NULL;
1869	pt_op->bind = true;
1870	pt_op->rebind = BIT(tile->id) & range->tile_present;
1871
1872	err = xe_pt_prepare_bind(tile, vma, range, pt_op->entries,
1873				 &pt_op->num_entries, false);
1874	if (!err) {
1875		xe_tile_assert(tile, pt_op->num_entries <=
1876			       ARRAY_SIZE(pt_op->entries));
1877		xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
1878					pt_op->num_entries, true);
1879
1880		xe_pt_update_ops_rfence_interval(pt_update_ops,
1881						 xe_svm_range_start(range),
1882						 xe_svm_range_end(range));
1883		++pt_update_ops->current_op;
1884		pt_update_ops->needs_svm_lock = true;
1885
1886		pt_op->vma = vma;
1887		xe_pt_commit_prepare_bind(vma, pt_op->entries,
1888					  pt_op->num_entries, pt_op->rebind);
1889	} else {
1890		xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
1891	}
1892
1893	return err;
1894}
1895
1896static int unbind_op_prepare(struct xe_tile *tile,
1897			     struct xe_vm_pgtable_update_ops *pt_update_ops,
1898			     struct xe_vma *vma)
1899{
1900	u32 current_op = pt_update_ops->current_op;
1901	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1902	int err;
1903
1904	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
1905		return 0;
1906
1907	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
1908	xe_bo_assert_held(xe_vma_bo(vma));
1909
1910	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1911	       "Preparing unbind, with range [%llx...%llx)\n",
1912	       xe_vma_start(vma), xe_vma_end(vma) - 1);
1913
1914	pt_op->vma = vma;
1915	pt_op->bind = false;
1916	pt_op->rebind = false;
1917
1918	err = vma_reserve_fences(tile_to_xe(tile), vma);
1919	if (err)
1920		return err;
1921
1922	pt_op->num_entries = xe_pt_stage_unbind(tile, xe_vma_vm(vma),
1923						vma, NULL, pt_op->entries);
1924
1925	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
1926				pt_op->num_entries, false);
1927	xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma),
1928					 xe_vma_end(vma));
1929	++pt_update_ops->current_op;
1930	pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
1931	pt_update_ops->needs_invalidation = true;
1932
1933	xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
1934
1935	return 0;
1936}
1937
1938static bool
1939xe_pt_op_check_range_skip_invalidation(struct xe_vm_pgtable_update_op *pt_op,
1940				       struct xe_svm_range *range)
1941{
1942	struct xe_vm_pgtable_update *update = pt_op->entries;
1943
1944	XE_WARN_ON(!pt_op->num_entries);
1945
1946	/*
1947	 * We can't skip the invalidation if we are removing PTEs that span more
1948	 * than the range, do some checks to ensure we are removing PTEs that
1949	 * are invalid.
1950	 */
1951
1952	if (pt_op->num_entries > 1)
1953		return false;
1954
1955	if (update->pt->level == 0)
1956		return true;
1957
1958	if (update->pt->level == 1)
1959		return xe_svm_range_size(range) >= SZ_2M;
1960
1961	return false;
1962}
1963
1964static int unbind_range_prepare(struct xe_vm *vm,
1965				struct xe_tile *tile,
1966				struct xe_vm_pgtable_update_ops *pt_update_ops,
1967				struct xe_svm_range *range)
1968{
1969	u32 current_op = pt_update_ops->current_op;
1970	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1971
1972	if (!(range->tile_present & BIT(tile->id)))
1973		return 0;
1974
1975	vm_dbg(&vm->xe->drm,
1976	       "Preparing unbind, with range [%lx...%lx)\n",
1977	       xe_svm_range_start(range), xe_svm_range_end(range) - 1);
1978
1979	pt_op->vma = XE_INVALID_VMA;
1980	pt_op->bind = false;
1981	pt_op->rebind = false;
1982
1983	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
1984						pt_op->entries);
1985
1986	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
1987				pt_op->num_entries, false);
1988	xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range),
1989					 xe_svm_range_end(range));
1990	++pt_update_ops->current_op;
1991	pt_update_ops->needs_svm_lock = true;
1992	pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) ||
1993		xe_vm_has_valid_gpu_mapping(tile, range->tile_present,
1994					    range->tile_invalidated) ||
1995		!xe_pt_op_check_range_skip_invalidation(pt_op, range);
1996
1997	xe_pt_commit_prepare_unbind(XE_INVALID_VMA, pt_op->entries,
1998				    pt_op->num_entries);
1999
2000	return 0;
2001}
2002
2003static int op_prepare(struct xe_vm *vm,
2004		      struct xe_tile *tile,
2005		      struct xe_vm_pgtable_update_ops *pt_update_ops,
2006		      struct xe_vma_op *op)
2007{
2008	int err = 0;
2009
2010	xe_vm_assert_held(vm);
2011
2012	switch (op->base.op) {
2013	case DRM_GPUVA_OP_MAP:
2014		if ((!op->map.immediate && xe_vm_in_fault_mode(vm) &&
2015		     !op->map.invalidate_on_bind) ||
2016		    (op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
2017			break;
2018
2019		err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma,
2020				      op->map.invalidate_on_bind);
2021		pt_update_ops->wait_vm_kernel = true;
2022		break;
2023	case DRM_GPUVA_OP_REMAP:
2024	{
2025		struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
2026
2027		if (xe_vma_is_cpu_addr_mirror(old))
2028			break;
2029
2030		err = unbind_op_prepare(tile, pt_update_ops, old);
2031
2032		if (!err && op->remap.prev && !op->remap.skip_prev) {
2033			err = bind_op_prepare(vm, tile, pt_update_ops,
2034					      op->remap.prev, false);
2035			pt_update_ops->wait_vm_bookkeep = true;
2036		}
2037		if (!err && op->remap.next && !op->remap.skip_next) {
2038			err = bind_op_prepare(vm, tile, pt_update_ops,
2039					      op->remap.next, false);
2040			pt_update_ops->wait_vm_bookkeep = true;
2041		}
2042		break;
2043	}
2044	case DRM_GPUVA_OP_UNMAP:
2045	{
2046		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
2047
2048		if (xe_vma_is_cpu_addr_mirror(vma))
2049			break;
2050
2051		err = unbind_op_prepare(tile, pt_update_ops, vma);
2052		break;
2053	}
2054	case DRM_GPUVA_OP_PREFETCH:
2055	{
2056		struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
2057
2058		if (xe_vma_is_cpu_addr_mirror(vma)) {
2059			struct xe_svm_range *range;
2060			unsigned long i;
2061
2062			xa_for_each(&op->prefetch_range.range, i, range) {
2063				err = bind_range_prepare(vm, tile, pt_update_ops,
2064							 vma, range);
2065				if (err)
2066					return err;
2067			}
2068		} else {
2069			err = bind_op_prepare(vm, tile, pt_update_ops, vma, false);
2070			pt_update_ops->wait_vm_kernel = true;
2071		}
2072		break;
2073	}
2074	case DRM_GPUVA_OP_DRIVER:
2075		if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
2076			xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
2077
2078			err = bind_range_prepare(vm, tile, pt_update_ops,
2079						 op->map_range.vma,
2080						 op->map_range.range);
2081		} else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) {
2082			err = unbind_range_prepare(vm, tile, pt_update_ops,
2083						   op->unmap_range.range);
2084		}
2085		break;
2086	default:
2087		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2088	}
2089
2090	return err;
2091}
2092
2093static void
2094xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
2095{
2096	init_llist_head(&pt_update_ops->deferred);
2097	pt_update_ops->start = ~0x0ull;
2098	pt_update_ops->last = 0x0ull;
2099}
2100
2101/**
2102 * xe_pt_update_ops_prepare() - Prepare PT update operations
2103 * @tile: Tile of PT update operations
2104 * @vops: VMA operationa
2105 *
2106 * Prepare PT update operations which includes updating internal PT state,
2107 * allocate memory for page tables, populate page table being pruned in, and
2108 * create PT update operations for leaf insertion / removal.
2109 *
2110 * Return: 0 on success, negative error code on error.
2111 */
2112int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
2113{
2114	struct xe_vm_pgtable_update_ops *pt_update_ops =
2115		&vops->pt_update_ops[tile->id];
2116	struct xe_vma_op *op;
2117	int shift = tile->media_gt ? 1 : 0;
2118	int err;
2119
2120	lockdep_assert_held(&vops->vm->lock);
2121	xe_vm_assert_held(vops->vm);
2122
2123	xe_pt_update_ops_init(pt_update_ops);
2124
2125	err = dma_resv_reserve_fences(xe_vm_resv(vops->vm),
2126				      tile_to_xe(tile)->info.tile_count << shift);
2127	if (err)
2128		return err;
2129
2130	list_for_each_entry(op, &vops->list, link) {
2131		err = op_prepare(vops->vm, tile, pt_update_ops, op);
2132
2133		if (err)
2134			return err;
2135	}
2136
2137	xe_tile_assert(tile, pt_update_ops->current_op <=
2138		       pt_update_ops->num_ops);
2139
2140#ifdef TEST_VM_OPS_ERROR
2141	if (vops->inject_error &&
2142	    vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
2143		return -ENOSPC;
2144#endif
2145
2146	return 0;
2147}
2148ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO);
2149
2150static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
2151			   struct xe_vm_pgtable_update_ops *pt_update_ops,
2152			   struct xe_vma *vma, struct dma_fence *fence,
2153			   struct dma_fence *fence2, bool invalidate_on_bind)
2154{
2155	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
2156
2157	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
2158		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
2159				   pt_update_ops->wait_vm_bookkeep ?
2160				   DMA_RESV_USAGE_KERNEL :
2161				   DMA_RESV_USAGE_BOOKKEEP);
2162		if (fence2)
2163			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
2164					   pt_update_ops->wait_vm_bookkeep ?
2165					   DMA_RESV_USAGE_KERNEL :
2166					   DMA_RESV_USAGE_BOOKKEEP);
2167	}
2168	/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
2169	WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id));
2170	if (invalidate_on_bind)
2171		WRITE_ONCE(vma->tile_invalidated,
2172			   vma->tile_invalidated | BIT(tile->id));
2173	else
2174		WRITE_ONCE(vma->tile_invalidated,
2175			   vma->tile_invalidated & ~BIT(tile->id));
2176	vma->tile_staged &= ~BIT(tile->id);
2177	if (xe_vma_is_userptr(vma)) {
2178		xe_svm_assert_held_read(vm);
2179		to_userptr_vma(vma)->userptr.initial_bind = true;
2180	}
2181
2182	/*
2183	 * Kick rebind worker if this bind triggers preempt fences and not in
2184	 * the rebind worker
2185	 */
2186	if (pt_update_ops->wait_vm_bookkeep &&
2187	    xe_vm_in_preempt_fence_mode(vm) &&
2188	    !current->mm)
2189		xe_vm_queue_rebind_worker(vm);
2190}
2191
2192static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
2193			     struct xe_vm_pgtable_update_ops *pt_update_ops,
2194			     struct xe_vma *vma, struct dma_fence *fence,
2195			     struct dma_fence *fence2)
2196{
2197	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
2198
2199	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
2200		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
2201				   pt_update_ops->wait_vm_bookkeep ?
2202				   DMA_RESV_USAGE_KERNEL :
2203				   DMA_RESV_USAGE_BOOKKEEP);
2204		if (fence2)
2205			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
2206					   pt_update_ops->wait_vm_bookkeep ?
2207					   DMA_RESV_USAGE_KERNEL :
2208					   DMA_RESV_USAGE_BOOKKEEP);
2209	}
2210	vma->tile_present &= ~BIT(tile->id);
2211	if (!vma->tile_present) {
2212		list_del_init(&vma->combined_links.rebind);
2213		if (xe_vma_is_userptr(vma)) {
2214			xe_svm_assert_held_read(vm);
2215
2216			spin_lock(&vm->userptr.invalidated_lock);
2217			list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link);
2218			spin_unlock(&vm->userptr.invalidated_lock);
2219		}
2220	}
2221}
2222
2223static void range_present_and_invalidated_tile(struct xe_vm *vm,
2224					       struct xe_svm_range *range,
2225					       u8 tile_id)
2226{
2227	/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
2228
2229	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
2230
2231	WRITE_ONCE(range->tile_present, range->tile_present | BIT(tile_id));
2232	WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~BIT(tile_id));
2233}
2234
2235static void op_commit(struct xe_vm *vm,
2236		      struct xe_tile *tile,
2237		      struct xe_vm_pgtable_update_ops *pt_update_ops,
2238		      struct xe_vma_op *op, struct dma_fence *fence,
2239		      struct dma_fence *fence2)
2240{
2241	xe_vm_assert_held(vm);
2242
2243	switch (op->base.op) {
2244	case DRM_GPUVA_OP_MAP:
2245		if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) ||
2246		    (op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
2247			break;
2248
2249		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,
2250			       fence2, op->map.invalidate_on_bind);
2251		break;
2252	case DRM_GPUVA_OP_REMAP:
2253	{
2254		struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
2255
2256		if (xe_vma_is_cpu_addr_mirror(old))
2257			break;
2258
2259		unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2);
2260
2261		if (op->remap.prev && !op->remap.skip_prev)
2262			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
2263				       fence, fence2, false);
2264		if (op->remap.next && !op->remap.skip_next)
2265			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
2266				       fence, fence2, false);
2267		break;
2268	}
2269	case DRM_GPUVA_OP_UNMAP:
2270	{
2271		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
2272
2273		if (!xe_vma_is_cpu_addr_mirror(vma))
2274			unbind_op_commit(vm, tile, pt_update_ops, vma, fence,
2275					 fence2);
2276		break;
2277	}
2278	case DRM_GPUVA_OP_PREFETCH:
2279	{
2280		struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
2281
2282		if (xe_vma_is_cpu_addr_mirror(vma)) {
2283			struct xe_svm_range *range = NULL;
2284			unsigned long i;
2285
2286			xa_for_each(&op->prefetch_range.range, i, range)
2287				range_present_and_invalidated_tile(vm, range, tile->id);
2288		} else {
2289			bind_op_commit(vm, tile, pt_update_ops, vma, fence,
2290				       fence2, false);
2291		}
2292		break;
2293	}
2294	case DRM_GPUVA_OP_DRIVER:
2295	{
2296		/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
2297		if (op->subop == XE_VMA_SUBOP_MAP_RANGE)
2298			range_present_and_invalidated_tile(vm, op->map_range.range, tile->id);
2299		else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE)
2300			WRITE_ONCE(op->unmap_range.range->tile_present,
2301				   op->unmap_range.range->tile_present &
2302				   ~BIT(tile->id));
2303
2304		break;
2305	}
2306	default:
2307		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2308	}
2309}
2310
2311static const struct xe_migrate_pt_update_ops migrate_ops = {
2312	.populate = xe_vm_populate_pgtable,
2313	.clear = xe_migrate_clear_pgtable_callback,
2314	.pre_commit = xe_pt_pre_commit,
2315};
2316
2317#if IS_ENABLED(CONFIG_DRM_GPUSVM)
2318static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops = {
2319	.populate = xe_vm_populate_pgtable,
2320	.clear = xe_migrate_clear_pgtable_callback,
2321	.pre_commit = xe_pt_svm_userptr_pre_commit,
2322};
2323#else
2324static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops;
2325#endif
2326
2327static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q,
2328						 struct xe_gt *gt)
2329{
2330	if (xe_gt_is_media_type(gt))
2331		return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler;
2332
2333	return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler;
2334}
2335
2336/**
2337 * xe_pt_update_ops_run() - Run PT update operations
2338 * @tile: Tile of PT update operations
2339 * @vops: VMA operationa
2340 *
2341 * Run PT update operations which includes committing internal PT state changes,
2342 * creating job for PT update operations for leaf insertion / removal, and
2343 * installing job fence in various places.
2344 *
2345 * Return: fence on success, negative ERR_PTR on error.
2346 */
2347struct dma_fence *
2348xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
2349{
2350	struct xe_vm *vm = vops->vm;
2351	struct xe_vm_pgtable_update_ops *pt_update_ops =
2352		&vops->pt_update_ops[tile->id];
2353	struct xe_exec_queue *q = pt_update_ops->q;
2354	struct dma_fence *fence, *ifence = NULL, *mfence = NULL;
2355	struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL;
2356	struct xe_range_fence *rfence;
2357	struct xe_vma_op *op;
2358	int err = 0, i;
2359	struct xe_migrate_pt_update update = {
2360		.ops = pt_update_ops->needs_svm_lock ?
2361			&svm_userptr_migrate_ops :
2362			&migrate_ops,
2363		.vops = vops,
2364		.tile_id = tile->id,
2365	};
2366
2367	lockdep_assert_held(&vm->lock);
2368	xe_vm_assert_held(vm);
2369
2370	if (!pt_update_ops->current_op) {
2371		xe_tile_assert(tile, xe_vm_in_fault_mode(vm));
2372
2373		return dma_fence_get_stub();
2374	}
2375
2376#ifdef TEST_VM_OPS_ERROR
2377	if (vops->inject_error &&
2378	    vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
2379		return ERR_PTR(-ENOSPC);
2380#endif
2381
2382	if (pt_update_ops->needs_invalidation) {
2383		struct xe_dep_scheduler *dep_scheduler =
2384			to_dep_scheduler(q, tile->primary_gt);
2385
2386		ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval,
2387					       dep_scheduler, vm,
2388					       pt_update_ops->start,
2389					       pt_update_ops->last,
2390					       XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
2391		if (IS_ERR(ijob)) {
2392			err = PTR_ERR(ijob);
2393			goto kill_vm_tile1;
2394		}
2395		update.ijob = ijob;
2396
2397		if (tile->media_gt) {
2398			dep_scheduler = to_dep_scheduler(q, tile->media_gt);
2399
2400			mjob = xe_tlb_inval_job_create(q,
2401						       &tile->media_gt->tlb_inval,
2402						       dep_scheduler, vm,
2403						       pt_update_ops->start,
2404						       pt_update_ops->last,
2405						       XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT);
2406			if (IS_ERR(mjob)) {
2407				err = PTR_ERR(mjob);
2408				goto free_ijob;
2409			}
2410			update.mjob = mjob;
2411		}
2412	}
2413
2414	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
2415	if (!rfence) {
2416		err = -ENOMEM;
2417		goto free_ijob;
2418	}
2419
2420	fence = xe_migrate_update_pgtables(tile->migrate, &update);
2421	if (IS_ERR(fence)) {
2422		err = PTR_ERR(fence);
2423		goto free_rfence;
2424	}
2425
2426	/* Point of no return - VM killed if failure after this */
2427	for (i = 0; i < pt_update_ops->current_op; ++i) {
2428		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
2429
2430		xe_pt_commit(pt_op->vma, pt_op->entries,
2431			     pt_op->num_entries, &pt_update_ops->deferred);
2432		pt_op->vma = NULL;	/* skip in xe_pt_update_ops_abort */
2433	}
2434
2435	if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,
2436				  &xe_range_fence_kfree_ops,
2437				  pt_update_ops->start,
2438				  pt_update_ops->last, fence))
2439		dma_fence_wait(fence, false);
2440
2441	if (ijob)
2442		ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence);
2443	if (mjob)
2444		mfence = xe_tlb_inval_job_push(mjob, tile->migrate, fence);
2445
2446	if (!mjob && !ijob) {
2447		dma_resv_add_fence(xe_vm_resv(vm), fence,
2448				   pt_update_ops->wait_vm_bookkeep ?
2449				   DMA_RESV_USAGE_KERNEL :
2450				   DMA_RESV_USAGE_BOOKKEEP);
2451
2452		list_for_each_entry(op, &vops->list, link)
2453			op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);
2454	} else if (ijob && !mjob) {
2455		dma_resv_add_fence(xe_vm_resv(vm), ifence,
2456				   pt_update_ops->wait_vm_bookkeep ?
2457				   DMA_RESV_USAGE_KERNEL :
2458				   DMA_RESV_USAGE_BOOKKEEP);
2459
2460		list_for_each_entry(op, &vops->list, link)
2461			op_commit(vops->vm, tile, pt_update_ops, op, ifence, NULL);
2462	} else {
2463		dma_resv_add_fence(xe_vm_resv(vm), ifence,
2464				   pt_update_ops->wait_vm_bookkeep ?
2465				   DMA_RESV_USAGE_KERNEL :
2466				   DMA_RESV_USAGE_BOOKKEEP);
2467
2468		dma_resv_add_fence(xe_vm_resv(vm), mfence,
2469				   pt_update_ops->wait_vm_bookkeep ?
2470				   DMA_RESV_USAGE_KERNEL :
2471				   DMA_RESV_USAGE_BOOKKEEP);
2472
2473		list_for_each_entry(op, &vops->list, link)
2474			op_commit(vops->vm, tile, pt_update_ops, op, ifence,
2475				  mfence);
2476	}
2477
2478	if (pt_update_ops->needs_svm_lock)
2479		xe_svm_notifier_unlock(vm);
2480
2481	/*
2482	 * The last fence is only used for zero bind queue idling; migrate
2483	 * queues are not exposed to user space.
2484	 */
2485	if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE))
2486		xe_exec_queue_last_fence_set(q, vm, fence);
2487
2488	xe_tlb_inval_job_put(mjob);
2489	xe_tlb_inval_job_put(ijob);
2490	dma_fence_put(ifence);
2491	dma_fence_put(mfence);
2492
2493	return fence;
2494
2495free_rfence:
2496	kfree(rfence);
2497free_ijob:
2498	xe_tlb_inval_job_put(mjob);
2499	xe_tlb_inval_job_put(ijob);
2500kill_vm_tile1:
2501	if (err != -EAGAIN && err != -ENODATA && tile->id)
2502		xe_vm_kill(vops->vm, false);
2503
2504	return ERR_PTR(err);
2505}
2506ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO);
2507
2508/**
2509 * xe_pt_update_ops_fini() - Finish PT update operations
2510 * @tile: Tile of PT update operations
2511 * @vops: VMA operations
2512 *
2513 * Finish PT update operations by committing to destroy page table memory
2514 */
2515void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
2516{
2517	struct xe_vm_pgtable_update_ops *pt_update_ops =
2518		&vops->pt_update_ops[tile->id];
2519	int i;
2520
2521	lockdep_assert_held(&vops->vm->lock);
2522	xe_vm_assert_held(vops->vm);
2523
2524	for (i = 0; i < pt_update_ops->current_op; ++i) {
2525		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
2526
2527		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
2528	}
2529	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
2530}
2531
2532/**
2533 * xe_pt_update_ops_abort() - Abort PT update operations
2534 * @tile: Tile of PT update operations
2535 * @vops: VMA operationa
2536 *
2537 *  Abort PT update operations by unwinding internal PT state
2538 */
2539void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
2540{
2541	struct xe_vm_pgtable_update_ops *pt_update_ops =
2542		&vops->pt_update_ops[tile->id];
2543	int i;
2544
2545	lockdep_assert_held(&vops->vm->lock);
2546	xe_vm_assert_held(vops->vm);
2547
2548	for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
2549		struct xe_vm_pgtable_update_op *pt_op =
2550			&pt_update_ops->ops[i];
2551
2552		if (!pt_op->vma || i >= pt_update_ops->current_op)
2553			continue;
2554
2555		if (pt_op->bind)
2556			xe_pt_abort_bind(pt_op->vma, pt_op->entries,
2557					 pt_op->num_entries,
2558					 pt_op->rebind);
2559		else
2560			xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
2561					   pt_op->num_entries);
2562	}
2563
2564	xe_pt_update_ops_fini(tile, vops);
2565}