xe_vm.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2021 Intel Corporation
   4 */
   5
   6#include "xe_vm.h"
   7
   8#include <linux/dma-fence-array.h>
   9#include <linux/nospec.h>
  10
  11#include <drm/drm_drv.h>
  12#include <drm/drm_exec.h>
  13#include <drm/drm_print.h>
  14#include <drm/ttm/ttm_tt.h>
  15#include <uapi/drm/xe_drm.h>
  16#include <linux/ascii85.h>
  17#include <linux/delay.h>
  18#include <linux/kthread.h>
  19#include <linux/mm.h>
  20#include <linux/swap.h>
  21
  22#include <generated/xe_wa_oob.h>
  23
  24#include "regs/xe_gtt_defs.h"
  25#include "xe_assert.h"
  26#include "xe_bo.h"
  27#include "xe_device.h"
  28#include "xe_drm_client.h"
  29#include "xe_exec_queue.h"
  30#include "xe_migrate.h"
  31#include "xe_pat.h"
  32#include "xe_pm.h"
  33#include "xe_preempt_fence.h"
  34#include "xe_pt.h"
  35#include "xe_pxp.h"
  36#include "xe_res_cursor.h"
  37#include "xe_sriov_vf.h"
  38#include "xe_svm.h"
  39#include "xe_sync.h"
  40#include "xe_tile.h"
  41#include "xe_tlb_inval.h"
  42#include "xe_trace_bo.h"
  43#include "xe_wa.h"
  44
  45static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm)
  46{
  47	return vm->gpuvm.r_obj;
  48}
  49
  50/**
  51 * xe_vm_drm_exec_lock() - Lock the vm's resv with a drm_exec transaction
  52 * @vm: The vm whose resv is to be locked.
  53 * @exec: The drm_exec transaction.
  54 *
  55 * Helper to lock the vm's resv as part of a drm_exec transaction.
  56 *
  57 * Return: %0 on success. See drm_exec_lock_obj() for error codes.
  58 */
  59int xe_vm_drm_exec_lock(struct xe_vm *vm, struct drm_exec *exec)
  60{
  61	return drm_exec_lock_obj(exec, xe_vm_obj(vm));
  62}
  63
  64static bool preempt_fences_waiting(struct xe_vm *vm)
  65{
  66	struct xe_exec_queue *q;
  67
  68	lockdep_assert_held(&vm->lock);
  69	xe_vm_assert_held(vm);
  70
  71	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
  72		if (!q->lr.pfence ||
  73		    test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
  74			     &q->lr.pfence->flags)) {
  75			return true;
  76		}
  77	}
  78
  79	return false;
  80}
  81
  82static void free_preempt_fences(struct list_head *list)
  83{
  84	struct list_head *link, *next;
  85
  86	list_for_each_safe(link, next, list)
  87		xe_preempt_fence_free(to_preempt_fence_from_link(link));
  88}
  89
  90static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list,
  91				unsigned int *count)
  92{
  93	lockdep_assert_held(&vm->lock);
  94	xe_vm_assert_held(vm);
  95
  96	if (*count >= vm->preempt.num_exec_queues)
  97		return 0;
  98
  99	for (; *count < vm->preempt.num_exec_queues; ++(*count)) {
 100		struct xe_preempt_fence *pfence = xe_preempt_fence_alloc();
 101
 102		if (IS_ERR(pfence))
 103			return PTR_ERR(pfence);
 104
 105		list_move_tail(xe_preempt_fence_link(pfence), list);
 106	}
 107
 108	return 0;
 109}
 110
 111static int wait_for_existing_preempt_fences(struct xe_vm *vm)
 112{
 113	struct xe_exec_queue *q;
 114	bool vf_migration = IS_SRIOV_VF(vm->xe) &&
 115		xe_sriov_vf_migration_supported(vm->xe);
 116	signed long wait_time = vf_migration ? HZ / 5 : MAX_SCHEDULE_TIMEOUT;
 117
 118	xe_vm_assert_held(vm);
 119
 120	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
 121		if (q->lr.pfence) {
 122			long timeout;
 123
 124			timeout = dma_fence_wait_timeout(q->lr.pfence, false,
 125							 wait_time);
 126			if (!timeout) {
 127				xe_assert(vm->xe, vf_migration);
 128				return -EAGAIN;
 129			}
 130
 131			/* Only -ETIME on fence indicates VM needs to be killed */
 132			if (timeout < 0 || q->lr.pfence->error == -ETIME)
 133				return -ETIME;
 134
 135			dma_fence_put(q->lr.pfence);
 136			q->lr.pfence = NULL;
 137		}
 138	}
 139
 140	return 0;
 141}
 142
 143static bool xe_vm_is_idle(struct xe_vm *vm)
 144{
 145	struct xe_exec_queue *q;
 146
 147	xe_vm_assert_held(vm);
 148	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
 149		if (!xe_exec_queue_is_idle(q))
 150			return false;
 151	}
 152
 153	return true;
 154}
 155
 156static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list)
 157{
 158	struct list_head *link;
 159	struct xe_exec_queue *q;
 160
 161	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
 162		struct dma_fence *fence;
 163
 164		link = list->next;
 165		xe_assert(vm->xe, link != list);
 166
 167		fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link),
 168					     q, q->lr.context,
 169					     ++q->lr.seqno);
 170		dma_fence_put(q->lr.pfence);
 171		q->lr.pfence = fence;
 172	}
 173}
 174
 175static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo)
 176{
 177	struct xe_exec_queue *q;
 178	int err;
 179
 180	xe_bo_assert_held(bo);
 181
 182	if (!vm->preempt.num_exec_queues)
 183		return 0;
 184
 185	err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues);
 186	if (err)
 187		return err;
 188
 189	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link)
 190		if (q->lr.pfence) {
 191			dma_resv_add_fence(bo->ttm.base.resv,
 192					   q->lr.pfence,
 193					   DMA_RESV_USAGE_BOOKKEEP);
 194		}
 195
 196	return 0;
 197}
 198
 199static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
 200						struct drm_exec *exec)
 201{
 202	struct xe_exec_queue *q;
 203
 204	lockdep_assert_held(&vm->lock);
 205	xe_vm_assert_held(vm);
 206
 207	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
 208		q->ops->resume(q);
 209
 210		drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence,
 211					 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
 212	}
 213}
 214
 215int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 216{
 217	struct drm_gpuvm_exec vm_exec = {
 218		.vm = &vm->gpuvm,
 219		.flags = DRM_EXEC_INTERRUPTIBLE_WAIT,
 220		.num_fences = 1,
 221	};
 222	struct drm_exec *exec = &vm_exec.exec;
 223	struct xe_validation_ctx ctx;
 224	struct dma_fence *pfence;
 225	int err;
 226	bool wait;
 227
 228	xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm));
 229
 230	down_write(&vm->lock);
 231	err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val);
 232	if (err)
 233		goto out_up_write;
 234
 235	pfence = xe_preempt_fence_create(q, q->lr.context,
 236					 ++q->lr.seqno);
 237	if (IS_ERR(pfence)) {
 238		err = PTR_ERR(pfence);
 239		goto out_fini;
 240	}
 241
 242	list_add(&q->lr.link, &vm->preempt.exec_queues);
 243	++vm->preempt.num_exec_queues;
 244	q->lr.pfence = pfence;
 245
 246	xe_svm_notifier_lock(vm);
 247
 248	drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence,
 249				 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
 250
 251	/*
 252	 * Check to see if a preemption on VM is in flight or userptr
 253	 * invalidation, if so trigger this preempt fence to sync state with
 254	 * other preempt fences on the VM.
 255	 */
 256	wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm);
 257	if (wait)
 258		dma_fence_enable_sw_signaling(pfence);
 259
 260	xe_svm_notifier_unlock(vm);
 261
 262out_fini:
 263	xe_validation_ctx_fini(&ctx);
 264out_up_write:
 265	up_write(&vm->lock);
 266
 267	return err;
 268}
 269ALLOW_ERROR_INJECTION(xe_vm_add_compute_exec_queue, ERRNO);
 270
 271/**
 272 * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM
 273 * @vm: The VM.
 274 * @q: The exec_queue
 275 *
 276 * Note that this function might be called multiple times on the same queue.
 277 */
 278void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 279{
 280	if (!xe_vm_in_preempt_fence_mode(vm))
 281		return;
 282
 283	down_write(&vm->lock);
 284	if (!list_empty(&q->lr.link)) {
 285		list_del_init(&q->lr.link);
 286		--vm->preempt.num_exec_queues;
 287	}
 288	if (q->lr.pfence) {
 289		dma_fence_enable_sw_signaling(q->lr.pfence);
 290		dma_fence_put(q->lr.pfence);
 291		q->lr.pfence = NULL;
 292	}
 293	up_write(&vm->lock);
 294}
 295
 296#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
 297
 298/**
 299 * xe_vm_kill() - VM Kill
 300 * @vm: The VM.
 301 * @unlocked: Flag indicates the VM's dma-resv is not held
 302 *
 303 * Kill the VM by setting banned flag indicated VM is no longer available for
 304 * use. If in preempt fence mode, also kill all exec queue attached to the VM.
 305 */
 306void xe_vm_kill(struct xe_vm *vm, bool unlocked)
 307{
 308	struct xe_exec_queue *q;
 309
 310	lockdep_assert_held(&vm->lock);
 311
 312	if (unlocked)
 313		xe_vm_lock(vm, false);
 314
 315	vm->flags |= XE_VM_FLAG_BANNED;
 316	trace_xe_vm_kill(vm);
 317
 318	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link)
 319		q->ops->kill(q);
 320
 321	if (unlocked)
 322		xe_vm_unlock(vm);
 323
 324	/* TODO: Inform user the VM is banned */
 325}
 326
 327static int xe_gpuvm_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
 328{
 329	struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
 330	struct drm_gpuva *gpuva;
 331	int ret;
 332
 333	lockdep_assert_held(&vm->lock);
 334	drm_gpuvm_bo_for_each_va(gpuva, vm_bo)
 335		list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind,
 336			       &vm->rebind_list);
 337
 338	if (!try_wait_for_completion(&vm->xe->pm_block))
 339		return -EAGAIN;
 340
 341	ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false, exec);
 342	if (ret)
 343		return ret;
 344
 345	vm_bo->evicted = false;
 346	return 0;
 347}
 348
 349/**
 350 * xe_vm_validate_rebind() - Validate buffer objects and rebind vmas
 351 * @vm: The vm for which we are rebinding.
 352 * @exec: The struct drm_exec with the locked GEM objects.
 353 * @num_fences: The number of fences to reserve for the operation, not
 354 * including rebinds and validations.
 355 *
 356 * Validates all evicted gem objects and rebinds their vmas. Note that
 357 * rebindings may cause evictions and hence the validation-rebind
 358 * sequence is rerun until there are no more objects to validate.
 359 *
 360 * Return: 0 on success, negative error code on error. In particular,
 361 * may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if
 362 * the drm_exec transaction needs to be restarted.
 363 */
 364int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec,
 365			  unsigned int num_fences)
 366{
 367	struct drm_gem_object *obj;
 368	unsigned long index;
 369	int ret;
 370
 371	do {
 372		ret = drm_gpuvm_validate(&vm->gpuvm, exec);
 373		if (ret)
 374			return ret;
 375
 376		ret = xe_vm_rebind(vm, false);
 377		if (ret)
 378			return ret;
 379	} while (!list_empty(&vm->gpuvm.evict.list));
 380
 381	drm_exec_for_each_locked_object(exec, index, obj) {
 382		ret = dma_resv_reserve_fences(obj->resv, num_fences);
 383		if (ret)
 384			return ret;
 385	}
 386
 387	return 0;
 388}
 389
 390static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm,
 391				 bool *done)
 392{
 393	int err;
 394
 395	err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0);
 396	if (err)
 397		return err;
 398
 399	if (xe_vm_is_idle(vm)) {
 400		vm->preempt.rebind_deactivated = true;
 401		*done = true;
 402		return 0;
 403	}
 404
 405	if (!preempt_fences_waiting(vm)) {
 406		*done = true;
 407		return 0;
 408	}
 409
 410	err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0);
 411	if (err)
 412		return err;
 413
 414	err = wait_for_existing_preempt_fences(vm);
 415	if (err)
 416		return err;
 417
 418	/*
 419	 * Add validation and rebinding to the locking loop since both can
 420	 * cause evictions which may require blocing dma_resv locks.
 421	 * The fence reservation here is intended for the new preempt fences
 422	 * we attach at the end of the rebind work.
 423	 */
 424	return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues);
 425}
 426
 427static bool vm_suspend_rebind_worker(struct xe_vm *vm)
 428{
 429	struct xe_device *xe = vm->xe;
 430	bool ret = false;
 431
 432	mutex_lock(&xe->rebind_resume_lock);
 433	if (!try_wait_for_completion(&vm->xe->pm_block)) {
 434		ret = true;
 435		list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list);
 436	}
 437	mutex_unlock(&xe->rebind_resume_lock);
 438
 439	return ret;
 440}
 441
 442/**
 443 * xe_vm_resume_rebind_worker() - Resume the rebind worker.
 444 * @vm: The vm whose preempt worker to resume.
 445 *
 446 * Resume a preempt worker that was previously suspended by
 447 * vm_suspend_rebind_worker().
 448 */
 449void xe_vm_resume_rebind_worker(struct xe_vm *vm)
 450{
 451	queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work);
 452}
 453
 454static void preempt_rebind_work_func(struct work_struct *w)
 455{
 456	struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work);
 457	struct xe_validation_ctx ctx;
 458	struct drm_exec exec;
 459	unsigned int fence_count = 0;
 460	LIST_HEAD(preempt_fences);
 461	int err = 0;
 462	long wait;
 463	int __maybe_unused tries = 0;
 464
 465	xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm));
 466	trace_xe_vm_rebind_worker_enter(vm);
 467
 468	down_write(&vm->lock);
 469
 470	if (xe_vm_is_closed_or_banned(vm)) {
 471		up_write(&vm->lock);
 472		trace_xe_vm_rebind_worker_exit(vm);
 473		return;
 474	}
 475
 476retry:
 477	if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) {
 478		up_write(&vm->lock);
 479		/* We don't actually block but don't make progress. */
 480		xe_pm_might_block_on_suspend();
 481		return;
 482	}
 483
 484	if (xe_vm_userptr_check_repin(vm)) {
 485		err = xe_vm_userptr_pin(vm);
 486		if (err)
 487			goto out_unlock_outer;
 488	}
 489
 490	err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec,
 491				     (struct xe_val_flags) {.interruptible = true});
 492	if (err)
 493		goto out_unlock_outer;
 494
 495	drm_exec_until_all_locked(&exec) {
 496		bool done = false;
 497
 498		err = xe_preempt_work_begin(&exec, vm, &done);
 499		drm_exec_retry_on_contention(&exec);
 500		xe_validation_retry_on_oom(&ctx, &err);
 501		if (err || done) {
 502			xe_validation_ctx_fini(&ctx);
 503			goto out_unlock_outer;
 504		}
 505	}
 506
 507	err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
 508	if (err)
 509		goto out_unlock;
 510
 511	xe_vm_set_validation_exec(vm, &exec);
 512	err = xe_vm_rebind(vm, true);
 513	xe_vm_set_validation_exec(vm, NULL);
 514	if (err)
 515		goto out_unlock;
 516
 517	/* Wait on rebinds and munmap style VM unbinds */
 518	wait = dma_resv_wait_timeout(xe_vm_resv(vm),
 519				     DMA_RESV_USAGE_KERNEL,
 520				     false, MAX_SCHEDULE_TIMEOUT);
 521	if (wait <= 0) {
 522		err = -ETIME;
 523		goto out_unlock;
 524	}
 525
 526#define retry_required(__tries, __vm) \
 527	(IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \
 528	(!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \
 529	__xe_vm_userptr_needs_repin(__vm))
 530
 531	xe_svm_notifier_lock(vm);
 532	if (retry_required(tries, vm)) {
 533		xe_svm_notifier_unlock(vm);
 534		err = -EAGAIN;
 535		goto out_unlock;
 536	}
 537
 538#undef retry_required
 539
 540	spin_lock(&vm->xe->ttm.lru_lock);
 541	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
 542	spin_unlock(&vm->xe->ttm.lru_lock);
 543
 544	/* Point of no return. */
 545	arm_preempt_fences(vm, &preempt_fences);
 546	resume_and_reinstall_preempt_fences(vm, &exec);
 547	xe_svm_notifier_unlock(vm);
 548
 549out_unlock:
 550	xe_validation_ctx_fini(&ctx);
 551out_unlock_outer:
 552	if (err == -EAGAIN) {
 553		trace_xe_vm_rebind_worker_retry(vm);
 554
 555		/*
 556		 * We can't block in workers on a VF which supports migration
 557		 * given this can block the VF post-migration workers from
 558		 * getting scheduled.
 559		 */
 560		if (IS_SRIOV_VF(vm->xe) &&
 561		    xe_sriov_vf_migration_supported(vm->xe)) {
 562			up_write(&vm->lock);
 563			xe_vm_queue_rebind_worker(vm);
 564			return;
 565		}
 566
 567		goto retry;
 568	}
 569
 570	if (err) {
 571		drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
 572		xe_vm_kill(vm, true);
 573	}
 574	up_write(&vm->lock);
 575
 576	free_preempt_fences(&preempt_fences);
 577
 578	trace_xe_vm_rebind_worker_exit(vm);
 579}
 580
 581static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
 582{
 583	int i;
 584
 585	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
 586		if (!vops->pt_update_ops[i].num_ops)
 587			continue;
 588
 589		vops->pt_update_ops[i].ops =
 590			kmalloc_array(vops->pt_update_ops[i].num_ops,
 591				      sizeof(*vops->pt_update_ops[i].ops),
 592				      GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 593		if (!vops->pt_update_ops[i].ops)
 594			return array_of_binds ? -ENOBUFS : -ENOMEM;
 595	}
 596
 597	return 0;
 598}
 599ALLOW_ERROR_INJECTION(xe_vma_ops_alloc, ERRNO);
 600
 601static void xe_vma_svm_prefetch_op_fini(struct xe_vma_op *op)
 602{
 603	struct xe_vma *vma;
 604
 605	vma = gpuva_to_vma(op->base.prefetch.va);
 606
 607	if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma))
 608		xa_destroy(&op->prefetch_range.range);
 609}
 610
 611static void xe_vma_svm_prefetch_ops_fini(struct xe_vma_ops *vops)
 612{
 613	struct xe_vma_op *op;
 614
 615	if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH))
 616		return;
 617
 618	list_for_each_entry(op, &vops->list, link)
 619		xe_vma_svm_prefetch_op_fini(op);
 620}
 621
 622static void xe_vma_ops_fini(struct xe_vma_ops *vops)
 623{
 624	int i;
 625
 626	xe_vma_svm_prefetch_ops_fini(vops);
 627
 628	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
 629		kfree(vops->pt_update_ops[i].ops);
 630}
 631
 632static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val)
 633{
 634	int i;
 635
 636	if (!inc_val)
 637		return;
 638
 639	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
 640		if (BIT(i) & tile_mask)
 641			vops->pt_update_ops[i].num_ops += inc_val;
 642}
 643
 644#define XE_VMA_CREATE_MASK (		    \
 645	XE_VMA_READ_ONLY |		    \
 646	XE_VMA_DUMPABLE |		    \
 647	XE_VMA_SYSTEM_ALLOCATOR |           \
 648	DRM_GPUVA_SPARSE |		    \
 649	XE_VMA_MADV_AUTORESET)
 650
 651static void xe_vm_populate_rebind(struct xe_vma_op *op, struct xe_vma *vma,
 652				  u8 tile_mask)
 653{
 654	INIT_LIST_HEAD(&op->link);
 655	op->tile_mask = tile_mask;
 656	op->base.op = DRM_GPUVA_OP_MAP;
 657	op->base.map.va.addr = vma->gpuva.va.addr;
 658	op->base.map.va.range = vma->gpuva.va.range;
 659	op->base.map.gem.obj = vma->gpuva.gem.obj;
 660	op->base.map.gem.offset = vma->gpuva.gem.offset;
 661	op->map.vma = vma;
 662	op->map.immediate = true;
 663	op->map.vma_flags = vma->gpuva.flags & XE_VMA_CREATE_MASK;
 664}
 665
 666static int xe_vm_ops_add_rebind(struct xe_vma_ops *vops, struct xe_vma *vma,
 667				u8 tile_mask)
 668{
 669	struct xe_vma_op *op;
 670
 671	op = kzalloc(sizeof(*op), GFP_KERNEL);
 672	if (!op)
 673		return -ENOMEM;
 674
 675	xe_vm_populate_rebind(op, vma, tile_mask);
 676	list_add_tail(&op->link, &vops->list);
 677	xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1);
 678
 679	return 0;
 680}
 681
 682static struct dma_fence *ops_execute(struct xe_vm *vm,
 683				     struct xe_vma_ops *vops);
 684static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
 685			    struct xe_exec_queue *q,
 686			    struct xe_sync_entry *syncs, u32 num_syncs);
 687
 688int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 689{
 690	struct dma_fence *fence;
 691	struct xe_vma *vma, *next;
 692	struct xe_vma_ops vops;
 693	struct xe_vma_op *op, *next_op;
 694	int err, i;
 695
 696	lockdep_assert_held(&vm->lock);
 697	if ((xe_vm_in_lr_mode(vm) && !rebind_worker) ||
 698	    list_empty(&vm->rebind_list))
 699		return 0;
 700
 701	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
 702	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
 703		vops.pt_update_ops[i].wait_vm_bookkeep = true;
 704
 705	xe_vm_assert_held(vm);
 706	list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
 707		xe_assert(vm->xe, vma->tile_present);
 708
 709		if (rebind_worker)
 710			trace_xe_vma_rebind_worker(vma);
 711		else
 712			trace_xe_vma_rebind_exec(vma);
 713
 714		err = xe_vm_ops_add_rebind(&vops, vma,
 715					   vma->tile_present);
 716		if (err)
 717			goto free_ops;
 718	}
 719
 720	err = xe_vma_ops_alloc(&vops, false);
 721	if (err)
 722		goto free_ops;
 723
 724	fence = ops_execute(vm, &vops);
 725	if (IS_ERR(fence)) {
 726		err = PTR_ERR(fence);
 727	} else {
 728		dma_fence_put(fence);
 729		list_for_each_entry_safe(vma, next, &vm->rebind_list,
 730					 combined_links.rebind)
 731			list_del_init(&vma->combined_links.rebind);
 732	}
 733free_ops:
 734	list_for_each_entry_safe(op, next_op, &vops.list, link) {
 735		list_del(&op->link);
 736		kfree(op);
 737	}
 738	xe_vma_ops_fini(&vops);
 739
 740	return err;
 741}
 742
 743struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask)
 744{
 745	struct dma_fence *fence = NULL;
 746	struct xe_vma_ops vops;
 747	struct xe_vma_op *op, *next_op;
 748	struct xe_tile *tile;
 749	u8 id;
 750	int err;
 751
 752	lockdep_assert_held(&vm->lock);
 753	xe_vm_assert_held(vm);
 754	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
 755
 756	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
 757	vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
 758	for_each_tile(tile, vm->xe, id) {
 759		vops.pt_update_ops[id].wait_vm_bookkeep = true;
 760		vops.pt_update_ops[tile->id].q =
 761			xe_migrate_exec_queue(tile->migrate);
 762	}
 763
 764	err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);
 765	if (err)
 766		return ERR_PTR(err);
 767
 768	err = xe_vma_ops_alloc(&vops, false);
 769	if (err) {
 770		fence = ERR_PTR(err);
 771		goto free_ops;
 772	}
 773
 774	fence = ops_execute(vm, &vops);
 775
 776free_ops:
 777	list_for_each_entry_safe(op, next_op, &vops.list, link) {
 778		list_del(&op->link);
 779		kfree(op);
 780	}
 781	xe_vma_ops_fini(&vops);
 782
 783	return fence;
 784}
 785
 786static void xe_vm_populate_range_rebind(struct xe_vma_op *op,
 787					struct xe_vma *vma,
 788					struct xe_svm_range *range,
 789					u8 tile_mask)
 790{
 791	INIT_LIST_HEAD(&op->link);
 792	op->tile_mask = tile_mask;
 793	op->base.op = DRM_GPUVA_OP_DRIVER;
 794	op->subop = XE_VMA_SUBOP_MAP_RANGE;
 795	op->map_range.vma = vma;
 796	op->map_range.range = range;
 797}
 798
 799static int
 800xe_vm_ops_add_range_rebind(struct xe_vma_ops *vops,
 801			   struct xe_vma *vma,
 802			   struct xe_svm_range *range,
 803			   u8 tile_mask)
 804{
 805	struct xe_vma_op *op;
 806
 807	op = kzalloc(sizeof(*op), GFP_KERNEL);
 808	if (!op)
 809		return -ENOMEM;
 810
 811	xe_vm_populate_range_rebind(op, vma, range, tile_mask);
 812	list_add_tail(&op->link, &vops->list);
 813	xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1);
 814
 815	return 0;
 816}
 817
 818/**
 819 * xe_vm_range_rebind() - VM range (re)bind
 820 * @vm: The VM which the range belongs to.
 821 * @vma: The VMA which the range belongs to.
 822 * @range: SVM range to rebind.
 823 * @tile_mask: Tile mask to bind the range to.
 824 *
 825 * (re)bind SVM range setting up GPU page tables for the range.
 826 *
 827 * Return: dma fence for rebind to signal completion on success, ERR_PTR on
 828 * failure
 829 */
 830struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm,
 831				     struct xe_vma *vma,
 832				     struct xe_svm_range *range,
 833				     u8 tile_mask)
 834{
 835	struct dma_fence *fence = NULL;
 836	struct xe_vma_ops vops;
 837	struct xe_vma_op *op, *next_op;
 838	struct xe_tile *tile;
 839	u8 id;
 840	int err;
 841
 842	lockdep_assert_held(&vm->lock);
 843	xe_vm_assert_held(vm);
 844	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
 845	xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma));
 846
 847	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
 848	vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
 849	for_each_tile(tile, vm->xe, id) {
 850		vops.pt_update_ops[id].wait_vm_bookkeep = true;
 851		vops.pt_update_ops[tile->id].q =
 852			xe_migrate_exec_queue(tile->migrate);
 853	}
 854
 855	err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask);
 856	if (err)
 857		return ERR_PTR(err);
 858
 859	err = xe_vma_ops_alloc(&vops, false);
 860	if (err) {
 861		fence = ERR_PTR(err);
 862		goto free_ops;
 863	}
 864
 865	fence = ops_execute(vm, &vops);
 866
 867free_ops:
 868	list_for_each_entry_safe(op, next_op, &vops.list, link) {
 869		list_del(&op->link);
 870		kfree(op);
 871	}
 872	xe_vma_ops_fini(&vops);
 873
 874	return fence;
 875}
 876
 877static void xe_vm_populate_range_unbind(struct xe_vma_op *op,
 878					struct xe_svm_range *range)
 879{
 880	INIT_LIST_HEAD(&op->link);
 881	op->tile_mask = range->tile_present;
 882	op->base.op = DRM_GPUVA_OP_DRIVER;
 883	op->subop = XE_VMA_SUBOP_UNMAP_RANGE;
 884	op->unmap_range.range = range;
 885}
 886
 887static int
 888xe_vm_ops_add_range_unbind(struct xe_vma_ops *vops,
 889			   struct xe_svm_range *range)
 890{
 891	struct xe_vma_op *op;
 892
 893	op = kzalloc(sizeof(*op), GFP_KERNEL);
 894	if (!op)
 895		return -ENOMEM;
 896
 897	xe_vm_populate_range_unbind(op, range);
 898	list_add_tail(&op->link, &vops->list);
 899	xe_vma_ops_incr_pt_update_ops(vops, range->tile_present, 1);
 900
 901	return 0;
 902}
 903
 904/**
 905 * xe_vm_range_unbind() - VM range unbind
 906 * @vm: The VM which the range belongs to.
 907 * @range: SVM range to rebind.
 908 *
 909 * Unbind SVM range removing the GPU page tables for the range.
 910 *
 911 * Return: dma fence for unbind to signal completion on success, ERR_PTR on
 912 * failure
 913 */
 914struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm,
 915				     struct xe_svm_range *range)
 916{
 917	struct dma_fence *fence = NULL;
 918	struct xe_vma_ops vops;
 919	struct xe_vma_op *op, *next_op;
 920	struct xe_tile *tile;
 921	u8 id;
 922	int err;
 923
 924	lockdep_assert_held(&vm->lock);
 925	xe_vm_assert_held(vm);
 926	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
 927
 928	if (!range->tile_present)
 929		return dma_fence_get_stub();
 930
 931	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
 932	for_each_tile(tile, vm->xe, id) {
 933		vops.pt_update_ops[id].wait_vm_bookkeep = true;
 934		vops.pt_update_ops[tile->id].q =
 935			xe_migrate_exec_queue(tile->migrate);
 936	}
 937
 938	err = xe_vm_ops_add_range_unbind(&vops, range);
 939	if (err)
 940		return ERR_PTR(err);
 941
 942	err = xe_vma_ops_alloc(&vops, false);
 943	if (err) {
 944		fence = ERR_PTR(err);
 945		goto free_ops;
 946	}
 947
 948	fence = ops_execute(vm, &vops);
 949
 950free_ops:
 951	list_for_each_entry_safe(op, next_op, &vops.list, link) {
 952		list_del(&op->link);
 953		kfree(op);
 954	}
 955	xe_vma_ops_fini(&vops);
 956
 957	return fence;
 958}
 959
 960static void xe_vma_free(struct xe_vma *vma)
 961{
 962	if (xe_vma_is_userptr(vma))
 963		kfree(to_userptr_vma(vma));
 964	else
 965		kfree(vma);
 966}
 967
 968static struct xe_vma *xe_vma_create(struct xe_vm *vm,
 969				    struct xe_bo *bo,
 970				    u64 bo_offset_or_userptr,
 971				    u64 start, u64 end,
 972				    struct xe_vma_mem_attr *attr,
 973				    unsigned int flags)
 974{
 975	struct xe_vma *vma;
 976	struct xe_tile *tile;
 977	u8 id;
 978	bool is_null = (flags & DRM_GPUVA_SPARSE);
 979	bool is_cpu_addr_mirror = (flags & XE_VMA_SYSTEM_ALLOCATOR);
 980
 981	xe_assert(vm->xe, start < end);
 982	xe_assert(vm->xe, end < vm->size);
 983
 984	/*
 985	 * Allocate and ensure that the xe_vma_is_userptr() return
 986	 * matches what was allocated.
 987	 */
 988	if (!bo && !is_null && !is_cpu_addr_mirror) {
 989		struct xe_userptr_vma *uvma = kzalloc(sizeof(*uvma), GFP_KERNEL);
 990
 991		if (!uvma)
 992			return ERR_PTR(-ENOMEM);
 993
 994		vma = &uvma->vma;
 995	} else {
 996		vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 997		if (!vma)
 998			return ERR_PTR(-ENOMEM);
 999
1000		if (bo)
1001			vma->gpuva.gem.obj = &bo->ttm.base;
1002	}
1003
1004	INIT_LIST_HEAD(&vma->combined_links.rebind);
1005
1006	INIT_LIST_HEAD(&vma->gpuva.gem.entry);
1007	vma->gpuva.vm = &vm->gpuvm;
1008	vma->gpuva.va.addr = start;
1009	vma->gpuva.va.range = end - start + 1;
1010	vma->gpuva.flags = flags;
1011
1012	for_each_tile(tile, vm->xe, id)
1013		vma->tile_mask |= 0x1 << id;
1014
1015	if (vm->xe->info.has_atomic_enable_pte_bit)
1016		vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT;
1017
1018	vma->attr = *attr;
1019
1020	if (bo) {
1021		struct drm_gpuvm_bo *vm_bo;
1022
1023		xe_bo_assert_held(bo);
1024
1025		vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base);
1026		if (IS_ERR(vm_bo)) {
1027			xe_vma_free(vma);
1028			return ERR_CAST(vm_bo);
1029		}
1030
1031		drm_gpuvm_bo_extobj_add(vm_bo);
1032		drm_gem_object_get(&bo->ttm.base);
1033		vma->gpuva.gem.offset = bo_offset_or_userptr;
1034		drm_gpuva_link(&vma->gpuva, vm_bo);
1035		drm_gpuvm_bo_put(vm_bo);
1036	} else /* userptr or null */ {
1037		if (!is_null && !is_cpu_addr_mirror) {
1038			struct xe_userptr_vma *uvma = to_userptr_vma(vma);
1039			u64 size = end - start + 1;
1040			int err;
1041
1042			vma->gpuva.gem.offset = bo_offset_or_userptr;
1043
1044			err = xe_userptr_setup(uvma, xe_vma_userptr(vma), size);
1045			if (err) {
1046				xe_vma_free(vma);
1047				return ERR_PTR(err);
1048			}
1049		}
1050
1051		xe_vm_get(vm);
1052	}
1053
1054	return vma;
1055}
1056
1057static void xe_vma_destroy_late(struct xe_vma *vma)
1058{
1059	struct xe_vm *vm = xe_vma_vm(vma);
1060
1061	if (vma->ufence) {
1062		xe_sync_ufence_put(vma->ufence);
1063		vma->ufence = NULL;
1064	}
1065
1066	if (xe_vma_is_userptr(vma)) {
1067		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
1068
1069		xe_userptr_remove(uvma);
1070		xe_vm_put(vm);
1071	} else if (xe_vma_is_null(vma) || xe_vma_is_cpu_addr_mirror(vma)) {
1072		xe_vm_put(vm);
1073	} else {
1074		xe_bo_put(xe_vma_bo(vma));
1075	}
1076
1077	xe_vma_free(vma);
1078}
1079
1080static void vma_destroy_work_func(struct work_struct *w)
1081{
1082	struct xe_vma *vma =
1083		container_of(w, struct xe_vma, destroy_work);
1084
1085	xe_vma_destroy_late(vma);
1086}
1087
1088static void vma_destroy_cb(struct dma_fence *fence,
1089			   struct dma_fence_cb *cb)
1090{
1091	struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb);
1092
1093	INIT_WORK(&vma->destroy_work, vma_destroy_work_func);
1094	queue_work(system_unbound_wq, &vma->destroy_work);
1095}
1096
1097static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
1098{
1099	struct xe_vm *vm = xe_vma_vm(vma);
1100
1101	lockdep_assert_held_write(&vm->lock);
1102	xe_assert(vm->xe, list_empty(&vma->combined_links.destroy));
1103
1104	if (xe_vma_is_userptr(vma)) {
1105		xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED);
1106		xe_userptr_destroy(to_userptr_vma(vma));
1107	} else if (!xe_vma_is_null(vma) && !xe_vma_is_cpu_addr_mirror(vma)) {
1108		xe_bo_assert_held(xe_vma_bo(vma));
1109
1110		drm_gpuva_unlink(&vma->gpuva);
1111	}
1112
1113	xe_vm_assert_held(vm);
1114	if (fence) {
1115		int ret = dma_fence_add_callback(fence, &vma->destroy_cb,
1116						 vma_destroy_cb);
1117
1118		if (ret) {
1119			XE_WARN_ON(ret != -ENOENT);
1120			xe_vma_destroy_late(vma);
1121		}
1122	} else {
1123		xe_vma_destroy_late(vma);
1124	}
1125}
1126
1127/**
1128 * xe_vm_lock_vma() - drm_exec utility to lock a vma
1129 * @exec: The drm_exec object we're currently locking for.
1130 * @vma: The vma for witch we want to lock the vm resv and any attached
1131 * object's resv.
1132 *
1133 * Return: 0 on success, negative error code on error. In particular
1134 * may return -EDEADLK on WW transaction contention and -EINTR if
1135 * an interruptible wait is terminated by a signal.
1136 */
1137int xe_vm_lock_vma(struct drm_exec *exec, struct xe_vma *vma)
1138{
1139	struct xe_vm *vm = xe_vma_vm(vma);
1140	struct xe_bo *bo = xe_vma_bo(vma);
1141	int err;
1142
1143	XE_WARN_ON(!vm);
1144
1145	err = drm_exec_lock_obj(exec, xe_vm_obj(vm));
1146	if (!err && bo && !bo->vm)
1147		err = drm_exec_lock_obj(exec, &bo->ttm.base);
1148
1149	return err;
1150}
1151
1152static void xe_vma_destroy_unlocked(struct xe_vma *vma)
1153{
1154	struct xe_device *xe = xe_vma_vm(vma)->xe;
1155	struct xe_validation_ctx ctx;
1156	struct drm_exec exec;
1157	int err = 0;
1158
1159	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) {
1160		err = xe_vm_lock_vma(&exec, vma);
1161		drm_exec_retry_on_contention(&exec);
1162		if (XE_WARN_ON(err))
1163			break;
1164		xe_vma_destroy(vma, NULL);
1165	}
1166	xe_assert(xe, !err);
1167}
1168
1169struct xe_vma *
1170xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range)
1171{
1172	struct drm_gpuva *gpuva;
1173
1174	lockdep_assert_held(&vm->lock);
1175
1176	if (xe_vm_is_closed_or_banned(vm))
1177		return NULL;
1178
1179	xe_assert(vm->xe, start + range <= vm->size);
1180
1181	gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range);
1182
1183	return gpuva ? gpuva_to_vma(gpuva) : NULL;
1184}
1185
1186static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma)
1187{
1188	int err;
1189
1190	xe_assert(vm->xe, xe_vma_vm(vma) == vm);
1191	lockdep_assert_held(&vm->lock);
1192
1193	mutex_lock(&vm->snap_mutex);
1194	err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva);
1195	mutex_unlock(&vm->snap_mutex);
1196	XE_WARN_ON(err);	/* Shouldn't be possible */
1197
1198	return err;
1199}
1200
1201static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma)
1202{
1203	xe_assert(vm->xe, xe_vma_vm(vma) == vm);
1204	lockdep_assert_held(&vm->lock);
1205
1206	mutex_lock(&vm->snap_mutex);
1207	drm_gpuva_remove(&vma->gpuva);
1208	mutex_unlock(&vm->snap_mutex);
1209	if (vm->usm.last_fault_vma == vma)
1210		vm->usm.last_fault_vma = NULL;
1211}
1212
1213static struct drm_gpuva_op *xe_vm_op_alloc(void)
1214{
1215	struct xe_vma_op *op;
1216
1217	op = kzalloc(sizeof(*op), GFP_KERNEL);
1218
1219	if (unlikely(!op))
1220		return NULL;
1221
1222	return &op->base;
1223}
1224
1225static void xe_vm_free(struct drm_gpuvm *gpuvm);
1226
1227static const struct drm_gpuvm_ops gpuvm_ops = {
1228	.op_alloc = xe_vm_op_alloc,
1229	.vm_bo_validate = xe_gpuvm_validate,
1230	.vm_free = xe_vm_free,
1231};
1232
1233static u64 pde_encode_pat_index(u16 pat_index)
1234{
1235	u64 pte = 0;
1236
1237	if (pat_index & BIT(0))
1238		pte |= XE_PPGTT_PTE_PAT0;
1239
1240	if (pat_index & BIT(1))
1241		pte |= XE_PPGTT_PTE_PAT1;
1242
1243	return pte;
1244}
1245
1246static u64 pte_encode_pat_index(u16 pat_index, u32 pt_level)
1247{
1248	u64 pte = 0;
1249
1250	if (pat_index & BIT(0))
1251		pte |= XE_PPGTT_PTE_PAT0;
1252
1253	if (pat_index & BIT(1))
1254		pte |= XE_PPGTT_PTE_PAT1;
1255
1256	if (pat_index & BIT(2)) {
1257		if (pt_level)
1258			pte |= XE_PPGTT_PDE_PDPE_PAT2;
1259		else
1260			pte |= XE_PPGTT_PTE_PAT2;
1261	}
1262
1263	if (pat_index & BIT(3))
1264		pte |= XELPG_PPGTT_PTE_PAT3;
1265
1266	if (pat_index & (BIT(4)))
1267		pte |= XE2_PPGTT_PTE_PAT4;
1268
1269	return pte;
1270}
1271
1272static u64 pte_encode_ps(u32 pt_level)
1273{
1274	XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL);
1275
1276	if (pt_level == 1)
1277		return XE_PDE_PS_2M;
1278	else if (pt_level == 2)
1279		return XE_PDPE_PS_1G;
1280
1281	return 0;
1282}
1283
1284static u16 pde_pat_index(struct xe_bo *bo)
1285{
1286	struct xe_device *xe = xe_bo_device(bo);
1287	u16 pat_index;
1288
1289	/*
1290	 * We only have two bits to encode the PAT index in non-leaf nodes, but
1291	 * these only point to other paging structures so we only need a minimal
1292	 * selection of options. The user PAT index is only for encoding leaf
1293	 * nodes, where we have use of more bits to do the encoding. The
1294	 * non-leaf nodes are instead under driver control so the chosen index
1295	 * here should be distinct from the user PAT index. Also the
1296	 * corresponding coherency of the PAT index should be tied to the
1297	 * allocation type of the page table (or at least we should pick
1298	 * something which is always safe).
1299	 */
1300	if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached)
1301		pat_index = xe->pat.idx[XE_CACHE_WB];
1302	else
1303		pat_index = xe->pat.idx[XE_CACHE_NONE];
1304
1305	xe_assert(xe, pat_index <= 3);
1306
1307	return pat_index;
1308}
1309
1310static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset)
1311{
1312	u64 pde;
1313
1314	pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
1315	pde |= XE_PAGE_PRESENT | XE_PAGE_RW;
1316	pde |= pde_encode_pat_index(pde_pat_index(bo));
1317
1318	return pde;
1319}
1320
1321static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
1322			      u16 pat_index, u32 pt_level)
1323{
1324	u64 pte;
1325
1326	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
1327	pte |= XE_PAGE_PRESENT | XE_PAGE_RW;
1328	pte |= pte_encode_pat_index(pat_index, pt_level);
1329	pte |= pte_encode_ps(pt_level);
1330
1331	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
1332		pte |= XE_PPGTT_PTE_DM;
1333
1334	return pte;
1335}
1336
1337static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma,
1338			       u16 pat_index, u32 pt_level)
1339{
1340	pte |= XE_PAGE_PRESENT;
1341
1342	if (likely(!xe_vma_read_only(vma)))
1343		pte |= XE_PAGE_RW;
1344
1345	pte |= pte_encode_pat_index(pat_index, pt_level);
1346	pte |= pte_encode_ps(pt_level);
1347
1348	if (unlikely(xe_vma_is_null(vma)))
1349		pte |= XE_PTE_NULL;
1350
1351	return pte;
1352}
1353
1354static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr,
1355				u16 pat_index,
1356				u32 pt_level, bool devmem, u64 flags)
1357{
1358	u64 pte;
1359
1360	/* Avoid passing random bits directly as flags */
1361	xe_assert(xe, !(flags & ~XE_PTE_PS64));
1362
1363	pte = addr;
1364	pte |= XE_PAGE_PRESENT | XE_PAGE_RW;
1365	pte |= pte_encode_pat_index(pat_index, pt_level);
1366	pte |= pte_encode_ps(pt_level);
1367
1368	if (devmem)
1369		pte |= XE_PPGTT_PTE_DM;
1370
1371	pte |= flags;
1372
1373	return pte;
1374}
1375
1376static const struct xe_pt_ops xelp_pt_ops = {
1377	.pte_encode_bo = xelp_pte_encode_bo,
1378	.pte_encode_vma = xelp_pte_encode_vma,
1379	.pte_encode_addr = xelp_pte_encode_addr,
1380	.pde_encode_bo = xelp_pde_encode_bo,
1381};
1382
1383static void vm_destroy_work_func(struct work_struct *w);
1384
1385/**
1386 * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the
1387 * given tile and vm.
1388 * @xe: xe device.
1389 * @tile: tile to set up for.
1390 * @vm: vm to set up for.
1391 * @exec: The struct drm_exec object used to lock the vm resv.
1392 *
1393 * Sets up a pagetable tree with one page-table per level and a single
1394 * leaf PTE. All pagetable entries point to the single page-table or,
1395 * for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and
1396 * writes become NOPs.
1397 *
1398 * Return: 0 on success, negative error code on error.
1399 */
1400static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile,
1401				struct xe_vm *vm, struct drm_exec *exec)
1402{
1403	u8 id = tile->id;
1404	int i;
1405
1406	for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) {
1407		vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i, exec);
1408		if (IS_ERR(vm->scratch_pt[id][i])) {
1409			int err = PTR_ERR(vm->scratch_pt[id][i]);
1410
1411			vm->scratch_pt[id][i] = NULL;
1412			return err;
1413		}
1414		xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]);
1415	}
1416
1417	return 0;
1418}
1419ALLOW_ERROR_INJECTION(xe_vm_create_scratch, ERRNO);
1420
1421static void xe_vm_free_scratch(struct xe_vm *vm)
1422{
1423	struct xe_tile *tile;
1424	u8 id;
1425
1426	if (!xe_vm_has_scratch(vm))
1427		return;
1428
1429	for_each_tile(tile, vm->xe, id) {
1430		u32 i;
1431
1432		if (!vm->pt_root[id])
1433			continue;
1434
1435		for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i)
1436			if (vm->scratch_pt[id][i])
1437				xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL);
1438	}
1439}
1440
1441static void xe_vm_pt_destroy(struct xe_vm *vm)
1442{
1443	struct xe_tile *tile;
1444	u8 id;
1445
1446	xe_vm_assert_held(vm);
1447
1448	for_each_tile(tile, vm->xe, id) {
1449		if (vm->pt_root[id]) {
1450			xe_pt_destroy(vm->pt_root[id], vm->flags, NULL);
1451			vm->pt_root[id] = NULL;
1452		}
1453	}
1454}
1455
1456struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
1457{
1458	struct drm_gem_object *vm_resv_obj;
1459	struct xe_validation_ctx ctx;
1460	struct drm_exec exec;
1461	struct xe_vm *vm;
1462	int err;
1463	struct xe_tile *tile;
1464	u8 id;
1465
1466	/*
1467	 * Since the GSCCS is not user-accessible, we don't expect a GSC VM to
1468	 * ever be in faulting mode.
1469	 */
1470	xe_assert(xe, !((flags & XE_VM_FLAG_GSC) && (flags & XE_VM_FLAG_FAULT_MODE)));
1471
1472	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1473	if (!vm)
1474		return ERR_PTR(-ENOMEM);
1475
1476	vm->xe = xe;
1477
1478	vm->size = 1ull << xe->info.va_bits;
1479	vm->flags = flags;
1480
1481	if (xef)
1482		vm->xef = xe_file_get(xef);
1483	/**
1484	 * GSC VMs are kernel-owned, only used for PXP ops and can sometimes be
1485	 * manipulated under the PXP mutex. However, the PXP mutex can be taken
1486	 * under a user-VM lock when the PXP session is started at exec_queue
1487	 * creation time. Those are different VMs and therefore there is no risk
1488	 * of deadlock, but we need to tell lockdep that this is the case or it
1489	 * will print a warning.
1490	 */
1491	if (flags & XE_VM_FLAG_GSC) {
1492		static struct lock_class_key gsc_vm_key;
1493
1494		__init_rwsem(&vm->lock, "gsc_vm", &gsc_vm_key);
1495	} else {
1496		init_rwsem(&vm->lock);
1497	}
1498	mutex_init(&vm->snap_mutex);
1499
1500	INIT_LIST_HEAD(&vm->rebind_list);
1501
1502	INIT_LIST_HEAD(&vm->userptr.repin_list);
1503	INIT_LIST_HEAD(&vm->userptr.invalidated);
1504	spin_lock_init(&vm->userptr.invalidated_lock);
1505
1506	ttm_lru_bulk_move_init(&vm->lru_bulk_move);
1507
1508	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
1509
1510	INIT_LIST_HEAD(&vm->preempt.exec_queues);
1511	if (flags & XE_VM_FLAG_FAULT_MODE)
1512		vm->preempt.min_run_period_ms = 0;
1513	else
1514		vm->preempt.min_run_period_ms = 5;
1515
1516	for_each_tile(tile, xe, id)
1517		xe_range_fence_tree_init(&vm->rftree[id]);
1518
1519	vm->pt_ops = &xelp_pt_ops;
1520
1521	/*
1522	 * Long-running workloads are not protected by the scheduler references.
1523	 * By design, run_job for long-running workloads returns NULL and the
1524	 * scheduler drops all the references of it, hence protecting the VM
1525	 * for this case is necessary.
1526	 */
1527	if (flags & XE_VM_FLAG_LR_MODE) {
1528		INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func);
1529		xe_pm_runtime_get_noresume(xe);
1530		INIT_LIST_HEAD(&vm->preempt.pm_activate_link);
1531	}
1532
1533	err = xe_svm_init(vm);
1534	if (err)
1535		goto err_no_resv;
1536
1537	vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
1538	if (!vm_resv_obj) {
1539		err = -ENOMEM;
1540		goto err_svm_fini;
1541	}
1542
1543	drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm,
1544		       vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops);
1545
1546	drm_gem_object_put(vm_resv_obj);
1547
1548	err = 0;
1549	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true},
1550			    err) {
1551		err = xe_vm_drm_exec_lock(vm, &exec);
1552		drm_exec_retry_on_contention(&exec);
1553
1554		if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
1555			vm->flags |= XE_VM_FLAG_64K;
1556
1557		for_each_tile(tile, xe, id) {
1558			if (flags & XE_VM_FLAG_MIGRATION &&
1559			    tile->id != XE_VM_FLAG_TILE_ID(flags))
1560				continue;
1561
1562			vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level,
1563						       &exec);
1564			if (IS_ERR(vm->pt_root[id])) {
1565				err = PTR_ERR(vm->pt_root[id]);
1566				vm->pt_root[id] = NULL;
1567				xe_vm_pt_destroy(vm);
1568				drm_exec_retry_on_contention(&exec);
1569				xe_validation_retry_on_oom(&ctx, &err);
1570				break;
1571			}
1572		}
1573		if (err)
1574			break;
1575
1576		if (xe_vm_has_scratch(vm)) {
1577			for_each_tile(tile, xe, id) {
1578				if (!vm->pt_root[id])
1579					continue;
1580
1581				err = xe_vm_create_scratch(xe, tile, vm, &exec);
1582				if (err) {
1583					xe_vm_free_scratch(vm);
1584					xe_vm_pt_destroy(vm);
1585					drm_exec_retry_on_contention(&exec);
1586					xe_validation_retry_on_oom(&ctx, &err);
1587					break;
1588				}
1589			}
1590			if (err)
1591				break;
1592			vm->batch_invalidate_tlb = true;
1593		}
1594
1595		if (vm->flags & XE_VM_FLAG_LR_MODE) {
1596			INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func);
1597			vm->batch_invalidate_tlb = false;
1598		}
1599
1600		/* Fill pt_root after allocating scratch tables */
1601		for_each_tile(tile, xe, id) {
1602			if (!vm->pt_root[id])
1603				continue;
1604
1605			xe_pt_populate_empty(tile, vm, vm->pt_root[id]);
1606		}
1607	}
1608	if (err)
1609		goto err_close;
1610
1611	/* Kernel migration VM shouldn't have a circular loop.. */
1612	if (!(flags & XE_VM_FLAG_MIGRATION)) {
1613		for_each_tile(tile, xe, id) {
1614			struct xe_exec_queue *q;
1615			u32 create_flags = EXEC_QUEUE_FLAG_VM;
1616
1617			if (!vm->pt_root[id])
1618				continue;
1619
1620			q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0);
1621			if (IS_ERR(q)) {
1622				err = PTR_ERR(q);
1623				goto err_close;
1624			}
1625			vm->q[id] = q;
1626		}
1627	}
1628
1629	if (xef && xe->info.has_asid) {
1630		u32 asid;
1631
1632		down_write(&xe->usm.lock);
1633		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm,
1634				      XA_LIMIT(1, XE_MAX_ASID - 1),
1635				      &xe->usm.next_asid, GFP_KERNEL);
1636		up_write(&xe->usm.lock);
1637		if (err < 0)
1638			goto err_close;
1639
1640		vm->usm.asid = asid;
1641	}
1642
1643	trace_xe_vm_create(vm);
1644
1645	return vm;
1646
1647err_close:
1648	xe_vm_close_and_put(vm);
1649	return ERR_PTR(err);
1650
1651err_svm_fini:
1652	if (flags & XE_VM_FLAG_FAULT_MODE) {
1653		vm->size = 0; /* close the vm */
1654		xe_svm_fini(vm);
1655	}
1656err_no_resv:
1657	mutex_destroy(&vm->snap_mutex);
1658	for_each_tile(tile, xe, id)
1659		xe_range_fence_tree_fini(&vm->rftree[id]);
1660	ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
1661	if (vm->xef)
1662		xe_file_put(vm->xef);
1663	kfree(vm);
1664	if (flags & XE_VM_FLAG_LR_MODE)
1665		xe_pm_runtime_put(xe);
1666	return ERR_PTR(err);
1667}
1668
1669static void xe_vm_close(struct xe_vm *vm)
1670{
1671	struct xe_device *xe = vm->xe;
1672	bool bound;
1673	int idx;
1674
1675	bound = drm_dev_enter(&xe->drm, &idx);
1676
1677	down_write(&vm->lock);
1678	if (xe_vm_in_fault_mode(vm))
1679		xe_svm_notifier_lock(vm);
1680
1681	vm->size = 0;
1682
1683	if (!((vm->flags & XE_VM_FLAG_MIGRATION))) {
1684		struct xe_tile *tile;
1685		struct xe_gt *gt;
1686		u8 id;
1687
1688		/* Wait for pending binds */
1689		dma_resv_wait_timeout(xe_vm_resv(vm),
1690				      DMA_RESV_USAGE_BOOKKEEP,
1691				      false, MAX_SCHEDULE_TIMEOUT);
1692
1693		if (bound) {
1694			for_each_tile(tile, xe, id)
1695				if (vm->pt_root[id])
1696					xe_pt_clear(xe, vm->pt_root[id]);
1697
1698			for_each_gt(gt, xe, id)
1699				xe_tlb_inval_vm(&gt->tlb_inval, vm);
1700		}
1701	}
1702
1703	if (xe_vm_in_fault_mode(vm))
1704		xe_svm_notifier_unlock(vm);
1705	up_write(&vm->lock);
1706
1707	if (bound)
1708		drm_dev_exit(idx);
1709}
1710
1711void xe_vm_close_and_put(struct xe_vm *vm)
1712{
1713	LIST_HEAD(contested);
1714	struct xe_device *xe = vm->xe;
1715	struct xe_tile *tile;
1716	struct xe_vma *vma, *next_vma;
1717	struct drm_gpuva *gpuva, *next;
1718	u8 id;
1719
1720	xe_assert(xe, !vm->preempt.num_exec_queues);
1721
1722	xe_vm_close(vm);
1723	if (xe_vm_in_preempt_fence_mode(vm)) {
1724		mutex_lock(&xe->rebind_resume_lock);
1725		list_del_init(&vm->preempt.pm_activate_link);
1726		mutex_unlock(&xe->rebind_resume_lock);
1727		flush_work(&vm->preempt.rebind_work);
1728	}
1729	if (xe_vm_in_fault_mode(vm))
1730		xe_svm_close(vm);
1731
1732	down_write(&vm->lock);
1733	for_each_tile(tile, xe, id) {
1734		if (vm->q[id]) {
1735			int i;
1736
1737			xe_exec_queue_last_fence_put(vm->q[id], vm);
1738			for_each_tlb_inval(i)
1739				xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i);
1740		}
1741	}
1742	up_write(&vm->lock);
1743
1744	for_each_tile(tile, xe, id) {
1745		if (vm->q[id]) {
1746			xe_exec_queue_kill(vm->q[id]);
1747			xe_exec_queue_put(vm->q[id]);
1748			vm->q[id] = NULL;
1749		}
1750	}
1751
1752	down_write(&vm->lock);
1753	xe_vm_lock(vm, false);
1754	drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) {
1755		vma = gpuva_to_vma(gpuva);
1756
1757		if (xe_vma_has_no_bo(vma)) {
1758			xe_svm_notifier_lock(vm);
1759			vma->gpuva.flags |= XE_VMA_DESTROYED;
1760			xe_svm_notifier_unlock(vm);
1761		}
1762
1763		xe_vm_remove_vma(vm, vma);
1764
1765		/* easy case, remove from VMA? */
1766		if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) {
1767			list_del_init(&vma->combined_links.rebind);
1768			xe_vma_destroy(vma, NULL);
1769			continue;
1770		}
1771
1772		list_move_tail(&vma->combined_links.destroy, &contested);
1773		vma->gpuva.flags |= XE_VMA_DESTROYED;
1774	}
1775
1776	/*
1777	 * All vm operations will add shared fences to resv.
1778	 * The only exception is eviction for a shared object,
1779	 * but even so, the unbind when evicted would still
1780	 * install a fence to resv. Hence it's safe to
1781	 * destroy the pagetables immediately.
1782	 */
1783	xe_vm_free_scratch(vm);
1784	xe_vm_pt_destroy(vm);
1785	xe_vm_unlock(vm);
1786
1787	/*
1788	 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL
1789	 * Since we hold a refcount to the bo, we can remove and free
1790	 * the members safely without locking.
1791	 */
1792	list_for_each_entry_safe(vma, next_vma, &contested,
1793				 combined_links.destroy) {
1794		list_del_init(&vma->combined_links.destroy);
1795		xe_vma_destroy_unlocked(vma);
1796	}
1797
1798	xe_svm_fini(vm);
1799
1800	up_write(&vm->lock);
1801
1802	down_write(&xe->usm.lock);
1803	if (vm->usm.asid) {
1804		void *lookup;
1805
1806		xe_assert(xe, xe->info.has_asid);
1807		xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION));
1808
1809		lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
1810		xe_assert(xe, lookup == vm);
1811	}
1812	up_write(&xe->usm.lock);
1813
1814	for_each_tile(tile, xe, id)
1815		xe_range_fence_tree_fini(&vm->rftree[id]);
1816
1817	xe_vm_put(vm);
1818}
1819
1820static void vm_destroy_work_func(struct work_struct *w)
1821{
1822	struct xe_vm *vm =
1823		container_of(w, struct xe_vm, destroy_work);
1824	struct xe_device *xe = vm->xe;
1825	struct xe_tile *tile;
1826	u8 id;
1827
1828	/* xe_vm_close_and_put was not called? */
1829	xe_assert(xe, !vm->size);
1830
1831	if (xe_vm_in_preempt_fence_mode(vm))
1832		flush_work(&vm->preempt.rebind_work);
1833
1834	mutex_destroy(&vm->snap_mutex);
1835
1836	if (vm->flags & XE_VM_FLAG_LR_MODE)
1837		xe_pm_runtime_put(xe);
1838
1839	for_each_tile(tile, xe, id)
1840		XE_WARN_ON(vm->pt_root[id]);
1841
1842	trace_xe_vm_free(vm);
1843
1844	ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
1845
1846	if (vm->xef)
1847		xe_file_put(vm->xef);
1848
1849	kfree(vm);
1850}
1851
1852static void xe_vm_free(struct drm_gpuvm *gpuvm)
1853{
1854	struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
1855
1856	/* To destroy the VM we need to be able to sleep */
1857	queue_work(system_unbound_wq, &vm->destroy_work);
1858}
1859
1860struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
1861{
1862	struct xe_vm *vm;
1863
1864	mutex_lock(&xef->vm.lock);
1865	vm = xa_load(&xef->vm.xa, id);
1866	if (vm)
1867		xe_vm_get(vm);
1868	mutex_unlock(&xef->vm.lock);
1869
1870	return vm;
1871}
1872
1873u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile)
1874{
1875	return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0);
1876}
1877
1878static struct xe_exec_queue *
1879to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
1880{
1881	return q ? q : vm->q[0];
1882}
1883
1884static struct xe_user_fence *
1885find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs)
1886{
1887	unsigned int i;
1888
1889	for (i = 0; i < num_syncs; i++) {
1890		struct xe_sync_entry *e = &syncs[i];
1891
1892		if (xe_sync_is_ufence(e))
1893			return xe_sync_ufence_get(e);
1894	}
1895
1896	return NULL;
1897}
1898
1899#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \
1900				    DRM_XE_VM_CREATE_FLAG_LR_MODE | \
1901				    DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
1902
1903int xe_vm_create_ioctl(struct drm_device *dev, void *data,
1904		       struct drm_file *file)
1905{
1906	struct xe_device *xe = to_xe_device(dev);
1907	struct xe_file *xef = to_xe_file(file);
1908	struct drm_xe_vm_create *args = data;
1909	struct xe_gt *wa_gt = xe_root_mmio_gt(xe);
1910	struct xe_vm *vm;
1911	u32 id;
1912	int err;
1913	u32 flags = 0;
1914
1915	if (XE_IOCTL_DBG(xe, args->extensions))
1916		return -EINVAL;
1917
1918	if (wa_gt && XE_GT_WA(wa_gt, 22014953428))
1919		args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
1920
1921	if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE &&
1922			 !xe->info.has_usm))
1923		return -EINVAL;
1924
1925	if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1926		return -EINVAL;
1927
1928	if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS))
1929		return -EINVAL;
1930
1931	if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE &&
1932			 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE &&
1933			 !xe->info.needs_scratch))
1934		return -EINVAL;
1935
1936	if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) &&
1937			 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE))
1938		return -EINVAL;
1939
1940	if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE)
1941		flags |= XE_VM_FLAG_SCRATCH_PAGE;
1942	if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE)
1943		flags |= XE_VM_FLAG_LR_MODE;
1944	if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
1945		flags |= XE_VM_FLAG_FAULT_MODE;
1946
1947	vm = xe_vm_create(xe, flags, xef);
1948	if (IS_ERR(vm))
1949		return PTR_ERR(vm);
1950
1951#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
1952	/* Warning: Security issue - never enable by default */
1953	args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE);
1954#endif
1955
1956	/* user id alloc must always be last in ioctl to prevent UAF */
1957	err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL);
1958	if (err)
1959		goto err_close_and_put;
1960
1961	args->vm_id = id;
1962
1963	return 0;
1964
1965err_close_and_put:
1966	xe_vm_close_and_put(vm);
1967
1968	return err;
1969}
1970
1971int xe_vm_destroy_ioctl(struct drm_device *dev, void *data,
1972			struct drm_file *file)
1973{
1974	struct xe_device *xe = to_xe_device(dev);
1975	struct xe_file *xef = to_xe_file(file);
1976	struct drm_xe_vm_destroy *args = data;
1977	struct xe_vm *vm;
1978	int err = 0;
1979
1980	if (XE_IOCTL_DBG(xe, args->pad) ||
1981	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1982		return -EINVAL;
1983
1984	mutex_lock(&xef->vm.lock);
1985	vm = xa_load(&xef->vm.xa, args->vm_id);
1986	if (XE_IOCTL_DBG(xe, !vm))
1987		err = -ENOENT;
1988	else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues))
1989		err = -EBUSY;
1990	else
1991		xa_erase(&xef->vm.xa, args->vm_id);
1992	mutex_unlock(&xef->vm.lock);
1993
1994	if (!err)
1995		xe_vm_close_and_put(vm);
1996
1997	return err;
1998}
1999
2000static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end)
2001{
2002	struct drm_gpuva *gpuva;
2003	u32 num_vmas = 0;
2004
2005	lockdep_assert_held(&vm->lock);
2006	drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end)
2007		num_vmas++;
2008
2009	return num_vmas;
2010}
2011
2012static int get_mem_attrs(struct xe_vm *vm, u32 *num_vmas, u64 start,
2013			 u64 end, struct drm_xe_mem_range_attr *attrs)
2014{
2015	struct drm_gpuva *gpuva;
2016	int i = 0;
2017
2018	lockdep_assert_held(&vm->lock);
2019
2020	drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) {
2021		struct xe_vma *vma = gpuva_to_vma(gpuva);
2022
2023		if (i == *num_vmas)
2024			return -ENOSPC;
2025
2026		attrs[i].start = xe_vma_start(vma);
2027		attrs[i].end = xe_vma_end(vma);
2028		attrs[i].atomic.val = vma->attr.atomic_access;
2029		attrs[i].pat_index.val = vma->attr.pat_index;
2030		attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd;
2031		attrs[i].preferred_mem_loc.migration_policy =
2032		vma->attr.preferred_loc.migration_policy;
2033
2034		i++;
2035	}
2036
2037	*num_vmas = i;
2038	return 0;
2039}
2040
2041int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2042{
2043	struct xe_device *xe = to_xe_device(dev);
2044	struct xe_file *xef = to_xe_file(file);
2045	struct drm_xe_mem_range_attr *mem_attrs;
2046	struct drm_xe_vm_query_mem_range_attr *args = data;
2047	u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr);
2048	struct xe_vm *vm;
2049	int err = 0;
2050
2051	if (XE_IOCTL_DBG(xe,
2052			 ((args->num_mem_ranges == 0 &&
2053			  (attrs_user || args->sizeof_mem_range_attr != 0)) ||
2054			 (args->num_mem_ranges > 0 &&
2055			  (!attrs_user ||
2056			   args->sizeof_mem_range_attr !=
2057			   sizeof(struct drm_xe_mem_range_attr))))))
2058		return -EINVAL;
2059
2060	vm = xe_vm_lookup(xef, args->vm_id);
2061	if (XE_IOCTL_DBG(xe, !vm))
2062		return -EINVAL;
2063
2064	err = down_read_interruptible(&vm->lock);
2065	if (err)
2066		goto put_vm;
2067
2068	attrs_user = u64_to_user_ptr(args->vector_of_mem_attr);
2069
2070	if (args->num_mem_ranges == 0 && !attrs_user) {
2071		args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range);
2072		args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr);
2073		goto unlock_vm;
2074	}
2075
2076	mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr,
2077				   GFP_KERNEL | __GFP_ACCOUNT |
2078				   __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
2079	if (!mem_attrs) {
2080		err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM;
2081		goto unlock_vm;
2082	}
2083
2084	memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr);
2085	err = get_mem_attrs(vm, &args->num_mem_ranges, args->start,
2086			    args->start + args->range, mem_attrs);
2087	if (err)
2088		goto free_mem_attrs;
2089
2090	err = copy_to_user(attrs_user, mem_attrs,
2091			   args->sizeof_mem_range_attr * args->num_mem_ranges);
2092	if (err)
2093		err = -EFAULT;
2094
2095free_mem_attrs:
2096	kvfree(mem_attrs);
2097unlock_vm:
2098	up_read(&vm->lock);
2099put_vm:
2100	xe_vm_put(vm);
2101	return err;
2102}
2103
2104static bool vma_matches(struct xe_vma *vma, u64 page_addr)
2105{
2106	if (page_addr > xe_vma_end(vma) - 1 ||
2107	    page_addr + SZ_4K - 1 < xe_vma_start(vma))
2108		return false;
2109
2110	return true;
2111}
2112
2113/**
2114 * xe_vm_find_vma_by_addr() - Find a VMA by its address
2115 *
2116 * @vm: the xe_vm the vma belongs to
2117 * @page_addr: address to look up
2118 */
2119struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr)
2120{
2121	struct xe_vma *vma = NULL;
2122
2123	if (vm->usm.last_fault_vma) {   /* Fast lookup */
2124		if (vma_matches(vm->usm.last_fault_vma, page_addr))
2125			vma = vm->usm.last_fault_vma;
2126	}
2127	if (!vma)
2128		vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
2129
2130	return vma;
2131}
2132
2133static const u32 region_to_mem_type[] = {
2134	XE_PL_TT,
2135	XE_PL_VRAM0,
2136	XE_PL_VRAM1,
2137};
2138
2139static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma,
2140			     bool post_commit)
2141{
2142	xe_svm_notifier_lock(vm);
2143	vma->gpuva.flags |= XE_VMA_DESTROYED;
2144	xe_svm_notifier_unlock(vm);
2145	if (post_commit)
2146		xe_vm_remove_vma(vm, vma);
2147}
2148
2149#undef ULL
2150#define ULL	unsigned long long
2151
2152#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
2153static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
2154{
2155	struct xe_vma *vma;
2156
2157	switch (op->op) {
2158	case DRM_GPUVA_OP_MAP:
2159		vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx",
2160		       (ULL)op->map.va.addr, (ULL)op->map.va.range);
2161		break;
2162	case DRM_GPUVA_OP_REMAP:
2163		vma = gpuva_to_vma(op->remap.unmap->va);
2164		vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2165		       (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2166		       op->remap.unmap->keep ? 1 : 0);
2167		if (op->remap.prev)
2168			vm_dbg(&xe->drm,
2169			       "REMAP:PREV: addr=0x%016llx, range=0x%016llx",
2170			       (ULL)op->remap.prev->va.addr,
2171			       (ULL)op->remap.prev->va.range);
2172		if (op->remap.next)
2173			vm_dbg(&xe->drm,
2174			       "REMAP:NEXT: addr=0x%016llx, range=0x%016llx",
2175			       (ULL)op->remap.next->va.addr,
2176			       (ULL)op->remap.next->va.range);
2177		break;
2178	case DRM_GPUVA_OP_UNMAP:
2179		vma = gpuva_to_vma(op->unmap.va);
2180		vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2181		       (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2182		       op->unmap.keep ? 1 : 0);
2183		break;
2184	case DRM_GPUVA_OP_PREFETCH:
2185		vma = gpuva_to_vma(op->prefetch.va);
2186		vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx",
2187		       (ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma));
2188		break;
2189	default:
2190		drm_warn(&xe->drm, "NOT POSSIBLE");
2191	}
2192}
2193#else
2194static void print_op(struct xe_device *xe, struct drm_gpuva_op *op)
2195{
2196}
2197#endif
2198
2199static bool __xe_vm_needs_clear_scratch_pages(struct xe_vm *vm, u32 bind_flags)
2200{
2201	if (!xe_vm_in_fault_mode(vm))
2202		return false;
2203
2204	if (!xe_vm_has_scratch(vm))
2205		return false;
2206
2207	if (bind_flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE)
2208		return false;
2209
2210	return true;
2211}
2212
2213static void xe_svm_prefetch_gpuva_ops_fini(struct drm_gpuva_ops *ops)
2214{
2215	struct drm_gpuva_op *__op;
2216
2217	drm_gpuva_for_each_op(__op, ops) {
2218		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2219
2220		xe_vma_svm_prefetch_op_fini(op);
2221	}
2222}
2223
2224/*
2225 * Create operations list from IOCTL arguments, setup operations fields so parse
2226 * and commit steps are decoupled from IOCTL arguments. This step can fail.
2227 */
2228static struct drm_gpuva_ops *
2229vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
2230			 struct xe_bo *bo, u64 bo_offset_or_userptr,
2231			 u64 addr, u64 range,
2232			 u32 operation, u32 flags,
2233			 u32 prefetch_region, u16 pat_index)
2234{
2235	struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL;
2236	struct drm_gpuva_ops *ops;
2237	struct drm_gpuva_op *__op;
2238	struct drm_gpuvm_bo *vm_bo;
2239	u64 range_end = addr + range;
2240	int err;
2241
2242	lockdep_assert_held_write(&vm->lock);
2243
2244	vm_dbg(&vm->xe->drm,
2245	       "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx",
2246	       operation, (ULL)addr, (ULL)range,
2247	       (ULL)bo_offset_or_userptr);
2248
2249	switch (operation) {
2250	case DRM_XE_VM_BIND_OP_MAP:
2251	case DRM_XE_VM_BIND_OP_MAP_USERPTR: {
2252		struct drm_gpuvm_map_req map_req = {
2253			.map.va.addr = addr,
2254			.map.va.range = range,
2255			.map.gem.obj = obj,
2256			.map.gem.offset = bo_offset_or_userptr,
2257		};
2258
2259		ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req);
2260		break;
2261	}
2262	case DRM_XE_VM_BIND_OP_UNMAP:
2263		ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range);
2264		break;
2265	case DRM_XE_VM_BIND_OP_PREFETCH:
2266		ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range);
2267		break;
2268	case DRM_XE_VM_BIND_OP_UNMAP_ALL:
2269		xe_assert(vm->xe, bo);
2270
2271		err = xe_bo_lock(bo, true);
2272		if (err)
2273			return ERR_PTR(err);
2274
2275		vm_bo = drm_gpuvm_bo_obtain(&vm->gpuvm, obj);
2276		if (IS_ERR(vm_bo)) {
2277			xe_bo_unlock(bo);
2278			return ERR_CAST(vm_bo);
2279		}
2280
2281		ops = drm_gpuvm_bo_unmap_ops_create(vm_bo);
2282		drm_gpuvm_bo_put(vm_bo);
2283		xe_bo_unlock(bo);
2284		break;
2285	default:
2286		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2287		ops = ERR_PTR(-EINVAL);
2288	}
2289	if (IS_ERR(ops))
2290		return ops;
2291
2292	drm_gpuva_for_each_op(__op, ops) {
2293		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2294
2295		if (__op->op == DRM_GPUVA_OP_MAP) {
2296			op->map.immediate =
2297				flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE;
2298			if (flags & DRM_XE_VM_BIND_FLAG_READONLY)
2299				op->map.vma_flags |= XE_VMA_READ_ONLY;
2300			if (flags & DRM_XE_VM_BIND_FLAG_NULL)
2301				op->map.vma_flags |= DRM_GPUVA_SPARSE;
2302			if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR)
2303				op->map.vma_flags |= XE_VMA_SYSTEM_ALLOCATOR;
2304			if (flags & DRM_XE_VM_BIND_FLAG_DUMPABLE)
2305				op->map.vma_flags |= XE_VMA_DUMPABLE;
2306			if (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET)
2307				op->map.vma_flags |= XE_VMA_MADV_AUTORESET;
2308			op->map.pat_index = pat_index;
2309			op->map.invalidate_on_bind =
2310				__xe_vm_needs_clear_scratch_pages(vm, flags);
2311		} else if (__op->op == DRM_GPUVA_OP_PREFETCH) {
2312			struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
2313			struct xe_tile *tile;
2314			struct xe_svm_range *svm_range;
2315			struct drm_gpusvm_ctx ctx = {};
2316			struct drm_pagemap *dpagemap;
2317			u8 id, tile_mask = 0;
2318			u32 i;
2319
2320			if (!xe_vma_is_cpu_addr_mirror(vma)) {
2321				op->prefetch.region = prefetch_region;
2322				break;
2323			}
2324
2325			ctx.read_only = xe_vma_read_only(vma);
2326			ctx.devmem_possible = IS_DGFX(vm->xe) &&
2327					      IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
2328
2329			for_each_tile(tile, vm->xe, id)
2330				tile_mask |= 0x1 << id;
2331
2332			xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC);
2333			op->prefetch_range.ranges_count = 0;
2334			tile = NULL;
2335
2336			if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) {
2337				dpagemap = xe_vma_resolve_pagemap(vma,
2338								  xe_device_get_root_tile(vm->xe));
2339				/*
2340				 * TODO: Once multigpu support is enabled will need
2341				 * something to dereference tile from dpagemap.
2342				 */
2343				if (dpagemap)
2344					tile = xe_device_get_root_tile(vm->xe);
2345			} else if (prefetch_region) {
2346				tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] -
2347						      XE_PL_VRAM0];
2348			}
2349
2350			op->prefetch_range.tile = tile;
2351alloc_next_range:
2352			svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx);
2353
2354			if (PTR_ERR(svm_range) == -ENOENT) {
2355				u64 ret = xe_svm_find_vma_start(vm, addr, range_end, vma);
2356
2357				addr = ret == ULONG_MAX ? 0 : ret;
2358				if (addr)
2359					goto alloc_next_range;
2360				else
2361					goto print_op_label;
2362			}
2363
2364			if (IS_ERR(svm_range)) {
2365				err = PTR_ERR(svm_range);
2366				goto unwind_prefetch_ops;
2367			}
2368
2369			if (xe_svm_range_validate(vm, svm_range, tile_mask, !!tile)) {
2370				xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID");
2371				goto check_next_range;
2372			}
2373
2374			err = xa_alloc(&op->prefetch_range.range,
2375				       &i, svm_range, xa_limit_32b,
2376				       GFP_KERNEL);
2377
2378			if (err)
2379				goto unwind_prefetch_ops;
2380
2381			op->prefetch_range.ranges_count++;
2382			vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH;
2383			xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED");
2384check_next_range:
2385			if (range_end > xe_svm_range_end(svm_range) &&
2386			    xe_svm_range_end(svm_range) < xe_vma_end(vma)) {
2387				addr = xe_svm_range_end(svm_range);
2388				goto alloc_next_range;
2389			}
2390		}
2391print_op_label:
2392		print_op(vm->xe, __op);
2393	}
2394
2395	return ops;
2396
2397unwind_prefetch_ops:
2398	xe_svm_prefetch_gpuva_ops_fini(ops);
2399	drm_gpuva_ops_free(&vm->gpuvm, ops);
2400	return ERR_PTR(err);
2401}
2402
2403ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_create, ERRNO);
2404
2405static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
2406			      struct xe_vma_mem_attr *attr, unsigned int flags)
2407{
2408	struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
2409	struct xe_validation_ctx ctx;
2410	struct drm_exec exec;
2411	struct xe_vma *vma;
2412	int err = 0;
2413
2414	lockdep_assert_held_write(&vm->lock);
2415
2416	if (bo) {
2417		err = 0;
2418		xe_validation_guard(&ctx, &vm->xe->val, &exec,
2419				    (struct xe_val_flags) {.interruptible = true}, err) {
2420			if (!bo->vm) {
2421				err = drm_exec_lock_obj(&exec, xe_vm_obj(vm));
2422				drm_exec_retry_on_contention(&exec);
2423			}
2424			if (!err) {
2425				err = drm_exec_lock_obj(&exec, &bo->ttm.base);
2426				drm_exec_retry_on_contention(&exec);
2427			}
2428			if (err)
2429				return ERR_PTR(err);
2430
2431			vma = xe_vma_create(vm, bo, op->gem.offset,
2432					    op->va.addr, op->va.addr +
2433					    op->va.range - 1, attr, flags);
2434			if (IS_ERR(vma))
2435				return vma;
2436
2437			if (!bo->vm) {
2438				err = add_preempt_fences(vm, bo);
2439				if (err) {
2440					prep_vma_destroy(vm, vma, false);
2441					xe_vma_destroy(vma, NULL);
2442				}
2443			}
2444		}
2445		if (err)
2446			return ERR_PTR(err);
2447	} else {
2448		vma = xe_vma_create(vm, NULL, op->gem.offset,
2449				    op->va.addr, op->va.addr +
2450				    op->va.range - 1, attr, flags);
2451		if (IS_ERR(vma))
2452			return vma;
2453
2454		if (xe_vma_is_userptr(vma)) {
2455			err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
2456			/*
2457			 * -EBUSY has dedicated meaning that a user fence
2458			 * attached to the VMA is busy, in practice
2459			 * xe_vma_userptr_pin_pages can only fail with -EBUSY if
2460			 * we are low on memory so convert this to -ENOMEM.
2461			 */
2462			if (err == -EBUSY)
2463				err = -ENOMEM;
2464		}
2465	}
2466	if (err) {
2467		prep_vma_destroy(vm, vma, false);
2468		xe_vma_destroy_unlocked(vma);
2469		vma = ERR_PTR(err);
2470	}
2471
2472	return vma;
2473}
2474
2475static u64 xe_vma_max_pte_size(struct xe_vma *vma)
2476{
2477	if (vma->gpuva.flags & XE_VMA_PTE_1G)
2478		return SZ_1G;
2479	else if (vma->gpuva.flags & (XE_VMA_PTE_2M | XE_VMA_PTE_COMPACT))
2480		return SZ_2M;
2481	else if (vma->gpuva.flags & XE_VMA_PTE_64K)
2482		return SZ_64K;
2483	else if (vma->gpuva.flags & XE_VMA_PTE_4K)
2484		return SZ_4K;
2485
2486	return SZ_1G;	/* Uninitialized, used max size */
2487}
2488
2489static void xe_vma_set_pte_size(struct xe_vma *vma, u64 size)
2490{
2491	switch (size) {
2492	case SZ_1G:
2493		vma->gpuva.flags |= XE_VMA_PTE_1G;
2494		break;
2495	case SZ_2M:
2496		vma->gpuva.flags |= XE_VMA_PTE_2M;
2497		break;
2498	case SZ_64K:
2499		vma->gpuva.flags |= XE_VMA_PTE_64K;
2500		break;
2501	case SZ_4K:
2502		vma->gpuva.flags |= XE_VMA_PTE_4K;
2503		break;
2504	}
2505}
2506
2507static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
2508{
2509	int err = 0;
2510
2511	lockdep_assert_held_write(&vm->lock);
2512
2513	switch (op->base.op) {
2514	case DRM_GPUVA_OP_MAP:
2515		err |= xe_vm_insert_vma(vm, op->map.vma);
2516		if (!err)
2517			op->flags |= XE_VMA_OP_COMMITTED;
2518		break;
2519	case DRM_GPUVA_OP_REMAP:
2520	{
2521		u8 tile_present =
2522			gpuva_to_vma(op->base.remap.unmap->va)->tile_present;
2523
2524		prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va),
2525				 true);
2526		op->flags |= XE_VMA_OP_COMMITTED;
2527
2528		if (op->remap.prev) {
2529			err |= xe_vm_insert_vma(vm, op->remap.prev);
2530			if (!err)
2531				op->flags |= XE_VMA_OP_PREV_COMMITTED;
2532			if (!err && op->remap.skip_prev) {
2533				op->remap.prev->tile_present =
2534					tile_present;
2535			}
2536		}
2537		if (op->remap.next) {
2538			err |= xe_vm_insert_vma(vm, op->remap.next);
2539			if (!err)
2540				op->flags |= XE_VMA_OP_NEXT_COMMITTED;
2541			if (!err && op->remap.skip_next) {
2542				op->remap.next->tile_present =
2543					tile_present;
2544			}
2545		}
2546
2547		/*
2548		 * Adjust for partial unbind after removing VMA from VM. In case
2549		 * of unwind we might need to undo this later.
2550		 */
2551		if (!err) {
2552			op->base.remap.unmap->va->va.addr = op->remap.start;
2553			op->base.remap.unmap->va->va.range = op->remap.range;
2554		}
2555		break;
2556	}
2557	case DRM_GPUVA_OP_UNMAP:
2558		prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true);
2559		op->flags |= XE_VMA_OP_COMMITTED;
2560		break;
2561	case DRM_GPUVA_OP_PREFETCH:
2562		op->flags |= XE_VMA_OP_COMMITTED;
2563		break;
2564	default:
2565		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2566	}
2567
2568	return err;
2569}
2570
2571/**
2572 * xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes
2573 * @vma: Pointer to the xe_vma structure to check
2574 *
2575 * This function determines whether the given VMA (Virtual Memory Area)
2576 * has its memory attributes set to their default values. Specifically,
2577 * it checks the following conditions:
2578 *
2579 * - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED`
2580 * - `pat_index` is equal to `default_pat_index`
2581 * - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE`
2582 * - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES`
2583 *
2584 * Return: true if all attributes are at their default values, false otherwise.
2585 */
2586bool xe_vma_has_default_mem_attrs(struct xe_vma *vma)
2587{
2588	return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED &&
2589		vma->attr.pat_index ==  vma->attr.default_pat_index &&
2590		vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE &&
2591		vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES);
2592}
2593
2594static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops,
2595				   struct xe_vma_ops *vops)
2596{
2597	struct xe_device *xe = vm->xe;
2598	struct drm_gpuva_op *__op;
2599	struct xe_tile *tile;
2600	u8 id, tile_mask = 0;
2601	int err = 0;
2602
2603	lockdep_assert_held_write(&vm->lock);
2604
2605	for_each_tile(tile, vm->xe, id)
2606		tile_mask |= 0x1 << id;
2607
2608	drm_gpuva_for_each_op(__op, ops) {
2609		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2610		struct xe_vma *vma;
2611		unsigned int flags = 0;
2612
2613		INIT_LIST_HEAD(&op->link);
2614		list_add_tail(&op->link, &vops->list);
2615		op->tile_mask = tile_mask;
2616
2617		switch (op->base.op) {
2618		case DRM_GPUVA_OP_MAP:
2619		{
2620			struct xe_vma_mem_attr default_attr = {
2621				.preferred_loc = {
2622					.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE,
2623					.migration_policy = DRM_XE_MIGRATE_ALL_PAGES,
2624				},
2625				.atomic_access = DRM_XE_ATOMIC_UNDEFINED,
2626				.default_pat_index = op->map.pat_index,
2627				.pat_index = op->map.pat_index,
2628			};
2629
2630			flags |= op->map.vma_flags & XE_VMA_CREATE_MASK;
2631
2632			vma = new_vma(vm, &op->base.map, &default_attr,
2633				      flags);
2634			if (IS_ERR(vma))
2635				return PTR_ERR(vma);
2636
2637			op->map.vma = vma;
2638			if (((op->map.immediate || !xe_vm_in_fault_mode(vm)) &&
2639			     !(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) ||
2640			    op->map.invalidate_on_bind)
2641				xe_vma_ops_incr_pt_update_ops(vops,
2642							      op->tile_mask, 1);
2643			break;
2644		}
2645		case DRM_GPUVA_OP_REMAP:
2646		{
2647			struct xe_vma *old =
2648				gpuva_to_vma(op->base.remap.unmap->va);
2649			bool skip = xe_vma_is_cpu_addr_mirror(old);
2650			u64 start = xe_vma_start(old), end = xe_vma_end(old);
2651			int num_remap_ops = 0;
2652
2653			if (op->base.remap.prev)
2654				start = op->base.remap.prev->va.addr +
2655					op->base.remap.prev->va.range;
2656			if (op->base.remap.next)
2657				end = op->base.remap.next->va.addr;
2658
2659			if (xe_vma_is_cpu_addr_mirror(old) &&
2660			    xe_svm_has_mapping(vm, start, end)) {
2661				if (vops->flags & XE_VMA_OPS_FLAG_MADVISE)
2662					xe_svm_unmap_address_range(vm, start, end);
2663				else
2664					return -EBUSY;
2665			}
2666
2667			op->remap.start = xe_vma_start(old);
2668			op->remap.range = xe_vma_size(old);
2669			op->remap.old_start = op->remap.start;
2670			op->remap.old_range = op->remap.range;
2671
2672			flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK;
2673			if (op->base.remap.prev) {
2674				vma = new_vma(vm, op->base.remap.prev,
2675					      &old->attr, flags);
2676				if (IS_ERR(vma))
2677					return PTR_ERR(vma);
2678
2679				op->remap.prev = vma;
2680
2681				/*
2682				 * Userptr creates a new SG mapping so
2683				 * we must also rebind.
2684				 */
2685				op->remap.skip_prev = skip ||
2686					(!xe_vma_is_userptr(old) &&
2687					IS_ALIGNED(xe_vma_end(vma),
2688						   xe_vma_max_pte_size(old)));
2689				if (op->remap.skip_prev) {
2690					xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old));
2691					op->remap.range -=
2692						xe_vma_end(vma) -
2693						xe_vma_start(old);
2694					op->remap.start = xe_vma_end(vma);
2695					vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx",
2696					       (ULL)op->remap.start,
2697					       (ULL)op->remap.range);
2698				} else {
2699					num_remap_ops++;
2700				}
2701			}
2702
2703			if (op->base.remap.next) {
2704				vma = new_vma(vm, op->base.remap.next,
2705					      &old->attr, flags);
2706				if (IS_ERR(vma))
2707					return PTR_ERR(vma);
2708
2709				op->remap.next = vma;
2710
2711				/*
2712				 * Userptr creates a new SG mapping so
2713				 * we must also rebind.
2714				 */
2715				op->remap.skip_next = skip ||
2716					(!xe_vma_is_userptr(old) &&
2717					IS_ALIGNED(xe_vma_start(vma),
2718						   xe_vma_max_pte_size(old)));
2719				if (op->remap.skip_next) {
2720					xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old));
2721					op->remap.range -=
2722						xe_vma_end(old) -
2723						xe_vma_start(vma);
2724					vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx",
2725					       (ULL)op->remap.start,
2726					       (ULL)op->remap.range);
2727				} else {
2728					num_remap_ops++;
2729				}
2730			}
2731			if (!skip)
2732				num_remap_ops++;
2733
2734			xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, num_remap_ops);
2735			break;
2736		}
2737		case DRM_GPUVA_OP_UNMAP:
2738			vma = gpuva_to_vma(op->base.unmap.va);
2739
2740			if (xe_vma_is_cpu_addr_mirror(vma) &&
2741			    xe_svm_has_mapping(vm, xe_vma_start(vma),
2742					       xe_vma_end(vma)))
2743				return -EBUSY;
2744
2745			if (!xe_vma_is_cpu_addr_mirror(vma))
2746				xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1);
2747			break;
2748		case DRM_GPUVA_OP_PREFETCH:
2749			vma = gpuva_to_vma(op->base.prefetch.va);
2750
2751			if (xe_vma_is_userptr(vma)) {
2752				err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
2753				if (err)
2754					return err;
2755			}
2756
2757			if (xe_vma_is_cpu_addr_mirror(vma))
2758				xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask,
2759							      op->prefetch_range.ranges_count);
2760			else
2761				xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1);
2762
2763			break;
2764		default:
2765			drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2766		}
2767
2768		err = xe_vma_op_commit(vm, op);
2769		if (err)
2770			return err;
2771	}
2772
2773	return 0;
2774}
2775
2776static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
2777			     bool post_commit, bool prev_post_commit,
2778			     bool next_post_commit)
2779{
2780	lockdep_assert_held_write(&vm->lock);
2781
2782	switch (op->base.op) {
2783	case DRM_GPUVA_OP_MAP:
2784		if (op->map.vma) {
2785			prep_vma_destroy(vm, op->map.vma, post_commit);
2786			xe_vma_destroy_unlocked(op->map.vma);
2787		}
2788		break;
2789	case DRM_GPUVA_OP_UNMAP:
2790	{
2791		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
2792
2793		if (vma) {
2794			xe_svm_notifier_lock(vm);
2795			vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2796			xe_svm_notifier_unlock(vm);
2797			if (post_commit)
2798				xe_vm_insert_vma(vm, vma);
2799		}
2800		break;
2801	}
2802	case DRM_GPUVA_OP_REMAP:
2803	{
2804		struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va);
2805
2806		if (op->remap.prev) {
2807			prep_vma_destroy(vm, op->remap.prev, prev_post_commit);
2808			xe_vma_destroy_unlocked(op->remap.prev);
2809		}
2810		if (op->remap.next) {
2811			prep_vma_destroy(vm, op->remap.next, next_post_commit);
2812			xe_vma_destroy_unlocked(op->remap.next);
2813		}
2814		if (vma) {
2815			xe_svm_notifier_lock(vm);
2816			vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2817			xe_svm_notifier_unlock(vm);
2818			if (post_commit) {
2819				/*
2820				 * Restore the old va range, in case of the
2821				 * prev/next skip optimisation. Otherwise what
2822				 * we re-insert here could be smaller than the
2823				 * original range.
2824				 */
2825				op->base.remap.unmap->va->va.addr =
2826					op->remap.old_start;
2827				op->base.remap.unmap->va->va.range =
2828					op->remap.old_range;
2829				xe_vm_insert_vma(vm, vma);
2830			}
2831		}
2832		break;
2833	}
2834	case DRM_GPUVA_OP_PREFETCH:
2835		/* Nothing to do */
2836		break;
2837	default:
2838		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2839	}
2840}
2841
2842static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
2843				     struct drm_gpuva_ops **ops,
2844				     int num_ops_list)
2845{
2846	int i;
2847
2848	for (i = num_ops_list - 1; i >= 0; --i) {
2849		struct drm_gpuva_ops *__ops = ops[i];
2850		struct drm_gpuva_op *__op;
2851
2852		if (!__ops)
2853			continue;
2854
2855		drm_gpuva_for_each_op_reverse(__op, __ops) {
2856			struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2857
2858			xe_vma_op_unwind(vm, op,
2859					 op->flags & XE_VMA_OP_COMMITTED,
2860					 op->flags & XE_VMA_OP_PREV_COMMITTED,
2861					 op->flags & XE_VMA_OP_NEXT_COMMITTED);
2862		}
2863	}
2864}
2865
2866static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma,
2867				 bool res_evict, bool validate)
2868{
2869	struct xe_bo *bo = xe_vma_bo(vma);
2870	struct xe_vm *vm = xe_vma_vm(vma);
2871	int err = 0;
2872
2873	if (bo) {
2874		if (!bo->vm)
2875			err = drm_exec_lock_obj(exec, &bo->ttm.base);
2876		if (!err && validate)
2877			err = xe_bo_validate(bo, vm,
2878					     !xe_vm_in_preempt_fence_mode(vm) &&
2879					     res_evict, exec);
2880	}
2881
2882	return err;
2883}
2884
2885static int check_ufence(struct xe_vma *vma)
2886{
2887	if (vma->ufence) {
2888		struct xe_user_fence * const f = vma->ufence;
2889
2890		if (!xe_sync_ufence_get_status(f))
2891			return -EBUSY;
2892
2893		vma->ufence = NULL;
2894		xe_sync_ufence_put(f);
2895	}
2896
2897	return 0;
2898}
2899
2900static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op)
2901{
2902	bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
2903	struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
2904	struct xe_tile *tile = op->prefetch_range.tile;
2905	int err = 0;
2906
2907	struct xe_svm_range *svm_range;
2908	struct drm_gpusvm_ctx ctx = {};
2909	unsigned long i;
2910
2911	if (!xe_vma_is_cpu_addr_mirror(vma))
2912		return 0;
2913
2914	ctx.read_only = xe_vma_read_only(vma);
2915	ctx.devmem_possible = devmem_possible;
2916	ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0;
2917	ctx.device_private_page_owner = xe_svm_devm_owner(vm->xe);
2918
2919	/* TODO: Threading the migration */
2920	xa_for_each(&op->prefetch_range.range, i, svm_range) {
2921		if (!tile)
2922			xe_svm_range_migrate_to_smem(vm, svm_range);
2923
2924		if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, !!tile)) {
2925			err = xe_svm_alloc_vram(tile, svm_range, &ctx);
2926			if (err) {
2927				drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n",
2928					vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
2929				return -ENODATA;
2930			}
2931			xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM");
2932		}
2933
2934		err = xe_svm_range_get_pages(vm, svm_range, &ctx);
2935		if (err) {
2936			drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n",
2937				vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
2938			if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM)
2939				err = -ENODATA;
2940			return err;
2941		}
2942		xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE");
2943	}
2944
2945	return err;
2946}
2947
2948static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
2949			    struct xe_vma_ops *vops, struct xe_vma_op *op)
2950{
2951	int err = 0;
2952	bool res_evict;
2953
2954	/*
2955	 * We only allow evicting a BO within the VM if it is not part of an
2956	 * array of binds, as an array of binds can evict another BO within the
2957	 * bind.
2958	 */
2959	res_evict = !(vops->flags & XE_VMA_OPS_ARRAY_OF_BINDS);
2960
2961	switch (op->base.op) {
2962	case DRM_GPUVA_OP_MAP:
2963		if (!op->map.invalidate_on_bind)
2964			err = vma_lock_and_validate(exec, op->map.vma,
2965						    res_evict,
2966						    !xe_vm_in_fault_mode(vm) ||
2967						    op->map.immediate);
2968		break;
2969	case DRM_GPUVA_OP_REMAP:
2970		err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va));
2971		if (err)
2972			break;
2973
2974		err = vma_lock_and_validate(exec,
2975					    gpuva_to_vma(op->base.remap.unmap->va),
2976					    res_evict, false);
2977		if (!err && op->remap.prev)
2978			err = vma_lock_and_validate(exec, op->remap.prev,
2979						    res_evict, true);
2980		if (!err && op->remap.next)
2981			err = vma_lock_and_validate(exec, op->remap.next,
2982						    res_evict, true);
2983		break;
2984	case DRM_GPUVA_OP_UNMAP:
2985		err = check_ufence(gpuva_to_vma(op->base.unmap.va));
2986		if (err)
2987			break;
2988
2989		err = vma_lock_and_validate(exec,
2990					    gpuva_to_vma(op->base.unmap.va),
2991					    res_evict, false);
2992		break;
2993	case DRM_GPUVA_OP_PREFETCH:
2994	{
2995		struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
2996		u32 region;
2997
2998		if (!xe_vma_is_cpu_addr_mirror(vma)) {
2999			region = op->prefetch.region;
3000			xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC ||
3001				  region <= ARRAY_SIZE(region_to_mem_type));
3002		}
3003
3004		err = vma_lock_and_validate(exec,
3005					    gpuva_to_vma(op->base.prefetch.va),
3006					    res_evict, false);
3007		if (!err && !xe_vma_has_no_bo(vma))
3008			err = xe_bo_migrate(xe_vma_bo(vma),
3009					    region_to_mem_type[region],
3010					    NULL,
3011					    exec);
3012		break;
3013	}
3014	default:
3015		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
3016	}
3017
3018	return err;
3019}
3020
3021static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops)
3022{
3023	struct xe_vma_op *op;
3024	int err;
3025
3026	if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH))
3027		return 0;
3028
3029	list_for_each_entry(op, &vops->list, link) {
3030		if (op->base.op  == DRM_GPUVA_OP_PREFETCH) {
3031			err = prefetch_ranges(vm, op);
3032			if (err)
3033				return err;
3034		}
3035	}
3036
3037	return 0;
3038}
3039
3040static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec,
3041					   struct xe_vm *vm,
3042					   struct xe_vma_ops *vops)
3043{
3044	struct xe_vma_op *op;
3045	int err;
3046
3047	err = drm_exec_lock_obj(exec, xe_vm_obj(vm));
3048	if (err)
3049		return err;
3050
3051	list_for_each_entry(op, &vops->list, link) {
3052		err = op_lock_and_prep(exec, vm, vops, op);
3053		if (err)
3054			return err;
3055	}
3056
3057#ifdef TEST_VM_OPS_ERROR
3058	if (vops->inject_error &&
3059	    vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK)
3060		return -ENOSPC;
3061#endif
3062
3063	return 0;
3064}
3065
3066static void op_trace(struct xe_vma_op *op)
3067{
3068	switch (op->base.op) {
3069	case DRM_GPUVA_OP_MAP:
3070		trace_xe_vma_bind(op->map.vma);
3071		break;
3072	case DRM_GPUVA_OP_REMAP:
3073		trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va));
3074		if (op->remap.prev)
3075			trace_xe_vma_bind(op->remap.prev);
3076		if (op->remap.next)
3077			trace_xe_vma_bind(op->remap.next);
3078		break;
3079	case DRM_GPUVA_OP_UNMAP:
3080		trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va));
3081		break;
3082	case DRM_GPUVA_OP_PREFETCH:
3083		trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va));
3084		break;
3085	case DRM_GPUVA_OP_DRIVER:
3086		break;
3087	default:
3088		XE_WARN_ON("NOT POSSIBLE");
3089	}
3090}
3091
3092static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops)
3093{
3094	struct xe_vma_op *op;
3095
3096	list_for_each_entry(op, &vops->list, link)
3097		op_trace(op);
3098}
3099
3100static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops)
3101{
3102	struct xe_exec_queue *q = vops->q;
3103	struct xe_tile *tile;
3104	int number_tiles = 0;
3105	u8 id;
3106
3107	for_each_tile(tile, vm->xe, id) {
3108		if (vops->pt_update_ops[id].num_ops)
3109			++number_tiles;
3110
3111		if (vops->pt_update_ops[id].q)
3112			continue;
3113
3114		if (q) {
3115			vops->pt_update_ops[id].q = q;
3116			if (vm->pt_root[id] && !list_empty(&q->multi_gt_list))
3117				q = list_next_entry(q, multi_gt_list);
3118		} else {
3119			vops->pt_update_ops[id].q = vm->q[id];
3120		}
3121	}
3122
3123	return number_tiles;
3124}
3125
3126static struct dma_fence *ops_execute(struct xe_vm *vm,
3127				     struct xe_vma_ops *vops)
3128{
3129	struct xe_tile *tile;
3130	struct dma_fence *fence = NULL;
3131	struct dma_fence **fences = NULL;
3132	struct dma_fence_array *cf = NULL;
3133	int number_tiles = 0, current_fence = 0, n_fence = 0, err;
3134	u8 id;
3135
3136	number_tiles = vm_ops_setup_tile_args(vm, vops);
3137	if (number_tiles == 0)
3138		return ERR_PTR(-ENODATA);
3139
3140	if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) {
3141		for_each_tile(tile, vm->xe, id)
3142			++n_fence;
3143	} else {
3144		for_each_tile(tile, vm->xe, id)
3145			n_fence += (1 + XE_MAX_GT_PER_TILE);
3146	}
3147
3148	fences = kmalloc_array(n_fence, sizeof(*fences), GFP_KERNEL);
3149	if (!fences) {
3150		fence = ERR_PTR(-ENOMEM);
3151		goto err_trace;
3152	}
3153
3154	cf = dma_fence_array_alloc(n_fence);
3155	if (!cf) {
3156		fence = ERR_PTR(-ENOMEM);
3157		goto err_out;
3158	}
3159
3160	for_each_tile(tile, vm->xe, id) {
3161		if (!vops->pt_update_ops[id].num_ops)
3162			continue;
3163
3164		err = xe_pt_update_ops_prepare(tile, vops);
3165		if (err) {
3166			fence = ERR_PTR(err);
3167			goto err_out;
3168		}
3169	}
3170
3171	trace_xe_vm_ops_execute(vops);
3172
3173	for_each_tile(tile, vm->xe, id) {
3174		struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
3175		int i;
3176
3177		fence = NULL;
3178		if (!vops->pt_update_ops[id].num_ops)
3179			goto collect_fences;
3180
3181		fence = xe_pt_update_ops_run(tile, vops);
3182		if (IS_ERR(fence))
3183			goto err_out;
3184
3185collect_fences:
3186		fences[current_fence++] = fence ?: dma_fence_get_stub();
3187		if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)
3188			continue;
3189
3190		xe_migrate_job_lock(tile->migrate, q);
3191		for_each_tlb_inval(i)
3192			fences[current_fence++] =
3193				xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
3194		xe_migrate_job_unlock(tile->migrate, q);
3195	}
3196
3197	xe_assert(vm->xe, current_fence == n_fence);
3198	dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1),
3199			     1, false);
3200	fence = &cf->base;
3201
3202	for_each_tile(tile, vm->xe, id) {
3203		if (!vops->pt_update_ops[id].num_ops)
3204			continue;
3205
3206		xe_pt_update_ops_fini(tile, vops);
3207	}
3208
3209	return fence;
3210
3211err_out:
3212	for_each_tile(tile, vm->xe, id) {
3213		if (!vops->pt_update_ops[id].num_ops)
3214			continue;
3215
3216		xe_pt_update_ops_abort(tile, vops);
3217	}
3218	while (current_fence)
3219		dma_fence_put(fences[--current_fence]);
3220	kfree(fences);
3221	kfree(cf);
3222
3223err_trace:
3224	trace_xe_vm_ops_fail(vm);
3225	return fence;
3226}
3227
3228static void vma_add_ufence(struct xe_vma *vma, struct xe_user_fence *ufence)
3229{
3230	if (vma->ufence)
3231		xe_sync_ufence_put(vma->ufence);
3232	vma->ufence = __xe_sync_ufence_get(ufence);
3233}
3234
3235static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op,
3236			  struct xe_user_fence *ufence)
3237{
3238	switch (op->base.op) {
3239	case DRM_GPUVA_OP_MAP:
3240		if (!xe_vma_is_cpu_addr_mirror(op->map.vma))
3241			vma_add_ufence(op->map.vma, ufence);
3242		break;
3243	case DRM_GPUVA_OP_REMAP:
3244		if (op->remap.prev)
3245			vma_add_ufence(op->remap.prev, ufence);
3246		if (op->remap.next)
3247			vma_add_ufence(op->remap.next, ufence);
3248		break;
3249	case DRM_GPUVA_OP_UNMAP:
3250		break;
3251	case DRM_GPUVA_OP_PREFETCH:
3252		vma_add_ufence(gpuva_to_vma(op->base.prefetch.va), ufence);
3253		break;
3254	default:
3255		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
3256	}
3257}
3258
3259static void vm_bind_ioctl_ops_fini(struct xe_vm *vm, struct xe_vma_ops *vops,
3260				   struct dma_fence *fence)
3261{
3262	struct xe_user_fence *ufence;
3263	struct xe_vma_op *op;
3264	int i;
3265
3266	ufence = find_ufence_get(vops->syncs, vops->num_syncs);
3267	list_for_each_entry(op, &vops->list, link) {
3268		if (ufence)
3269			op_add_ufence(vm, op, ufence);
3270
3271		if (op->base.op == DRM_GPUVA_OP_UNMAP)
3272			xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence);
3273		else if (op->base.op == DRM_GPUVA_OP_REMAP)
3274			xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va),
3275				       fence);
3276	}
3277	if (ufence)
3278		xe_sync_ufence_put(ufence);
3279	if (fence) {
3280		for (i = 0; i < vops->num_syncs; i++)
3281			xe_sync_entry_signal(vops->syncs + i, fence);
3282	}
3283}
3284
3285static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm,
3286						   struct xe_vma_ops *vops)
3287{
3288	struct xe_validation_ctx ctx;
3289	struct drm_exec exec;
3290	struct dma_fence *fence;
3291	int err = 0;
3292
3293	lockdep_assert_held_write(&vm->lock);
3294
3295	xe_validation_guard(&ctx, &vm->xe->val, &exec,
3296			    ((struct xe_val_flags) {
3297				    .interruptible = true,
3298				    .exec_ignore_duplicates = true,
3299			    }), err) {
3300		err = vm_bind_ioctl_ops_lock_and_prep(&exec, vm, vops);
3301		drm_exec_retry_on_contention(&exec);
3302		xe_validation_retry_on_oom(&ctx, &err);
3303		if (err)
3304			return ERR_PTR(err);
3305
3306		xe_vm_set_validation_exec(vm, &exec);
3307		fence = ops_execute(vm, vops);
3308		xe_vm_set_validation_exec(vm, NULL);
3309		if (IS_ERR(fence)) {
3310			if (PTR_ERR(fence) == -ENODATA)
3311				vm_bind_ioctl_ops_fini(vm, vops, NULL);
3312			return fence;
3313		}
3314
3315		vm_bind_ioctl_ops_fini(vm, vops, fence);
3316	}
3317
3318	return err ? ERR_PTR(err) : fence;
3319}
3320ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO);
3321
3322#define SUPPORTED_FLAGS_STUB  \
3323	(DRM_XE_VM_BIND_FLAG_READONLY | \
3324	 DRM_XE_VM_BIND_FLAG_IMMEDIATE | \
3325	 DRM_XE_VM_BIND_FLAG_NULL | \
3326	 DRM_XE_VM_BIND_FLAG_DUMPABLE | \
3327	 DRM_XE_VM_BIND_FLAG_CHECK_PXP | \
3328	 DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR | \
3329	 DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET)
3330
3331#ifdef TEST_VM_OPS_ERROR
3332#define SUPPORTED_FLAGS	(SUPPORTED_FLAGS_STUB | FORCE_OP_ERROR)
3333#else
3334#define SUPPORTED_FLAGS	SUPPORTED_FLAGS_STUB
3335#endif
3336
3337#define XE_64K_PAGE_MASK 0xffffull
3338#define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)
3339
3340static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm,
3341				    struct drm_xe_vm_bind *args,
3342				    struct drm_xe_vm_bind_op **bind_ops)
3343{
3344	int err;
3345	int i;
3346
3347	if (XE_IOCTL_DBG(xe, args->pad || args->pad2) ||
3348	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
3349		return -EINVAL;
3350
3351	if (XE_IOCTL_DBG(xe, args->extensions))
3352		return -EINVAL;
3353
3354	if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
3355		return -EINVAL;
3356
3357	if (args->num_binds > 1) {
3358		u64 __user *bind_user =
3359			u64_to_user_ptr(args->vector_of_binds);
3360
3361		*bind_ops = kvmalloc_array(args->num_binds,
3362					   sizeof(struct drm_xe_vm_bind_op),
3363					   GFP_KERNEL | __GFP_ACCOUNT |
3364					   __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
3365		if (!*bind_ops)
3366			return args->num_binds > 1 ? -ENOBUFS : -ENOMEM;
3367
3368		err = copy_from_user(*bind_ops, bind_user,
3369				     sizeof(struct drm_xe_vm_bind_op) *
3370				     args->num_binds);
3371		if (XE_IOCTL_DBG(xe, err)) {
3372			err = -EFAULT;
3373			goto free_bind_ops;
3374		}
3375	} else {
3376		*bind_ops = &args->bind;
3377	}
3378
3379	for (i = 0; i < args->num_binds; ++i) {
3380		u64 range = (*bind_ops)[i].range;
3381		u64 addr = (*bind_ops)[i].addr;
3382		u32 op = (*bind_ops)[i].op;
3383		u32 flags = (*bind_ops)[i].flags;
3384		u32 obj = (*bind_ops)[i].obj;
3385		u64 obj_offset = (*bind_ops)[i].obj_offset;
3386		u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance;
3387		bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL;
3388		bool is_cpu_addr_mirror = flags &
3389			DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR;
3390		u16 pat_index = (*bind_ops)[i].pat_index;
3391		u16 coh_mode;
3392
3393		if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror &&
3394				 (!xe_vm_in_fault_mode(vm) ||
3395				 !IS_ENABLED(CONFIG_DRM_XE_GPUSVM)))) {
3396			err = -EINVAL;
3397			goto free_bind_ops;
3398		}
3399
3400		if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) {
3401			err = -EINVAL;
3402			goto free_bind_ops;
3403		}
3404
3405		pat_index = array_index_nospec(pat_index, xe->pat.n_entries);
3406		(*bind_ops)[i].pat_index = pat_index;
3407		coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
3408		if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */
3409			err = -EINVAL;
3410			goto free_bind_ops;
3411		}
3412
3413		if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) {
3414			err = -EINVAL;
3415			goto free_bind_ops;
3416		}
3417
3418		if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) ||
3419		    XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) ||
3420		    XE_IOCTL_DBG(xe, obj && (is_null || is_cpu_addr_mirror)) ||
3421		    XE_IOCTL_DBG(xe, obj_offset && (is_null ||
3422						    is_cpu_addr_mirror)) ||
3423		    XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP &&
3424				 (is_null || is_cpu_addr_mirror)) ||
3425		    XE_IOCTL_DBG(xe, !obj &&
3426				 op == DRM_XE_VM_BIND_OP_MAP &&
3427				 !is_null && !is_cpu_addr_mirror) ||
3428		    XE_IOCTL_DBG(xe, !obj &&
3429				 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) ||
3430		    XE_IOCTL_DBG(xe, addr &&
3431				 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) ||
3432		    XE_IOCTL_DBG(xe, range &&
3433				 op == DRM_XE_VM_BIND_OP_UNMAP_ALL) ||
3434		    XE_IOCTL_DBG(xe, obj &&
3435				 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) ||
3436		    XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
3437				 op == DRM_XE_VM_BIND_OP_MAP_USERPTR) ||
3438		    XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR &&
3439				 !IS_ENABLED(CONFIG_DRM_GPUSVM)) ||
3440		    XE_IOCTL_DBG(xe, obj &&
3441				 op == DRM_XE_VM_BIND_OP_PREFETCH) ||
3442		    XE_IOCTL_DBG(xe, prefetch_region &&
3443				 op != DRM_XE_VM_BIND_OP_PREFETCH) ||
3444		    XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC &&
3445				      /* Guard against undefined shift in BIT(prefetch_region) */
3446				      (prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) ||
3447				      !(BIT(prefetch_region) & xe->info.mem_region_mask)))) ||
3448		    XE_IOCTL_DBG(xe, obj &&
3449				 op == DRM_XE_VM_BIND_OP_UNMAP) ||
3450		    XE_IOCTL_DBG(xe, (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) &&
3451				 (!is_cpu_addr_mirror || op != DRM_XE_VM_BIND_OP_MAP))) {
3452			err = -EINVAL;
3453			goto free_bind_ops;
3454		}
3455
3456		if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) ||
3457		    XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) ||
3458		    XE_IOCTL_DBG(xe, range & ~PAGE_MASK) ||
3459		    XE_IOCTL_DBG(xe, !range &&
3460				 op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) {
3461			err = -EINVAL;
3462			goto free_bind_ops;
3463		}
3464	}
3465
3466	return 0;
3467
3468free_bind_ops:
3469	if (args->num_binds > 1)
3470		kvfree(*bind_ops);
3471	*bind_ops = NULL;
3472	return err;
3473}
3474
3475static int vm_bind_ioctl_signal_fences(struct xe_vm *vm,
3476				       struct xe_exec_queue *q,
3477				       struct xe_sync_entry *syncs,
3478				       int num_syncs)
3479{
3480	struct dma_fence *fence = NULL;
3481	int i, err = 0;
3482
3483	if (num_syncs) {
3484		fence = xe_sync_in_fence_get(syncs, num_syncs,
3485					     to_wait_exec_queue(vm, q), vm);
3486		if (IS_ERR(fence))
3487			return PTR_ERR(fence);
3488
3489		for (i = 0; i < num_syncs; i++)
3490			xe_sync_entry_signal(&syncs[i], fence);
3491	}
3492
3493	dma_fence_put(fence);
3494
3495	return err;
3496}
3497
3498static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
3499			    struct xe_exec_queue *q,
3500			    struct xe_sync_entry *syncs, u32 num_syncs)
3501{
3502	memset(vops, 0, sizeof(*vops));
3503	INIT_LIST_HEAD(&vops->list);
3504	vops->vm = vm;
3505	vops->q = q;
3506	vops->syncs = syncs;
3507	vops->num_syncs = num_syncs;
3508	vops->flags = 0;
3509}
3510
3511static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo,
3512					u64 addr, u64 range, u64 obj_offset,
3513					u16 pat_index, u32 op, u32 bind_flags)
3514{
3515	u16 coh_mode;
3516
3517	if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) ||
3518	    XE_IOCTL_DBG(xe, obj_offset >
3519			 xe_bo_size(bo) - range)) {
3520		return -EINVAL;
3521	}
3522
3523	/*
3524	 * Some platforms require 64k VM_BIND alignment,
3525	 * specifically those with XE_VRAM_FLAGS_NEED64K.
3526	 *
3527	 * Other platforms may have BO's set to 64k physical placement,
3528	 * but can be mapped at 4k offsets anyway. This check is only
3529	 * there for the former case.
3530	 */
3531	if ((bo->flags & XE_BO_FLAG_INTERNAL_64K) &&
3532	    (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) {
3533		if (XE_IOCTL_DBG(xe, obj_offset &
3534				 XE_64K_PAGE_MASK) ||
3535		    XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) ||
3536		    XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) {
3537			return -EINVAL;
3538		}
3539	}
3540
3541	coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
3542	if (bo->cpu_caching) {
3543		if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
3544				 bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) {
3545			return -EINVAL;
3546		}
3547	} else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) {
3548		/*
3549		 * Imported dma-buf from a different device should
3550		 * require 1way or 2way coherency since we don't know
3551		 * how it was mapped on the CPU. Just assume is it
3552		 * potentially cached on CPU side.
3553		 */
3554		return -EINVAL;
3555	}
3556
3557	/* If a BO is protected it can only be mapped if the key is still valid */
3558	if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) &&
3559	    op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL)
3560		if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0))
3561			return -ENOEXEC;
3562
3563	return 0;
3564}
3565
3566int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3567{
3568	struct xe_device *xe = to_xe_device(dev);
3569	struct xe_file *xef = to_xe_file(file);
3570	struct drm_xe_vm_bind *args = data;
3571	struct drm_xe_sync __user *syncs_user;
3572	struct xe_bo **bos = NULL;
3573	struct drm_gpuva_ops **ops = NULL;
3574	struct xe_vm *vm;
3575	struct xe_exec_queue *q = NULL;
3576	u32 num_syncs, num_ufence = 0;
3577	struct xe_sync_entry *syncs = NULL;
3578	struct drm_xe_vm_bind_op *bind_ops = NULL;
3579	struct xe_vma_ops vops;
3580	struct dma_fence *fence;
3581	int err;
3582	int i;
3583
3584	vm = xe_vm_lookup(xef, args->vm_id);
3585	if (XE_IOCTL_DBG(xe, !vm))
3586		return -EINVAL;
3587
3588	err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops);
3589	if (err)
3590		goto put_vm;
3591
3592	if (args->exec_queue_id) {
3593		q = xe_exec_queue_lookup(xef, args->exec_queue_id);
3594		if (XE_IOCTL_DBG(xe, !q)) {
3595			err = -ENOENT;
3596			goto free_bind_ops;
3597		}
3598
3599		if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) {
3600			err = -EINVAL;
3601			goto put_exec_queue;
3602		}
3603	}
3604
3605	if (XE_IOCTL_DBG(xe, q && vm != q->user_vm)) {
3606		err = -EINVAL;
3607		goto put_exec_queue;
3608	}
3609
3610	/* Ensure all UNMAPs visible */
3611	xe_svm_flush(vm);
3612
3613	err = down_write_killable(&vm->lock);
3614	if (err)
3615		goto put_exec_queue;
3616
3617	if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
3618		err = -ENOENT;
3619		goto release_vm_lock;
3620	}
3621
3622	for (i = 0; i < args->num_binds; ++i) {
3623		u64 range = bind_ops[i].range;
3624		u64 addr = bind_ops[i].addr;
3625
3626		if (XE_IOCTL_DBG(xe, range > vm->size) ||
3627		    XE_IOCTL_DBG(xe, addr > vm->size - range)) {
3628			err = -EINVAL;
3629			goto release_vm_lock;
3630		}
3631	}
3632
3633	if (args->num_binds) {
3634		bos = kvcalloc(args->num_binds, sizeof(*bos),
3635			       GFP_KERNEL | __GFP_ACCOUNT |
3636			       __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
3637		if (!bos) {
3638			err = -ENOMEM;
3639			goto release_vm_lock;
3640		}
3641
3642		ops = kvcalloc(args->num_binds, sizeof(*ops),
3643			       GFP_KERNEL | __GFP_ACCOUNT |
3644			       __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
3645		if (!ops) {
3646			err = -ENOMEM;
3647			goto free_bos;
3648		}
3649	}
3650
3651	for (i = 0; i < args->num_binds; ++i) {
3652		struct drm_gem_object *gem_obj;
3653		u64 range = bind_ops[i].range;
3654		u64 addr = bind_ops[i].addr;
3655		u32 obj = bind_ops[i].obj;
3656		u64 obj_offset = bind_ops[i].obj_offset;
3657		u16 pat_index = bind_ops[i].pat_index;
3658		u32 op = bind_ops[i].op;
3659		u32 bind_flags = bind_ops[i].flags;
3660
3661		if (!obj)
3662			continue;
3663
3664		gem_obj = drm_gem_object_lookup(file, obj);
3665		if (XE_IOCTL_DBG(xe, !gem_obj)) {
3666			err = -ENOENT;
3667			goto put_obj;
3668		}
3669		bos[i] = gem_to_xe_bo(gem_obj);
3670
3671		err = xe_vm_bind_ioctl_validate_bo(xe, bos[i], addr, range,
3672						   obj_offset, pat_index, op,
3673						   bind_flags);
3674		if (err)
3675			goto put_obj;
3676	}
3677
3678	if (args->num_syncs) {
3679		syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL);
3680		if (!syncs) {
3681			err = -ENOMEM;
3682			goto put_obj;
3683		}
3684	}
3685
3686	syncs_user = u64_to_user_ptr(args->syncs);
3687	for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
3688		struct xe_exec_queue *__q = q ?: vm->q[0];
3689
3690		err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
3691					  &syncs_user[num_syncs],
3692					  __q->ufence_syncobj,
3693					  ++__q->ufence_timeline_value,
3694					  (xe_vm_in_lr_mode(vm) ?
3695					   SYNC_PARSE_FLAG_LR_MODE : 0) |
3696					  (!args->num_binds ?
3697					   SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0));
3698		if (err)
3699			goto free_syncs;
3700
3701		if (xe_sync_is_ufence(&syncs[num_syncs]))
3702			num_ufence++;
3703	}
3704
3705	if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
3706		err = -EINVAL;
3707		goto free_syncs;
3708	}
3709
3710	if (!args->num_binds) {
3711		err = -ENODATA;
3712		goto free_syncs;
3713	}
3714
3715	xe_vma_ops_init(&vops, vm, q, syncs, num_syncs);
3716	if (args->num_binds > 1)
3717		vops.flags |= XE_VMA_OPS_ARRAY_OF_BINDS;
3718	for (i = 0; i < args->num_binds; ++i) {
3719		u64 range = bind_ops[i].range;
3720		u64 addr = bind_ops[i].addr;
3721		u32 op = bind_ops[i].op;
3722		u32 flags = bind_ops[i].flags;
3723		u64 obj_offset = bind_ops[i].obj_offset;
3724		u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance;
3725		u16 pat_index = bind_ops[i].pat_index;
3726
3727		ops[i] = vm_bind_ioctl_ops_create(vm, &vops, bos[i], obj_offset,
3728						  addr, range, op, flags,
3729						  prefetch_region, pat_index);
3730		if (IS_ERR(ops[i])) {
3731			err = PTR_ERR(ops[i]);
3732			ops[i] = NULL;
3733			goto unwind_ops;
3734		}
3735
3736		err = vm_bind_ioctl_ops_parse(vm, ops[i], &vops);
3737		if (err)
3738			goto unwind_ops;
3739
3740#ifdef TEST_VM_OPS_ERROR
3741		if (flags & FORCE_OP_ERROR) {
3742			vops.inject_error = true;
3743			vm->xe->vm_inject_error_position =
3744				(vm->xe->vm_inject_error_position + 1) %
3745				FORCE_OP_ERROR_COUNT;
3746		}
3747#endif
3748	}
3749
3750	/* Nothing to do */
3751	if (list_empty(&vops.list)) {
3752		err = -ENODATA;
3753		goto unwind_ops;
3754	}
3755
3756	err = xe_vma_ops_alloc(&vops, args->num_binds > 1);
3757	if (err)
3758		goto unwind_ops;
3759
3760	err = vm_bind_ioctl_ops_prefetch_ranges(vm, &vops);
3761	if (err)
3762		goto unwind_ops;
3763
3764	fence = vm_bind_ioctl_ops_execute(vm, &vops);
3765	if (IS_ERR(fence))
3766		err = PTR_ERR(fence);
3767	else
3768		dma_fence_put(fence);
3769
3770unwind_ops:
3771	if (err && err != -ENODATA)
3772		vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
3773	xe_vma_ops_fini(&vops);
3774	for (i = args->num_binds - 1; i >= 0; --i)
3775		if (ops[i])
3776			drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
3777free_syncs:
3778	if (err == -ENODATA)
3779		err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs);
3780	while (num_syncs--)
3781		xe_sync_entry_cleanup(&syncs[num_syncs]);
3782
3783	kfree(syncs);
3784put_obj:
3785	for (i = 0; i < args->num_binds; ++i)
3786		xe_bo_put(bos[i]);
3787
3788	kvfree(ops);
3789free_bos:
3790	kvfree(bos);
3791release_vm_lock:
3792	up_write(&vm->lock);
3793put_exec_queue:
3794	if (q)
3795		xe_exec_queue_put(q);
3796free_bind_ops:
3797	if (args->num_binds > 1)
3798		kvfree(bind_ops);
3799put_vm:
3800	xe_vm_put(vm);
3801	return err;
3802}
3803
3804/**
3805 * xe_vm_bind_kernel_bo - bind a kernel BO to a VM
3806 * @vm: VM to bind the BO to
3807 * @bo: BO to bind
3808 * @q: exec queue to use for the bind (optional)
3809 * @addr: address at which to bind the BO
3810 * @cache_lvl: PAT cache level to use
3811 *
3812 * Execute a VM bind map operation on a kernel-owned BO to bind it into a
3813 * kernel-owned VM.
3814 *
3815 * Returns a dma_fence to track the binding completion if the job to do so was
3816 * successfully submitted, an error pointer otherwise.
3817 */
3818struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo,
3819				       struct xe_exec_queue *q, u64 addr,
3820				       enum xe_cache_level cache_lvl)
3821{
3822	struct xe_vma_ops vops;
3823	struct drm_gpuva_ops *ops = NULL;
3824	struct dma_fence *fence;
3825	int err;
3826
3827	xe_bo_get(bo);
3828	xe_vm_get(vm);
3829	if (q)
3830		xe_exec_queue_get(q);
3831
3832	down_write(&vm->lock);
3833
3834	xe_vma_ops_init(&vops, vm, q, NULL, 0);
3835
3836	ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo),
3837				       DRM_XE_VM_BIND_OP_MAP, 0, 0,
3838				       vm->xe->pat.idx[cache_lvl]);
3839	if (IS_ERR(ops)) {
3840		err = PTR_ERR(ops);
3841		goto release_vm_lock;
3842	}
3843
3844	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
3845	if (err)
3846		goto release_vm_lock;
3847
3848	xe_assert(vm->xe, !list_empty(&vops.list));
3849
3850	err = xe_vma_ops_alloc(&vops, false);
3851	if (err)
3852		goto unwind_ops;
3853
3854	fence = vm_bind_ioctl_ops_execute(vm, &vops);
3855	if (IS_ERR(fence))
3856		err = PTR_ERR(fence);
3857
3858unwind_ops:
3859	if (err && err != -ENODATA)
3860		vm_bind_ioctl_ops_unwind(vm, &ops, 1);
3861
3862	xe_vma_ops_fini(&vops);
3863	drm_gpuva_ops_free(&vm->gpuvm, ops);
3864
3865release_vm_lock:
3866	up_write(&vm->lock);
3867
3868	if (q)
3869		xe_exec_queue_put(q);
3870	xe_vm_put(vm);
3871	xe_bo_put(bo);
3872
3873	if (err)
3874		fence = ERR_PTR(err);
3875
3876	return fence;
3877}
3878
3879/**
3880 * xe_vm_lock() - Lock the vm's dma_resv object
3881 * @vm: The struct xe_vm whose lock is to be locked
3882 * @intr: Whether to perform any wait interruptible
3883 *
3884 * Return: 0 on success, -EINTR if @intr is true and the wait for a
3885 * contended lock was interrupted. If @intr is false, the function
3886 * always returns 0.
3887 */
3888int xe_vm_lock(struct xe_vm *vm, bool intr)
3889{
3890	int ret;
3891
3892	if (intr)
3893		ret = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
3894	else
3895		ret = dma_resv_lock(xe_vm_resv(vm), NULL);
3896
3897	return ret;
3898}
3899
3900/**
3901 * xe_vm_unlock() - Unlock the vm's dma_resv object
3902 * @vm: The struct xe_vm whose lock is to be released.
3903 *
3904 * Unlock a buffer object lock that was locked by xe_vm_lock().
3905 */
3906void xe_vm_unlock(struct xe_vm *vm)
3907{
3908	dma_resv_unlock(xe_vm_resv(vm));
3909}
3910
3911/**
3912 * xe_vm_range_tilemask_tlb_inval - Issue a TLB invalidation on this tilemask for an
3913 * address range
3914 * @vm: The VM
3915 * @start: start address
3916 * @end: end address
3917 * @tile_mask: mask for which gt's issue tlb invalidation
3918 *
3919 * Issue a range based TLB invalidation for gt's in tilemask
3920 *
3921 * Returns 0 for success, negative error code otherwise.
3922 */
3923int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start,
3924				   u64 end, u8 tile_mask)
3925{
3926	struct xe_tlb_inval_fence
3927		fence[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
3928	struct xe_tile *tile;
3929	u32 fence_id = 0;
3930	u8 id;
3931	int err;
3932
3933	if (!tile_mask)
3934		return 0;
3935
3936	for_each_tile(tile, vm->xe, id) {
3937		if (!(tile_mask & BIT(id)))
3938			continue;
3939
3940		xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval,
3941					&fence[fence_id], true);
3942
3943		err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
3944					 &fence[fence_id], start, end,
3945					 vm->usm.asid);
3946		if (err)
3947			goto wait;
3948		++fence_id;
3949
3950		if (!tile->media_gt)
3951			continue;
3952
3953		xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval,
3954					&fence[fence_id], true);
3955
3956		err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
3957					 &fence[fence_id], start, end,
3958					 vm->usm.asid);
3959		if (err)
3960			goto wait;
3961		++fence_id;
3962	}
3963
3964wait:
3965	for (id = 0; id < fence_id; ++id)
3966		xe_tlb_inval_fence_wait(&fence[id]);
3967
3968	return err;
3969}
3970
3971/**
3972 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
3973 * @vma: VMA to invalidate
3974 *
3975 * Walks a list of page tables leaves which it memset the entries owned by this
3976 * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
3977 * complete.
3978 *
3979 * Returns 0 for success, negative error code otherwise.
3980 */
3981int xe_vm_invalidate_vma(struct xe_vma *vma)
3982{
3983	struct xe_device *xe = xe_vma_vm(vma)->xe;
3984	struct xe_vm *vm = xe_vma_vm(vma);
3985	struct xe_tile *tile;
3986	u8 tile_mask = 0;
3987	int ret = 0;
3988	u8 id;
3989
3990	xe_assert(xe, !xe_vma_is_null(vma));
3991	xe_assert(xe, !xe_vma_is_cpu_addr_mirror(vma));
3992	trace_xe_vma_invalidate(vma);
3993
3994	vm_dbg(&vm->xe->drm,
3995	       "INVALIDATE: addr=0x%016llx, range=0x%016llx",
3996		xe_vma_start(vma), xe_vma_size(vma));
3997
3998	/*
3999	 * Check that we don't race with page-table updates, tile_invalidated
4000	 * update is safe
4001	 */
4002	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
4003		if (xe_vma_is_userptr(vma)) {
4004			lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
4005				       (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
4006					lockdep_is_held(&xe_vm_resv(vm)->lock.base)));
4007
4008			WARN_ON_ONCE(!mmu_interval_check_retry
4009				     (&to_userptr_vma(vma)->userptr.notifier,
4010				      to_userptr_vma(vma)->userptr.pages.notifier_seq));
4011			WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(vm),
4012							     DMA_RESV_USAGE_BOOKKEEP));
4013
4014		} else {
4015			xe_bo_assert_held(xe_vma_bo(vma));
4016		}
4017	}
4018
4019	for_each_tile(tile, xe, id)
4020		if (xe_pt_zap_ptes(tile, vma))
4021			tile_mask |= BIT(id);
4022
4023	xe_device_wmb(xe);
4024
4025	ret = xe_vm_range_tilemask_tlb_inval(xe_vma_vm(vma), xe_vma_start(vma),
4026					     xe_vma_end(vma), tile_mask);
4027
4028	/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
4029	WRITE_ONCE(vma->tile_invalidated, vma->tile_mask);
4030
4031	return ret;
4032}
4033
4034int xe_vm_validate_protected(struct xe_vm *vm)
4035{
4036	struct drm_gpuva *gpuva;
4037	int err = 0;
4038
4039	if (!vm)
4040		return -ENODEV;
4041
4042	mutex_lock(&vm->snap_mutex);
4043
4044	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
4045		struct xe_vma *vma = gpuva_to_vma(gpuva);
4046		struct xe_bo *bo = vma->gpuva.gem.obj ?
4047			gem_to_xe_bo(vma->gpuva.gem.obj) : NULL;
4048
4049		if (!bo)
4050			continue;
4051
4052		if (xe_bo_is_protected(bo)) {
4053			err = xe_pxp_bo_key_check(vm->xe->pxp, bo);
4054			if (err)
4055				break;
4056		}
4057	}
4058
4059	mutex_unlock(&vm->snap_mutex);
4060	return err;
4061}
4062
4063struct xe_vm_snapshot {
4064	unsigned long num_snaps;
4065	struct {
4066		u64 ofs, bo_ofs;
4067		unsigned long len;
4068		struct xe_bo *bo;
4069		void *data;
4070		struct mm_struct *mm;
4071	} snap[];
4072};
4073
4074struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm)
4075{
4076	unsigned long num_snaps = 0, i;
4077	struct xe_vm_snapshot *snap = NULL;
4078	struct drm_gpuva *gpuva;
4079
4080	if (!vm)
4081		return NULL;
4082
4083	mutex_lock(&vm->snap_mutex);
4084	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
4085		if (gpuva->flags & XE_VMA_DUMPABLE)
4086			num_snaps++;
4087	}
4088
4089	if (num_snaps)
4090		snap = kvzalloc(offsetof(struct xe_vm_snapshot, snap[num_snaps]), GFP_NOWAIT);
4091	if (!snap) {
4092		snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV);
4093		goto out_unlock;
4094	}
4095
4096	snap->num_snaps = num_snaps;
4097	i = 0;
4098	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
4099		struct xe_vma *vma = gpuva_to_vma(gpuva);
4100		struct xe_bo *bo = vma->gpuva.gem.obj ?
4101			gem_to_xe_bo(vma->gpuva.gem.obj) : NULL;
4102
4103		if (!(gpuva->flags & XE_VMA_DUMPABLE))
4104			continue;
4105
4106		snap->snap[i].ofs = xe_vma_start(vma);
4107		snap->snap[i].len = xe_vma_size(vma);
4108		if (bo) {
4109			snap->snap[i].bo = xe_bo_get(bo);
4110			snap->snap[i].bo_ofs = xe_vma_bo_offset(vma);
4111		} else if (xe_vma_is_userptr(vma)) {
4112			struct mm_struct *mm =
4113				to_userptr_vma(vma)->userptr.notifier.mm;
4114
4115			if (mmget_not_zero(mm))
4116				snap->snap[i].mm = mm;
4117			else
4118				snap->snap[i].data = ERR_PTR(-EFAULT);
4119
4120			snap->snap[i].bo_ofs = xe_vma_userptr(vma);
4121		} else {
4122			snap->snap[i].data = ERR_PTR(-ENOENT);
4123		}
4124		i++;
4125	}
4126
4127out_unlock:
4128	mutex_unlock(&vm->snap_mutex);
4129	return snap;
4130}
4131
4132void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap)
4133{
4134	if (IS_ERR_OR_NULL(snap))
4135		return;
4136
4137	for (int i = 0; i < snap->num_snaps; i++) {
4138		struct xe_bo *bo = snap->snap[i].bo;
4139		int err;
4140
4141		if (IS_ERR(snap->snap[i].data))
4142			continue;
4143
4144		snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER);
4145		if (!snap->snap[i].data) {
4146			snap->snap[i].data = ERR_PTR(-ENOMEM);
4147			goto cleanup_bo;
4148		}
4149
4150		if (bo) {
4151			err = xe_bo_read(bo, snap->snap[i].bo_ofs,
4152					 snap->snap[i].data, snap->snap[i].len);
4153		} else {
4154			void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs;
4155
4156			kthread_use_mm(snap->snap[i].mm);
4157			if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len))
4158				err = 0;
4159			else
4160				err = -EFAULT;
4161			kthread_unuse_mm(snap->snap[i].mm);
4162
4163			mmput(snap->snap[i].mm);
4164			snap->snap[i].mm = NULL;
4165		}
4166
4167		if (err) {
4168			kvfree(snap->snap[i].data);
4169			snap->snap[i].data = ERR_PTR(err);
4170		}
4171
4172cleanup_bo:
4173		xe_bo_put(bo);
4174		snap->snap[i].bo = NULL;
4175	}
4176}
4177
4178void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p)
4179{
4180	unsigned long i, j;
4181
4182	if (IS_ERR_OR_NULL(snap)) {
4183		drm_printf(p, "[0].error: %li\n", PTR_ERR(snap));
4184		return;
4185	}
4186
4187	for (i = 0; i < snap->num_snaps; i++) {
4188		drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);
4189
4190		if (IS_ERR(snap->snap[i].data)) {
4191			drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,
4192				   PTR_ERR(snap->snap[i].data));
4193			continue;
4194		}
4195
4196		drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);
4197
4198		for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {
4199			u32 *val = snap->snap[i].data + j;
4200			char dumped[ASCII85_BUFSZ];
4201
4202			drm_puts(p, ascii85_encode(*val, dumped));
4203		}
4204
4205		drm_puts(p, "\n");
4206
4207		if (drm_coredump_printer_is_full(p))
4208			return;
4209	}
4210}
4211
4212void xe_vm_snapshot_free(struct xe_vm_snapshot *snap)
4213{
4214	unsigned long i;
4215
4216	if (IS_ERR_OR_NULL(snap))
4217		return;
4218
4219	for (i = 0; i < snap->num_snaps; i++) {
4220		if (!IS_ERR(snap->snap[i].data))
4221			kvfree(snap->snap[i].data);
4222		xe_bo_put(snap->snap[i].bo);
4223		if (snap->snap[i].mm)
4224			mmput(snap->snap[i].mm);
4225	}
4226	kvfree(snap);
4227}
4228
4229/**
4230 * xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations
4231 * @xe: Pointer to the Xe device structure
4232 * @vma: Pointer to the virtual memory area (VMA) structure
4233 * @is_atomic: In pagefault path and atomic operation
4234 *
4235 * This function determines whether the given VMA needs to be migrated to
4236 * VRAM in order to do atomic GPU operation.
4237 *
4238 * Return:
4239 *   1        - Migration to VRAM is required
4240 *   0        - Migration is not required
4241 *   -EACCES  - Invalid access for atomic memory attr
4242 *
4243 */
4244int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic)
4245{
4246	u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access :
4247					     vma->attr.atomic_access;
4248
4249	if (!IS_DGFX(xe) || !is_atomic)
4250		return false;
4251
4252	/*
4253	 * NOTE: The checks implemented here are platform-specific. For
4254	 * instance, on a device supporting CXL atomics, these would ideally
4255	 * work universally without additional handling.
4256	 */
4257	switch (atomic_access) {
4258	case DRM_XE_ATOMIC_DEVICE:
4259		return !xe->info.has_device_atomics_on_smem;
4260
4261	case DRM_XE_ATOMIC_CPU:
4262		return -EACCES;
4263
4264	case DRM_XE_ATOMIC_UNDEFINED:
4265	case DRM_XE_ATOMIC_GLOBAL:
4266	default:
4267		return 1;
4268	}
4269}
4270
4271static int xe_vm_alloc_vma(struct xe_vm *vm,
4272			   struct drm_gpuvm_map_req *map_req,
4273			   bool is_madvise)
4274{
4275	struct xe_vma_ops vops;
4276	struct drm_gpuva_ops *ops = NULL;
4277	struct drm_gpuva_op *__op;
4278	unsigned int vma_flags = 0;
4279	bool remap_op = false;
4280	struct xe_vma_mem_attr tmp_attr;
4281	u16 default_pat;
4282	int err;
4283
4284	lockdep_assert_held_write(&vm->lock);
4285
4286	if (is_madvise)
4287		ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req);
4288	else
4289		ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req);
4290
4291	if (IS_ERR(ops))
4292		return PTR_ERR(ops);
4293
4294	if (list_empty(&ops->list)) {
4295		err = 0;
4296		goto free_ops;
4297	}
4298
4299	drm_gpuva_for_each_op(__op, ops) {
4300		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
4301		struct xe_vma *vma = NULL;
4302
4303		if (!is_madvise) {
4304			if (__op->op == DRM_GPUVA_OP_UNMAP) {
4305				vma = gpuva_to_vma(op->base.unmap.va);
4306				XE_WARN_ON(!xe_vma_has_default_mem_attrs(vma));
4307				default_pat = vma->attr.default_pat_index;
4308				vma_flags = vma->gpuva.flags;
4309			}
4310
4311			if (__op->op == DRM_GPUVA_OP_REMAP) {
4312				vma = gpuva_to_vma(op->base.remap.unmap->va);
4313				default_pat = vma->attr.default_pat_index;
4314				vma_flags = vma->gpuva.flags;
4315			}
4316
4317			if (__op->op == DRM_GPUVA_OP_MAP) {
4318				op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK;
4319				op->map.pat_index = default_pat;
4320			}
4321		} else {
4322			if (__op->op == DRM_GPUVA_OP_REMAP) {
4323				vma = gpuva_to_vma(op->base.remap.unmap->va);
4324				xe_assert(vm->xe, !remap_op);
4325				xe_assert(vm->xe, xe_vma_has_no_bo(vma));
4326				remap_op = true;
4327				vma_flags = vma->gpuva.flags;
4328			}
4329
4330			if (__op->op == DRM_GPUVA_OP_MAP) {
4331				xe_assert(vm->xe, remap_op);
4332				remap_op = false;
4333				/*
4334				 * In case of madvise ops DRM_GPUVA_OP_MAP is
4335				 * always after DRM_GPUVA_OP_REMAP, so ensure
4336				 * to propagate the flags from the vma we're
4337				 * unmapping.
4338				 */
4339				op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK;
4340			}
4341		}
4342		print_op(vm->xe, __op);
4343	}
4344
4345	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
4346
4347	if (is_madvise)
4348		vops.flags |= XE_VMA_OPS_FLAG_MADVISE;
4349
4350	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
4351	if (err)
4352		goto unwind_ops;
4353
4354	xe_vm_lock(vm, false);
4355
4356	drm_gpuva_for_each_op(__op, ops) {
4357		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
4358		struct xe_vma *vma;
4359
4360		if (__op->op == DRM_GPUVA_OP_UNMAP) {
4361			vma = gpuva_to_vma(op->base.unmap.va);
4362			/* There should be no unmap for madvise */
4363			if (is_madvise)
4364				XE_WARN_ON("UNEXPECTED UNMAP");
4365
4366			xe_vma_destroy(vma, NULL);
4367		} else if (__op->op == DRM_GPUVA_OP_REMAP) {
4368			vma = gpuva_to_vma(op->base.remap.unmap->va);
4369			/* In case of madvise ops Store attributes for REMAP UNMAPPED
4370			 * VMA, so they can be assigned to newly MAP created vma.
4371			 */
4372			if (is_madvise)
4373				tmp_attr = vma->attr;
4374
4375			xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL);
4376		} else if (__op->op == DRM_GPUVA_OP_MAP) {
4377			vma = op->map.vma;
4378			/* In case of madvise call, MAP will always be followed by REMAP.
4379			 * Therefore temp_attr will always have sane values, making it safe to
4380			 * copy them to new vma.
4381			 */
4382			if (is_madvise)
4383				vma->attr = tmp_attr;
4384		}
4385	}
4386
4387	xe_vm_unlock(vm);
4388	drm_gpuva_ops_free(&vm->gpuvm, ops);
4389	return 0;
4390
4391unwind_ops:
4392	vm_bind_ioctl_ops_unwind(vm, &ops, 1);
4393free_ops:
4394	drm_gpuva_ops_free(&vm->gpuvm, ops);
4395	return err;
4396}
4397
4398/**
4399 * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops
4400 * @vm: Pointer to the xe_vm structure
4401 * @start: Starting input address
4402 * @range: Size of the input range
4403 *
4404 * This function splits existing vma to create new vma for user provided input range
4405 *
4406 * Return: 0 if success
4407 */
4408int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range)
4409{
4410	struct drm_gpuvm_map_req map_req = {
4411		.map.va.addr = start,
4412		.map.va.range = range,
4413	};
4414
4415	lockdep_assert_held_write(&vm->lock);
4416
4417	vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range);
4418
4419	return xe_vm_alloc_vma(vm, &map_req, true);
4420}
4421
4422/**
4423 * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
4424 * @vm: Pointer to the xe_vm structure
4425 * @start: Starting input address
4426 * @range: Size of the input range
4427 *
4428 * This function splits/merges existing vma to create new vma for user provided input range
4429 *
4430 * Return: 0 if success
4431 */
4432int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t range)
4433{
4434	struct drm_gpuvm_map_req map_req = {
4435		.map.va.addr = start,
4436		.map.va.range = range,
4437	};
4438
4439	lockdep_assert_held_write(&vm->lock);
4440
4441	vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx",
4442	       start, range);
4443
4444	return xe_vm_alloc_vma(vm, &map_req, false);
4445}