xe_exec_queue.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2021 Intel Corporation
   4 */
   5
   6#include "xe_exec_queue.h"
   7
   8#include <linux/nospec.h>
   9
  10#include <drm/drm_device.h>
  11#include <drm/drm_drv.h>
  12#include <drm/drm_file.h>
  13#include <drm/drm_syncobj.h>
  14#include <uapi/drm/xe_drm.h>
  15
  16#include "xe_dep_scheduler.h"
  17#include "xe_device.h"
  18#include "xe_gt.h"
  19#include "xe_gt_sriov_vf.h"
  20#include "xe_hw_engine_class_sysfs.h"
  21#include "xe_hw_engine_group.h"
  22#include "xe_hw_fence.h"
  23#include "xe_irq.h"
  24#include "xe_lrc.h"
  25#include "xe_macros.h"
  26#include "xe_migrate.h"
  27#include "xe_pm.h"
  28#include "xe_ring_ops_types.h"
  29#include "xe_trace.h"
  30#include "xe_vm.h"
  31#include "xe_pxp.h"
  32
  33/**
  34 * DOC: Execution Queue
  35 *
  36 * An Execution queue is an interface for the HW context of execution.
  37 * The user creates an execution queue, submits the GPU jobs through those
  38 * queues and in the end destroys them.
  39 *
  40 * Execution queues can also be created by XeKMD itself for driver internal
  41 * operations like object migration etc.
  42 *
  43 * An execution queue is associated with a specified HW engine or a group of
  44 * engines (belonging to the same tile and engine class) and any GPU job
  45 * submitted on the queue will be run on one of these engines.
  46 *
  47 * An execution queue is tied to an address space (VM). It holds a reference
  48 * of the associated VM and the underlying Logical Ring Context/s (LRC/s)
  49 * until the queue is destroyed.
  50 *
  51 * The execution queue sits on top of the submission backend. It opaquely
  52 * handles the GuC and Execlist backends whichever the platform uses, and
  53 * the ring operations the different engine classes support.
  54 */
  55
  56enum xe_exec_queue_sched_prop {
  57	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
  58	XE_EXEC_QUEUE_TIMESLICE = 1,
  59	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,
  60	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,
  61};
  62
  63static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
  64				      u64 extensions, int ext_number);
  65
  66static void __xe_exec_queue_free(struct xe_exec_queue *q)
  67{
  68	int i;
  69
  70	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i)
  71		if (q->tlb_inval[i].dep_scheduler)
  72			xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);
  73
  74	if (xe_exec_queue_uses_pxp(q))
  75		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
  76	if (q->vm)
  77		xe_vm_put(q->vm);
  78
  79	if (q->xef)
  80		xe_file_put(q->xef);
  81
  82	kfree(q);
  83}
  84
  85static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q)
  86{
  87	struct xe_tile *tile = gt_to_tile(q->gt);
  88	int i;
  89
  90	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) {
  91		struct xe_dep_scheduler *dep_scheduler;
  92		struct xe_gt *gt;
  93		struct workqueue_struct *wq;
  94
  95		if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
  96			gt = tile->primary_gt;
  97		else
  98			gt = tile->media_gt;
  99
 100		if (!gt)
 101			continue;
 102
 103		wq = gt->tlb_inval.job_wq;
 104
 105#define MAX_TLB_INVAL_JOBS	16	/* Picking a reasonable value */
 106		dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
 107							MAX_TLB_INVAL_JOBS);
 108		if (IS_ERR(dep_scheduler))
 109			return PTR_ERR(dep_scheduler);
 110
 111		q->tlb_inval[i].dep_scheduler = dep_scheduler;
 112	}
 113#undef MAX_TLB_INVAL_JOBS
 114
 115	return 0;
 116}
 117
 118static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 119						   struct xe_vm *vm,
 120						   u32 logical_mask,
 121						   u16 width, struct xe_hw_engine *hwe,
 122						   u32 flags, u64 extensions)
 123{
 124	struct xe_exec_queue *q;
 125	struct xe_gt *gt = hwe->gt;
 126	int err;
 127
 128	/* only kernel queues can be permanent */
 129	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));
 130
 131	q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL);
 132	if (!q)
 133		return ERR_PTR(-ENOMEM);
 134
 135	kref_init(&q->refcount);
 136	q->flags = flags;
 137	q->hwe = hwe;
 138	q->gt = gt;
 139	q->class = hwe->class;
 140	q->width = width;
 141	q->msix_vec = XE_IRQ_DEFAULT_MSIX;
 142	q->logical_mask = logical_mask;
 143	q->fence_irq = &gt->fence_irq[hwe->class];
 144	q->ring_ops = gt->ring_ops[hwe->class];
 145	q->ops = gt->exec_queue_ops;
 146	INIT_LIST_HEAD(&q->lr.link);
 147	INIT_LIST_HEAD(&q->multi_gt_link);
 148	INIT_LIST_HEAD(&q->hw_engine_group_link);
 149	INIT_LIST_HEAD(&q->pxp.link);
 150
 151	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
 152	q->sched_props.preempt_timeout_us =
 153				hwe->eclass->sched_props.preempt_timeout_us;
 154	q->sched_props.job_timeout_ms =
 155				hwe->eclass->sched_props.job_timeout_ms;
 156	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&
 157	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)
 158		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;
 159	else
 160		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;
 161
 162	if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) {
 163		err = alloc_dep_schedulers(xe, q);
 164		if (err) {
 165			__xe_exec_queue_free(q);
 166			return ERR_PTR(err);
 167		}
 168	}
 169
 170	if (vm)
 171		q->vm = xe_vm_get(vm);
 172
 173	if (extensions) {
 174		/*
 175		 * may set q->usm, must come before xe_lrc_create(),
 176		 * may overwrite q->sched_props, must come before q->ops->init()
 177		 */
 178		err = exec_queue_user_extensions(xe, q, extensions, 0);
 179		if (err) {
 180			__xe_exec_queue_free(q);
 181			return ERR_PTR(err);
 182		}
 183	}
 184
 185	return q;
 186}
 187
 188static void __xe_exec_queue_fini(struct xe_exec_queue *q)
 189{
 190	int i;
 191
 192	q->ops->fini(q);
 193
 194	for (i = 0; i < q->width; ++i)
 195		xe_lrc_put(q->lrc[i]);
 196}
 197
 198static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
 199{
 200	int i, err;
 201	u32 flags = 0;
 202
 203	/*
 204	 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no
 205	 * other workload can use the EUs at the same time). On MTL this is done
 206	 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there
 207	 * is a dedicated bit for it.
 208	 */
 209	if (xe_exec_queue_uses_pxp(q) &&
 210	    (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) {
 211		if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20)
 212			flags |= XE_LRC_CREATE_PXP;
 213		else
 214			flags |= XE_LRC_CREATE_RUNALONE;
 215	}
 216
 217	if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL))
 218		flags |= XE_LRC_CREATE_USER_CTX;
 219
 220	err = q->ops->init(q);
 221	if (err)
 222		return err;
 223
 224	/*
 225	 * This must occur after q->ops->init to avoid race conditions during VF
 226	 * post-migration recovery, as the fixups for the LRC GGTT addresses
 227	 * depend on the queue being present in the backend tracking structure.
 228	 *
 229	 * In addition to above, we must wait on inflight GGTT changes to avoid
 230	 * writing out stale values here. Such wait provides a solid solution
 231	 * (without a race) only if the function can detect migration instantly
 232	 * from the moment vCPU resumes execution.
 233	 */
 234	for (i = 0; i < q->width; ++i) {
 235		struct xe_lrc *lrc;
 236
 237		xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
 238		lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(),
 239				    q->msix_vec, flags);
 240		if (IS_ERR(lrc)) {
 241			err = PTR_ERR(lrc);
 242			goto err_lrc;
 243		}
 244
 245		/* Pairs with READ_ONCE to xe_exec_queue_contexts_hwsp_rebase */
 246		WRITE_ONCE(q->lrc[i], lrc);
 247	}
 248
 249	return 0;
 250
 251err_lrc:
 252	__xe_exec_queue_fini(q);
 253	return err;
 254}
 255
 256struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
 257					   u32 logical_mask, u16 width,
 258					   struct xe_hw_engine *hwe, u32 flags,
 259					   u64 extensions)
 260{
 261	struct xe_exec_queue *q;
 262	int err;
 263
 264	/* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */
 265	xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0)));
 266
 267	q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,
 268				  extensions);
 269	if (IS_ERR(q))
 270		return q;
 271
 272	err = __xe_exec_queue_init(q, flags);
 273	if (err)
 274		goto err_post_alloc;
 275
 276	/*
 277	 * We can only add the queue to the PXP list after the init is complete,
 278	 * because the PXP termination can call exec_queue_kill and that will
 279	 * go bad if the queue is only half-initialized. This means that we
 280	 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc
 281	 * and we need to do it here instead.
 282	 */
 283	if (xe_exec_queue_uses_pxp(q)) {
 284		err = xe_pxp_exec_queue_add(xe->pxp, q);
 285		if (err)
 286			goto err_post_init;
 287	}
 288
 289	return q;
 290
 291err_post_init:
 292	__xe_exec_queue_fini(q);
 293err_post_alloc:
 294	__xe_exec_queue_free(q);
 295	return ERR_PTR(err);
 296}
 297ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO);
 298
 299struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,
 300						 struct xe_vm *vm,
 301						 enum xe_engine_class class,
 302						 u32 flags, u64 extensions)
 303{
 304	struct xe_hw_engine *hwe, *hwe0 = NULL;
 305	enum xe_hw_engine_id id;
 306	u32 logical_mask = 0;
 307
 308	for_each_hw_engine(hwe, gt, id) {
 309		if (xe_hw_engine_is_reserved(hwe))
 310			continue;
 311
 312		if (hwe->class == class) {
 313			logical_mask |= BIT(hwe->logical_instance);
 314			if (!hwe0)
 315				hwe0 = hwe;
 316		}
 317	}
 318
 319	if (!logical_mask)
 320		return ERR_PTR(-ENODEV);
 321
 322	return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions);
 323}
 324
 325/**
 326 * xe_exec_queue_create_bind() - Create bind exec queue.
 327 * @xe: Xe device.
 328 * @tile: tile which bind exec queue belongs to.
 329 * @flags: exec queue creation flags
 330 * @user_vm: The user VM which this exec queue belongs to
 331 * @extensions: exec queue creation extensions
 332 *
 333 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM
 334 * for access to physical memory required for page table programming. On a
 335 * faulting devices the reserved copy engine instance must be used to avoid
 336 * deadlocking (user binds cannot get stuck behind faults as kernel binds which
 337 * resolve faults depend on user binds). On non-faulting devices any copy engine
 338 * can be used.
 339 *
 340 * Returns exec queue on success, ERR_PTR on failure
 341 */
 342struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
 343						struct xe_tile *tile,
 344						struct xe_vm *user_vm,
 345						u32 flags, u64 extensions)
 346{
 347	struct xe_gt *gt = tile->primary_gt;
 348	struct xe_exec_queue *q;
 349	struct xe_vm *migrate_vm;
 350
 351	migrate_vm = xe_migrate_get_vm(tile->migrate);
 352	if (xe->info.has_usm) {
 353		struct xe_hw_engine *hwe = xe_gt_hw_engine(gt,
 354							   XE_ENGINE_CLASS_COPY,
 355							   gt->usm.reserved_bcs_instance,
 356							   false);
 357
 358		if (!hwe) {
 359			xe_vm_put(migrate_vm);
 360			return ERR_PTR(-EINVAL);
 361		}
 362
 363		q = xe_exec_queue_create(xe, migrate_vm,
 364					 BIT(hwe->logical_instance), 1, hwe,
 365					 flags, extensions);
 366	} else {
 367		q = xe_exec_queue_create_class(xe, gt, migrate_vm,
 368					       XE_ENGINE_CLASS_COPY, flags,
 369					       extensions);
 370	}
 371	xe_vm_put(migrate_vm);
 372
 373	if (!IS_ERR(q)) {
 374		int err = drm_syncobj_create(&q->ufence_syncobj,
 375					     DRM_SYNCOBJ_CREATE_SIGNALED,
 376					     NULL);
 377		if (err) {
 378			xe_exec_queue_put(q);
 379			return ERR_PTR(err);
 380		}
 381
 382		if (user_vm)
 383			q->user_vm = xe_vm_get(user_vm);
 384	}
 385
 386	return q;
 387}
 388ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
 389
 390void xe_exec_queue_destroy(struct kref *ref)
 391{
 392	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
 393	struct xe_exec_queue *eq, *next;
 394	int i;
 395
 396	xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0);
 397
 398	if (q->ufence_syncobj)
 399		drm_syncobj_put(q->ufence_syncobj);
 400
 401	if (xe_exec_queue_uses_pxp(q))
 402		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
 403
 404	xe_exec_queue_last_fence_put_unlocked(q);
 405	for_each_tlb_inval(i)
 406		xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i);
 407
 408	if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {
 409		list_for_each_entry_safe(eq, next, &q->multi_gt_list,
 410					 multi_gt_link)
 411			xe_exec_queue_put(eq);
 412	}
 413
 414	if (q->user_vm) {
 415		xe_vm_put(q->user_vm);
 416		q->user_vm = NULL;
 417	}
 418
 419	q->ops->destroy(q);
 420}
 421
 422void xe_exec_queue_fini(struct xe_exec_queue *q)
 423{
 424	/*
 425	 * Before releasing our ref to lrc and xef, accumulate our run ticks
 426	 * and wakeup any waiters.
 427	 */
 428	xe_exec_queue_update_run_ticks(q);
 429	if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal))
 430		wake_up_var(&q->xef->exec_queue.pending_removal);
 431
 432	__xe_exec_queue_fini(q);
 433	__xe_exec_queue_free(q);
 434}
 435
 436void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
 437{
 438	switch (q->class) {
 439	case XE_ENGINE_CLASS_RENDER:
 440		snprintf(q->name, sizeof(q->name), "rcs%d", instance);
 441		break;
 442	case XE_ENGINE_CLASS_VIDEO_DECODE:
 443		snprintf(q->name, sizeof(q->name), "vcs%d", instance);
 444		break;
 445	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
 446		snprintf(q->name, sizeof(q->name), "vecs%d", instance);
 447		break;
 448	case XE_ENGINE_CLASS_COPY:
 449		snprintf(q->name, sizeof(q->name), "bcs%d", instance);
 450		break;
 451	case XE_ENGINE_CLASS_COMPUTE:
 452		snprintf(q->name, sizeof(q->name), "ccs%d", instance);
 453		break;
 454	case XE_ENGINE_CLASS_OTHER:
 455		snprintf(q->name, sizeof(q->name), "gsccs%d", instance);
 456		break;
 457	default:
 458		XE_WARN_ON(q->class);
 459	}
 460}
 461
 462struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id)
 463{
 464	struct xe_exec_queue *q;
 465
 466	mutex_lock(&xef->exec_queue.lock);
 467	q = xa_load(&xef->exec_queue.xa, id);
 468	if (q)
 469		xe_exec_queue_get(q);
 470	mutex_unlock(&xef->exec_queue.lock);
 471
 472	return q;
 473}
 474
 475enum xe_exec_queue_priority
 476xe_exec_queue_device_get_max_priority(struct xe_device *xe)
 477{
 478	return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :
 479				       XE_EXEC_QUEUE_PRIORITY_NORMAL;
 480}
 481
 482static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,
 483				   u64 value)
 484{
 485	if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))
 486		return -EINVAL;
 487
 488	if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))
 489		return -EPERM;
 490
 491	q->sched_props.priority = value;
 492	return 0;
 493}
 494
 495static bool xe_exec_queue_enforce_schedule_limit(void)
 496{
 497#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
 498	return true;
 499#else
 500	return !capable(CAP_SYS_NICE);
 501#endif
 502}
 503
 504static void
 505xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,
 506			      enum xe_exec_queue_sched_prop prop,
 507			      u32 *min, u32 *max)
 508{
 509	switch (prop) {
 510	case XE_EXEC_QUEUE_JOB_TIMEOUT:
 511		*min = eclass->sched_props.job_timeout_min;
 512		*max = eclass->sched_props.job_timeout_max;
 513		break;
 514	case XE_EXEC_QUEUE_TIMESLICE:
 515		*min = eclass->sched_props.timeslice_min;
 516		*max = eclass->sched_props.timeslice_max;
 517		break;
 518	case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
 519		*min = eclass->sched_props.preempt_timeout_min;
 520		*max = eclass->sched_props.preempt_timeout_max;
 521		break;
 522	default:
 523		break;
 524	}
 525#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
 526	if (capable(CAP_SYS_NICE)) {
 527		switch (prop) {
 528		case XE_EXEC_QUEUE_JOB_TIMEOUT:
 529			*min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;
 530			*max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;
 531			break;
 532		case XE_EXEC_QUEUE_TIMESLICE:
 533			*min = XE_HW_ENGINE_TIMESLICE_MIN;
 534			*max = XE_HW_ENGINE_TIMESLICE_MAX;
 535			break;
 536		case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
 537			*min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;
 538			*max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;
 539			break;
 540		default:
 541			break;
 542		}
 543	}
 544#endif
 545}
 546
 547static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,
 548				    u64 value)
 549{
 550	u32 min = 0, max = 0;
 551
 552	xe_exec_queue_get_prop_minmax(q->hwe->eclass,
 553				      XE_EXEC_QUEUE_TIMESLICE, &min, &max);
 554
 555	if (xe_exec_queue_enforce_schedule_limit() &&
 556	    !xe_hw_engine_timeout_in_range(value, min, max))
 557		return -EINVAL;
 558
 559	q->sched_props.timeslice_us = value;
 560	return 0;
 561}
 562
 563static int
 564exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value)
 565{
 566	if (value == DRM_XE_PXP_TYPE_NONE)
 567		return 0;
 568
 569	/* we only support HWDRM sessions right now */
 570	if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM))
 571		return -EINVAL;
 572
 573	if (!xe_pxp_is_enabled(xe->pxp))
 574		return -ENODEV;
 575
 576	return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);
 577}
 578
 579typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
 580					     struct xe_exec_queue *q,
 581					     u64 value);
 582
 583static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
 584	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
 585	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
 586	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,
 587};
 588
 589static int exec_queue_user_ext_set_property(struct xe_device *xe,
 590					    struct xe_exec_queue *q,
 591					    u64 extension)
 592{
 593	u64 __user *address = u64_to_user_ptr(extension);
 594	struct drm_xe_ext_set_property ext;
 595	int err;
 596	u32 idx;
 597
 598	err = copy_from_user(&ext, address, sizeof(ext));
 599	if (XE_IOCTL_DBG(xe, err))
 600		return -EFAULT;
 601
 602	if (XE_IOCTL_DBG(xe, ext.property >=
 603			 ARRAY_SIZE(exec_queue_set_property_funcs)) ||
 604	    XE_IOCTL_DBG(xe, ext.pad) ||
 605	    XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
 606			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&
 607			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE))
 608		return -EINVAL;
 609
 610	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
 611	if (!exec_queue_set_property_funcs[idx])
 612		return -EINVAL;
 613
 614	return exec_queue_set_property_funcs[idx](xe, q, ext.value);
 615}
 616
 617typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,
 618					       struct xe_exec_queue *q,
 619					       u64 extension);
 620
 621static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
 622	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
 623};
 624
 625#define MAX_USER_EXTENSIONS	16
 626static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
 627				      u64 extensions, int ext_number)
 628{
 629	u64 __user *address = u64_to_user_ptr(extensions);
 630	struct drm_xe_user_extension ext;
 631	int err;
 632	u32 idx;
 633
 634	if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
 635		return -E2BIG;
 636
 637	err = copy_from_user(&ext, address, sizeof(ext));
 638	if (XE_IOCTL_DBG(xe, err))
 639		return -EFAULT;
 640
 641	if (XE_IOCTL_DBG(xe, ext.pad) ||
 642	    XE_IOCTL_DBG(xe, ext.name >=
 643			 ARRAY_SIZE(exec_queue_user_extension_funcs)))
 644		return -EINVAL;
 645
 646	idx = array_index_nospec(ext.name,
 647				 ARRAY_SIZE(exec_queue_user_extension_funcs));
 648	err = exec_queue_user_extension_funcs[idx](xe, q, extensions);
 649	if (XE_IOCTL_DBG(xe, err))
 650		return err;
 651
 652	if (ext.next_extension)
 653		return exec_queue_user_extensions(xe, q, ext.next_extension,
 654						  ++ext_number);
 655
 656	return 0;
 657}
 658
 659static u32 calc_validate_logical_mask(struct xe_device *xe,
 660				      struct drm_xe_engine_class_instance *eci,
 661				      u16 width, u16 num_placements)
 662{
 663	int len = width * num_placements;
 664	int i, j, n;
 665	u16 class;
 666	u16 gt_id;
 667	u32 return_mask = 0, prev_mask;
 668
 669	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) &&
 670			 len > 1))
 671		return 0;
 672
 673	for (i = 0; i < width; ++i) {
 674		u32 current_mask = 0;
 675
 676		for (j = 0; j < num_placements; ++j) {
 677			struct xe_hw_engine *hwe;
 678
 679			n = j * width + i;
 680
 681			hwe = xe_hw_engine_lookup(xe, eci[n]);
 682			if (XE_IOCTL_DBG(xe, !hwe))
 683				return 0;
 684
 685			if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe)))
 686				return 0;
 687
 688			if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) ||
 689			    XE_IOCTL_DBG(xe, n && eci[n].engine_class != class))
 690				return 0;
 691
 692			class = eci[n].engine_class;
 693			gt_id = eci[n].gt_id;
 694
 695			if (width == 1 || !i)
 696				return_mask |= BIT(eci[n].engine_instance);
 697			current_mask |= BIT(eci[n].engine_instance);
 698		}
 699
 700		/* Parallel submissions must be logically contiguous */
 701		if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1))
 702			return 0;
 703
 704		prev_mask = current_mask;
 705	}
 706
 707	return return_mask;
 708}
 709
 710int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 711			       struct drm_file *file)
 712{
 713	struct xe_device *xe = to_xe_device(dev);
 714	struct xe_file *xef = to_xe_file(file);
 715	struct drm_xe_exec_queue_create *args = data;
 716	struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE];
 717	struct drm_xe_engine_class_instance __user *user_eci =
 718		u64_to_user_ptr(args->instances);
 719	struct xe_hw_engine *hwe;
 720	struct xe_vm *vm;
 721	struct xe_tile *tile;
 722	struct xe_exec_queue *q = NULL;
 723	u32 logical_mask;
 724	u32 flags = 0;
 725	u32 id;
 726	u32 len;
 727	int err;
 728
 729	if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) ||
 730	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
 731		return -EINVAL;
 732
 733	len = args->width * args->num_placements;
 734	if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE))
 735		return -EINVAL;
 736
 737	err = copy_from_user(eci, user_eci,
 738			     sizeof(struct drm_xe_engine_class_instance) * len);
 739	if (XE_IOCTL_DBG(xe, err))
 740		return -EFAULT;
 741
 742	if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id)))
 743		return -EINVAL;
 744
 745	if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT)
 746		flags |= EXEC_QUEUE_FLAG_LOW_LATENCY;
 747
 748	if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) {
 749		if (XE_IOCTL_DBG(xe, args->width != 1) ||
 750		    XE_IOCTL_DBG(xe, args->num_placements != 1) ||
 751		    XE_IOCTL_DBG(xe, eci[0].engine_instance != 0))
 752			return -EINVAL;
 753
 754		vm = xe_vm_lookup(xef, args->vm_id);
 755		if (XE_IOCTL_DBG(xe, !vm))
 756			return -ENOENT;
 757
 758		err = down_read_interruptible(&vm->lock);
 759		if (err) {
 760			xe_vm_put(vm);
 761			return err;
 762		}
 763
 764		if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
 765			up_read(&vm->lock);
 766			xe_vm_put(vm);
 767			return -ENOENT;
 768		}
 769
 770		for_each_tile(tile, xe, id) {
 771			struct xe_exec_queue *new;
 772
 773			flags |= EXEC_QUEUE_FLAG_VM;
 774			if (id)
 775				flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD;
 776
 777			new = xe_exec_queue_create_bind(xe, tile, vm, flags,
 778							args->extensions);
 779			if (IS_ERR(new)) {
 780				up_read(&vm->lock);
 781				xe_vm_put(vm);
 782				err = PTR_ERR(new);
 783				if (q)
 784					goto put_exec_queue;
 785				return err;
 786			}
 787			if (id == 0)
 788				q = new;
 789			else
 790				list_add_tail(&new->multi_gt_list,
 791					      &q->multi_gt_link);
 792		}
 793		up_read(&vm->lock);
 794		xe_vm_put(vm);
 795	} else {
 796		logical_mask = calc_validate_logical_mask(xe, eci,
 797							  args->width,
 798							  args->num_placements);
 799		if (XE_IOCTL_DBG(xe, !logical_mask))
 800			return -EINVAL;
 801
 802		hwe = xe_hw_engine_lookup(xe, eci[0]);
 803		if (XE_IOCTL_DBG(xe, !hwe))
 804			return -EINVAL;
 805
 806		vm = xe_vm_lookup(xef, args->vm_id);
 807		if (XE_IOCTL_DBG(xe, !vm))
 808			return -ENOENT;
 809
 810		err = down_read_interruptible(&vm->lock);
 811		if (err) {
 812			xe_vm_put(vm);
 813			return err;
 814		}
 815
 816		if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
 817			up_read(&vm->lock);
 818			xe_vm_put(vm);
 819			return -ENOENT;
 820		}
 821
 822		q = xe_exec_queue_create(xe, vm, logical_mask,
 823					 args->width, hwe, flags,
 824					 args->extensions);
 825		up_read(&vm->lock);
 826		xe_vm_put(vm);
 827		if (IS_ERR(q))
 828			return PTR_ERR(q);
 829
 830		if (xe_vm_in_preempt_fence_mode(vm)) {
 831			q->lr.context = dma_fence_context_alloc(1);
 832
 833			err = xe_vm_add_compute_exec_queue(vm, q);
 834			if (XE_IOCTL_DBG(xe, err))
 835				goto put_exec_queue;
 836		}
 837
 838		if (q->vm && q->hwe->hw_engine_group) {
 839			err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q);
 840			if (err)
 841				goto put_exec_queue;
 842		}
 843	}
 844
 845	q->xef = xe_file_get(xef);
 846
 847	/* user id alloc must always be last in ioctl to prevent UAF */
 848	err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL);
 849	if (err)
 850		goto kill_exec_queue;
 851
 852	args->exec_queue_id = id;
 853
 854	return 0;
 855
 856kill_exec_queue:
 857	xe_exec_queue_kill(q);
 858put_exec_queue:
 859	xe_exec_queue_put(q);
 860	return err;
 861}
 862
 863int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
 864				     struct drm_file *file)
 865{
 866	struct xe_device *xe = to_xe_device(dev);
 867	struct xe_file *xef = to_xe_file(file);
 868	struct drm_xe_exec_queue_get_property *args = data;
 869	struct xe_exec_queue *q;
 870	int ret;
 871
 872	if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
 873		return -EINVAL;
 874
 875	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
 876	if (XE_IOCTL_DBG(xe, !q))
 877		return -ENOENT;
 878
 879	switch (args->property) {
 880	case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN:
 881		args->value = q->ops->reset_status(q);
 882		ret = 0;
 883		break;
 884	default:
 885		ret = -EINVAL;
 886	}
 887
 888	xe_exec_queue_put(q);
 889
 890	return ret;
 891}
 892
 893/**
 894 * xe_exec_queue_lrc() - Get the LRC from exec queue.
 895 * @q: The exec_queue.
 896 *
 897 * Retrieves the primary LRC for the exec queue. Note that this function
 898 * returns only the first LRC instance, even when multiple parallel LRCs
 899 * are configured.
 900 *
 901 * Return: Pointer to LRC on success, error on failure
 902 */
 903struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q)
 904{
 905	return q->lrc[0];
 906}
 907
 908/**
 909 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running
 910 * @q: The exec_queue
 911 *
 912 * Return: True if the exec_queue is long-running, false otherwise.
 913 */
 914bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
 915{
 916	return q->vm && xe_vm_in_lr_mode(q->vm) &&
 917		!(q->flags & EXEC_QUEUE_FLAG_VM);
 918}
 919
 920/**
 921 * xe_exec_queue_is_idle() - Whether an exec_queue is idle.
 922 * @q: The exec_queue
 923 *
 924 * FIXME: Need to determine what to use as the short-lived
 925 * timeline lock for the exec_queues, so that the return value
 926 * of this function becomes more than just an advisory
 927 * snapshot in time. The timeline lock must protect the
 928 * seqno from racing submissions on the same exec_queue.
 929 * Typically vm->resv, but user-created timeline locks use the migrate vm
 930 * and never grabs the migrate vm->resv so we have a race there.
 931 *
 932 * Return: True if the exec_queue is idle, false otherwise.
 933 */
 934bool xe_exec_queue_is_idle(struct xe_exec_queue *q)
 935{
 936	if (xe_exec_queue_is_parallel(q)) {
 937		int i;
 938
 939		for (i = 0; i < q->width; ++i) {
 940			if (xe_lrc_seqno(q->lrc[i]) !=
 941			    q->lrc[i]->fence_ctx.next_seqno - 1)
 942				return false;
 943		}
 944
 945		return true;
 946	}
 947
 948	return xe_lrc_seqno(q->lrc[0]) ==
 949		q->lrc[0]->fence_ctx.next_seqno - 1;
 950}
 951
 952/**
 953 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue
 954 * from hw
 955 * @q: The exec queue
 956 *
 957 * Update the timestamp saved by HW for this exec queue and save run ticks
 958 * calculated by using the delta from last update.
 959 */
 960void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
 961{
 962	struct xe_device *xe = gt_to_xe(q->gt);
 963	struct xe_lrc *lrc;
 964	u64 old_ts, new_ts;
 965	int idx;
 966
 967	/*
 968	 * Jobs that are executed by kernel doesn't have a corresponding xe_file
 969	 * and thus are not accounted.
 970	 */
 971	if (!q->xef)
 972		return;
 973
 974	/* Synchronize with unbind while holding the xe file open */
 975	if (!drm_dev_enter(&xe->drm, &idx))
 976		return;
 977	/*
 978	 * Only sample the first LRC. For parallel submission, all of them are
 979	 * scheduled together and we compensate that below by multiplying by
 980	 * width - this may introduce errors if that premise is not true and
 981	 * they don't exit 100% aligned. On the other hand, looping through
 982	 * the LRCs and reading them in different time could also introduce
 983	 * errors.
 984	 */
 985	lrc = q->lrc[0];
 986	new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
 987	q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;
 988
 989	drm_dev_exit(idx);
 990}
 991
 992/**
 993 * xe_exec_queue_kill - permanently stop all execution from an exec queue
 994 * @q: The exec queue
 995 *
 996 * This function permanently stops all activity on an exec queue. If the queue
 997 * is actively executing on the HW, it will be kicked off the engine; any
 998 * pending jobs are discarded and all future submissions are rejected.
 999 * This function is safe to call multiple times.
1000 */
1001void xe_exec_queue_kill(struct xe_exec_queue *q)
1002{
1003	struct xe_exec_queue *eq = q, *next;
1004
1005	list_for_each_entry_safe(eq, next, &eq->multi_gt_list,
1006				 multi_gt_link) {
1007		q->ops->kill(eq);
1008		xe_vm_remove_compute_exec_queue(q->vm, eq);
1009	}
1010
1011	q->ops->kill(q);
1012	xe_vm_remove_compute_exec_queue(q->vm, q);
1013}
1014
1015int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
1016				struct drm_file *file)
1017{
1018	struct xe_device *xe = to_xe_device(dev);
1019	struct xe_file *xef = to_xe_file(file);
1020	struct drm_xe_exec_queue_destroy *args = data;
1021	struct xe_exec_queue *q;
1022
1023	if (XE_IOCTL_DBG(xe, args->pad) ||
1024	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1025		return -EINVAL;
1026
1027	mutex_lock(&xef->exec_queue.lock);
1028	q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id);
1029	if (q)
1030		atomic_inc(&xef->exec_queue.pending_removal);
1031	mutex_unlock(&xef->exec_queue.lock);
1032
1033	if (XE_IOCTL_DBG(xe, !q))
1034		return -ENOENT;
1035
1036	if (q->vm && q->hwe->hw_engine_group)
1037		xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
1038
1039	xe_exec_queue_kill(q);
1040
1041	trace_xe_exec_queue_close(q);
1042	xe_exec_queue_put(q);
1043
1044	return 0;
1045}
1046
1047static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
1048						    struct xe_vm *vm)
1049{
1050	if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) {
1051		xe_migrate_job_lock_assert(q);
1052	} else if (q->flags & EXEC_QUEUE_FLAG_VM) {
1053		lockdep_assert_held(&vm->lock);
1054	} else {
1055		xe_vm_assert_held(vm);
1056		lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem);
1057	}
1058}
1059
1060/**
1061 * xe_exec_queue_last_fence_put() - Drop ref to last fence
1062 * @q: The exec queue
1063 * @vm: The VM the engine does a bind or exec for
1064 */
1065void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm)
1066{
1067	xe_exec_queue_last_fence_lockdep_assert(q, vm);
1068
1069	xe_exec_queue_last_fence_put_unlocked(q);
1070}
1071
1072/**
1073 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked
1074 * @q: The exec queue
1075 *
1076 * Only safe to be called from xe_exec_queue_destroy().
1077 */
1078void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q)
1079{
1080	if (q->last_fence) {
1081		dma_fence_put(q->last_fence);
1082		q->last_fence = NULL;
1083	}
1084}
1085
1086/**
1087 * xe_exec_queue_last_fence_get() - Get last fence
1088 * @q: The exec queue
1089 * @vm: The VM the engine does a bind or exec for
1090 *
1091 * Get last fence, takes a ref
1092 *
1093 * Returns: last fence if not signaled, dma fence stub if signaled
1094 */
1095struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q,
1096					       struct xe_vm *vm)
1097{
1098	struct dma_fence *fence;
1099
1100	xe_exec_queue_last_fence_lockdep_assert(q, vm);
1101
1102	if (q->last_fence &&
1103	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
1104		xe_exec_queue_last_fence_put(q, vm);
1105
1106	fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
1107	dma_fence_get(fence);
1108	return fence;
1109}
1110
1111/**
1112 * xe_exec_queue_last_fence_get_for_resume() - Get last fence
1113 * @q: The exec queue
1114 * @vm: The VM the engine does a bind or exec for
1115 *
1116 * Get last fence, takes a ref. Only safe to be called in the context of
1117 * resuming the hw engine group's long-running exec queue, when the group
1118 * semaphore is held.
1119 *
1120 * Returns: last fence if not signaled, dma fence stub if signaled
1121 */
1122struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q,
1123							  struct xe_vm *vm)
1124{
1125	struct dma_fence *fence;
1126
1127	lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem);
1128
1129	if (q->last_fence &&
1130	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
1131		xe_exec_queue_last_fence_put_unlocked(q);
1132
1133	fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
1134	dma_fence_get(fence);
1135	return fence;
1136}
1137
1138/**
1139 * xe_exec_queue_last_fence_set() - Set last fence
1140 * @q: The exec queue
1141 * @vm: The VM the engine does a bind or exec for
1142 * @fence: The fence
1143 *
1144 * Set the last fence for the engine. Increases reference count for fence, when
1145 * closing engine xe_exec_queue_last_fence_put should be called.
1146 */
1147void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm,
1148				  struct dma_fence *fence)
1149{
1150	xe_exec_queue_last_fence_lockdep_assert(q, vm);
1151	xe_assert(vm->xe, !dma_fence_is_container(fence));
1152
1153	xe_exec_queue_last_fence_put(q, vm);
1154	q->last_fence = dma_fence_get(fence);
1155}
1156
1157/**
1158 * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence
1159 * @q: The exec queue
1160 * @vm: The VM the engine does a bind for
1161 * @type: Either primary or media GT
1162 */
1163void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
1164					    struct xe_vm *vm,
1165					    unsigned int type)
1166{
1167	xe_exec_queue_last_fence_lockdep_assert(q, vm);
1168	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1169		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1170
1171	xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type);
1172}
1173
1174/**
1175 * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB
1176 * invalidation fence unlocked
1177 * @q: The exec queue
1178 * @type: Either primary or media GT
1179 *
1180 * Only safe to be called from xe_exec_queue_destroy().
1181 */
1182void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
1183						     unsigned int type)
1184{
1185	xe_assert(q->vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1186		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1187
1188	dma_fence_put(q->tlb_inval[type].last_fence);
1189	q->tlb_inval[type].last_fence = NULL;
1190}
1191
1192/**
1193 * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation
1194 * @q: The exec queue
1195 * @vm: The VM the engine does a bind for
1196 * @type: Either primary or media GT
1197 *
1198 * Get last fence, takes a ref
1199 *
1200 * Returns: last fence if not signaled, dma fence stub if signaled
1201 */
1202struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
1203							 struct xe_vm *vm,
1204							 unsigned int type)
1205{
1206	struct dma_fence *fence;
1207
1208	xe_exec_queue_last_fence_lockdep_assert(q, vm);
1209	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1210		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1211	xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
1212				      EXEC_QUEUE_FLAG_MIGRATE));
1213
1214	if (q->tlb_inval[type].last_fence &&
1215	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1216		     &q->tlb_inval[type].last_fence->flags))
1217		xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
1218
1219	fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub();
1220	dma_fence_get(fence);
1221	return fence;
1222}
1223
1224/**
1225 * xe_exec_queue_tlb_inval_last_fence_set() - Set last fence for TLB invalidation
1226 * @q: The exec queue
1227 * @vm: The VM the engine does a bind for
1228 * @fence: The fence
1229 * @type: Either primary or media GT
1230 *
1231 * Set the last fence for the tlb invalidation type on the queue. Increases
1232 * reference count for fence, when closing queue
1233 * xe_exec_queue_tlb_inval_last_fence_put should be called.
1234 */
1235void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
1236					    struct xe_vm *vm,
1237					    struct dma_fence *fence,
1238					    unsigned int type)
1239{
1240	xe_exec_queue_last_fence_lockdep_assert(q, vm);
1241	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1242		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1243	xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
1244				      EXEC_QUEUE_FLAG_MIGRATE));
1245	xe_assert(vm->xe, !dma_fence_is_container(fence));
1246
1247	xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
1248	q->tlb_inval[type].last_fence = dma_fence_get(fence);
1249}
1250
1251/**
1252 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references
1253 * within all LRCs of a queue.
1254 * @q: the &xe_exec_queue struct instance containing target LRCs
1255 * @scratch: scratch buffer to be used as temporary storage
1256 *
1257 * Returns: zero on success, negative error code on failure
1258 */
1259int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch)
1260{
1261	int i;
1262	int err = 0;
1263
1264	for (i = 0; i < q->width; ++i) {
1265		struct xe_lrc *lrc;
1266
1267		/* Pairs with WRITE_ONCE in __xe_exec_queue_init  */
1268		lrc = READ_ONCE(q->lrc[i]);
1269		if (!lrc)
1270			continue;
1271
1272		xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch);
1273		xe_lrc_update_hwctx_regs_with_address(lrc);
1274		err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch);
1275		if (err)
1276			break;
1277	}
1278
1279	return err;
1280}