xe_tlb_inval_job.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2025 Intel Corporation
  4 */
  5
  6#include "xe_assert.h"
  7#include "xe_dep_job_types.h"
  8#include "xe_dep_scheduler.h"
  9#include "xe_exec_queue.h"
 10#include "xe_gt_types.h"
 11#include "xe_tlb_inval.h"
 12#include "xe_tlb_inval_job.h"
 13#include "xe_migrate.h"
 14#include "xe_pm.h"
 15#include "xe_vm.h"
 16
 17/** struct xe_tlb_inval_job - TLB invalidation job */
 18struct xe_tlb_inval_job {
 19	/** @dep: base generic dependency Xe job */
 20	struct xe_dep_job dep;
 21	/** @tlb_inval: TLB invalidation client */
 22	struct xe_tlb_inval *tlb_inval;
 23	/** @q: exec queue issuing the invalidate */
 24	struct xe_exec_queue *q;
 25	/** @vm: VM which TLB invalidation is being issued for */
 26	struct xe_vm *vm;
 27	/** @refcount: ref count of this job */
 28	struct kref refcount;
 29	/**
 30	 * @fence: dma fence to indicate completion. 1 way relationship - job
 31	 * can safely reference fence, fence cannot safely reference job.
 32	 */
 33	struct dma_fence *fence;
 34	/** @start: Start address to invalidate */
 35	u64 start;
 36	/** @end: End address to invalidate */
 37	u64 end;
 38	/** @type: GT type */
 39	int type;
 40	/** @fence_armed: Fence has been armed */
 41	bool fence_armed;
 42};
 43
 44static struct dma_fence *xe_tlb_inval_job_run(struct xe_dep_job *dep_job)
 45{
 46	struct xe_tlb_inval_job *job =
 47		container_of(dep_job, typeof(*job), dep);
 48	struct xe_tlb_inval_fence *ifence =
 49		container_of(job->fence, typeof(*ifence), base);
 50
 51	xe_tlb_inval_range(job->tlb_inval, ifence, job->start,
 52			   job->end, job->vm->usm.asid);
 53
 54	return job->fence;
 55}
 56
 57static void xe_tlb_inval_job_free(struct xe_dep_job *dep_job)
 58{
 59	struct xe_tlb_inval_job *job =
 60		container_of(dep_job, typeof(*job), dep);
 61
 62	/* Pairs with get in xe_tlb_inval_job_push */
 63	xe_tlb_inval_job_put(job);
 64}
 65
 66static const struct xe_dep_job_ops dep_job_ops = {
 67	.run_job = xe_tlb_inval_job_run,
 68	.free_job = xe_tlb_inval_job_free,
 69};
 70
 71/**
 72 * xe_tlb_inval_job_create() - TLB invalidation job create
 73 * @q: exec queue issuing the invalidate
 74 * @tlb_inval: TLB invalidation client
 75 * @dep_scheduler: Dependency scheduler for job
 76 * @vm: VM which TLB invalidation is being issued for
 77 * @start: Start address to invalidate
 78 * @end: End address to invalidate
 79 * @type: GT type
 80 *
 81 * Create a TLB invalidation job and initialize internal fields. The caller is
 82 * responsible for releasing the creation reference.
 83 *
 84 * Return: TLB invalidation job object on success, ERR_PTR failure
 85 */
 86struct xe_tlb_inval_job *
 87xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 88			struct xe_dep_scheduler *dep_scheduler,
 89			struct xe_vm *vm, u64 start, u64 end, int type)
 90{
 91	struct xe_tlb_inval_job *job;
 92	struct drm_sched_entity *entity =
 93		xe_dep_scheduler_entity(dep_scheduler);
 94	struct xe_tlb_inval_fence *ifence;
 95	int err;
 96
 97	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
 98		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
 99
100	job = kmalloc(sizeof(*job), GFP_KERNEL);
101	if (!job)
102		return ERR_PTR(-ENOMEM);
103
104	job->q = q;
105	job->vm = vm;
106	job->tlb_inval = tlb_inval;
107	job->start = start;
108	job->end = end;
109	job->fence_armed = false;
110	job->dep.ops = &dep_job_ops;
111	job->type = type;
112	kref_init(&job->refcount);
113	xe_exec_queue_get(q);	/* Pairs with put in xe_tlb_inval_job_destroy */
114	xe_vm_get(vm);		/* Pairs with put in xe_tlb_inval_job_destroy */
115
116	ifence = kmalloc(sizeof(*ifence), GFP_KERNEL);
117	if (!ifence) {
118		err = -ENOMEM;
119		goto err_job;
120	}
121	job->fence = &ifence->base;
122
123	err = drm_sched_job_init(&job->dep.drm, entity, 1, NULL,
124				 q->xef ? q->xef->drm->client_id : 0);
125	if (err)
126		goto err_fence;
127
128	/* Pairs with put in xe_tlb_inval_job_destroy */
129	xe_pm_runtime_get_noresume(gt_to_xe(q->gt));
130
131	return job;
132
133err_fence:
134	kfree(ifence);
135err_job:
136	xe_vm_put(vm);
137	xe_exec_queue_put(q);
138	kfree(job);
139
140	return ERR_PTR(err);
141}
142
143static void xe_tlb_inval_job_destroy(struct kref *ref)
144{
145	struct xe_tlb_inval_job *job = container_of(ref, typeof(*job),
146						    refcount);
147	struct xe_tlb_inval_fence *ifence =
148		container_of(job->fence, typeof(*ifence), base);
149	struct xe_exec_queue *q = job->q;
150	struct xe_device *xe = gt_to_xe(q->gt);
151	struct xe_vm *vm = job->vm;
152
153	if (!job->fence_armed)
154		kfree(ifence);
155	else
156		/* Ref from xe_tlb_inval_fence_init */
157		dma_fence_put(job->fence);
158
159	drm_sched_job_cleanup(&job->dep.drm);
160	kfree(job);
161	xe_vm_put(vm);		/* Pairs with get from xe_tlb_inval_job_create */
162	xe_exec_queue_put(q);	/* Pairs with get from xe_tlb_inval_job_create */
163	xe_pm_runtime_put(xe);	/* Pairs with get from xe_tlb_inval_job_create */
164}
165
166/**
167 * xe_tlb_inval_job_alloc_dep() - TLB invalidation job alloc dependency
168 * @job: TLB invalidation job to alloc dependency for
169 *
170 * Allocate storage for a dependency in the TLB invalidation fence. This
171 * function should be called at most once per job and must be paired with
172 * xe_tlb_inval_job_push being called with a real fence.
173 *
174 * Return: 0 on success, -errno on failure
175 */
176int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job)
177{
178	xe_assert(gt_to_xe(job->q->gt), !xa_load(&job->dep.drm.dependencies, 0));
179	might_alloc(GFP_KERNEL);
180
181	return drm_sched_job_add_dependency(&job->dep.drm,
182					    dma_fence_get_stub());
183}
184
185/**
186 * xe_tlb_inval_job_push() - TLB invalidation job push
187 * @job: TLB invalidation job to push
188 * @m: The migration object being used
189 * @fence: Dependency for TLB invalidation job
190 *
191 * Pushes a TLB invalidation job for execution, using @fence as a dependency.
192 * Storage for @fence must be preallocated with xe_tlb_inval_job_alloc_dep
193 * prior to this call if @fence is not signaled. Takes a reference to the job’s
194 * finished fence, which the caller is responsible for releasing, and return it
195 * to the caller. This function is safe to be called in the path of reclaim.
196 *
197 * Return: Job's finished fence on success, cannot fail
198 */
199struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
200					struct xe_migrate *m,
201					struct dma_fence *fence)
202{
203	struct xe_tlb_inval_fence *ifence =
204		container_of(job->fence, typeof(*ifence), base);
205
206	if (!dma_fence_is_signaled(fence)) {
207		void *ptr;
208
209		/*
210		 * Can be in path of reclaim, hence the preallocation of fence
211		 * storage in xe_tlb_inval_job_alloc_dep. Verify caller did
212		 * this correctly.
213		 */
214		xe_assert(gt_to_xe(job->q->gt),
215			  xa_load(&job->dep.drm.dependencies, 0) ==
216			  dma_fence_get_stub());
217
218		dma_fence_get(fence);	/* ref released once dependency processed by scheduler */
219		ptr = xa_store(&job->dep.drm.dependencies, 0, fence,
220			       GFP_ATOMIC);
221		xe_assert(gt_to_xe(job->q->gt), !xa_is_err(ptr));
222	}
223
224	xe_tlb_inval_job_get(job);	/* Pairs with put in free_job */
225	job->fence_armed = true;
226
227	/*
228	 * We need the migration lock to protect the job's seqno and the spsc
229	 * queue, only taken on migration queue, user queues protected dma-resv
230	 * VM lock.
231	 */
232	xe_migrate_job_lock(m, job->q);
233
234	/* Creation ref pairs with put in xe_tlb_inval_job_destroy */
235	xe_tlb_inval_fence_init(job->tlb_inval, ifence, false);
236	dma_fence_get(job->fence);	/* Pairs with put in DRM scheduler */
237
238	drm_sched_job_arm(&job->dep.drm);
239	/*
240	 * caller ref, get must be done before job push as it could immediately
241	 * signal and free.
242	 */
243	dma_fence_get(&job->dep.drm.s_fence->finished);
244	drm_sched_entity_push_job(&job->dep.drm);
245
246	/* Let the upper layers fish this out */
247	xe_exec_queue_tlb_inval_last_fence_set(job->q, job->vm,
248					       &job->dep.drm.s_fence->finished,
249					       job->type);
250
251	xe_migrate_job_unlock(m, job->q);
252
253	/*
254	 * Not using job->fence, as it has its own dma-fence context, which does
255	 * not allow TLB invalidation fences on the same queue, GT tuple to
256	 * be squashed in dma-resv/DRM scheduler. Instead, we use the DRM scheduler
257	 * context and job's finished fence, which enables squashing.
258	 */
259	return &job->dep.drm.s_fence->finished;
260}
261
262/**
263 * xe_tlb_inval_job_get() - Get a reference to TLB invalidation job
264 * @job: TLB invalidation job object
265 *
266 * Increment the TLB invalidation job's reference count
267 */
268void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job)
269{
270	kref_get(&job->refcount);
271}
272
273/**
274 * xe_tlb_inval_job_put() - Put a reference to TLB invalidation job
275 * @job: TLB invalidation job object
276 *
277 * Decrement the TLB invalidation job's reference count, call
278 * xe_tlb_inval_job_destroy when reference count == 0. Skips decrement if
279 * input @job is NULL or IS_ERR.
280 */
281void xe_tlb_inval_job_put(struct xe_tlb_inval_job *job)
282{
283	if (!IS_ERR_OR_NULL(job))
284		kref_put(&job->refcount, xe_tlb_inval_job_destroy);
285}