xe_hw_fence.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021 Intel Corporation
  4 */
  5
  6#include "xe_hw_fence.h"
  7
  8#include <linux/device.h>
  9#include <linux/slab.h>
 10
 11#include "xe_bo.h"
 12#include "xe_device.h"
 13#include "xe_gt.h"
 14#include "xe_hw_engine.h"
 15#include "xe_macros.h"
 16#include "xe_map.h"
 17#include "xe_trace.h"
 18
 19static struct kmem_cache *xe_hw_fence_slab;
 20
 21int __init xe_hw_fence_module_init(void)
 22{
 23	xe_hw_fence_slab = kmem_cache_create("xe_hw_fence",
 24					     sizeof(struct xe_hw_fence), 0,
 25					     SLAB_HWCACHE_ALIGN, NULL);
 26	if (!xe_hw_fence_slab)
 27		return -ENOMEM;
 28
 29	return 0;
 30}
 31
 32void xe_hw_fence_module_exit(void)
 33{
 34	rcu_barrier();
 35	kmem_cache_destroy(xe_hw_fence_slab);
 36}
 37
 38static struct xe_hw_fence *fence_alloc(void)
 39{
 40	return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL);
 41}
 42
 43static void fence_free(struct rcu_head *rcu)
 44{
 45	struct xe_hw_fence *fence =
 46		container_of(rcu, struct xe_hw_fence, dma.rcu);
 47
 48	if (!WARN_ON_ONCE(!fence))
 49		kmem_cache_free(xe_hw_fence_slab, fence);
 50}
 51
 52static void hw_fence_irq_run_cb(struct irq_work *work)
 53{
 54	struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work);
 55	struct xe_hw_fence *fence, *next;
 56	bool tmp;
 57
 58	tmp = dma_fence_begin_signalling();
 59	spin_lock(&irq->lock);
 60	if (irq->enabled) {
 61		list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
 62			struct dma_fence *dma_fence = &fence->dma;
 63
 64			trace_xe_hw_fence_try_signal(fence);
 65			if (dma_fence_is_signaled_locked(dma_fence)) {
 66				trace_xe_hw_fence_signal(fence);
 67				list_del_init(&fence->irq_link);
 68				dma_fence_put(dma_fence);
 69			}
 70		}
 71	}
 72	spin_unlock(&irq->lock);
 73	dma_fence_end_signalling(tmp);
 74}
 75
 76void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq)
 77{
 78	spin_lock_init(&irq->lock);
 79	init_irq_work(&irq->work, hw_fence_irq_run_cb);
 80	INIT_LIST_HEAD(&irq->pending);
 81	irq->enabled = true;
 82}
 83
 84void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)
 85{
 86	struct xe_hw_fence *fence, *next;
 87	unsigned long flags;
 88	int err;
 89	bool tmp;
 90
 91	if (XE_WARN_ON(!list_empty(&irq->pending))) {
 92		tmp = dma_fence_begin_signalling();
 93		spin_lock_irqsave(&irq->lock, flags);
 94		list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
 95			list_del_init(&fence->irq_link);
 96			err = dma_fence_signal_locked(&fence->dma);
 97			dma_fence_put(&fence->dma);
 98			XE_WARN_ON(err);
 99		}
100		spin_unlock_irqrestore(&irq->lock, flags);
101		dma_fence_end_signalling(tmp);
102	}
103
104	/* Safe release of the irq->lock used in dma_fence_init. */
105	synchronize_rcu();
106}
107
108void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq)
109{
110	irq_work_queue(&irq->work);
111}
112
113void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq)
114{
115	spin_lock_irq(&irq->lock);
116	irq->enabled = false;
117	spin_unlock_irq(&irq->lock);
118}
119
120void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq)
121{
122	spin_lock_irq(&irq->lock);
123	irq->enabled = true;
124	spin_unlock_irq(&irq->lock);
125
126	irq_work_queue(&irq->work);
127}
128
129void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt,
130			  struct xe_hw_fence_irq *irq, const char *name)
131{
132	ctx->gt = gt;
133	ctx->irq = irq;
134	ctx->dma_fence_ctx = dma_fence_context_alloc(1);
135	ctx->next_seqno = XE_FENCE_INITIAL_SEQNO;
136	snprintf(ctx->name, sizeof(ctx->name), "%s", name);
137}
138
139void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx)
140{
141}
142
143static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence);
144
145static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)
146{
147	return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);
148}
149
150static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)
151{
152	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
153
154	return dev_name(fence->xe->drm.dev);
155}
156
157static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence)
158{
159	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
160
161	return fence->name;
162}
163
164static bool xe_hw_fence_signaled(struct dma_fence *dma_fence)
165{
166	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
167	struct xe_device *xe = fence->xe;
168	u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32);
169
170	return dma_fence->error ||
171		!__dma_fence_is_later(dma_fence, dma_fence->seqno, seqno);
172}
173
174static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence)
175{
176	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
177	struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence);
178
179	dma_fence_get(dma_fence);
180	list_add_tail(&fence->irq_link, &irq->pending);
181
182	/* SW completed (no HW IRQ) so kick handler to signal fence */
183	if (xe_hw_fence_signaled(dma_fence))
184		xe_hw_fence_irq_run(irq);
185
186	return true;
187}
188
189static void xe_hw_fence_release(struct dma_fence *dma_fence)
190{
191	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
192
193	XE_WARN_ON(!list_empty(&fence->irq_link));
194	call_rcu(&dma_fence->rcu, fence_free);
195}
196
197static const struct dma_fence_ops xe_hw_fence_ops = {
198	.get_driver_name = xe_hw_fence_get_driver_name,
199	.get_timeline_name = xe_hw_fence_get_timeline_name,
200	.enable_signaling = xe_hw_fence_enable_signaling,
201	.signaled = xe_hw_fence_signaled,
202	.release = xe_hw_fence_release,
203};
204
205static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence)
206{
207	if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops))
208		return NULL;
209
210	return container_of(fence, struct xe_hw_fence, dma);
211}
212
213/**
214 * xe_hw_fence_alloc() -  Allocate an hw fence.
215 *
216 * Allocate but don't initialize an hw fence.
217 *
218 * Return: Pointer to the allocated fence or
219 * negative error pointer on error.
220 */
221struct dma_fence *xe_hw_fence_alloc(void)
222{
223	struct xe_hw_fence *hw_fence = fence_alloc();
224
225	if (!hw_fence)
226		return ERR_PTR(-ENOMEM);
227
228	return &hw_fence->dma;
229}
230
231/**
232 * xe_hw_fence_free() - Free an hw fence.
233 * @fence: Pointer to the fence to free.
234 *
235 * Frees an hw fence that hasn't yet been
236 * initialized.
237 */
238void xe_hw_fence_free(struct dma_fence *fence)
239{
240	fence_free(&fence->rcu);
241}
242
243/**
244 * xe_hw_fence_init() - Initialize an hw fence.
245 * @fence: Pointer to the fence to initialize.
246 * @ctx: Pointer to the struct xe_hw_fence_ctx fence context.
247 * @seqno_map: Pointer to the map into where the seqno is blitted.
248 *
249 * Initializes a pre-allocated hw fence.
250 * After initialization, the fence is subject to normal
251 * dma-fence refcounting.
252 */
253void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx,
254		      struct iosys_map seqno_map)
255{
256	struct  xe_hw_fence *hw_fence =
257		container_of(fence, typeof(*hw_fence), dma);
258
259	hw_fence->xe = gt_to_xe(ctx->gt);
260	snprintf(hw_fence->name, sizeof(hw_fence->name), "%s", ctx->name);
261	hw_fence->seqno_map = seqno_map;
262	INIT_LIST_HEAD(&hw_fence->irq_link);
263
264	dma_fence_init(fence, &xe_hw_fence_ops, &ctx->irq->lock,
265		       ctx->dma_fence_ctx, ctx->next_seqno++);
266
267	trace_xe_hw_fence_create(hw_fence);
268}