msm_fence.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2013-2016 Red Hat
  4 * Author: Rob Clark <robdclark@gmail.com>
  5 */
  6
  7#include <linux/dma-fence.h>
  8
  9#include "msm_drv.h"
 10#include "msm_fence.h"
 11#include "msm_gpu.h"
 12
 13static struct msm_gpu *fctx2gpu(struct msm_fence_context *fctx)
 14{
 15	struct msm_drm_private *priv = fctx->dev->dev_private;
 16	return priv->gpu;
 17}
 18
 19static enum hrtimer_restart deadline_timer(struct hrtimer *t)
 20{
 21	struct msm_fence_context *fctx = container_of(t,
 22			struct msm_fence_context, deadline_timer);
 23
 24	kthread_queue_work(fctx2gpu(fctx)->worker, &fctx->deadline_work);
 25
 26	return HRTIMER_NORESTART;
 27}
 28
 29static void deadline_work(struct kthread_work *work)
 30{
 31	struct msm_fence_context *fctx = container_of(work,
 32			struct msm_fence_context, deadline_work);
 33
 34	/* If deadline fence has already passed, nothing to do: */
 35	if (msm_fence_completed(fctx, fctx->next_deadline_fence))
 36		return;
 37
 38	msm_devfreq_boost(fctx2gpu(fctx), 2);
 39}
 40
 41
 42struct msm_fence_context *
 43msm_fence_context_alloc(struct drm_device *dev, volatile uint32_t *fenceptr,
 44		const char *name)
 45{
 46	struct msm_fence_context *fctx;
 47	static int index = 0;
 48
 49	fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
 50	if (!fctx)
 51		return ERR_PTR(-ENOMEM);
 52
 53	fctx->dev = dev;
 54	strscpy(fctx->name, name, sizeof(fctx->name));
 55	fctx->context = dma_fence_context_alloc(1);
 56	fctx->index = index++;
 57	fctx->fenceptr = fenceptr;
 58	spin_lock_init(&fctx->spinlock);
 59
 60	/*
 61	 * Start out close to the 32b fence rollover point, so we can
 62	 * catch bugs with fence comparisons.
 63	 */
 64	fctx->last_fence = 0xffffff00;
 65	fctx->completed_fence = fctx->last_fence;
 66	*fctx->fenceptr = fctx->last_fence;
 67
 68	hrtimer_setup(&fctx->deadline_timer, deadline_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 69
 70	kthread_init_work(&fctx->deadline_work, deadline_work);
 71
 72	fctx->next_deadline = ktime_get();
 73
 74	return fctx;
 75}
 76
 77void msm_fence_context_free(struct msm_fence_context *fctx)
 78{
 79	kfree(fctx);
 80}
 81
 82bool msm_fence_completed(struct msm_fence_context *fctx, uint32_t fence)
 83{
 84	/*
 85	 * Note: Check completed_fence first, as fenceptr is in a write-combine
 86	 * mapping, so it will be more expensive to read.
 87	 */
 88	return (int32_t)(fctx->completed_fence - fence) >= 0 ||
 89		(int32_t)(*fctx->fenceptr - fence) >= 0;
 90}
 91
 92/* called from irq handler and workqueue (in recover path) */
 93void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
 94{
 95	unsigned long flags;
 96
 97	spin_lock_irqsave(&fctx->spinlock, flags);
 98	if (fence_after(fence, fctx->completed_fence))
 99		fctx->completed_fence = fence;
100	if (msm_fence_completed(fctx, fctx->next_deadline_fence))
101		hrtimer_cancel(&fctx->deadline_timer);
102	spin_unlock_irqrestore(&fctx->spinlock, flags);
103}
104
105struct msm_fence {
106	struct dma_fence base;
107	struct msm_fence_context *fctx;
108};
109
110static inline struct msm_fence *to_msm_fence(struct dma_fence *fence)
111{
112	return container_of(fence, struct msm_fence, base);
113}
114
115static const char *msm_fence_get_driver_name(struct dma_fence *fence)
116{
117	return "msm";
118}
119
120static const char *msm_fence_get_timeline_name(struct dma_fence *fence)
121{
122	struct msm_fence *f = to_msm_fence(fence);
123	return f->fctx->name;
124}
125
126static bool msm_fence_signaled(struct dma_fence *fence)
127{
128	struct msm_fence *f = to_msm_fence(fence);
129	return msm_fence_completed(f->fctx, f->base.seqno);
130}
131
132static void msm_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
133{
134	struct msm_fence *f = to_msm_fence(fence);
135	struct msm_fence_context *fctx = f->fctx;
136	unsigned long flags;
137	ktime_t now;
138
139	spin_lock_irqsave(&fctx->spinlock, flags);
140	now = ktime_get();
141
142	if (ktime_after(now, fctx->next_deadline) ||
143			ktime_before(deadline, fctx->next_deadline)) {
144		fctx->next_deadline = deadline;
145		fctx->next_deadline_fence =
146			max(fctx->next_deadline_fence, (uint32_t)fence->seqno);
147
148		/*
149		 * Set timer to trigger boost 3ms before deadline, or
150		 * if we are already less than 3ms before the deadline
151		 * schedule boost work immediately.
152		 */
153		deadline = ktime_sub(deadline, ms_to_ktime(3));
154
155		if (ktime_after(now, deadline)) {
156			kthread_queue_work(fctx2gpu(fctx)->worker,
157					&fctx->deadline_work);
158		} else {
159			hrtimer_start(&fctx->deadline_timer, deadline,
160					HRTIMER_MODE_ABS);
161		}
162	}
163
164	spin_unlock_irqrestore(&fctx->spinlock, flags);
165}
166
167static const struct dma_fence_ops msm_fence_ops = {
168	.get_driver_name = msm_fence_get_driver_name,
169	.get_timeline_name = msm_fence_get_timeline_name,
170	.signaled = msm_fence_signaled,
171	.set_deadline = msm_fence_set_deadline,
172};
173
174struct dma_fence *
175msm_fence_alloc(void)
176{
177	struct msm_fence *f;
178
179	f = kzalloc(sizeof(*f), GFP_KERNEL);
180	if (!f)
181		return ERR_PTR(-ENOMEM);
182
183	return &f->base;
184}
185
186void
187msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
188{
189	struct msm_fence *f = to_msm_fence(fence);
190
191	f->fctx = fctx;
192
193	/*
194	 * Until this point, the fence was just some pre-allocated memory,
195	 * no-one should have taken a reference to it yet.
196	 */
197	WARN_ON(kref_read(&fence->refcount));
198
199	dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
200		       fctx->context, ++fctx->last_fence);
201}