sw_sync.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Sync File validation framework
  4 *
  5 * Copyright (C) 2012 Google, Inc.
  6 */
  7
  8#include <linux/file.h>
  9#include <linux/fs.h>
 10#include <linux/uaccess.h>
 11#include <linux/panic.h>
 12#include <linux/slab.h>
 13#include <linux/sync_file.h>
 14
 15#include "sync_debug.h"
 16
 17#define CREATE_TRACE_POINTS
 18#include "sync_trace.h"
 19
 20/*
 21 * SW SYNC validation framework
 22 *
 23 * A sync object driver that uses a 32bit counter to coordinate
 24 * synchronization.  Useful when there is no hardware primitive backing
 25 * the synchronization.
 26 *
 27 * To start the framework just open:
 28 *
 29 * <debugfs>/sync/sw_sync
 30 *
 31 * That will create a sync timeline, all fences created under this timeline
 32 * file descriptor will belong to the this timeline.
 33 *
 34 * The 'sw_sync' file can be opened many times as to create different
 35 * timelines.
 36 *
 37 * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct
 38 * sw_sync_create_fence_data as parameter.
 39 *
 40 * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used
 41 * with the increment as u32. This will update the last signaled value
 42 * from the timeline and signal any fence that has a seqno smaller or equal
 43 * to it.
 44 *
 45 * struct sw_sync_create_fence_data
 46 * @value:	the seqno to initialise the fence with
 47 * @name:	the name of the new sync point
 48 * @fence:	return the fd of the new sync_file with the created fence
 49 */
 50struct sw_sync_create_fence_data {
 51	__u32	value;
 52	char	name[32];
 53	__s32	fence; /* fd of new fence */
 54};
 55
 56/**
 57 * struct sw_sync_get_deadline - get the deadline hint of a sw_sync fence
 58 * @deadline_ns: absolute time of the deadline
 59 * @pad:	must be zero
 60 * @fence_fd:	the sw_sync fence fd (in)
 61 *
 62 * Return the earliest deadline set on the fence.  The timebase for the
 63 * deadline is CLOCK_MONOTONIC (same as vblank).  If there is no deadline
 64 * set on the fence, this ioctl will return -ENOENT.
 65 */
 66struct sw_sync_get_deadline {
 67	__u64	deadline_ns;
 68	__u32	pad;
 69	__s32	fence_fd;
 70};
 71
 72#define SW_SYNC_IOC_MAGIC	'W'
 73
 74#define SW_SYNC_IOC_CREATE_FENCE	_IOWR(SW_SYNC_IOC_MAGIC, 0,\
 75		struct sw_sync_create_fence_data)
 76
 77#define SW_SYNC_IOC_INC			_IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
 78#define SW_SYNC_GET_DEADLINE		_IOWR(SW_SYNC_IOC_MAGIC, 2, \
 79		struct sw_sync_get_deadline)
 80
 81
 82#define SW_SYNC_HAS_DEADLINE_BIT	DMA_FENCE_FLAG_USER_BITS
 83
 84static const struct dma_fence_ops timeline_fence_ops;
 85
 86static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence)
 87{
 88	if (fence->ops != &timeline_fence_ops)
 89		return NULL;
 90	return container_of(fence, struct sync_pt, base);
 91}
 92
 93/**
 94 * sync_timeline_create() - creates a sync object
 95 * @name:	sync_timeline name
 96 *
 97 * Creates a new sync_timeline. Returns the sync_timeline object or NULL in
 98 * case of error.
 99 */
100static struct sync_timeline *sync_timeline_create(const char *name)
101{
102	struct sync_timeline *obj;
103
104	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
105	if (!obj)
106		return NULL;
107
108	kref_init(&obj->kref);
109	obj->context = dma_fence_context_alloc(1);
110	strscpy(obj->name, name, sizeof(obj->name));
111
112	obj->pt_tree = RB_ROOT;
113	INIT_LIST_HEAD(&obj->pt_list);
114	spin_lock_init(&obj->lock);
115
116	sync_timeline_debug_add(obj);
117
118	return obj;
119}
120
121static void sync_timeline_free(struct kref *kref)
122{
123	struct sync_timeline *obj =
124		container_of(kref, struct sync_timeline, kref);
125
126	sync_timeline_debug_remove(obj);
127
128	kfree(obj);
129}
130
131static void sync_timeline_get(struct sync_timeline *obj)
132{
133	kref_get(&obj->kref);
134}
135
136static void sync_timeline_put(struct sync_timeline *obj)
137{
138	kref_put(&obj->kref, sync_timeline_free);
139}
140
141static const char *timeline_fence_get_driver_name(struct dma_fence *fence)
142{
143	return "sw_sync";
144}
145
146static const char *timeline_fence_get_timeline_name(struct dma_fence *fence)
147{
148	struct sync_timeline *parent = dma_fence_parent(fence);
149
150	return parent->name;
151}
152
153static void timeline_fence_release(struct dma_fence *fence)
154{
155	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
156	struct sync_timeline *parent = dma_fence_parent(fence);
157	unsigned long flags;
158
159	spin_lock_irqsave(fence->lock, flags);
160	if (!list_empty(&pt->link)) {
161		list_del(&pt->link);
162		rb_erase(&pt->node, &parent->pt_tree);
163	}
164	spin_unlock_irqrestore(fence->lock, flags);
165
166	sync_timeline_put(parent);
167	dma_fence_free(fence);
168}
169
170static bool timeline_fence_signaled(struct dma_fence *fence)
171{
172	struct sync_timeline *parent = dma_fence_parent(fence);
173
174	return !__dma_fence_is_later(fence, fence->seqno, parent->value);
175}
176
177static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
178{
179	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
180	unsigned long flags;
181
182	spin_lock_irqsave(fence->lock, flags);
183	if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {
184		if (ktime_before(deadline, pt->deadline))
185			pt->deadline = deadline;
186	} else {
187		pt->deadline = deadline;
188		__set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags);
189	}
190	spin_unlock_irqrestore(fence->lock, flags);
191}
192
193static const struct dma_fence_ops timeline_fence_ops = {
194	.get_driver_name = timeline_fence_get_driver_name,
195	.get_timeline_name = timeline_fence_get_timeline_name,
196	.signaled = timeline_fence_signaled,
197	.release = timeline_fence_release,
198	.set_deadline = timeline_fence_set_deadline,
199};
200
201/**
202 * sync_timeline_signal() - signal a status change on a sync_timeline
203 * @obj:	sync_timeline to signal
204 * @inc:	num to increment on timeline->value
205 *
206 * A sync implementation should call this any time one of it's fences
207 * has signaled or has an error condition.
208 */
209static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
210{
211	LIST_HEAD(signalled);
212	struct sync_pt *pt, *next;
213
214	trace_sync_timeline(obj);
215
216	spin_lock_irq(&obj->lock);
217
218	obj->value += inc;
219
220	list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
221		if (!timeline_fence_signaled(&pt->base))
222			break;
223
224		dma_fence_get(&pt->base);
225
226		list_move_tail(&pt->link, &signalled);
227		rb_erase(&pt->node, &obj->pt_tree);
228
229		dma_fence_signal_locked(&pt->base);
230	}
231
232	spin_unlock_irq(&obj->lock);
233
234	list_for_each_entry_safe(pt, next, &signalled, link) {
235		list_del_init(&pt->link);
236		dma_fence_put(&pt->base);
237	}
238}
239
240/**
241 * sync_pt_create() - creates a sync pt
242 * @obj:	parent sync_timeline
243 * @value:	value of the fence
244 *
245 * Creates a new sync_pt (fence) as a child of @parent.  @size bytes will be
246 * allocated allowing for implementation specific data to be kept after
247 * the generic sync_timeline struct. Returns the sync_pt object or
248 * NULL in case of error.
249 */
250static struct sync_pt *sync_pt_create(struct sync_timeline *obj,
251				      unsigned int value)
252{
253	struct sync_pt *pt;
254
255	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
256	if (!pt)
257		return NULL;
258
259	sync_timeline_get(obj);
260	dma_fence_init(&pt->base, &timeline_fence_ops, &obj->lock,
261		       obj->context, value);
262	INIT_LIST_HEAD(&pt->link);
263
264	spin_lock_irq(&obj->lock);
265	if (!dma_fence_is_signaled_locked(&pt->base)) {
266		struct rb_node **p = &obj->pt_tree.rb_node;
267		struct rb_node *parent = NULL;
268
269		while (*p) {
270			struct sync_pt *other;
271			int cmp;
272
273			parent = *p;
274			other = rb_entry(parent, typeof(*pt), node);
275			cmp = value - other->base.seqno;
276			if (cmp > 0) {
277				p = &parent->rb_right;
278			} else if (cmp < 0) {
279				p = &parent->rb_left;
280			} else {
281				if (dma_fence_get_rcu(&other->base)) {
282					sync_timeline_put(obj);
283					kfree(pt);
284					pt = other;
285					goto unlock;
286				}
287				p = &parent->rb_left;
288			}
289		}
290		rb_link_node(&pt->node, parent, p);
291		rb_insert_color(&pt->node, &obj->pt_tree);
292
293		parent = rb_next(&pt->node);
294		list_add_tail(&pt->link,
295			      parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list);
296	}
297unlock:
298	spin_unlock_irq(&obj->lock);
299
300	return pt;
301}
302
303/*
304 * *WARNING*
305 *
306 * improper use of this can result in deadlocking kernel drivers from userspace.
307 */
308
309/* opening sw_sync create a new sync obj */
310static int sw_sync_debugfs_open(struct inode *inode, struct file *file)
311{
312	struct sync_timeline *obj;
313	char task_comm[TASK_COMM_LEN];
314
315	get_task_comm(task_comm, current);
316
317	obj = sync_timeline_create(task_comm);
318	if (!obj)
319		return -ENOMEM;
320
321	file->private_data = obj;
322
323	return 0;
324}
325
326static int sw_sync_debugfs_release(struct inode *inode, struct file *file)
327{
328	struct sync_timeline *obj = file->private_data;
329	struct sync_pt *pt, *next;
330
331	spin_lock_irq(&obj->lock);
332
333	list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
334		dma_fence_set_error(&pt->base, -ENOENT);
335		dma_fence_signal_locked(&pt->base);
336	}
337
338	spin_unlock_irq(&obj->lock);
339
340	sync_timeline_put(obj);
341	return 0;
342}
343
344static long sw_sync_ioctl_create_fence(struct sync_timeline *obj,
345				       unsigned long arg)
346{
347	int fd = get_unused_fd_flags(O_CLOEXEC);
348	int err;
349	struct sync_pt *pt;
350	struct sync_file *sync_file;
351	struct sw_sync_create_fence_data data;
352
353	/* SW sync fence are inherently unsafe and can deadlock the kernel */
354	add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
355
356	if (fd < 0)
357		return fd;
358
359	if (copy_from_user(&data, (void __user *)arg, sizeof(data))) {
360		err = -EFAULT;
361		goto err;
362	}
363
364	pt = sync_pt_create(obj, data.value);
365	if (!pt) {
366		err = -ENOMEM;
367		goto err;
368	}
369
370	sync_file = sync_file_create(&pt->base);
371	dma_fence_put(&pt->base);
372	if (!sync_file) {
373		err = -ENOMEM;
374		goto err;
375	}
376
377	data.fence = fd;
378	if (copy_to_user((void __user *)arg, &data, sizeof(data))) {
379		fput(sync_file->file);
380		err = -EFAULT;
381		goto err;
382	}
383
384	fd_install(fd, sync_file->file);
385
386	return 0;
387
388err:
389	put_unused_fd(fd);
390	return err;
391}
392
393static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg)
394{
395	u32 value;
396
397	if (copy_from_user(&value, (void __user *)arg, sizeof(value)))
398		return -EFAULT;
399
400	while (value > INT_MAX)  {
401		sync_timeline_signal(obj, INT_MAX);
402		value -= INT_MAX;
403	}
404
405	sync_timeline_signal(obj, value);
406
407	return 0;
408}
409
410static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long arg)
411{
412	struct sw_sync_get_deadline data;
413	struct dma_fence *fence;
414	unsigned long flags;
415	struct sync_pt *pt;
416	int ret = 0;
417
418	if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
419		return -EFAULT;
420
421	if (data.deadline_ns || data.pad)
422		return -EINVAL;
423
424	fence = sync_file_get_fence(data.fence_fd);
425	if (!fence)
426		return -EINVAL;
427
428	pt = dma_fence_to_sync_pt(fence);
429	if (!pt) {
430		ret = -EINVAL;
431		goto put_fence;
432	}
433
434	spin_lock_irqsave(fence->lock, flags);
435	if (!test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {
436		ret = -ENOENT;
437		goto unlock;
438	}
439	data.deadline_ns = ktime_to_ns(pt->deadline);
440	spin_unlock_irqrestore(fence->lock, flags);
441
442	dma_fence_put(fence);
443
444	if (ret)
445		return ret;
446
447	if (copy_to_user((void __user *)arg, &data, sizeof(data)))
448		return -EFAULT;
449
450	return 0;
451
452unlock:
453	spin_unlock_irqrestore(fence->lock, flags);
454put_fence:
455	dma_fence_put(fence);
456
457	return ret;
458}
459
460static long sw_sync_ioctl(struct file *file, unsigned int cmd,
461			  unsigned long arg)
462{
463	struct sync_timeline *obj = file->private_data;
464
465	switch (cmd) {
466	case SW_SYNC_IOC_CREATE_FENCE:
467		return sw_sync_ioctl_create_fence(obj, arg);
468
469	case SW_SYNC_IOC_INC:
470		return sw_sync_ioctl_inc(obj, arg);
471
472	case SW_SYNC_GET_DEADLINE:
473		return sw_sync_ioctl_get_deadline(obj, arg);
474
475	default:
476		return -ENOTTY;
477	}
478}
479
480const struct file_operations sw_sync_debugfs_fops = {
481	.open           = sw_sync_debugfs_open,
482	.release        = sw_sync_debugfs_release,
483	.unlocked_ioctl = sw_sync_ioctl,
484	.compat_ioctl	= compat_ptr_ioctl,
485};