drm_syncobj.c

   1/*
   2 * Copyright 2017 Red Hat
   3 * Parts ported from amdgpu (fence wait code).
   4 * Copyright 2016 Advanced Micro Devices, Inc.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice (including the next
  14 * paragraph) shall be included in all copies or substantial portions of the
  15 * Software.
  16 *
  17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23 * IN THE SOFTWARE.
  24 *
  25 * Authors:
  26 *
  27 */
  28
  29/**
  30 * DOC: Overview
  31 *
  32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
  33 * container for a synchronization primitive which can be used by userspace
  34 * to explicitly synchronize GPU commands, can be shared between userspace
  35 * processes, and can be shared between different DRM drivers.
  36 * Their primary use-case is to implement Vulkan fences and semaphores.
  37 * The syncobj userspace API provides ioctls for several operations:
  38 *
  39 *  - Creation and destruction of syncobjs
  40 *  - Import and export of syncobjs to/from a syncobj file descriptor
  41 *  - Import and export a syncobj's underlying fence to/from a sync file
  42 *  - Reset a syncobj (set its fence to NULL)
  43 *  - Signal a syncobj (set a trivially signaled fence)
  44 *  - Wait for a syncobj's fence to appear and be signaled
  45 *
  46 * The syncobj userspace API also provides operations to manipulate a syncobj
  47 * in terms of a timeline of struct &dma_fence_chain rather than a single
  48 * struct &dma_fence, through the following operations:
  49 *
  50 *   - Signal a given point on the timeline
  51 *   - Wait for a given point to appear and/or be signaled
  52 *   - Import and export from/to a given point of a timeline
  53 *
  54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct
  55 * &dma_fence which may be NULL.
  56 * When a syncobj is first created, its pointer is either NULL or a pointer
  57 * to an already signaled fence depending on whether the
  58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
  59 * &DRM_IOCTL_SYNCOBJ_CREATE.
  60 *
  61 * If the syncobj is considered as a binary (its state is either signaled or
  62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
  63 * the syncobj, the syncobj's fence is replaced with a fence which will be
  64 * signaled by the completion of that work.
  65 * If the syncobj is considered as a timeline primitive, when GPU work is
  66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new
  67 * struct &dma_fence_chain pointing to the DRM driver's fence and also
  68 * pointing to the previous fence that was in the syncobj. The new struct
  69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
  70 * completion of the DRM driver's work and also any work associated with the
  71 * fence previously in the syncobj.
  72 *
  73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
  74 * time the work is enqueued, it waits on the syncobj's fence before
  75 * submitting the work to hardware. That fence is either :
  76 *
  77 *    - The syncobj's current fence if the syncobj is considered as a binary
  78 *      primitive.
  79 *    - The struct &dma_fence associated with a given point if the syncobj is
  80 *      considered as a timeline primitive.
  81 *
  82 * If the syncobj's fence is NULL or not present in the syncobj's timeline,
  83 * the enqueue operation is expected to fail.
  84 *
  85 * With binary syncobj, all manipulation of the syncobjs's fence happens in
  86 * terms of the current fence at the time the ioctl is called by userspace
  87 * regardless of whether that operation is an immediate host-side operation
  88 * (signal or reset) or or an operation which is enqueued in some driver
  89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
  90 * to manipulate a syncobj from the host by resetting its pointer to NULL or
  91 * setting its pointer to a fence which is already signaled.
  92 *
  93 * With a timeline syncobj, all manipulation of the synobj's fence happens in
  94 * terms of a u64 value referring to point in the timeline. See
  95 * dma_fence_chain_find_seqno() to see how a given point is found in the
  96 * timeline.
  97 *
  98 * Note that applications should be careful to always use timeline set of
  99 * ioctl() when dealing with syncobj considered as timeline. Using a binary
 100 * set of ioctl() with a syncobj considered as timeline could result incorrect
 101 * synchronization. The use of binary syncobj is supported through the
 102 * timeline set of ioctl() by using a point value of 0, this will reproduce
 103 * the behavior of the binary set of ioctl() (for example replace the
 104 * syncobj's fence when signaling).
 105 *
 106 *
 107 * Host-side wait on syncobjs
 108 * --------------------------
 109 *
 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
 111 * host-side wait on all of the syncobj fences simultaneously.
 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
 113 * all of the syncobj fences to be signaled before it returns.
 114 * Otherwise, it returns once at least one syncobj fence has been signaled
 115 * and the index of a signaled fence is written back to the client.
 116 *
 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL
 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
 119 * the host-side wait will first wait for the syncobj to receive a non-NULL
 120 * fence and then wait on that fence.
 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned.
 123 * Assuming the syncobj starts off with a NULL fence, this allows a client
 124 * to do a host wait in one thread (or process) which waits on GPU work
 125 * submitted in another thread (or process) without having to manually
 126 * synchronize between the two.
 127 * This requirement is inherited from the Vulkan fence API.
 128 *
 129 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set
 130 * a fence deadline hint on the backing fences before waiting, to provide the
 131 * fence signaler with an appropriate sense of urgency.  The deadline is
 132 * specified as an absolute &CLOCK_MONOTONIC value in units of ns.
 133 *
 134 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
 135 * handles as well as an array of u64 points and does a host-side wait on all
 136 * of syncobj fences at the given points simultaneously.
 137 *
 138 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
 139 * fence to materialize on the timeline without waiting for the fence to be
 140 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
 141 * requirement is inherited from the wait-before-signal behavior required by
 142 * the Vulkan timeline semaphore API.
 143 *
 144 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
 145 * blocking: an eventfd will be signaled when the syncobj is. This is useful to
 146 * integrate the wait in an event loop.
 147 *
 148 *
 149 * Import/export of syncobjs
 150 * -------------------------
 151 *
 152 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
 153 * provide two mechanisms for import/export of syncobjs.
 154 *
 155 * The first lets the client import or export an entire syncobj to a file
 156 * descriptor.
 157 * These fd's are opaque and have no other use case, except passing the
 158 * syncobj between processes.
 159 * All exported file descriptors and any syncobj handles created as a
 160 * result of importing those file descriptors own a reference to the
 161 * same underlying struct &drm_syncobj and the syncobj can be used
 162 * persistently across all the processes with which it is shared.
 163 * The syncobj is freed only once the last reference is dropped.
 164 * Unlike dma-buf, importing a syncobj creates a new handle (with its own
 165 * reference) for every import instead of de-duplicating.
 166 * The primary use-case of this persistent import/export is for shared
 167 * Vulkan fences and semaphores.
 168 *
 169 * The second import/export mechanism, which is indicated by
 170 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
 171 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
 172 * import/export the syncobj's current fence from/to a &sync_file.
 173 * When a syncobj is exported to a sync file, that sync file wraps the
 174 * sycnobj's fence at the time of export and any later signal or reset
 175 * operations on the syncobj will not affect the exported sync file.
 176 * When a sync file is imported into a syncobj, the syncobj's fence is set
 177 * to the fence wrapped by that sync file.
 178 * Because sync files are immutable, resetting or signaling the syncobj
 179 * will not affect any sync files whose fences have been imported into the
 180 * syncobj.
 181 *
 182 *
 183 * Import/export of timeline points in timeline syncobjs
 184 * -----------------------------------------------------
 185 *
 186 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
 187 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
 188 * into another syncobj.
 189 *
 190 * Note that if you want to transfer a struct &dma_fence_chain from a given
 191 * point on a timeline syncobj from/into a binary syncobj, you can use the
 192 * point 0 to mean take/replace the fence in the syncobj.
 193 */
 194
 195#include <linux/anon_inodes.h>
 196#include <linux/dma-fence-unwrap.h>
 197#include <linux/eventfd.h>
 198#include <linux/export.h>
 199#include <linux/file.h>
 200#include <linux/fs.h>
 201#include <linux/sched/signal.h>
 202#include <linux/sync_file.h>
 203#include <linux/uaccess.h>
 204
 205#include <drm/drm.h>
 206#include <drm/drm_drv.h>
 207#include <drm/drm_file.h>
 208#include <drm/drm_gem.h>
 209#include <drm/drm_print.h>
 210#include <drm/drm_syncobj.h>
 211#include <drm/drm_utils.h>
 212
 213#include "drm_internal.h"
 214
 215struct syncobj_wait_entry {
 216	struct list_head node;
 217	struct task_struct *task;
 218	struct dma_fence *fence;
 219	struct dma_fence_cb fence_cb;
 220	u64    point;
 221};
 222
 223static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 224				      struct syncobj_wait_entry *wait);
 225
 226struct syncobj_eventfd_entry {
 227	struct list_head node;
 228	struct dma_fence *fence;
 229	struct dma_fence_cb fence_cb;
 230	struct drm_syncobj *syncobj;
 231	struct eventfd_ctx *ev_fd_ctx;
 232	u64 point;
 233	u32 flags;
 234};
 235
 236static void
 237syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
 238			   struct syncobj_eventfd_entry *entry);
 239
 240/**
 241 * drm_syncobj_find - lookup and reference a sync object.
 242 * @file_private: drm file private pointer
 243 * @handle: sync object handle to lookup.
 244 *
 245 * Returns a reference to the syncobj pointed to by handle or NULL. The
 246 * reference must be released by calling drm_syncobj_put().
 247 */
 248struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 249				     u32 handle)
 250{
 251	struct drm_syncobj *syncobj;
 252
 253	spin_lock(&file_private->syncobj_table_lock);
 254
 255	/* Check if we currently have a reference on the object */
 256	syncobj = idr_find(&file_private->syncobj_idr, handle);
 257	if (syncobj)
 258		drm_syncobj_get(syncobj);
 259
 260	spin_unlock(&file_private->syncobj_table_lock);
 261
 262	return syncobj;
 263}
 264EXPORT_SYMBOL(drm_syncobj_find);
 265
 266static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 267				       struct syncobj_wait_entry *wait)
 268{
 269	struct dma_fence *fence;
 270
 271	if (wait->fence)
 272		return;
 273
 274	spin_lock(&syncobj->lock);
 275	/* We've already tried once to get a fence and failed.  Now that we
 276	 * have the lock, try one more time just to be sure we don't add a
 277	 * callback when a fence has already been set.
 278	 */
 279	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 280	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 281		dma_fence_put(fence);
 282		list_add_tail(&wait->node, &syncobj->cb_list);
 283	} else if (!fence) {
 284		wait->fence = dma_fence_get_stub();
 285	} else {
 286		wait->fence = fence;
 287	}
 288	spin_unlock(&syncobj->lock);
 289}
 290
 291static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 292				    struct syncobj_wait_entry *wait)
 293{
 294	if (!wait->node.next)
 295		return;
 296
 297	spin_lock(&syncobj->lock);
 298	list_del_init(&wait->node);
 299	spin_unlock(&syncobj->lock);
 300}
 301
 302static void
 303syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
 304{
 305	eventfd_ctx_put(entry->ev_fd_ctx);
 306	dma_fence_put(entry->fence);
 307	/* This happens either inside the syncobj lock, or after the node has
 308	 * already been removed from the list.
 309	 */
 310	list_del(&entry->node);
 311	kfree(entry);
 312}
 313
 314static void
 315drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
 316			struct syncobj_eventfd_entry *entry)
 317{
 318	spin_lock(&syncobj->lock);
 319	list_add_tail(&entry->node, &syncobj->ev_fd_list);
 320	syncobj_eventfd_entry_func(syncobj, entry);
 321	spin_unlock(&syncobj->lock);
 322}
 323
 324/**
 325 * drm_syncobj_add_point - add new timeline point to the syncobj
 326 * @syncobj: sync object to add timeline point do
 327 * @chain: chain node to use to add the point
 328 * @fence: fence to encapsulate in the chain node
 329 * @point: sequence number to use for the point
 330 *
 331 * Add the chain node as new timeline point to the syncobj.
 332 */
 333void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 334			   struct dma_fence_chain *chain,
 335			   struct dma_fence *fence,
 336			   uint64_t point)
 337{
 338	struct syncobj_wait_entry *wait_cur, *wait_tmp;
 339	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 340	struct dma_fence *prev;
 341
 342	dma_fence_get(fence);
 343
 344	spin_lock(&syncobj->lock);
 345
 346	prev = drm_syncobj_fence_get(syncobj);
 347	/* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 348	if (prev && prev->seqno >= point)
 349		DRM_DEBUG("You are adding an unorder point to timeline!\n");
 350	dma_fence_chain_init(chain, prev, fence, point);
 351	rcu_assign_pointer(syncobj->fence, &chain->base);
 352
 353	list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
 354		syncobj_wait_syncobj_func(syncobj, wait_cur);
 355	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 356		syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
 357	spin_unlock(&syncobj->lock);
 358
 359	/* Walk the chain once to trigger garbage collection */
 360	dma_fence_chain_for_each(fence, prev);
 361	dma_fence_put(prev);
 362}
 363EXPORT_SYMBOL(drm_syncobj_add_point);
 364
 365/**
 366 * drm_syncobj_replace_fence - replace fence in a sync object.
 367 * @syncobj: Sync object to replace fence in
 368 * @fence: fence to install in sync file.
 369 *
 370 * This replaces the fence on a sync object.
 371 */
 372void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 373			       struct dma_fence *fence)
 374{
 375	struct dma_fence *old_fence;
 376	struct syncobj_wait_entry *wait_cur, *wait_tmp;
 377	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 378
 379	if (fence)
 380		dma_fence_get(fence);
 381
 382	spin_lock(&syncobj->lock);
 383
 384	old_fence = rcu_dereference_protected(syncobj->fence,
 385					      lockdep_is_held(&syncobj->lock));
 386	rcu_assign_pointer(syncobj->fence, fence);
 387
 388	if (fence != old_fence) {
 389		list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
 390			syncobj_wait_syncobj_func(syncobj, wait_cur);
 391		list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 392			syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
 393	}
 394
 395	spin_unlock(&syncobj->lock);
 396
 397	dma_fence_put(old_fence);
 398}
 399EXPORT_SYMBOL(drm_syncobj_replace_fence);
 400
 401/**
 402 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 403 * @syncobj: sync object to assign the fence on
 404 *
 405 * Assign a already signaled stub fence to the sync object.
 406 */
 407static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 408{
 409	struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());
 410
 411	if (!fence)
 412		return -ENOMEM;
 413
 414	drm_syncobj_replace_fence(syncobj, fence);
 415	dma_fence_put(fence);
 416	return 0;
 417}
 418
 419/* 5s default for wait submission */
 420#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 421/**
 422 * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 423 * @file_private: drm file private pointer
 424 * @handle: sync object handle to lookup.
 425 * @point: timeline point
 426 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 427 * @fence: out parameter for the fence
 428 *
 429 * This is just a convenience function that combines drm_syncobj_find() and
 430 * drm_syncobj_fence_get().
 431 *
 432 * Returns 0 on success or a negative error value on failure. On success @fence
 433 * contains a reference to the fence, which must be released by calling
 434 * dma_fence_put().
 435 */
 436int drm_syncobj_find_fence(struct drm_file *file_private,
 437			   u32 handle, u64 point, u64 flags,
 438			   struct dma_fence **fence)
 439{
 440	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 441	struct syncobj_wait_entry wait;
 442	u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 443	int ret;
 444
 445	if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
 446		return -EINVAL;
 447
 448	if (!syncobj)
 449		return -ENOENT;
 450
 451	/* Waiting for userspace with locks help is illegal cause that can
 452	 * trivial deadlock with page faults for example. Make lockdep complain
 453	 * about it early on.
 454	 */
 455	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 456		might_sleep();
 457		lockdep_assert_none_held_once();
 458	}
 459
 460	*fence = drm_syncobj_fence_get(syncobj);
 461
 462	if (*fence) {
 463		ret = dma_fence_chain_find_seqno(fence, point);
 464		if (!ret) {
 465			/* If the requested seqno is already signaled
 466			 * drm_syncobj_find_fence may return a NULL
 467			 * fence. To make sure the recipient gets
 468			 * signalled, use a new fence instead.
 469			 */
 470			if (!*fence)
 471				*fence = dma_fence_get_stub();
 472
 473			goto out;
 474		}
 475		dma_fence_put(*fence);
 476	} else {
 477		ret = -EINVAL;
 478	}
 479
 480	if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 481		goto out;
 482
 483	memset(&wait, 0, sizeof(wait));
 484	wait.task = current;
 485	wait.point = point;
 486	drm_syncobj_fence_add_wait(syncobj, &wait);
 487
 488	do {
 489		set_current_state(TASK_INTERRUPTIBLE);
 490		if (wait.fence) {
 491			ret = 0;
 492			break;
 493		}
 494                if (timeout == 0) {
 495                        ret = -ETIME;
 496                        break;
 497                }
 498
 499		if (signal_pending(current)) {
 500			ret = -ERESTARTSYS;
 501			break;
 502		}
 503
 504                timeout = schedule_timeout(timeout);
 505	} while (1);
 506
 507	__set_current_state(TASK_RUNNING);
 508	*fence = wait.fence;
 509
 510	if (wait.node.next)
 511		drm_syncobj_remove_wait(syncobj, &wait);
 512
 513out:
 514	drm_syncobj_put(syncobj);
 515
 516	return ret;
 517}
 518EXPORT_SYMBOL(drm_syncobj_find_fence);
 519
 520/**
 521 * drm_syncobj_free - free a sync object.
 522 * @kref: kref to free.
 523 *
 524 * Only to be called from kref_put in drm_syncobj_put.
 525 */
 526void drm_syncobj_free(struct kref *kref)
 527{
 528	struct drm_syncobj *syncobj = container_of(kref,
 529						   struct drm_syncobj,
 530						   refcount);
 531	struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 532
 533	drm_syncobj_replace_fence(syncobj, NULL);
 534
 535	list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 536		syncobj_eventfd_entry_free(ev_fd_cur);
 537
 538	kfree(syncobj);
 539}
 540EXPORT_SYMBOL(drm_syncobj_free);
 541
 542/**
 543 * drm_syncobj_create - create a new syncobj
 544 * @out_syncobj: returned syncobj
 545 * @flags: DRM_SYNCOBJ_* flags
 546 * @fence: if non-NULL, the syncobj will represent this fence
 547 *
 548 * This is the first function to create a sync object. After creating, drivers
 549 * probably want to make it available to userspace, either through
 550 * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 551 *
 552 * Returns 0 on success or a negative error value on failure.
 553 */
 554int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 555		       struct dma_fence *fence)
 556{
 557	int ret;
 558	struct drm_syncobj *syncobj;
 559
 560	syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 561	if (!syncobj)
 562		return -ENOMEM;
 563
 564	kref_init(&syncobj->refcount);
 565	INIT_LIST_HEAD(&syncobj->cb_list);
 566	INIT_LIST_HEAD(&syncobj->ev_fd_list);
 567	spin_lock_init(&syncobj->lock);
 568
 569	if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 570		ret = drm_syncobj_assign_null_handle(syncobj);
 571		if (ret < 0) {
 572			drm_syncobj_put(syncobj);
 573			return ret;
 574		}
 575	}
 576
 577	if (fence)
 578		drm_syncobj_replace_fence(syncobj, fence);
 579
 580	*out_syncobj = syncobj;
 581	return 0;
 582}
 583EXPORT_SYMBOL(drm_syncobj_create);
 584
 585/**
 586 * drm_syncobj_get_handle - get a handle from a syncobj
 587 * @file_private: drm file private pointer
 588 * @syncobj: Sync object to export
 589 * @handle: out parameter with the new handle
 590 *
 591 * Exports a sync object created with drm_syncobj_create() as a handle on
 592 * @file_private to userspace.
 593 *
 594 * Returns 0 on success or a negative error value on failure.
 595 */
 596int drm_syncobj_get_handle(struct drm_file *file_private,
 597			   struct drm_syncobj *syncobj, u32 *handle)
 598{
 599	int ret;
 600
 601	/* take a reference to put in the idr */
 602	drm_syncobj_get(syncobj);
 603
 604	idr_preload(GFP_KERNEL);
 605	spin_lock(&file_private->syncobj_table_lock);
 606	ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 607	spin_unlock(&file_private->syncobj_table_lock);
 608
 609	idr_preload_end();
 610
 611	if (ret < 0) {
 612		drm_syncobj_put(syncobj);
 613		return ret;
 614	}
 615
 616	*handle = ret;
 617	return 0;
 618}
 619EXPORT_SYMBOL(drm_syncobj_get_handle);
 620
 621static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 622					u32 *handle, uint32_t flags)
 623{
 624	int ret;
 625	struct drm_syncobj *syncobj;
 626
 627	ret = drm_syncobj_create(&syncobj, flags, NULL);
 628	if (ret)
 629		return ret;
 630
 631	ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 632	drm_syncobj_put(syncobj);
 633	return ret;
 634}
 635
 636static int drm_syncobj_destroy(struct drm_file *file_private,
 637			       u32 handle)
 638{
 639	struct drm_syncobj *syncobj;
 640
 641	spin_lock(&file_private->syncobj_table_lock);
 642	syncobj = idr_remove(&file_private->syncobj_idr, handle);
 643	spin_unlock(&file_private->syncobj_table_lock);
 644
 645	if (!syncobj)
 646		return -EINVAL;
 647
 648	drm_syncobj_put(syncobj);
 649	return 0;
 650}
 651
 652static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 653{
 654	struct drm_syncobj *syncobj = file->private_data;
 655
 656	drm_syncobj_put(syncobj);
 657	return 0;
 658}
 659
 660static const struct file_operations drm_syncobj_file_fops = {
 661	.release = drm_syncobj_file_release,
 662};
 663
 664/**
 665 * drm_syncobj_get_fd - get a file descriptor from a syncobj
 666 * @syncobj: Sync object to export
 667 * @p_fd: out parameter with the new file descriptor
 668 *
 669 * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 670 *
 671 * Returns 0 on success or a negative error value on failure.
 672 */
 673int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 674{
 675	struct file *file;
 676	int fd;
 677
 678	fd = get_unused_fd_flags(O_CLOEXEC);
 679	if (fd < 0)
 680		return fd;
 681
 682	file = anon_inode_getfile("syncobj_file",
 683				  &drm_syncobj_file_fops,
 684				  syncobj, 0);
 685	if (IS_ERR(file)) {
 686		put_unused_fd(fd);
 687		return PTR_ERR(file);
 688	}
 689
 690	drm_syncobj_get(syncobj);
 691	fd_install(fd, file);
 692
 693	*p_fd = fd;
 694	return 0;
 695}
 696EXPORT_SYMBOL(drm_syncobj_get_fd);
 697
 698static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 699				    u32 handle, int *p_fd)
 700{
 701	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 702	int ret;
 703
 704	if (!syncobj)
 705		return -EINVAL;
 706
 707	ret = drm_syncobj_get_fd(syncobj, p_fd);
 708	drm_syncobj_put(syncobj);
 709	return ret;
 710}
 711
 712static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 713				    int fd, u32 *handle)
 714{
 715	struct drm_syncobj *syncobj;
 716	CLASS(fd, f)(fd);
 717	int ret;
 718
 719	if (fd_empty(f))
 720		return -EINVAL;
 721
 722	if (fd_file(f)->f_op != &drm_syncobj_file_fops)
 723		return -EINVAL;
 724
 725	/* take a reference to put in the idr */
 726	syncobj = fd_file(f)->private_data;
 727	drm_syncobj_get(syncobj);
 728
 729	idr_preload(GFP_KERNEL);
 730	spin_lock(&file_private->syncobj_table_lock);
 731	ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 732	spin_unlock(&file_private->syncobj_table_lock);
 733	idr_preload_end();
 734
 735	if (ret > 0) {
 736		*handle = ret;
 737		ret = 0;
 738	} else
 739		drm_syncobj_put(syncobj);
 740
 741	return ret;
 742}
 743
 744static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 745					      int fd, int handle, u64 point)
 746{
 747	struct dma_fence *fence = sync_file_get_fence(fd);
 748	struct drm_syncobj *syncobj;
 749
 750	if (!fence)
 751		return -EINVAL;
 752
 753	syncobj = drm_syncobj_find(file_private, handle);
 754	if (!syncobj) {
 755		dma_fence_put(fence);
 756		return -ENOENT;
 757	}
 758
 759	if (point) {
 760		struct dma_fence_chain *chain = dma_fence_chain_alloc();
 761
 762		if (!chain)
 763			return -ENOMEM;
 764
 765		drm_syncobj_add_point(syncobj, chain, fence, point);
 766	} else {
 767		drm_syncobj_replace_fence(syncobj, fence);
 768	}
 769
 770	dma_fence_put(fence);
 771	drm_syncobj_put(syncobj);
 772	return 0;
 773}
 774
 775static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 776					int handle, u64 point, int *p_fd)
 777{
 778	int ret;
 779	struct dma_fence *fence;
 780	struct sync_file *sync_file;
 781	int fd = get_unused_fd_flags(O_CLOEXEC);
 782
 783	if (fd < 0)
 784		return fd;
 785
 786	ret = drm_syncobj_find_fence(file_private, handle, point, 0, &fence);
 787	if (ret)
 788		goto err_put_fd;
 789
 790	sync_file = sync_file_create(fence);
 791
 792	dma_fence_put(fence);
 793
 794	if (!sync_file) {
 795		ret = -EINVAL;
 796		goto err_put_fd;
 797	}
 798
 799	fd_install(fd, sync_file->file);
 800
 801	*p_fd = fd;
 802	return 0;
 803err_put_fd:
 804	put_unused_fd(fd);
 805	return ret;
 806}
 807/**
 808 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time
 809 * @file_private: drm file-private structure to set up
 810 *
 811 * Called at device open time, sets up the structure for handling refcounting
 812 * of sync objects.
 813 */
 814void
 815drm_syncobj_open(struct drm_file *file_private)
 816{
 817	idr_init_base(&file_private->syncobj_idr, 1);
 818	spin_lock_init(&file_private->syncobj_table_lock);
 819}
 820
 821static int
 822drm_syncobj_release_handle(int id, void *ptr, void *data)
 823{
 824	struct drm_syncobj *syncobj = ptr;
 825
 826	drm_syncobj_put(syncobj);
 827	return 0;
 828}
 829
 830/**
 831 * drm_syncobj_release - release file-private sync object resources
 832 * @file_private: drm file-private structure to clean up
 833 *
 834 * Called at close time when the filp is going away.
 835 *
 836 * Releases any remaining references on objects by this filp.
 837 */
 838void
 839drm_syncobj_release(struct drm_file *file_private)
 840{
 841	idr_for_each(&file_private->syncobj_idr,
 842		     &drm_syncobj_release_handle, file_private);
 843	idr_destroy(&file_private->syncobj_idr);
 844}
 845
 846int
 847drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 848			 struct drm_file *file_private)
 849{
 850	struct drm_syncobj_create *args = data;
 851
 852	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 853		return -EOPNOTSUPP;
 854
 855	/* no valid flags yet */
 856	if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 857		return -EINVAL;
 858
 859	return drm_syncobj_create_as_handle(file_private,
 860					    &args->handle, args->flags);
 861}
 862
 863int
 864drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 865			  struct drm_file *file_private)
 866{
 867	struct drm_syncobj_destroy *args = data;
 868
 869	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 870		return -EOPNOTSUPP;
 871
 872	/* make sure padding is empty */
 873	if (args->pad)
 874		return -EINVAL;
 875	return drm_syncobj_destroy(file_private, args->handle);
 876}
 877
 878int
 879drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 880				   struct drm_file *file_private)
 881{
 882	struct drm_syncobj_handle *args = data;
 883	unsigned int valid_flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE |
 884				   DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE;
 885	u64 point = 0;
 886
 887	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 888		return -EOPNOTSUPP;
 889
 890	if (args->pad)
 891		return -EINVAL;
 892
 893	if (args->flags & ~valid_flags)
 894		return -EINVAL;
 895
 896	if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_TIMELINE)
 897		point = args->point;
 898
 899	if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 900		return drm_syncobj_export_sync_file(file_private, args->handle,
 901						    point, &args->fd);
 902
 903	if (point)
 904		return -EINVAL;
 905
 906	return drm_syncobj_handle_to_fd(file_private, args->handle,
 907					&args->fd);
 908}
 909
 910int
 911drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 912				   struct drm_file *file_private)
 913{
 914	struct drm_syncobj_handle *args = data;
 915	unsigned int valid_flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE |
 916				   DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE;
 917	u64 point = 0;
 918
 919	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 920		return -EOPNOTSUPP;
 921
 922	if (args->pad)
 923		return -EINVAL;
 924
 925	if (args->flags & ~valid_flags)
 926		return -EINVAL;
 927
 928	if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_TIMELINE)
 929		point = args->point;
 930
 931	if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 932		return drm_syncobj_import_sync_file_fence(file_private,
 933							  args->fd,
 934							  args->handle,
 935							  point);
 936
 937	if (point)
 938		return -EINVAL;
 939
 940	return drm_syncobj_fd_to_handle(file_private, args->fd,
 941					&args->handle);
 942}
 943
 944static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
 945					    struct drm_syncobj_transfer *args)
 946{
 947	struct drm_syncobj *timeline_syncobj = NULL;
 948	struct dma_fence *fence, *tmp;
 949	struct dma_fence_chain *chain;
 950	int ret;
 951
 952	timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 953	if (!timeline_syncobj) {
 954		return -ENOENT;
 955	}
 956	ret = drm_syncobj_find_fence(file_private, args->src_handle,
 957				     args->src_point, args->flags,
 958				     &tmp);
 959	if (ret)
 960		goto err_put_timeline;
 961
 962	fence = dma_fence_unwrap_merge(tmp);
 963	dma_fence_put(tmp);
 964	if (!fence) {
 965		ret = -ENOMEM;
 966		goto err_put_timeline;
 967	}
 968
 969	chain = dma_fence_chain_alloc();
 970	if (!chain) {
 971		ret = -ENOMEM;
 972		goto err_free_fence;
 973	}
 974
 975	drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
 976err_free_fence:
 977	dma_fence_put(fence);
 978err_put_timeline:
 979	drm_syncobj_put(timeline_syncobj);
 980
 981	return ret;
 982}
 983
 984static int
 985drm_syncobj_transfer_to_binary(struct drm_file *file_private,
 986			       struct drm_syncobj_transfer *args)
 987{
 988	struct drm_syncobj *binary_syncobj = NULL;
 989	struct dma_fence *fence;
 990	int ret;
 991
 992	binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 993	if (!binary_syncobj)
 994		return -ENOENT;
 995	ret = drm_syncobj_find_fence(file_private, args->src_handle,
 996				     args->src_point, args->flags, &fence);
 997	if (ret)
 998		goto err;
 999	drm_syncobj_replace_fence(binary_syncobj, fence);
1000	dma_fence_put(fence);
1001err:
1002	drm_syncobj_put(binary_syncobj);
1003
1004	return ret;
1005}
1006int
1007drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
1008			   struct drm_file *file_private)
1009{
1010	struct drm_syncobj_transfer *args = data;
1011	int ret;
1012
1013	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1014		return -EOPNOTSUPP;
1015
1016	if (args->pad)
1017		return -EINVAL;
1018
1019	if (args->dst_point)
1020		ret = drm_syncobj_transfer_to_timeline(file_private, args);
1021	else
1022		ret = drm_syncobj_transfer_to_binary(file_private, args);
1023
1024	return ret;
1025}
1026
1027static void syncobj_wait_fence_func(struct dma_fence *fence,
1028				    struct dma_fence_cb *cb)
1029{
1030	struct syncobj_wait_entry *wait =
1031		container_of(cb, struct syncobj_wait_entry, fence_cb);
1032
1033	wake_up_process(wait->task);
1034}
1035
1036static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1037				      struct syncobj_wait_entry *wait)
1038{
1039	struct dma_fence *fence;
1040
1041	/* This happens inside the syncobj lock */
1042	fence = rcu_dereference_protected(syncobj->fence,
1043					  lockdep_is_held(&syncobj->lock));
1044	dma_fence_get(fence);
1045	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
1046		dma_fence_put(fence);
1047		return;
1048	} else if (!fence) {
1049		wait->fence = dma_fence_get_stub();
1050	} else {
1051		wait->fence = fence;
1052	}
1053
1054	wake_up_process(wait->task);
1055	list_del_init(&wait->node);
1056}
1057
1058static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1059						  void __user *user_points,
1060						  uint32_t count,
1061						  uint32_t flags,
1062						  signed long timeout,
1063						  uint32_t *idx,
1064						  ktime_t *deadline)
1065{
1066	struct syncobj_wait_entry *entries;
1067	struct dma_fence *fence;
1068	uint64_t *points;
1069	uint32_t signaled_count, i;
1070
1071	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1072		     DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1073		might_sleep();
1074		lockdep_assert_none_held_once();
1075	}
1076
1077	points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1078	if (points == NULL)
1079		return -ENOMEM;
1080
1081	if (!user_points) {
1082		memset(points, 0, count * sizeof(uint64_t));
1083
1084	} else if (copy_from_user(points, user_points,
1085				  sizeof(uint64_t) * count)) {
1086		timeout = -EFAULT;
1087		goto err_free_points;
1088	}
1089
1090	entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1091	if (!entries) {
1092		timeout = -ENOMEM;
1093		goto err_free_points;
1094	}
1095	/* Walk the list of sync objects and initialize entries.  We do
1096	 * this up-front so that we can properly return -EINVAL if there is
1097	 * a syncobj with a missing fence and then never have the chance of
1098	 * returning -EINVAL again.
1099	 */
1100	signaled_count = 0;
1101	for (i = 0; i < count; ++i) {
1102		struct dma_fence *fence;
1103
1104		entries[i].task = current;
1105		entries[i].point = points[i];
1106		fence = drm_syncobj_fence_get(syncobjs[i]);
1107		if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
1108			dma_fence_put(fence);
1109			if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1110				     DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1111				continue;
1112			} else {
1113				timeout = -EINVAL;
1114				goto cleanup_entries;
1115			}
1116		}
1117
1118		if (fence)
1119			entries[i].fence = fence;
1120		else
1121			entries[i].fence = dma_fence_get_stub();
1122
1123		if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1124		    dma_fence_is_signaled(entries[i].fence)) {
1125			if (signaled_count == 0 && idx)
1126				*idx = i;
1127			signaled_count++;
1128		}
1129	}
1130
1131	if (signaled_count == count ||
1132	    (signaled_count > 0 &&
1133	     !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1134		goto cleanup_entries;
1135
1136	/* There's a very annoying laxness in the dma_fence API here, in
1137	 * that backends are not required to automatically report when a
1138	 * fence is signaled prior to fence->ops->enable_signaling() being
1139	 * called.  So here if we fail to match signaled_count, we need to
1140	 * fallthough and try a 0 timeout wait!
1141	 */
1142
1143	if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1144		     DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1145		for (i = 0; i < count; ++i)
1146			drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1147	}
1148
1149	if (deadline) {
1150		for (i = 0; i < count; ++i) {
1151			fence = entries[i].fence;
1152			if (!fence)
1153				continue;
1154			dma_fence_set_deadline(fence, *deadline);
1155		}
1156	}
1157
1158	do {
1159		set_current_state(TASK_INTERRUPTIBLE);
1160
1161		signaled_count = 0;
1162		for (i = 0; i < count; ++i) {
1163			fence = entries[i].fence;
1164			if (!fence)
1165				continue;
1166
1167			if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1168			    dma_fence_is_signaled(fence) ||
1169			    (!entries[i].fence_cb.func &&
1170			     dma_fence_add_callback(fence,
1171						    &entries[i].fence_cb,
1172						    syncobj_wait_fence_func))) {
1173				/* The fence has been signaled */
1174				if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1175					signaled_count++;
1176				} else {
1177					if (idx)
1178						*idx = i;
1179					goto done_waiting;
1180				}
1181			}
1182		}
1183
1184		if (signaled_count == count)
1185			goto done_waiting;
1186
1187		if (timeout == 0) {
1188			timeout = -ETIME;
1189			goto done_waiting;
1190		}
1191
1192		if (signal_pending(current)) {
1193			timeout = -ERESTARTSYS;
1194			goto done_waiting;
1195		}
1196
1197		timeout = schedule_timeout(timeout);
1198	} while (1);
1199
1200done_waiting:
1201	__set_current_state(TASK_RUNNING);
1202
1203cleanup_entries:
1204	for (i = 0; i < count; ++i) {
1205		drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1206		if (entries[i].fence_cb.func)
1207			dma_fence_remove_callback(entries[i].fence,
1208						  &entries[i].fence_cb);
1209		dma_fence_put(entries[i].fence);
1210	}
1211	kfree(entries);
1212
1213err_free_points:
1214	kfree(points);
1215
1216	return timeout;
1217}
1218
1219/**
1220 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1221 *
1222 * @timeout_nsec: timeout nsec component in ns, 0 for poll
1223 *
1224 * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1225 */
1226signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1227{
1228	ktime_t abs_timeout, now;
1229	u64 timeout_ns, timeout_jiffies64;
1230
1231	/* make 0 timeout means poll - absolute 0 doesn't seem valid */
1232	if (timeout_nsec == 0)
1233		return 0;
1234
1235	abs_timeout = ns_to_ktime(timeout_nsec);
1236	now = ktime_get();
1237
1238	if (!ktime_after(abs_timeout, now))
1239		return 0;
1240
1241	timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1242
1243	timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1244	/*  clamp timeout to avoid infinite timeout */
1245	if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1246		return MAX_SCHEDULE_TIMEOUT - 1;
1247
1248	return timeout_jiffies64 + 1;
1249}
1250EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1251
1252static int drm_syncobj_array_wait(struct drm_device *dev,
1253				  struct drm_file *file_private,
1254				  struct drm_syncobj_wait *wait,
1255				  struct drm_syncobj_timeline_wait *timeline_wait,
1256				  struct drm_syncobj **syncobjs, bool timeline,
1257				  ktime_t *deadline)
1258{
1259	signed long timeout = 0;
1260	uint32_t first = ~0;
1261
1262	if (!timeline) {
1263		timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1264		timeout = drm_syncobj_array_wait_timeout(syncobjs,
1265							 NULL,
1266							 wait->count_handles,
1267							 wait->flags,
1268							 timeout, &first,
1269							 deadline);
1270		if (timeout < 0)
1271			return timeout;
1272		wait->first_signaled = first;
1273	} else {
1274		timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1275		timeout = drm_syncobj_array_wait_timeout(syncobjs,
1276							 u64_to_user_ptr(timeline_wait->points),
1277							 timeline_wait->count_handles,
1278							 timeline_wait->flags,
1279							 timeout, &first,
1280							 deadline);
1281		if (timeout < 0)
1282			return timeout;
1283		timeline_wait->first_signaled = first;
1284	}
1285	return 0;
1286}
1287
1288static int drm_syncobj_array_find(struct drm_file *file_private,
1289				  void __user *user_handles,
1290				  uint32_t count_handles,
1291				  struct drm_syncobj ***syncobjs_out)
1292{
1293	uint32_t i, *handles;
1294	struct drm_syncobj **syncobjs;
1295	int ret;
1296
1297	handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1298	if (handles == NULL)
1299		return -ENOMEM;
1300
1301	if (copy_from_user(handles, user_handles,
1302			   sizeof(uint32_t) * count_handles)) {
1303		ret = -EFAULT;
1304		goto err_free_handles;
1305	}
1306
1307	syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1308	if (syncobjs == NULL) {
1309		ret = -ENOMEM;
1310		goto err_free_handles;
1311	}
1312
1313	for (i = 0; i < count_handles; i++) {
1314		syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1315		if (!syncobjs[i]) {
1316			ret = -ENOENT;
1317			goto err_put_syncobjs;
1318		}
1319	}
1320
1321	kfree(handles);
1322	*syncobjs_out = syncobjs;
1323	return 0;
1324
1325err_put_syncobjs:
1326	while (i-- > 0)
1327		drm_syncobj_put(syncobjs[i]);
1328	kfree(syncobjs);
1329err_free_handles:
1330	kfree(handles);
1331
1332	return ret;
1333}
1334
1335static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1336				   uint32_t count)
1337{
1338	uint32_t i;
1339
1340	for (i = 0; i < count; i++)
1341		drm_syncobj_put(syncobjs[i]);
1342	kfree(syncobjs);
1343}
1344
1345int
1346drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1347		       struct drm_file *file_private)
1348{
1349	struct drm_syncobj_wait *args = data;
1350	struct drm_syncobj **syncobjs;
1351	unsigned int possible_flags;
1352	ktime_t t, *tp = NULL;
1353	int ret = 0;
1354
1355	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1356		return -EOPNOTSUPP;
1357
1358	possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1359			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1360			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1361
1362	if (args->flags & ~possible_flags)
1363		return -EINVAL;
1364
1365	if (args->count_handles == 0)
1366		return 0;
1367
1368	ret = drm_syncobj_array_find(file_private,
1369				     u64_to_user_ptr(args->handles),
1370				     args->count_handles,
1371				     &syncobjs);
1372	if (ret < 0)
1373		return ret;
1374
1375	if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1376		t = ns_to_ktime(args->deadline_nsec);
1377		tp = &t;
1378	}
1379
1380	ret = drm_syncobj_array_wait(dev, file_private,
1381				     args, NULL, syncobjs, false, tp);
1382
1383	drm_syncobj_array_free(syncobjs, args->count_handles);
1384
1385	return ret;
1386}
1387
1388int
1389drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1390				struct drm_file *file_private)
1391{
1392	struct drm_syncobj_timeline_wait *args = data;
1393	struct drm_syncobj **syncobjs;
1394	unsigned int possible_flags;
1395	ktime_t t, *tp = NULL;
1396	int ret = 0;
1397
1398	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1399		return -EOPNOTSUPP;
1400
1401	possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1402			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1403			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE |
1404			 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;
1405
1406	if (args->flags & ~possible_flags)
1407		return -EINVAL;
1408
1409	if (args->count_handles == 0)
1410		return 0;
1411
1412	ret = drm_syncobj_array_find(file_private,
1413				     u64_to_user_ptr(args->handles),
1414				     args->count_handles,
1415				     &syncobjs);
1416	if (ret < 0)
1417		return ret;
1418
1419	if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {
1420		t = ns_to_ktime(args->deadline_nsec);
1421		tp = &t;
1422	}
1423
1424	ret = drm_syncobj_array_wait(dev, file_private,
1425				     NULL, args, syncobjs, true, tp);
1426
1427	drm_syncobj_array_free(syncobjs, args->count_handles);
1428
1429	return ret;
1430}
1431
1432static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1433					     struct dma_fence_cb *cb)
1434{
1435	struct syncobj_eventfd_entry *entry =
1436		container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1437
1438	eventfd_signal(entry->ev_fd_ctx);
1439	syncobj_eventfd_entry_free(entry);
1440}
1441
1442static void
1443syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1444			   struct syncobj_eventfd_entry *entry)
1445{
1446	int ret;
1447	struct dma_fence *fence;
1448
1449	/* This happens inside the syncobj lock */
1450	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1451	if (!fence)
1452		return;
1453
1454	ret = dma_fence_chain_find_seqno(&fence, entry->point);
1455	if (ret != 0) {
1456		/* The given seqno has not been submitted yet. */
1457		dma_fence_put(fence);
1458		return;
1459	} else if (!fence) {
1460		/* If dma_fence_chain_find_seqno returns 0 but sets the fence
1461		 * to NULL, it implies that the given seqno is signaled and a
1462		 * later seqno has already been submitted. Assign a stub fence
1463		 * so that the eventfd still gets signaled below.
1464		 */
1465		fence = dma_fence_get_stub();
1466	}
1467
1468	list_del_init(&entry->node);
1469	entry->fence = fence;
1470
1471	if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1472		eventfd_signal(entry->ev_fd_ctx);
1473		syncobj_eventfd_entry_free(entry);
1474	} else {
1475		ret = dma_fence_add_callback(fence, &entry->fence_cb,
1476					     syncobj_eventfd_entry_fence_func);
1477		if (ret == -ENOENT) {
1478			eventfd_signal(entry->ev_fd_ctx);
1479			syncobj_eventfd_entry_free(entry);
1480		}
1481	}
1482}
1483
1484int
1485drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1486			  struct drm_file *file_private)
1487{
1488	struct drm_syncobj_eventfd *args = data;
1489	struct drm_syncobj *syncobj;
1490	struct eventfd_ctx *ev_fd_ctx;
1491	struct syncobj_eventfd_entry *entry;
1492	int ret;
1493
1494	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1495		return -EOPNOTSUPP;
1496
1497	if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1498		return -EINVAL;
1499
1500	if (args->pad)
1501		return -EINVAL;
1502
1503	syncobj = drm_syncobj_find(file_private, args->handle);
1504	if (!syncobj)
1505		return -ENOENT;
1506
1507	ev_fd_ctx = eventfd_ctx_fdget(args->fd);
1508	if (IS_ERR(ev_fd_ctx)) {
1509		ret = PTR_ERR(ev_fd_ctx);
1510		goto err_fdget;
1511	}
1512
1513	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1514	if (!entry) {
1515		ret = -ENOMEM;
1516		goto err_kzalloc;
1517	}
1518	entry->syncobj = syncobj;
1519	entry->ev_fd_ctx = ev_fd_ctx;
1520	entry->point = args->point;
1521	entry->flags = args->flags;
1522
1523	drm_syncobj_add_eventfd(syncobj, entry);
1524	drm_syncobj_put(syncobj);
1525
1526	return 0;
1527
1528err_kzalloc:
1529	eventfd_ctx_put(ev_fd_ctx);
1530err_fdget:
1531	drm_syncobj_put(syncobj);
1532	return ret;
1533}
1534
1535int
1536drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1537			struct drm_file *file_private)
1538{
1539	struct drm_syncobj_array *args = data;
1540	struct drm_syncobj **syncobjs;
1541	uint32_t i;
1542	int ret;
1543
1544	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1545		return -EOPNOTSUPP;
1546
1547	if (args->pad != 0)
1548		return -EINVAL;
1549
1550	if (args->count_handles == 0)
1551		return -EINVAL;
1552
1553	ret = drm_syncobj_array_find(file_private,
1554				     u64_to_user_ptr(args->handles),
1555				     args->count_handles,
1556				     &syncobjs);
1557	if (ret < 0)
1558		return ret;
1559
1560	for (i = 0; i < args->count_handles; i++)
1561		drm_syncobj_replace_fence(syncobjs[i], NULL);
1562
1563	drm_syncobj_array_free(syncobjs, args->count_handles);
1564
1565	return 0;
1566}
1567
1568int
1569drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1570			 struct drm_file *file_private)
1571{
1572	struct drm_syncobj_array *args = data;
1573	struct drm_syncobj **syncobjs;
1574	uint32_t i;
1575	int ret;
1576
1577	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1578		return -EOPNOTSUPP;
1579
1580	if (args->pad != 0)
1581		return -EINVAL;
1582
1583	if (args->count_handles == 0)
1584		return -EINVAL;
1585
1586	ret = drm_syncobj_array_find(file_private,
1587				     u64_to_user_ptr(args->handles),
1588				     args->count_handles,
1589				     &syncobjs);
1590	if (ret < 0)
1591		return ret;
1592
1593	for (i = 0; i < args->count_handles; i++) {
1594		ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1595		if (ret < 0)
1596			break;
1597	}
1598
1599	drm_syncobj_array_free(syncobjs, args->count_handles);
1600
1601	return ret;
1602}
1603
1604int
1605drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1606				  struct drm_file *file_private)
1607{
1608	struct drm_syncobj_timeline_array *args = data;
1609	struct drm_syncobj **syncobjs;
1610	struct dma_fence_chain **chains;
1611	uint64_t *points;
1612	uint32_t i, j;
1613	int ret;
1614
1615	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1616		return -EOPNOTSUPP;
1617
1618	if (args->flags != 0)
1619		return -EINVAL;
1620
1621	if (args->count_handles == 0)
1622		return -EINVAL;
1623
1624	ret = drm_syncobj_array_find(file_private,
1625				     u64_to_user_ptr(args->handles),
1626				     args->count_handles,
1627				     &syncobjs);
1628	if (ret < 0)
1629		return ret;
1630
1631	points = kmalloc_array(args->count_handles, sizeof(*points),
1632			       GFP_KERNEL);
1633	if (!points) {
1634		ret = -ENOMEM;
1635		goto out;
1636	}
1637	if (!u64_to_user_ptr(args->points)) {
1638		memset(points, 0, args->count_handles * sizeof(uint64_t));
1639	} else if (copy_from_user(points, u64_to_user_ptr(args->points),
1640				  sizeof(uint64_t) * args->count_handles)) {
1641		ret = -EFAULT;
1642		goto err_points;
1643	}
1644
1645	chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1646	if (!chains) {
1647		ret = -ENOMEM;
1648		goto err_points;
1649	}
1650	for (i = 0; i < args->count_handles; i++) {
1651		chains[i] = dma_fence_chain_alloc();
1652		if (!chains[i]) {
1653			for (j = 0; j < i; j++)
1654				dma_fence_chain_free(chains[j]);
1655			ret = -ENOMEM;
1656			goto err_chains;
1657		}
1658	}
1659
1660	for (i = 0; i < args->count_handles; i++) {
1661		struct dma_fence *fence = dma_fence_get_stub();
1662
1663		drm_syncobj_add_point(syncobjs[i], chains[i],
1664				      fence, points[i]);
1665		dma_fence_put(fence);
1666	}
1667err_chains:
1668	kfree(chains);
1669err_points:
1670	kfree(points);
1671out:
1672	drm_syncobj_array_free(syncobjs, args->count_handles);
1673
1674	return ret;
1675}
1676
1677int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1678			    struct drm_file *file_private)
1679{
1680	struct drm_syncobj_timeline_array *args = data;
1681	struct drm_syncobj **syncobjs;
1682	uint64_t __user *points = u64_to_user_ptr(args->points);
1683	uint32_t i;
1684	int ret;
1685
1686	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1687		return -EOPNOTSUPP;
1688
1689	if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1690		return -EINVAL;
1691
1692	if (args->count_handles == 0)
1693		return -EINVAL;
1694
1695	ret = drm_syncobj_array_find(file_private,
1696				     u64_to_user_ptr(args->handles),
1697				     args->count_handles,
1698				     &syncobjs);
1699	if (ret < 0)
1700		return ret;
1701
1702	for (i = 0; i < args->count_handles; i++) {
1703		struct dma_fence_chain *chain;
1704		struct dma_fence *fence;
1705		uint64_t point;
1706
1707		fence = drm_syncobj_fence_get(syncobjs[i]);
1708		chain = to_dma_fence_chain(fence);
1709		if (chain) {
1710			struct dma_fence *iter, *last_signaled =
1711				dma_fence_get(fence);
1712
1713			if (args->flags &
1714			    DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1715				point = fence->seqno;
1716			} else {
1717				dma_fence_chain_for_each(iter, fence) {
1718					if (iter->context != fence->context) {
1719						dma_fence_put(iter);
1720						/* It is most likely that timeline has
1721						* unorder points. */
1722						break;
1723					}
1724					dma_fence_put(last_signaled);
1725					last_signaled = dma_fence_get(iter);
1726				}
1727				point = dma_fence_is_signaled(last_signaled) ?
1728					last_signaled->seqno :
1729					to_dma_fence_chain(last_signaled)->prev_seqno;
1730			}
1731			dma_fence_put(last_signaled);
1732		} else {
1733			point = 0;
1734		}
1735		dma_fence_put(fence);
1736		ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1737		ret = ret ? -EFAULT : 0;
1738		if (ret)
1739			break;
1740	}
1741	drm_syncobj_array_free(syncobjs, args->count_handles);
1742
1743	return ret;
1744}