i915_request.c

   1/*
   2 * Copyright © 2008-2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/dma-fence-array.h>
  26#include <linux/dma-fence-chain.h>
  27#include <linux/irq_work.h>
  28#include <linux/prefetch.h>
  29#include <linux/sched.h>
  30#include <linux/sched/clock.h>
  31#include <linux/sched/signal.h>
  32#include <linux/sched/mm.h>
  33
  34#include <drm/drm_print.h>
  35
  36#include "gem/i915_gem_context.h"
  37#include "gt/intel_breadcrumbs.h"
  38#include "gt/intel_context.h"
  39#include "gt/intel_engine.h"
  40#include "gt/intel_engine_heartbeat.h"
  41#include "gt/intel_engine_regs.h"
  42#include "gt/intel_gpu_commands.h"
  43#include "gt/intel_reset.h"
  44#include "gt/intel_ring.h"
  45#include "gt/intel_rps.h"
  46
  47#include "i915_active.h"
  48#include "i915_config.h"
  49#include "i915_deps.h"
  50#include "i915_driver.h"
  51#include "i915_drv.h"
  52#include "i915_trace.h"
  53
  54struct execute_cb {
  55	struct irq_work work;
  56	struct i915_sw_fence *fence;
  57};
  58
  59static struct kmem_cache *slab_requests;
  60static struct kmem_cache *slab_execute_cbs;
  61
  62static const char *i915_fence_get_driver_name(struct dma_fence *fence)
  63{
  64	return dev_name(to_request(fence)->i915->drm.dev);
  65}
  66
  67static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
  68{
  69	const struct i915_gem_context *ctx;
  70
  71	/*
  72	 * The timeline struct (as part of the ppgtt underneath a context)
  73	 * may be freed when the request is no longer in use by the GPU.
  74	 * We could extend the life of a context to beyond that of all
  75	 * fences, possibly keeping the hw resource around indefinitely,
  76	 * or we just give them a false name. Since
  77	 * dma_fence_ops.get_timeline_name is a debug feature, the occasional
  78	 * lie seems justifiable.
  79	 */
  80	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  81		return "signaled";
  82
  83	ctx = i915_request_gem_context(to_request(fence));
  84	if (!ctx)
  85		return "[" DRIVER_NAME "]";
  86
  87	return ctx->name;
  88}
  89
  90static bool i915_fence_signaled(struct dma_fence *fence)
  91{
  92	return i915_request_completed(to_request(fence));
  93}
  94
  95static bool i915_fence_enable_signaling(struct dma_fence *fence)
  96{
  97	return i915_request_enable_breadcrumb(to_request(fence));
  98}
  99
 100static signed long i915_fence_wait(struct dma_fence *fence,
 101				   bool interruptible,
 102				   signed long timeout)
 103{
 104	return i915_request_wait_timeout(to_request(fence),
 105					 interruptible | I915_WAIT_PRIORITY,
 106					 timeout);
 107}
 108
 109struct kmem_cache *i915_request_slab_cache(void)
 110{
 111	return slab_requests;
 112}
 113
 114static void i915_fence_release(struct dma_fence *fence)
 115{
 116	struct i915_request *rq = to_request(fence);
 117
 118	GEM_BUG_ON(rq->guc_prio != GUC_PRIO_INIT &&
 119		   rq->guc_prio != GUC_PRIO_FINI);
 120
 121	i915_request_free_capture_list(fetch_and_zero(&rq->capture_list));
 122	if (rq->batch_res) {
 123		i915_vma_resource_put(rq->batch_res);
 124		rq->batch_res = NULL;
 125	}
 126
 127	/*
 128	 * The request is put onto a RCU freelist (i.e. the address
 129	 * is immediately reused), mark the fences as being freed now.
 130	 * Otherwise the debugobjects for the fences are only marked as
 131	 * freed when the slab cache itself is freed, and so we would get
 132	 * caught trying to reuse dead objects.
 133	 */
 134	i915_sw_fence_fini(&rq->submit);
 135	i915_sw_fence_fini(&rq->semaphore);
 136
 137	/*
 138	 * Keep one request on each engine for reserved use under mempressure.
 139	 *
 140	 * We do not hold a reference to the engine here and so have to be
 141	 * very careful in what rq->engine we poke. The virtual engine is
 142	 * referenced via the rq->context and we released that ref during
 143	 * i915_request_retire(), ergo we must not dereference a virtual
 144	 * engine here. Not that we would want to, as the only consumer of
 145	 * the reserved engine->request_pool is the power management parking,
 146	 * which must-not-fail, and that is only run on the physical engines.
 147	 *
 148	 * Since the request must have been executed to be have completed,
 149	 * we know that it will have been processed by the HW and will
 150	 * not be unsubmitted again, so rq->engine and rq->execution_mask
 151	 * at this point is stable. rq->execution_mask will be a single
 152	 * bit if the last and _only_ engine it could execution on was a
 153	 * physical engine, if it's multiple bits then it started on and
 154	 * could still be on a virtual engine. Thus if the mask is not a
 155	 * power-of-two we assume that rq->engine may still be a virtual
 156	 * engine and so a dangling invalid pointer that we cannot dereference
 157	 *
 158	 * For example, consider the flow of a bonded request through a virtual
 159	 * engine. The request is created with a wide engine mask (all engines
 160	 * that we might execute on). On processing the bond, the request mask
 161	 * is reduced to one or more engines. If the request is subsequently
 162	 * bound to a single engine, it will then be constrained to only
 163	 * execute on that engine and never returned to the virtual engine
 164	 * after timeslicing away, see __unwind_incomplete_requests(). Thus we
 165	 * know that if the rq->execution_mask is a single bit, rq->engine
 166	 * can be a physical engine with the exact corresponding mask.
 167	 */
 168	if (is_power_of_2(rq->execution_mask) &&
 169	    !cmpxchg(&rq->engine->request_pool, NULL, rq))
 170		return;
 171
 172	kmem_cache_free(slab_requests, rq);
 173}
 174
 175const struct dma_fence_ops i915_fence_ops = {
 176	.get_driver_name = i915_fence_get_driver_name,
 177	.get_timeline_name = i915_fence_get_timeline_name,
 178	.enable_signaling = i915_fence_enable_signaling,
 179	.signaled = i915_fence_signaled,
 180	.wait = i915_fence_wait,
 181	.release = i915_fence_release,
 182};
 183
 184static void irq_execute_cb(struct irq_work *wrk)
 185{
 186	struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
 187
 188	i915_sw_fence_complete(cb->fence);
 189	kmem_cache_free(slab_execute_cbs, cb);
 190}
 191
 192static __always_inline void
 193__notify_execute_cb(struct i915_request *rq, bool (*fn)(struct irq_work *wrk))
 194{
 195	struct execute_cb *cb, *cn;
 196
 197	if (llist_empty(&rq->execute_cb))
 198		return;
 199
 200	llist_for_each_entry_safe(cb, cn,
 201				  llist_del_all(&rq->execute_cb),
 202				  work.node.llist)
 203		fn(&cb->work);
 204}
 205
 206static void __notify_execute_cb_irq(struct i915_request *rq)
 207{
 208	__notify_execute_cb(rq, irq_work_queue);
 209}
 210
 211static bool irq_work_imm(struct irq_work *wrk)
 212{
 213	wrk->func(wrk);
 214	return false;
 215}
 216
 217void i915_request_notify_execute_cb_imm(struct i915_request *rq)
 218{
 219	__notify_execute_cb(rq, irq_work_imm);
 220}
 221
 222static void __i915_request_fill(struct i915_request *rq, u8 val)
 223{
 224	void *vaddr = rq->ring->vaddr;
 225	u32 head;
 226
 227	head = rq->infix;
 228	if (rq->postfix < head) {
 229		memset(vaddr + head, val, rq->ring->size - head);
 230		head = 0;
 231	}
 232	memset(vaddr + head, val, rq->postfix - head);
 233}
 234
 235/**
 236 * i915_request_active_engine
 237 * @rq: request to inspect
 238 * @active: pointer in which to return the active engine
 239 *
 240 * Fills the currently active engine to the @active pointer if the request
 241 * is active and still not completed.
 242 *
 243 * Returns true if request was active or false otherwise.
 244 */
 245bool
 246i915_request_active_engine(struct i915_request *rq,
 247			   struct intel_engine_cs **active)
 248{
 249	struct intel_engine_cs *engine, *locked;
 250	bool ret = false;
 251
 252	/*
 253	 * Serialise with __i915_request_submit() so that it sees
 254	 * is-banned?, or we know the request is already inflight.
 255	 *
 256	 * Note that rq->engine is unstable, and so we double
 257	 * check that we have acquired the lock on the final engine.
 258	 */
 259	locked = READ_ONCE(rq->engine);
 260	spin_lock_irq(&locked->sched_engine->lock);
 261	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
 262		spin_unlock(&locked->sched_engine->lock);
 263		locked = engine;
 264		spin_lock(&locked->sched_engine->lock);
 265	}
 266
 267	if (i915_request_is_active(rq)) {
 268		if (!__i915_request_is_complete(rq))
 269			*active = locked;
 270		ret = true;
 271	}
 272
 273	spin_unlock_irq(&locked->sched_engine->lock);
 274
 275	return ret;
 276}
 277
 278static enum hrtimer_restart __rq_watchdog_expired(struct hrtimer *hrtimer)
 279{
 280	struct i915_request *rq =
 281		container_of(hrtimer, struct i915_request, watchdog.timer);
 282	struct intel_gt *gt = rq->engine->gt;
 283
 284	if (!i915_request_completed(rq)) {
 285		if (llist_add(&rq->watchdog.link, &gt->watchdog.list))
 286			queue_work(gt->i915->unordered_wq, &gt->watchdog.work);
 287	} else {
 288		i915_request_put(rq);
 289	}
 290
 291	return HRTIMER_NORESTART;
 292}
 293
 294static void __rq_init_watchdog(struct i915_request *rq)
 295{
 296	struct i915_request_watchdog *wdg = &rq->watchdog;
 297
 298	hrtimer_setup(&wdg->timer, __rq_watchdog_expired, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 299}
 300
 301static void __rq_arm_watchdog(struct i915_request *rq)
 302{
 303	struct i915_request_watchdog *wdg = &rq->watchdog;
 304	struct intel_context *ce = rq->context;
 305
 306	if (!ce->watchdog.timeout_us)
 307		return;
 308
 309	i915_request_get(rq);
 310
 311	hrtimer_start_range_ns(&wdg->timer,
 312			       ns_to_ktime(ce->watchdog.timeout_us *
 313					   NSEC_PER_USEC),
 314			       NSEC_PER_MSEC,
 315			       HRTIMER_MODE_REL);
 316}
 317
 318static void __rq_cancel_watchdog(struct i915_request *rq)
 319{
 320	struct i915_request_watchdog *wdg = &rq->watchdog;
 321
 322	if (hrtimer_try_to_cancel(&wdg->timer) > 0)
 323		i915_request_put(rq);
 324}
 325
 326#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 327
 328/**
 329 * i915_request_free_capture_list - Free a capture list
 330 * @capture: Pointer to the first list item or NULL
 331 *
 332 */
 333void i915_request_free_capture_list(struct i915_capture_list *capture)
 334{
 335	while (capture) {
 336		struct i915_capture_list *next = capture->next;
 337
 338		i915_vma_resource_put(capture->vma_res);
 339		kfree(capture);
 340		capture = next;
 341	}
 342}
 343
 344#define assert_capture_list_is_null(_rq) GEM_BUG_ON((_rq)->capture_list)
 345
 346#define clear_capture_list(_rq) ((_rq)->capture_list = NULL)
 347
 348#else
 349
 350#define i915_request_free_capture_list(_a) do {} while (0)
 351
 352#define assert_capture_list_is_null(_a) do {} while (0)
 353
 354#define clear_capture_list(_rq) do {} while (0)
 355
 356#endif
 357
 358bool i915_request_retire(struct i915_request *rq)
 359{
 360	if (!__i915_request_is_complete(rq))
 361		return false;
 362
 363	RQ_TRACE(rq, "\n");
 364
 365	GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
 366	trace_i915_request_retire(rq);
 367	i915_request_mark_complete(rq);
 368
 369	__rq_cancel_watchdog(rq);
 370
 371	/*
 372	 * We know the GPU must have read the request to have
 373	 * sent us the seqno + interrupt, so use the position
 374	 * of tail of the request to update the last known position
 375	 * of the GPU head.
 376	 *
 377	 * Note this requires that we are always called in request
 378	 * completion order.
 379	 */
 380	GEM_BUG_ON(!list_is_first(&rq->link,
 381				  &i915_request_timeline(rq)->requests));
 382	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 383		/* Poison before we release our space in the ring */
 384		__i915_request_fill(rq, POISON_FREE);
 385	rq->ring->head = rq->postfix;
 386
 387	if (!i915_request_signaled(rq)) {
 388		spin_lock_irq(&rq->lock);
 389		dma_fence_signal_locked(&rq->fence);
 390		spin_unlock_irq(&rq->lock);
 391	}
 392
 393	if (test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags))
 394		intel_rps_dec_waiters(&rq->engine->gt->rps);
 395
 396	/*
 397	 * We only loosely track inflight requests across preemption,
 398	 * and so we may find ourselves attempting to retire a _completed_
 399	 * request that we have removed from the HW and put back on a run
 400	 * queue.
 401	 *
 402	 * As we set I915_FENCE_FLAG_ACTIVE on the request, this should be
 403	 * after removing the breadcrumb and signaling it, so that we do not
 404	 * inadvertently attach the breadcrumb to a completed request.
 405	 */
 406	rq->engine->remove_active_request(rq);
 407	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
 408
 409	__list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */
 410
 411	intel_context_exit(rq->context);
 412	intel_context_unpin(rq->context);
 413
 414	i915_sched_node_fini(&rq->sched);
 415	i915_request_put(rq);
 416
 417	return true;
 418}
 419
 420void i915_request_retire_upto(struct i915_request *rq)
 421{
 422	struct intel_timeline * const tl = i915_request_timeline(rq);
 423	struct i915_request *tmp;
 424
 425	RQ_TRACE(rq, "\n");
 426	GEM_BUG_ON(!__i915_request_is_complete(rq));
 427
 428	do {
 429		tmp = list_first_entry(&tl->requests, typeof(*tmp), link);
 430		GEM_BUG_ON(!i915_request_completed(tmp));
 431	} while (i915_request_retire(tmp) && tmp != rq);
 432}
 433
 434static struct i915_request * const *
 435__engine_active(struct intel_engine_cs *engine)
 436{
 437	return READ_ONCE(engine->execlists.active);
 438}
 439
 440static bool __request_in_flight(const struct i915_request *signal)
 441{
 442	struct i915_request * const *port, *rq;
 443	bool inflight = false;
 444
 445	if (!i915_request_is_ready(signal))
 446		return false;
 447
 448	/*
 449	 * Even if we have unwound the request, it may still be on
 450	 * the GPU (preempt-to-busy). If that request is inside an
 451	 * unpreemptible critical section, it will not be removed. Some
 452	 * GPU functions may even be stuck waiting for the paired request
 453	 * (__await_execution) to be submitted and cannot be preempted
 454	 * until the bond is executing.
 455	 *
 456	 * As we know that there are always preemption points between
 457	 * requests, we know that only the currently executing request
 458	 * may be still active even though we have cleared the flag.
 459	 * However, we can't rely on our tracking of ELSP[0] to know
 460	 * which request is currently active and so maybe stuck, as
 461	 * the tracking maybe an event behind. Instead assume that
 462	 * if the context is still inflight, then it is still active
 463	 * even if the active flag has been cleared.
 464	 *
 465	 * To further complicate matters, if there a pending promotion, the HW
 466	 * may either perform a context switch to the second inflight execlists,
 467	 * or it may switch to the pending set of execlists. In the case of the
 468	 * latter, it may send the ACK and we process the event copying the
 469	 * pending[] over top of inflight[], _overwriting_ our *active. Since
 470	 * this implies the HW is arbitrating and not struck in *active, we do
 471	 * not worry about complete accuracy, but we do require no read/write
 472	 * tearing of the pointer [the read of the pointer must be valid, even
 473	 * as the array is being overwritten, for which we require the writes
 474	 * to avoid tearing.]
 475	 *
 476	 * Note that the read of *execlists->active may race with the promotion
 477	 * of execlists->pending[] to execlists->inflight[], overwriting
 478	 * the value at *execlists->active. This is fine. The promotion implies
 479	 * that we received an ACK from the HW, and so the context is not
 480	 * stuck -- if we do not see ourselves in *active, the inflight status
 481	 * is valid. If instead we see ourselves being copied into *active,
 482	 * we are inflight and may signal the callback.
 483	 */
 484	if (!intel_context_inflight(signal->context))
 485		return false;
 486
 487	rcu_read_lock();
 488	for (port = __engine_active(signal->engine);
 489	     (rq = READ_ONCE(*port)); /* may race with promotion of pending[] */
 490	     port++) {
 491		if (rq->context == signal->context) {
 492			inflight = i915_seqno_passed(rq->fence.seqno,
 493						     signal->fence.seqno);
 494			break;
 495		}
 496	}
 497	rcu_read_unlock();
 498
 499	return inflight;
 500}
 501
 502static int
 503__await_execution(struct i915_request *rq,
 504		  struct i915_request *signal,
 505		  gfp_t gfp)
 506{
 507	struct execute_cb *cb;
 508
 509	if (i915_request_is_active(signal))
 510		return 0;
 511
 512	cb = kmem_cache_alloc(slab_execute_cbs, gfp);
 513	if (!cb)
 514		return -ENOMEM;
 515
 516	cb->fence = &rq->submit;
 517	i915_sw_fence_await(cb->fence);
 518	init_irq_work(&cb->work, irq_execute_cb);
 519
 520	/*
 521	 * Register the callback first, then see if the signaler is already
 522	 * active. This ensures that if we race with the
 523	 * __notify_execute_cb from i915_request_submit() and we are not
 524	 * included in that list, we get a second bite of the cherry and
 525	 * execute it ourselves. After this point, a future
 526	 * i915_request_submit() will notify us.
 527	 *
 528	 * In i915_request_retire() we set the ACTIVE bit on a completed
 529	 * request (then flush the execute_cb). So by registering the
 530	 * callback first, then checking the ACTIVE bit, we serialise with
 531	 * the completed/retired request.
 532	 */
 533	if (llist_add(&cb->work.node.llist, &signal->execute_cb)) {
 534		if (i915_request_is_active(signal) ||
 535		    __request_in_flight(signal))
 536			i915_request_notify_execute_cb_imm(signal);
 537	}
 538
 539	return 0;
 540}
 541
 542static bool fatal_error(int error)
 543{
 544	switch (error) {
 545	case 0: /* not an error! */
 546	case -EAGAIN: /* innocent victim of a GT reset (__i915_request_reset) */
 547	case -ETIMEDOUT: /* waiting for Godot (timer_i915_sw_fence_wake) */
 548		return false;
 549	default:
 550		return true;
 551	}
 552}
 553
 554void __i915_request_skip(struct i915_request *rq)
 555{
 556	GEM_BUG_ON(!fatal_error(rq->fence.error));
 557
 558	if (rq->infix == rq->postfix)
 559		return;
 560
 561	RQ_TRACE(rq, "error: %d\n", rq->fence.error);
 562
 563	/*
 564	 * As this request likely depends on state from the lost
 565	 * context, clear out all the user operations leaving the
 566	 * breadcrumb at the end (so we get the fence notifications).
 567	 */
 568	__i915_request_fill(rq, 0);
 569	rq->infix = rq->postfix;
 570}
 571
 572bool i915_request_set_error_once(struct i915_request *rq, int error)
 573{
 574	int old;
 575
 576	GEM_BUG_ON(!IS_ERR_VALUE((long)error));
 577
 578	if (i915_request_signaled(rq))
 579		return false;
 580
 581	old = READ_ONCE(rq->fence.error);
 582	do {
 583		if (fatal_error(old))
 584			return false;
 585	} while (!try_cmpxchg(&rq->fence.error, &old, error));
 586
 587	return true;
 588}
 589
 590struct i915_request *i915_request_mark_eio(struct i915_request *rq)
 591{
 592	if (__i915_request_is_complete(rq))
 593		return NULL;
 594
 595	GEM_BUG_ON(i915_request_signaled(rq));
 596
 597	/* As soon as the request is completed, it may be retired */
 598	rq = i915_request_get(rq);
 599
 600	i915_request_set_error_once(rq, -EIO);
 601	i915_request_mark_complete(rq);
 602
 603	return rq;
 604}
 605
 606bool __i915_request_submit(struct i915_request *request)
 607{
 608	struct intel_engine_cs *engine = request->engine;
 609	bool result = false;
 610
 611	RQ_TRACE(request, "\n");
 612
 613	GEM_BUG_ON(!irqs_disabled());
 614	lockdep_assert_held(&engine->sched_engine->lock);
 615
 616	/*
 617	 * With the advent of preempt-to-busy, we frequently encounter
 618	 * requests that we have unsubmitted from HW, but left running
 619	 * until the next ack and so have completed in the meantime. On
 620	 * resubmission of that completed request, we can skip
 621	 * updating the payload, and execlists can even skip submitting
 622	 * the request.
 623	 *
 624	 * We must remove the request from the caller's priority queue,
 625	 * and the caller must only call us when the request is in their
 626	 * priority queue, under the sched_engine->lock. This ensures that the
 627	 * request has *not* yet been retired and we can safely move
 628	 * the request into the engine->active.list where it will be
 629	 * dropped upon retiring. (Otherwise if resubmit a *retired*
 630	 * request, this would be a horrible use-after-free.)
 631	 */
 632	if (__i915_request_is_complete(request)) {
 633		list_del_init(&request->sched.link);
 634		goto active;
 635	}
 636
 637	if (unlikely(!intel_context_is_schedulable(request->context)))
 638		i915_request_set_error_once(request, -EIO);
 639
 640	if (unlikely(fatal_error(request->fence.error)))
 641		__i915_request_skip(request);
 642
 643	/*
 644	 * Are we using semaphores when the gpu is already saturated?
 645	 *
 646	 * Using semaphores incurs a cost in having the GPU poll a
 647	 * memory location, busywaiting for it to change. The continual
 648	 * memory reads can have a noticeable impact on the rest of the
 649	 * system with the extra bus traffic, stalling the cpu as it too
 650	 * tries to access memory across the bus (perf stat -e bus-cycles).
 651	 *
 652	 * If we installed a semaphore on this request and we only submit
 653	 * the request after the signaler completed, that indicates the
 654	 * system is overloaded and using semaphores at this time only
 655	 * increases the amount of work we are doing. If so, we disable
 656	 * further use of semaphores until we are idle again, whence we
 657	 * optimistically try again.
 658	 */
 659	if (request->sched.semaphores &&
 660	    i915_sw_fence_signaled(&request->semaphore))
 661		engine->saturated |= request->sched.semaphores;
 662
 663	engine->emit_fini_breadcrumb(request,
 664				     request->ring->vaddr + request->postfix);
 665
 666	trace_i915_request_execute(request);
 667	if (engine->bump_serial)
 668		engine->bump_serial(engine);
 669	else
 670		engine->serial++;
 671
 672	result = true;
 673
 674	GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 675	engine->add_active_request(request);
 676active:
 677	clear_bit(I915_FENCE_FLAG_PQUEUE, &request->fence.flags);
 678	set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
 679
 680	/*
 681	 * XXX Rollback bonded-execution on __i915_request_unsubmit()?
 682	 *
 683	 * In the future, perhaps when we have an active time-slicing scheduler,
 684	 * it will be interesting to unsubmit parallel execution and remove
 685	 * busywaits from the GPU until their master is restarted. This is
 686	 * quite hairy, we have to carefully rollback the fence and do a
 687	 * preempt-to-idle cycle on the target engine, all the while the
 688	 * master execute_cb may refire.
 689	 */
 690	__notify_execute_cb_irq(request);
 691
 692	/* We may be recursing from the signal callback of another i915 fence */
 693	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
 694		i915_request_enable_breadcrumb(request);
 695
 696	return result;
 697}
 698
 699void i915_request_submit(struct i915_request *request)
 700{
 701	struct intel_engine_cs *engine = request->engine;
 702	unsigned long flags;
 703
 704	/* Will be called from irq-context when using foreign fences. */
 705	spin_lock_irqsave(&engine->sched_engine->lock, flags);
 706
 707	__i915_request_submit(request);
 708
 709	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 710}
 711
 712void __i915_request_unsubmit(struct i915_request *request)
 713{
 714	struct intel_engine_cs *engine = request->engine;
 715
 716	/*
 717	 * Only unwind in reverse order, required so that the per-context list
 718	 * is kept in seqno/ring order.
 719	 */
 720	RQ_TRACE(request, "\n");
 721
 722	GEM_BUG_ON(!irqs_disabled());
 723	lockdep_assert_held(&engine->sched_engine->lock);
 724
 725	/*
 726	 * Before we remove this breadcrumb from the signal list, we have
 727	 * to ensure that a concurrent dma_fence_enable_signaling() does not
 728	 * attach itself. We first mark the request as no longer active and
 729	 * make sure that is visible to other cores, and then remove the
 730	 * breadcrumb if attached.
 731	 */
 732	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 733	clear_bit_unlock(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
 734	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
 735		i915_request_cancel_breadcrumb(request);
 736
 737	/* We've already spun, don't charge on resubmitting. */
 738	if (request->sched.semaphores && __i915_request_has_started(request))
 739		request->sched.semaphores = 0;
 740
 741	/*
 742	 * We don't need to wake_up any waiters on request->execute, they
 743	 * will get woken by any other event or us re-adding this request
 744	 * to the engine timeline (__i915_request_submit()). The waiters
 745	 * should be quite adapt at finding that the request now has a new
 746	 * global_seqno to the one they went to sleep on.
 747	 */
 748}
 749
 750void i915_request_unsubmit(struct i915_request *request)
 751{
 752	struct intel_engine_cs *engine = request->engine;
 753	unsigned long flags;
 754
 755	/* Will be called from irq-context when using foreign fences. */
 756	spin_lock_irqsave(&engine->sched_engine->lock, flags);
 757
 758	__i915_request_unsubmit(request);
 759
 760	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 761}
 762
 763void i915_request_cancel(struct i915_request *rq, int error)
 764{
 765	if (!i915_request_set_error_once(rq, error))
 766		return;
 767
 768	set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
 769
 770	intel_context_cancel_request(rq->context, rq);
 771}
 772
 773static int
 774submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 775{
 776	struct i915_request *request =
 777		container_of(fence, typeof(*request), submit);
 778
 779	switch (state) {
 780	case FENCE_COMPLETE:
 781		trace_i915_request_submit(request);
 782
 783		if (unlikely(fence->error))
 784			i915_request_set_error_once(request, fence->error);
 785		else
 786			__rq_arm_watchdog(request);
 787
 788		/*
 789		 * We need to serialize use of the submit_request() callback
 790		 * with its hotplugging performed during an emergency
 791		 * i915_gem_set_wedged().  We use the RCU mechanism to mark the
 792		 * critical section in order to force i915_gem_set_wedged() to
 793		 * wait until the submit_request() is completed before
 794		 * proceeding.
 795		 */
 796		rcu_read_lock();
 797		request->engine->submit_request(request);
 798		rcu_read_unlock();
 799		break;
 800
 801	case FENCE_FREE:
 802		i915_request_put(request);
 803		break;
 804	}
 805
 806	return NOTIFY_DONE;
 807}
 808
 809static int
 810semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 811{
 812	struct i915_request *rq = container_of(fence, typeof(*rq), semaphore);
 813
 814	switch (state) {
 815	case FENCE_COMPLETE:
 816		break;
 817
 818	case FENCE_FREE:
 819		i915_request_put(rq);
 820		break;
 821	}
 822
 823	return NOTIFY_DONE;
 824}
 825
 826static void retire_requests(struct intel_timeline *tl)
 827{
 828	struct i915_request *rq, *rn;
 829
 830	list_for_each_entry_safe(rq, rn, &tl->requests, link)
 831		if (!i915_request_retire(rq))
 832			break;
 833}
 834
 835static noinline struct i915_request *
 836request_alloc_slow(struct intel_timeline *tl,
 837		   struct i915_request **rsvd,
 838		   gfp_t gfp)
 839{
 840	struct i915_request *rq;
 841
 842	/* If we cannot wait, dip into our reserves */
 843	if (!gfpflags_allow_blocking(gfp)) {
 844		rq = xchg(rsvd, NULL);
 845		if (!rq) /* Use the normal failure path for one final WARN */
 846			goto out;
 847
 848		return rq;
 849	}
 850
 851	if (list_empty(&tl->requests))
 852		goto out;
 853
 854	/* Move our oldest request to the slab-cache (if not in use!) */
 855	rq = list_first_entry(&tl->requests, typeof(*rq), link);
 856	i915_request_retire(rq);
 857
 858	rq = kmem_cache_alloc(slab_requests,
 859			      gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 860	if (rq)
 861		return rq;
 862
 863	/* Ratelimit ourselves to prevent oom from malicious clients */
 864	rq = list_last_entry(&tl->requests, typeof(*rq), link);
 865	cond_synchronize_rcu(rq->rcustate);
 866
 867	/* Retire our old requests in the hope that we free some */
 868	retire_requests(tl);
 869
 870out:
 871	return kmem_cache_alloc(slab_requests, gfp);
 872}
 873
 874static void __i915_request_ctor(void *arg)
 875{
 876	struct i915_request *rq = arg;
 877
 878	spin_lock_init(&rq->lock);
 879	i915_sched_node_init(&rq->sched);
 880	i915_sw_fence_init(&rq->submit, submit_notify);
 881	i915_sw_fence_init(&rq->semaphore, semaphore_notify);
 882
 883	clear_capture_list(rq);
 884	rq->batch_res = NULL;
 885
 886	init_llist_head(&rq->execute_cb);
 887}
 888
 889#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 890#define clear_batch_ptr(_rq) ((_rq)->batch = NULL)
 891#else
 892#define clear_batch_ptr(_a) do {} while (0)
 893#endif
 894
 895struct i915_request *
 896__i915_request_create(struct intel_context *ce, gfp_t gfp)
 897{
 898	struct intel_timeline *tl = ce->timeline;
 899	struct i915_request *rq;
 900	u32 seqno;
 901	int ret;
 902
 903	might_alloc(gfp);
 904
 905	/* Check that the caller provided an already pinned context */
 906	__intel_context_pin(ce);
 907
 908	/*
 909	 * Beware: Dragons be flying overhead.
 910	 *
 911	 * We use RCU to look up requests in flight. The lookups may
 912	 * race with the request being allocated from the slab freelist.
 913	 * That is the request we are writing to here, may be in the process
 914	 * of being read by __i915_active_request_get_rcu(). As such,
 915	 * we have to be very careful when overwriting the contents. During
 916	 * the RCU lookup, we change chase the request->engine pointer,
 917	 * read the request->global_seqno and increment the reference count.
 918	 *
 919	 * The reference count is incremented atomically. If it is zero,
 920	 * the lookup knows the request is unallocated and complete. Otherwise,
 921	 * it is either still in use, or has been reallocated and reset
 922	 * with dma_fence_init(). This increment is safe for release as we
 923	 * check that the request we have a reference to and matches the active
 924	 * request.
 925	 *
 926	 * Before we increment the refcount, we chase the request->engine
 927	 * pointer. We must not call kmem_cache_zalloc() or else we set
 928	 * that pointer to NULL and cause a crash during the lookup. If
 929	 * we see the request is completed (based on the value of the
 930	 * old engine and seqno), the lookup is complete and reports NULL.
 931	 * If we decide the request is not completed (new engine or seqno),
 932	 * then we grab a reference and double check that it is still the
 933	 * active request - which it won't be and restart the lookup.
 934	 *
 935	 * Do not use kmem_cache_zalloc() here!
 936	 */
 937	rq = kmem_cache_alloc(slab_requests,
 938			      gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 939	if (unlikely(!rq)) {
 940		rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp);
 941		if (!rq) {
 942			ret = -ENOMEM;
 943			goto err_unreserve;
 944		}
 945	}
 946
 947	rq->context = ce;
 948	rq->engine = ce->engine;
 949	rq->ring = ce->ring;
 950	rq->execution_mask = ce->engine->mask;
 951	rq->i915 = ce->engine->i915;
 952
 953	ret = intel_timeline_get_seqno(tl, rq, &seqno);
 954	if (ret)
 955		goto err_free;
 956
 957	dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock,
 958		       tl->fence_context, seqno);
 959
 960	RCU_INIT_POINTER(rq->timeline, tl);
 961	rq->hwsp_seqno = tl->hwsp_seqno;
 962	GEM_BUG_ON(__i915_request_is_complete(rq));
 963
 964	rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */
 965
 966	rq->guc_prio = GUC_PRIO_INIT;
 967
 968	/* We bump the ref for the fence chain */
 969	i915_sw_fence_reinit(&i915_request_get(rq)->submit);
 970	i915_sw_fence_reinit(&i915_request_get(rq)->semaphore);
 971
 972	i915_sched_node_reinit(&rq->sched);
 973
 974	/* No zalloc, everything must be cleared after use */
 975	clear_batch_ptr(rq);
 976	__rq_init_watchdog(rq);
 977	assert_capture_list_is_null(rq);
 978	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
 979	GEM_BUG_ON(rq->batch_res);
 980
 981	/*
 982	 * Reserve space in the ring buffer for all the commands required to
 983	 * eventually emit this request. This is to guarantee that the
 984	 * i915_request_add() call can't fail. Note that the reserve may need
 985	 * to be redone if the request is not actually submitted straight
 986	 * away, e.g. because a GPU scheduler has deferred it.
 987	 *
 988	 * Note that due to how we add reserved_space to intel_ring_begin()
 989	 * we need to double our request to ensure that if we need to wrap
 990	 * around inside i915_request_add() there is sufficient space at
 991	 * the beginning of the ring as well.
 992	 */
 993	rq->reserved_space =
 994		2 * rq->engine->emit_fini_breadcrumb_dw * sizeof(u32);
 995
 996	/*
 997	 * Record the position of the start of the request so that
 998	 * should we detect the updated seqno part-way through the
 999	 * GPU processing the request, we never over-estimate the
1000	 * position of the head.
1001	 */
1002	rq->head = rq->ring->emit;
1003
1004	ret = rq->engine->request_alloc(rq);
1005	if (ret)
1006		goto err_unwind;
1007
1008	rq->infix = rq->ring->emit; /* end of header; start of user payload */
1009
1010	intel_context_mark_active(ce);
1011	list_add_tail_rcu(&rq->link, &tl->requests);
1012
1013	return rq;
1014
1015err_unwind:
1016	ce->ring->emit = rq->head;
1017
1018	/* Make sure we didn't add ourselves to external state before freeing */
1019	GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
1020	GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
1021
1022err_free:
1023	kmem_cache_free(slab_requests, rq);
1024err_unreserve:
1025	intel_context_unpin(ce);
1026	return ERR_PTR(ret);
1027}
1028
1029struct i915_request *
1030i915_request_create(struct intel_context *ce)
1031{
1032	struct i915_request *rq;
1033	struct intel_timeline *tl;
1034
1035	tl = intel_context_timeline_lock(ce);
1036	if (IS_ERR(tl))
1037		return ERR_CAST(tl);
1038
1039	/* Move our oldest request to the slab-cache (if not in use!) */
1040	rq = list_first_entry(&tl->requests, typeof(*rq), link);
1041	if (!list_is_last(&rq->link, &tl->requests))
1042		i915_request_retire(rq);
1043
1044	intel_context_enter(ce);
1045	rq = __i915_request_create(ce, GFP_KERNEL);
1046	intel_context_exit(ce); /* active reference transferred to request */
1047	if (IS_ERR(rq))
1048		goto err_unlock;
1049
1050	/* Check that we do not interrupt ourselves with a new request */
1051	rq->cookie = lockdep_pin_lock(&tl->mutex);
1052
1053	return rq;
1054
1055err_unlock:
1056	intel_context_timeline_unlock(tl);
1057	return rq;
1058}
1059
1060static int
1061i915_request_await_start(struct i915_request *rq, struct i915_request *signal)
1062{
1063	struct dma_fence *fence;
1064	int err;
1065
1066	if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline))
1067		return 0;
1068
1069	if (i915_request_started(signal))
1070		return 0;
1071
1072	/*
1073	 * The caller holds a reference on @signal, but we do not serialise
1074	 * against it being retired and removed from the lists.
1075	 *
1076	 * We do not hold a reference to the request before @signal, and
1077	 * so must be very careful to ensure that it is not _recycled_ as
1078	 * we follow the link backwards.
1079	 */
1080	fence = NULL;
1081	rcu_read_lock();
1082	do {
1083		struct list_head *pos = READ_ONCE(signal->link.prev);
1084		struct i915_request *prev;
1085
1086		/* Confirm signal has not been retired, the link is valid */
1087		if (unlikely(__i915_request_has_started(signal)))
1088			break;
1089
1090		/* Is signal the earliest request on its timeline? */
1091		if (pos == &rcu_dereference(signal->timeline)->requests)
1092			break;
1093
1094		/*
1095		 * Peek at the request before us in the timeline. That
1096		 * request will only be valid before it is retired, so
1097		 * after acquiring a reference to it, confirm that it is
1098		 * still part of the signaler's timeline.
1099		 */
1100		prev = list_entry(pos, typeof(*prev), link);
1101		if (!i915_request_get_rcu(prev))
1102			break;
1103
1104		/* After the strong barrier, confirm prev is still attached */
1105		if (unlikely(READ_ONCE(prev->link.next) != &signal->link)) {
1106			i915_request_put(prev);
1107			break;
1108		}
1109
1110		fence = &prev->fence;
1111	} while (0);
1112	rcu_read_unlock();
1113	if (!fence)
1114		return 0;
1115
1116	err = 0;
1117	if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence))
1118		err = i915_sw_fence_await_dma_fence(&rq->submit,
1119						    fence, 0,
1120						    I915_FENCE_GFP);
1121	dma_fence_put(fence);
1122
1123	return err;
1124}
1125
1126static intel_engine_mask_t
1127already_busywaiting(struct i915_request *rq)
1128{
1129	/*
1130	 * Polling a semaphore causes bus traffic, delaying other users of
1131	 * both the GPU and CPU. We want to limit the impact on others,
1132	 * while taking advantage of early submission to reduce GPU
1133	 * latency. Therefore we restrict ourselves to not using more
1134	 * than one semaphore from each source, and not using a semaphore
1135	 * if we have detected the engine is saturated (i.e. would not be
1136	 * submitted early and cause bus traffic reading an already passed
1137	 * semaphore).
1138	 *
1139	 * See the are-we-too-late? check in __i915_request_submit().
1140	 */
1141	return rq->sched.semaphores | READ_ONCE(rq->engine->saturated);
1142}
1143
1144static int
1145__emit_semaphore_wait(struct i915_request *to,
1146		      struct i915_request *from,
1147		      u32 seqno)
1148{
1149	const int has_token = GRAPHICS_VER(to->engine->i915) >= 12;
1150	u32 hwsp_offset;
1151	int len, err;
1152	u32 *cs;
1153
1154	GEM_BUG_ON(GRAPHICS_VER(to->engine->i915) < 8);
1155	GEM_BUG_ON(i915_request_has_initial_breadcrumb(to));
1156
1157	/* We need to pin the signaler's HWSP until we are finished reading. */
1158	err = intel_timeline_read_hwsp(from, to, &hwsp_offset);
1159	if (err)
1160		return err;
1161
1162	len = 4;
1163	if (has_token)
1164		len += 2;
1165
1166	cs = intel_ring_begin(to, len);
1167	if (IS_ERR(cs))
1168		return PTR_ERR(cs);
1169
1170	/*
1171	 * Using greater-than-or-equal here means we have to worry
1172	 * about seqno wraparound. To side step that issue, we swap
1173	 * the timeline HWSP upon wrapping, so that everyone listening
1174	 * for the old (pre-wrap) values do not see the much smaller
1175	 * (post-wrap) values than they were expecting (and so wait
1176	 * forever).
1177	 */
1178	*cs++ = (MI_SEMAPHORE_WAIT |
1179		 MI_SEMAPHORE_GLOBAL_GTT |
1180		 MI_SEMAPHORE_POLL |
1181		 MI_SEMAPHORE_SAD_GTE_SDD) +
1182		has_token;
1183	*cs++ = seqno;
1184	*cs++ = hwsp_offset;
1185	*cs++ = 0;
1186	if (has_token) {
1187		*cs++ = 0;
1188		*cs++ = MI_NOOP;
1189	}
1190
1191	intel_ring_advance(to, cs);
1192	return 0;
1193}
1194
1195static bool
1196can_use_semaphore_wait(struct i915_request *to, struct i915_request *from)
1197{
1198	return to->engine->gt->ggtt == from->engine->gt->ggtt;
1199}
1200
1201static int
1202emit_semaphore_wait(struct i915_request *to,
1203		    struct i915_request *from,
1204		    gfp_t gfp)
1205{
1206	const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask;
1207	struct i915_sw_fence *wait = &to->submit;
1208
1209	if (!can_use_semaphore_wait(to, from))
1210		goto await_fence;
1211
1212	if (!intel_context_use_semaphores(to->context))
1213		goto await_fence;
1214
1215	if (i915_request_has_initial_breadcrumb(to))
1216		goto await_fence;
1217
1218	/*
1219	 * If this or its dependents are waiting on an external fence
1220	 * that may fail catastrophically, then we want to avoid using
1221	 * semaphores as they bypass the fence signaling metadata, and we
1222	 * lose the fence->error propagation.
1223	 */
1224	if (from->sched.flags & I915_SCHED_HAS_EXTERNAL_CHAIN)
1225		goto await_fence;
1226
1227	/* Just emit the first semaphore we see as request space is limited. */
1228	if (already_busywaiting(to) & mask)
1229		goto await_fence;
1230
1231	if (i915_request_await_start(to, from) < 0)
1232		goto await_fence;
1233
1234	/* Only submit our spinner after the signaler is running! */
1235	if (__await_execution(to, from, gfp))
1236		goto await_fence;
1237
1238	if (__emit_semaphore_wait(to, from, from->fence.seqno))
1239		goto await_fence;
1240
1241	to->sched.semaphores |= mask;
1242	wait = &to->semaphore;
1243
1244await_fence:
1245	return i915_sw_fence_await_dma_fence(wait,
1246					     &from->fence, 0,
1247					     I915_FENCE_GFP);
1248}
1249
1250static bool intel_timeline_sync_has_start(struct intel_timeline *tl,
1251					  struct dma_fence *fence)
1252{
1253	return __intel_timeline_sync_is_later(tl,
1254					      fence->context,
1255					      fence->seqno - 1);
1256}
1257
1258static int intel_timeline_sync_set_start(struct intel_timeline *tl,
1259					 const struct dma_fence *fence)
1260{
1261	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);
1262}
1263
1264static int
1265__i915_request_await_execution(struct i915_request *to,
1266			       struct i915_request *from)
1267{
1268	int err;
1269
1270	GEM_BUG_ON(intel_context_is_barrier(from->context));
1271
1272	/* Submit both requests at the same time */
1273	err = __await_execution(to, from, I915_FENCE_GFP);
1274	if (err)
1275		return err;
1276
1277	/* Squash repeated depenendices to the same timelines */
1278	if (intel_timeline_sync_has_start(i915_request_timeline(to),
1279					  &from->fence))
1280		return 0;
1281
1282	/*
1283	 * Wait until the start of this request.
1284	 *
1285	 * The execution cb fires when we submit the request to HW. But in
1286	 * many cases this may be long before the request itself is ready to
1287	 * run (consider that we submit 2 requests for the same context, where
1288	 * the request of interest is behind an indefinite spinner). So we hook
1289	 * up to both to reduce our queues and keep the execution lag minimised
1290	 * in the worst case, though we hope that the await_start is elided.
1291	 */
1292	err = i915_request_await_start(to, from);
1293	if (err < 0)
1294		return err;
1295
1296	/*
1297	 * Ensure both start together [after all semaphores in signal]
1298	 *
1299	 * Now that we are queued to the HW at roughly the same time (thanks
1300	 * to the execute cb) and are ready to run at roughly the same time
1301	 * (thanks to the await start), our signaler may still be indefinitely
1302	 * delayed by waiting on a semaphore from a remote engine. If our
1303	 * signaler depends on a semaphore, so indirectly do we, and we do not
1304	 * want to start our payload until our signaler also starts theirs.
1305	 * So we wait.
1306	 *
1307	 * However, there is also a second condition for which we need to wait
1308	 * for the precise start of the signaler. Consider that the signaler
1309	 * was submitted in a chain of requests following another context
1310	 * (with just an ordinary intra-engine fence dependency between the
1311	 * two). In this case the signaler is queued to HW, but not for
1312	 * immediate execution, and so we must wait until it reaches the
1313	 * active slot.
1314	 */
1315	if (can_use_semaphore_wait(to, from) &&
1316	    intel_engine_has_semaphores(to->engine) &&
1317	    !i915_request_has_initial_breadcrumb(to)) {
1318		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);
1319		if (err < 0)
1320			return err;
1321	}
1322
1323	/* Couple the dependency tree for PI on this exposed to->fence */
1324	if (to->engine->sched_engine->schedule) {
1325		err = i915_sched_node_add_dependency(&to->sched,
1326						     &from->sched,
1327						     I915_DEPENDENCY_WEAK);
1328		if (err < 0)
1329			return err;
1330	}
1331
1332	return intel_timeline_sync_set_start(i915_request_timeline(to),
1333					     &from->fence);
1334}
1335
1336static void mark_external(struct i915_request *rq)
1337{
1338	/*
1339	 * The downside of using semaphores is that we lose metadata passing
1340	 * along the signaling chain. This is particularly nasty when we
1341	 * need to pass along a fatal error such as EFAULT or EDEADLK. For
1342	 * fatal errors we want to scrub the request before it is executed,
1343	 * which means that we cannot preload the request onto HW and have
1344	 * it wait upon a semaphore.
1345	 */
1346	rq->sched.flags |= I915_SCHED_HAS_EXTERNAL_CHAIN;
1347}
1348
1349static int
1350__i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1351{
1352	mark_external(rq);
1353	return i915_sw_fence_await_dma_fence(&rq->submit, fence,
1354					     i915_fence_context_timeout(rq->i915,
1355									fence->context),
1356					     I915_FENCE_GFP);
1357}
1358
1359static int
1360i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
1361{
1362	struct dma_fence *iter;
1363	int err = 0;
1364
1365	if (!to_dma_fence_chain(fence))
1366		return __i915_request_await_external(rq, fence);
1367
1368	dma_fence_chain_for_each(iter, fence) {
1369		struct dma_fence_chain *chain = to_dma_fence_chain(iter);
1370
1371		if (!dma_fence_is_i915(chain->fence)) {
1372			err = __i915_request_await_external(rq, iter);
1373			break;
1374		}
1375
1376		err = i915_request_await_dma_fence(rq, chain->fence);
1377		if (err < 0)
1378			break;
1379	}
1380
1381	dma_fence_put(iter);
1382	return err;
1383}
1384
1385static inline bool is_parallel_rq(struct i915_request *rq)
1386{
1387	return intel_context_is_parallel(rq->context);
1388}
1389
1390static inline struct intel_context *request_to_parent(struct i915_request *rq)
1391{
1392	return intel_context_to_parent(rq->context);
1393}
1394
1395static bool is_same_parallel_context(struct i915_request *to,
1396				     struct i915_request *from)
1397{
1398	if (is_parallel_rq(to))
1399		return request_to_parent(to) == request_to_parent(from);
1400
1401	return false;
1402}
1403
1404int
1405i915_request_await_execution(struct i915_request *rq,
1406			     struct dma_fence *fence)
1407{
1408	struct dma_fence **child = &fence;
1409	unsigned int nchild = 1;
1410	int ret;
1411
1412	if (dma_fence_is_array(fence)) {
1413		struct dma_fence_array *array = to_dma_fence_array(fence);
1414
1415		/* XXX Error for signal-on-any fence arrays */
1416
1417		child = array->fences;
1418		nchild = array->num_fences;
1419		GEM_BUG_ON(!nchild);
1420	}
1421
1422	do {
1423		fence = *child++;
1424		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1425			continue;
1426
1427		if (fence->context == rq->fence.context)
1428			continue;
1429
1430		/*
1431		 * We don't squash repeated fence dependencies here as we
1432		 * want to run our callback in all cases.
1433		 */
1434
1435		if (dma_fence_is_i915(fence)) {
1436			if (is_same_parallel_context(rq, to_request(fence)))
1437				continue;
1438			ret = __i915_request_await_execution(rq,
1439							     to_request(fence));
1440		} else {
1441			ret = i915_request_await_external(rq, fence);
1442		}
1443		if (ret < 0)
1444			return ret;
1445	} while (--nchild);
1446
1447	return 0;
1448}
1449
1450static int
1451await_request_submit(struct i915_request *to, struct i915_request *from)
1452{
1453	/*
1454	 * If we are waiting on a virtual engine, then it may be
1455	 * constrained to execute on a single engine *prior* to submission.
1456	 * When it is submitted, it will be first submitted to the virtual
1457	 * engine and then passed to the physical engine. We cannot allow
1458	 * the waiter to be submitted immediately to the physical engine
1459	 * as it may then bypass the virtual request.
1460	 */
1461	if (to->engine == READ_ONCE(from->engine))
1462		return i915_sw_fence_await_sw_fence_gfp(&to->submit,
1463							&from->submit,
1464							I915_FENCE_GFP);
1465	else
1466		return __i915_request_await_execution(to, from);
1467}
1468
1469static int
1470i915_request_await_request(struct i915_request *to, struct i915_request *from)
1471{
1472	int ret;
1473
1474	GEM_BUG_ON(to == from);
1475	GEM_BUG_ON(to->timeline == from->timeline);
1476
1477	if (i915_request_completed(from)) {
1478		i915_sw_fence_set_error_once(&to->submit, from->fence.error);
1479		return 0;
1480	}
1481
1482	if (to->engine->sched_engine->schedule) {
1483		ret = i915_sched_node_add_dependency(&to->sched,
1484						     &from->sched,
1485						     I915_DEPENDENCY_EXTERNAL);
1486		if (ret < 0)
1487			return ret;
1488	}
1489
1490	if (!intel_engine_uses_guc(to->engine) &&
1491	    is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask)))
1492		ret = await_request_submit(to, from);
1493	else
1494		ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
1495	if (ret < 0)
1496		return ret;
1497
1498	return 0;
1499}
1500
1501int
1502i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
1503{
1504	struct dma_fence **child = &fence;
1505	unsigned int nchild = 1;
1506	int ret;
1507
1508	/*
1509	 * Note that if the fence-array was created in signal-on-any mode,
1510	 * we should *not* decompose it into its individual fences. However,
1511	 * we don't currently store which mode the fence-array is operating
1512	 * in. Fortunately, the only user of signal-on-any is private to
1513	 * amdgpu and we should not see any incoming fence-array from
1514	 * sync-file being in signal-on-any mode.
1515	 */
1516	if (dma_fence_is_array(fence)) {
1517		struct dma_fence_array *array = to_dma_fence_array(fence);
1518
1519		child = array->fences;
1520		nchild = array->num_fences;
1521		GEM_BUG_ON(!nchild);
1522	}
1523
1524	do {
1525		fence = *child++;
1526		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1527			continue;
1528
1529		/*
1530		 * Requests on the same timeline are explicitly ordered, along
1531		 * with their dependencies, by i915_request_add() which ensures
1532		 * that requests are submitted in-order through each ring.
1533		 */
1534		if (fence->context == rq->fence.context)
1535			continue;
1536
1537		/* Squash repeated waits to the same timelines */
1538		if (fence->context &&
1539		    intel_timeline_sync_is_later(i915_request_timeline(rq),
1540						 fence))
1541			continue;
1542
1543		if (dma_fence_is_i915(fence)) {
1544			if (is_same_parallel_context(rq, to_request(fence)))
1545				continue;
1546			ret = i915_request_await_request(rq, to_request(fence));
1547		} else {
1548			ret = i915_request_await_external(rq, fence);
1549		}
1550		if (ret < 0)
1551			return ret;
1552
1553		/* Record the latest fence used against each timeline */
1554		if (fence->context)
1555			intel_timeline_sync_set(i915_request_timeline(rq),
1556						fence);
1557	} while (--nchild);
1558
1559	return 0;
1560}
1561
1562/**
1563 * i915_request_await_deps - set this request to (async) wait upon a struct
1564 * i915_deps dma_fence collection
1565 * @rq: request we are wishing to use
1566 * @deps: The struct i915_deps containing the dependencies.
1567 *
1568 * Returns 0 if successful, negative error code on error.
1569 */
1570int i915_request_await_deps(struct i915_request *rq, const struct i915_deps *deps)
1571{
1572	int i, err;
1573
1574	for (i = 0; i < deps->num_deps; ++i) {
1575		err = i915_request_await_dma_fence(rq, deps->fences[i]);
1576		if (err)
1577			return err;
1578	}
1579
1580	return 0;
1581}
1582
1583/**
1584 * i915_request_await_object - set this request to (async) wait upon a bo
1585 * @to: request we are wishing to use
1586 * @obj: object which may be in use on another ring.
1587 * @write: whether the wait is on behalf of a writer
1588 *
1589 * This code is meant to abstract object synchronization with the GPU.
1590 * Conceptually we serialise writes between engines inside the GPU.
1591 * We only allow one engine to write into a buffer at any time, but
1592 * multiple readers. To ensure each has a coherent view of memory, we must:
1593 *
1594 * - If there is an outstanding write request to the object, the new
1595 *   request must wait for it to complete (either CPU or in hw, requests
1596 *   on the same ring will be naturally ordered).
1597 *
1598 * - If we are a write request (pending_write_domain is set), the new
1599 *   request must wait for outstanding read requests to complete.
1600 *
1601 * Returns 0 if successful, else propagates up the lower layer error.
1602 */
1603int
1604i915_request_await_object(struct i915_request *to,
1605			  struct drm_i915_gem_object *obj,
1606			  bool write)
1607{
1608	struct dma_resv_iter cursor;
1609	struct dma_fence *fence;
1610	int ret = 0;
1611
1612	dma_resv_for_each_fence(&cursor, obj->base.resv,
1613				dma_resv_usage_rw(write), fence) {
1614		ret = i915_request_await_dma_fence(to, fence);
1615		if (ret)
1616			break;
1617	}
1618
1619	return ret;
1620}
1621
1622static void i915_request_await_huc(struct i915_request *rq)
1623{
1624	struct intel_huc *huc = &rq->context->engine->gt->uc.huc;
1625
1626	/* don't stall kernel submissions! */
1627	if (!rcu_access_pointer(rq->context->gem_context))
1628		return;
1629
1630	if (intel_huc_wait_required(huc))
1631		i915_sw_fence_await_sw_fence(&rq->submit,
1632					     &huc->delayed_load.fence,
1633					     &rq->hucq);
1634}
1635
1636static struct i915_request *
1637__i915_request_ensure_parallel_ordering(struct i915_request *rq,
1638					struct intel_timeline *timeline)
1639{
1640	struct i915_request *prev;
1641
1642	GEM_BUG_ON(!is_parallel_rq(rq));
1643
1644	prev = request_to_parent(rq)->parallel.last_rq;
1645	if (prev) {
1646		if (!__i915_request_is_complete(prev)) {
1647			i915_sw_fence_await_sw_fence(&rq->submit,
1648						     &prev->submit,
1649						     &rq->submitq);
1650
1651			if (rq->engine->sched_engine->schedule)
1652				__i915_sched_node_add_dependency(&rq->sched,
1653								 &prev->sched,
1654								 &rq->dep,
1655								 0);
1656		}
1657		i915_request_put(prev);
1658	}
1659
1660	request_to_parent(rq)->parallel.last_rq = i915_request_get(rq);
1661
1662	/*
1663	 * Users have to put a reference potentially got by
1664	 * __i915_active_fence_set() to the returned request
1665	 * when no longer needed
1666	 */
1667	return to_request(__i915_active_fence_set(&timeline->last_request,
1668						  &rq->fence));
1669}
1670
1671static struct i915_request *
1672__i915_request_ensure_ordering(struct i915_request *rq,
1673			       struct intel_timeline *timeline)
1674{
1675	struct i915_request *prev;
1676
1677	GEM_BUG_ON(is_parallel_rq(rq));
1678
1679	prev = to_request(__i915_active_fence_set(&timeline->last_request,
1680						  &rq->fence));
1681
1682	if (prev && !__i915_request_is_complete(prev)) {
1683		bool uses_guc = intel_engine_uses_guc(rq->engine);
1684		bool pow2 = is_power_of_2(READ_ONCE(prev->engine)->mask |
1685					  rq->engine->mask);
1686		bool same_context = prev->context == rq->context;
1687
1688		/*
1689		 * The requests are supposed to be kept in order. However,
1690		 * we need to be wary in case the timeline->last_request
1691		 * is used as a barrier for external modification to this
1692		 * context.
1693		 */
1694		GEM_BUG_ON(same_context &&
1695			   i915_seqno_passed(prev->fence.seqno,
1696					     rq->fence.seqno));
1697
1698		if ((same_context && uses_guc) || (!uses_guc && pow2))
1699			i915_sw_fence_await_sw_fence(&rq->submit,
1700						     &prev->submit,
1701						     &rq->submitq);
1702		else
1703			__i915_sw_fence_await_dma_fence(&rq->submit,
1704							&prev->fence,
1705							&rq->dmaq);
1706		if (rq->engine->sched_engine->schedule)
1707			__i915_sched_node_add_dependency(&rq->sched,
1708							 &prev->sched,
1709							 &rq->dep,
1710							 0);
1711	}
1712
1713	/*
1714	 * Users have to put the reference to prev potentially got
1715	 * by __i915_active_fence_set() when no longer needed
1716	 */
1717	return prev;
1718}
1719
1720static struct i915_request *
1721__i915_request_add_to_timeline(struct i915_request *rq)
1722{
1723	struct intel_timeline *timeline = i915_request_timeline(rq);
1724	struct i915_request *prev;
1725
1726	/*
1727	 * Media workloads may require HuC, so stall them until HuC loading is
1728	 * complete. Note that HuC not being loaded when a user submission
1729	 * arrives can only happen when HuC is loaded via GSC and in that case
1730	 * we still expect the window between us starting to accept submissions
1731	 * and HuC loading completion to be small (a few hundred ms).
1732	 */
1733	if (rq->engine->class == VIDEO_DECODE_CLASS)
1734		i915_request_await_huc(rq);
1735
1736	/*
1737	 * Dependency tracking and request ordering along the timeline
1738	 * is special cased so that we can eliminate redundant ordering
1739	 * operations while building the request (we know that the timeline
1740	 * itself is ordered, and here we guarantee it).
1741	 *
1742	 * As we know we will need to emit tracking along the timeline,
1743	 * we embed the hooks into our request struct -- at the cost of
1744	 * having to have specialised no-allocation interfaces (which will
1745	 * be beneficial elsewhere).
1746	 *
1747	 * A second benefit to open-coding i915_request_await_request is
1748	 * that we can apply a slight variant of the rules specialised
1749	 * for timelines that jump between engines (such as virtual engines).
1750	 * If we consider the case of virtual engine, we must emit a dma-fence
1751	 * to prevent scheduling of the second request until the first is
1752	 * complete (to maximise our greedy late load balancing) and this
1753	 * precludes optimising to use semaphores serialisation of a single
1754	 * timeline across engines.
1755	 *
1756	 * We do not order parallel submission requests on the timeline as each
1757	 * parallel submission context has its own timeline and the ordering
1758	 * rules for parallel requests are that they must be submitted in the
1759	 * order received from the execbuf IOCTL. So rather than using the
1760	 * timeline we store a pointer to last request submitted in the
1761	 * relationship in the gem context and insert a submission fence
1762	 * between that request and request passed into this function or
1763	 * alternatively we use completion fence if gem context has a single
1764	 * timeline and this is the first submission of an execbuf IOCTL.
1765	 */
1766	if (likely(!is_parallel_rq(rq)))
1767		prev = __i915_request_ensure_ordering(rq, timeline);
1768	else
1769		prev = __i915_request_ensure_parallel_ordering(rq, timeline);
1770	if (prev)
1771		i915_request_put(prev);
1772
1773	/*
1774	 * Make sure that no request gazumped us - if it was allocated after
1775	 * our i915_request_alloc() and called __i915_request_add() before
1776	 * us, the timeline will hold its seqno which is later than ours.
1777	 */
1778	GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
1779
1780	return prev;
1781}
1782
1783/*
1784 * NB: This function is not allowed to fail. Doing so would mean the the
1785 * request is not being tracked for completion but the work itself is
1786 * going to happen on the hardware. This would be a Bad Thing(tm).
1787 */
1788struct i915_request *__i915_request_commit(struct i915_request *rq)
1789{
1790	struct intel_engine_cs *engine = rq->engine;
1791	struct intel_ring *ring = rq->ring;
1792	u32 *cs;
1793
1794	RQ_TRACE(rq, "\n");
1795
1796	/*
1797	 * To ensure that this call will not fail, space for its emissions
1798	 * should already have been reserved in the ring buffer. Let the ring
1799	 * know that it is time to use that space up.
1800	 */
1801	GEM_BUG_ON(rq->reserved_space > ring->space);
1802	rq->reserved_space = 0;
1803	rq->emitted_jiffies = jiffies;
1804
1805	/*
1806	 * Record the position of the start of the breadcrumb so that
1807	 * should we detect the updated seqno part-way through the
1808	 * GPU processing the request, we never over-estimate the
1809	 * position of the ring's HEAD.
1810	 */
1811	cs = intel_ring_begin(rq, engine->emit_fini_breadcrumb_dw);
1812	GEM_BUG_ON(IS_ERR(cs));
1813	rq->postfix = intel_ring_offset(rq, cs);
1814
1815	return __i915_request_add_to_timeline(rq);
1816}
1817
1818void __i915_request_queue_bh(struct i915_request *rq)
1819{
1820	i915_sw_fence_commit(&rq->semaphore);
1821	i915_sw_fence_commit(&rq->submit);
1822}
1823
1824void __i915_request_queue(struct i915_request *rq,
1825			  const struct i915_sched_attr *attr)
1826{
1827	/*
1828	 * Let the backend know a new request has arrived that may need
1829	 * to adjust the existing execution schedule due to a high priority
1830	 * request - i.e. we may want to preempt the current request in order
1831	 * to run a high priority dependency chain *before* we can execute this
1832	 * request.
1833	 *
1834	 * This is called before the request is ready to run so that we can
1835	 * decide whether to preempt the entire chain so that it is ready to
1836	 * run at the earliest possible convenience.
1837	 */
1838	if (attr && rq->engine->sched_engine->schedule)
1839		rq->engine->sched_engine->schedule(rq, attr);
1840
1841	local_bh_disable();
1842	__i915_request_queue_bh(rq);
1843	local_bh_enable(); /* kick tasklets */
1844}
1845
1846void i915_request_add(struct i915_request *rq)
1847{
1848	struct intel_timeline * const tl = i915_request_timeline(rq);
1849	struct i915_sched_attr attr = {};
1850	struct i915_gem_context *ctx;
1851
1852	lockdep_assert_held(&tl->mutex);
1853	lockdep_unpin_lock(&tl->mutex, rq->cookie);
1854
1855	trace_i915_request_add(rq);
1856	__i915_request_commit(rq);
1857
1858	/* XXX placeholder for selftests */
1859	rcu_read_lock();
1860	ctx = rcu_dereference(rq->context->gem_context);
1861	if (ctx)
1862		attr = ctx->sched;
1863	rcu_read_unlock();
1864
1865	__i915_request_queue(rq, &attr);
1866
1867	mutex_unlock(&tl->mutex);
1868}
1869
1870static unsigned long local_clock_ns(unsigned int *cpu)
1871{
1872	unsigned long t;
1873
1874	/*
1875	 * Cheaply and approximately convert from nanoseconds to microseconds.
1876	 * The result and subsequent calculations are also defined in the same
1877	 * approximate microseconds units. The principal source of timing
1878	 * error here is from the simple truncation.
1879	 *
1880	 * Note that local_clock() is only defined wrt to the current CPU;
1881	 * the comparisons are no longer valid if we switch CPUs. Instead of
1882	 * blocking preemption for the entire busywait, we can detect the CPU
1883	 * switch and use that as indicator of system load and a reason to
1884	 * stop busywaiting, see busywait_stop().
1885	 */
1886	*cpu = get_cpu();
1887	t = local_clock();
1888	put_cpu();
1889
1890	return t;
1891}
1892
1893static bool busywait_stop(unsigned long timeout, unsigned int cpu)
1894{
1895	unsigned int this_cpu;
1896
1897	if (time_after(local_clock_ns(&this_cpu), timeout))
1898		return true;
1899
1900	return this_cpu != cpu;
1901}
1902
1903static bool __i915_spin_request(struct i915_request * const rq, int state)
1904{
1905	unsigned long timeout_ns;
1906	unsigned int cpu;
1907
1908	/*
1909	 * Only wait for the request if we know it is likely to complete.
1910	 *
1911	 * We don't track the timestamps around requests, nor the average
1912	 * request length, so we do not have a good indicator that this
1913	 * request will complete within the timeout. What we do know is the
1914	 * order in which requests are executed by the context and so we can
1915	 * tell if the request has been started. If the request is not even
1916	 * running yet, it is a fair assumption that it will not complete
1917	 * within our relatively short timeout.
1918	 */
1919	if (!i915_request_is_running(rq))
1920		return false;
1921
1922	/*
1923	 * When waiting for high frequency requests, e.g. during synchronous
1924	 * rendering split between the CPU and GPU, the finite amount of time
1925	 * required to set up the irq and wait upon it limits the response
1926	 * rate. By busywaiting on the request completion for a short while we
1927	 * can service the high frequency waits as quick as possible. However,
1928	 * if it is a slow request, we want to sleep as quickly as possible.
1929	 * The tradeoff between waiting and sleeping is roughly the time it
1930	 * takes to sleep on a request, on the order of a microsecond.
1931	 */
1932
1933	timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns);
1934	timeout_ns += local_clock_ns(&cpu);
1935	do {
1936		if (dma_fence_is_signaled(&rq->fence))
1937			return true;
1938
1939		if (signal_pending_state(state, current))
1940			break;
1941
1942		if (busywait_stop(timeout_ns, cpu))
1943			break;
1944
1945		cpu_relax();
1946	} while (!need_resched());
1947
1948	return false;
1949}
1950
1951struct request_wait {
1952	struct dma_fence_cb cb;
1953	struct task_struct *tsk;
1954};
1955
1956static void request_wait_wake(struct dma_fence *fence, struct dma_fence_cb *cb)
1957{
1958	struct request_wait *wait = container_of(cb, typeof(*wait), cb);
1959
1960	wake_up_process(fetch_and_zero(&wait->tsk));
1961}
1962
1963/**
1964 * i915_request_wait_timeout - wait until execution of request has finished
1965 * @rq: the request to wait upon
1966 * @flags: how to wait
1967 * @timeout: how long to wait in jiffies
1968 *
1969 * i915_request_wait_timeout() waits for the request to be completed, for a
1970 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
1971 * unbounded wait).
1972 *
1973 * Returns the remaining time (in jiffies) if the request completed, which may
1974 * be zero if the request is unfinished after the timeout expires.
1975 * If the timeout is 0, it will return 1 if the fence is signaled.
1976 *
1977 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
1978 * pending before the request completes.
1979 *
1980 * NOTE: This function has the same wait semantics as dma-fence.
1981 */
1982long i915_request_wait_timeout(struct i915_request *rq,
1983			       unsigned int flags,
1984			       long timeout)
1985{
1986	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
1987		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1988	struct request_wait wait;
1989
1990	might_sleep();
1991	GEM_BUG_ON(timeout < 0);
1992
1993	if (dma_fence_is_signaled(&rq->fence))
1994		return timeout ?: 1;
1995
1996	if (!timeout)
1997		return -ETIME;
1998
1999	trace_i915_request_wait_begin(rq, flags);
2000
2001	/*
2002	 * We must never wait on the GPU while holding a lock as we
2003	 * may need to perform a GPU reset. So while we don't need to
2004	 * serialise wait/reset with an explicit lock, we do want
2005	 * lockdep to detect potential dependency cycles.
2006	 */
2007	mutex_acquire(&rq->engine->gt->reset.mutex.dep_map, 0, 0, _THIS_IP_);
2008
2009	/*
2010	 * Optimistic spin before touching IRQs.
2011	 *
2012	 * We may use a rather large value here to offset the penalty of
2013	 * switching away from the active task. Frequently, the client will
2014	 * wait upon an old swapbuffer to throttle itself to remain within a
2015	 * frame of the gpu. If the client is running in lockstep with the gpu,
2016	 * then it should not be waiting long at all, and a sleep now will incur
2017	 * extra scheduler latency in producing the next frame. To try to
2018	 * avoid adding the cost of enabling/disabling the interrupt to the
2019	 * short wait, we first spin to see if the request would have completed
2020	 * in the time taken to setup the interrupt.
2021	 *
2022	 * We need upto 5us to enable the irq, and upto 20us to hide the
2023	 * scheduler latency of a context switch, ignoring the secondary
2024	 * impacts from a context switch such as cache eviction.
2025	 *
2026	 * The scheme used for low-latency IO is called "hybrid interrupt
2027	 * polling". The suggestion there is to sleep until just before you
2028	 * expect to be woken by the device interrupt and then poll for its
2029	 * completion. That requires having a good predictor for the request
2030	 * duration, which we currently lack.
2031	 */
2032	if (CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT &&
2033	    __i915_spin_request(rq, state))
2034		goto out;
2035
2036	/*
2037	 * This client is about to stall waiting for the GPU. In many cases
2038	 * this is undesirable and limits the throughput of the system, as
2039	 * many clients cannot continue processing user input/output whilst
2040	 * blocked. RPS autotuning may take tens of milliseconds to respond
2041	 * to the GPU load and thus incurs additional latency for the client.
2042	 * We can circumvent that by promoting the GPU frequency to maximum
2043	 * before we sleep. This makes the GPU throttle up much more quickly
2044	 * (good for benchmarks and user experience, e.g. window animations),
2045	 * but at a cost of spending more power processing the workload
2046	 * (bad for battery).
2047	 */
2048	if (flags & I915_WAIT_PRIORITY && !i915_request_started(rq))
2049		intel_rps_boost(rq);
2050
2051	wait.tsk = current;
2052	if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))
2053		goto out;
2054
2055	/*
2056	 * Flush the submission tasklet, but only if it may help this request.
2057	 *
2058	 * We sometimes experience some latency between the HW interrupts and
2059	 * tasklet execution (mostly due to ksoftirqd latency, but it can also
2060	 * be due to lazy CS events), so lets run the tasklet manually if there
2061	 * is a chance it may submit this request. If the request is not ready
2062	 * to run, as it is waiting for other fences to be signaled, flushing
2063	 * the tasklet is busy work without any advantage for this client.
2064	 *
2065	 * If the HW is being lazy, this is the last chance before we go to
2066	 * sleep to catch any pending events. We will check periodically in
2067	 * the heartbeat to flush the submission tasklets as a last resort
2068	 * for unhappy HW.
2069	 */
2070	if (i915_request_is_ready(rq))
2071		__intel_engine_flush_submission(rq->engine, false);
2072
2073	for (;;) {
2074		set_current_state(state);
2075
2076		if (dma_fence_is_signaled(&rq->fence))
2077			break;
2078
2079		if (signal_pending_state(state, current)) {
2080			timeout = -ERESTARTSYS;
2081			break;
2082		}
2083
2084		if (!timeout) {
2085			timeout = -ETIME;
2086			break;
2087		}
2088
2089		timeout = io_schedule_timeout(timeout);
2090	}
2091	__set_current_state(TASK_RUNNING);
2092
2093	if (READ_ONCE(wait.tsk))
2094		dma_fence_remove_callback(&rq->fence, &wait.cb);
2095	GEM_BUG_ON(!list_empty(&wait.cb.node));
2096
2097out:
2098	mutex_release(&rq->engine->gt->reset.mutex.dep_map, _THIS_IP_);
2099	trace_i915_request_wait_end(rq);
2100	return timeout;
2101}
2102
2103/**
2104 * i915_request_wait - wait until execution of request has finished
2105 * @rq: the request to wait upon
2106 * @flags: how to wait
2107 * @timeout: how long to wait in jiffies
2108 *
2109 * i915_request_wait() waits for the request to be completed, for a
2110 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
2111 * unbounded wait).
2112 *
2113 * Returns the remaining time (in jiffies) if the request completed, which may
2114 * be zero or -ETIME if the request is unfinished after the timeout expires.
2115 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
2116 * pending before the request completes.
2117 *
2118 * NOTE: This function behaves differently from dma-fence wait semantics for
2119 * timeout = 0. It returns 0 on success, and -ETIME if not signaled.
2120 */
2121long i915_request_wait(struct i915_request *rq,
2122		       unsigned int flags,
2123		       long timeout)
2124{
2125	long ret = i915_request_wait_timeout(rq, flags, timeout);
2126
2127	if (!ret)
2128		return -ETIME;
2129
2130	if (ret > 0 && !timeout)
2131		return 0;
2132
2133	return ret;
2134}
2135
2136static int print_sched_attr(const struct i915_sched_attr *attr,
2137			    char *buf, int x, int len)
2138{
2139	if (attr->priority == I915_PRIORITY_INVALID)
2140		return x;
2141
2142	x += snprintf(buf + x, len - x,
2143		      " prio=%d", attr->priority);
2144
2145	return x;
2146}
2147
2148static char queue_status(const struct i915_request *rq)
2149{
2150	if (i915_request_is_active(rq))
2151		return 'E';
2152
2153	if (i915_request_is_ready(rq))
2154		return intel_engine_is_virtual(rq->engine) ? 'V' : 'R';
2155
2156	return 'U';
2157}
2158
2159static const char *run_status(const struct i915_request *rq)
2160{
2161	if (__i915_request_is_complete(rq))
2162		return "!";
2163
2164	if (__i915_request_has_started(rq))
2165		return "*";
2166
2167	if (!i915_sw_fence_signaled(&rq->semaphore))
2168		return "&";
2169
2170	return "";
2171}
2172
2173static const char *fence_status(const struct i915_request *rq)
2174{
2175	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
2176		return "+";
2177
2178	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
2179		return "-";
2180
2181	return "";
2182}
2183
2184void i915_request_show(struct drm_printer *m,
2185		       const struct i915_request *rq,
2186		       const char *prefix,
2187		       int indent)
2188{
2189	const char __rcu *timeline;
2190	char buf[80] = "";
2191	int x = 0;
2192
2193	/*
2194	 * The prefix is used to show the queue status, for which we use
2195	 * the following flags:
2196	 *
2197	 *  U [Unready]
2198	 *    - initial status upon being submitted by the user
2199	 *
2200	 *    - the request is not ready for execution as it is waiting
2201	 *      for external fences
2202	 *
2203	 *  R [Ready]
2204	 *    - all fences the request was waiting on have been signaled,
2205	 *      and the request is now ready for execution and will be
2206	 *      in a backend queue
2207	 *
2208	 *    - a ready request may still need to wait on semaphores
2209	 *      [internal fences]
2210	 *
2211	 *  V [Ready/virtual]
2212	 *    - same as ready, but queued over multiple backends
2213	 *
2214	 *  E [Executing]
2215	 *    - the request has been transferred from the backend queue and
2216	 *      submitted for execution on HW
2217	 *
2218	 *    - a completed request may still be regarded as executing, its
2219	 *      status may not be updated until it is retired and removed
2220	 *      from the lists
2221	 */
2222
2223	x = print_sched_attr(&rq->sched.attr, buf, x, sizeof(buf));
2224
2225	rcu_read_lock();
2226	timeline = dma_fence_timeline_name((struct dma_fence *)&rq->fence);
2227	drm_printf(m, "%s%.*s%c %llx:%lld%s%s %s @ %dms: %s\n",
2228		   prefix, indent, "                ",
2229		   queue_status(rq),
2230		   rq->fence.context, rq->fence.seqno,
2231		   run_status(rq),
2232		   fence_status(rq),
2233		   buf,
2234		   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
2235		   rcu_dereference(timeline));
2236	rcu_read_unlock();
2237}
2238
2239static bool engine_match_ring(struct intel_engine_cs *engine, struct i915_request *rq)
2240{
2241	u32 ring = ENGINE_READ(engine, RING_START);
2242
2243	return ring == i915_ggtt_offset(rq->ring->vma);
2244}
2245
2246static bool match_ring(struct i915_request *rq)
2247{
2248	struct intel_engine_cs *engine;
2249	bool found;
2250	int i;
2251
2252	if (!intel_engine_is_virtual(rq->engine))
2253		return engine_match_ring(rq->engine, rq);
2254
2255	found = false;
2256	i = 0;
2257	while ((engine = intel_engine_get_sibling(rq->engine, i++))) {
2258		found = engine_match_ring(engine, rq);
2259		if (found)
2260			break;
2261	}
2262
2263	return found;
2264}
2265
2266enum i915_request_state i915_test_request_state(struct i915_request *rq)
2267{
2268	if (i915_request_completed(rq))
2269		return I915_REQUEST_COMPLETE;
2270
2271	if (!i915_request_started(rq))
2272		return I915_REQUEST_PENDING;
2273
2274	if (match_ring(rq))
2275		return I915_REQUEST_ACTIVE;
2276
2277	return I915_REQUEST_QUEUED;
2278}
2279
2280#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2281#include "selftests/mock_request.c"
2282#include "selftests/i915_request.c"
2283#endif
2284
2285void i915_request_module_exit(void)
2286{
2287	kmem_cache_destroy(slab_execute_cbs);
2288	kmem_cache_destroy(slab_requests);
2289}
2290
2291int __init i915_request_module_init(void)
2292{
2293	slab_requests =
2294		kmem_cache_create("i915_request",
2295				  sizeof(struct i915_request),
2296				  __alignof__(struct i915_request),
2297				  SLAB_HWCACHE_ALIGN |
2298				  SLAB_RECLAIM_ACCOUNT |
2299				  SLAB_TYPESAFE_BY_RCU,
2300				  __i915_request_ctor);
2301	if (!slab_requests)
2302		return -ENOMEM;
2303
2304	slab_execute_cbs = KMEM_CACHE(execute_cb,
2305					     SLAB_HWCACHE_ALIGN |
2306					     SLAB_RECLAIM_ACCOUNT |
2307					     SLAB_TYPESAFE_BY_RCU);
2308	if (!slab_execute_cbs)
2309		goto err_requests;
2310
2311	return 0;
2312
2313err_requests:
2314	kmem_cache_destroy(slab_requests);
2315	return -ENOMEM;
2316}