vc4_gem.c

   1/*
   2 * Copyright © 2014 Broadcom
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 */
  23
  24#include <linux/module.h>
  25#include <linux/platform_device.h>
  26#include <linux/pm_runtime.h>
  27#include <linux/device.h>
  28#include <linux/io.h>
  29#include <linux/sched/signal.h>
  30#include <linux/dma-fence-array.h>
  31
  32#include <drm/drm_exec.h>
  33#include <drm/drm_print.h>
  34#include <drm/drm_syncobj.h>
  35
  36#include "uapi/drm/vc4_drm.h"
  37#include "vc4_drv.h"
  38#include "vc4_regs.h"
  39#include "vc4_trace.h"
  40
  41static void
  42vc4_queue_hangcheck(struct drm_device *dev)
  43{
  44	struct vc4_dev *vc4 = to_vc4_dev(dev);
  45
  46	mod_timer(&vc4->hangcheck.timer,
  47		  round_jiffies_up(jiffies + msecs_to_jiffies(100)));
  48}
  49
  50struct vc4_hang_state {
  51	struct drm_vc4_get_hang_state user_state;
  52
  53	u32 bo_count;
  54	struct drm_gem_object **bo;
  55};
  56
  57static void
  58vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state)
  59{
  60	unsigned int i;
  61
  62	for (i = 0; i < state->user_state.bo_count; i++)
  63		drm_gem_object_put(state->bo[i]);
  64
  65	kfree(state->bo);
  66	kfree(state);
  67}
  68
  69int
  70vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
  71			 struct drm_file *file_priv)
  72{
  73	struct drm_vc4_get_hang_state *get_state = data;
  74	struct drm_vc4_get_hang_state_bo *bo_state;
  75	struct vc4_hang_state *kernel_state;
  76	struct drm_vc4_get_hang_state *state;
  77	struct vc4_dev *vc4 = to_vc4_dev(dev);
  78	unsigned long irqflags;
  79	u32 i;
  80	int ret = 0;
  81
  82	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
  83		return -ENODEV;
  84
  85	if (!vc4->v3d) {
  86		DRM_DEBUG("VC4_GET_HANG_STATE with no VC4 V3D probed\n");
  87		return -ENODEV;
  88	}
  89
  90	spin_lock_irqsave(&vc4->job_lock, irqflags);
  91	kernel_state = vc4->hang_state;
  92	if (!kernel_state) {
  93		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
  94		return -ENOENT;
  95	}
  96	state = &kernel_state->user_state;
  97
  98	/* If the user's array isn't big enough, just return the
  99	 * required array size.
 100	 */
 101	if (get_state->bo_count < state->bo_count) {
 102		get_state->bo_count = state->bo_count;
 103		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 104		return 0;
 105	}
 106
 107	vc4->hang_state = NULL;
 108	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 109
 110	/* Save the user's BO pointer, so we don't stomp it with the memcpy. */
 111	state->bo = get_state->bo;
 112	memcpy(get_state, state, sizeof(*state));
 113
 114	bo_state = kcalloc(state->bo_count, sizeof(*bo_state), GFP_KERNEL);
 115	if (!bo_state) {
 116		ret = -ENOMEM;
 117		goto err_free;
 118	}
 119
 120	for (i = 0; i < state->bo_count; i++) {
 121		struct vc4_bo *vc4_bo = to_vc4_bo(kernel_state->bo[i]);
 122		u32 handle;
 123
 124		ret = drm_gem_handle_create(file_priv, kernel_state->bo[i],
 125					    &handle);
 126
 127		if (ret) {
 128			state->bo_count = i;
 129			goto err_delete_handle;
 130		}
 131		bo_state[i].handle = handle;
 132		bo_state[i].paddr = vc4_bo->base.dma_addr;
 133		bo_state[i].size = vc4_bo->base.base.size;
 134	}
 135
 136	if (copy_to_user(u64_to_user_ptr(get_state->bo),
 137			 bo_state,
 138			 state->bo_count * sizeof(*bo_state)))
 139		ret = -EFAULT;
 140
 141err_delete_handle:
 142	if (ret) {
 143		for (i = 0; i < state->bo_count; i++)
 144			drm_gem_handle_delete(file_priv, bo_state[i].handle);
 145	}
 146
 147err_free:
 148	vc4_free_hang_state(dev, kernel_state);
 149	kfree(bo_state);
 150
 151	return ret;
 152}
 153
 154static void
 155vc4_save_hang_state(struct drm_device *dev)
 156{
 157	struct vc4_dev *vc4 = to_vc4_dev(dev);
 158	struct drm_vc4_get_hang_state *state;
 159	struct vc4_hang_state *kernel_state;
 160	struct vc4_exec_info *exec[2];
 161	struct vc4_bo *bo;
 162	unsigned long irqflags;
 163	unsigned int i, j, k, unref_list_count;
 164
 165	kernel_state = kcalloc(1, sizeof(*kernel_state), GFP_KERNEL);
 166	if (!kernel_state)
 167		return;
 168
 169	state = &kernel_state->user_state;
 170
 171	spin_lock_irqsave(&vc4->job_lock, irqflags);
 172	exec[0] = vc4_first_bin_job(vc4);
 173	exec[1] = vc4_first_render_job(vc4);
 174	if (!exec[0] && !exec[1])
 175		goto err_free_state;
 176
 177	/* Get the bos from both binner and renderer into hang state. */
 178	state->bo_count = 0;
 179	for (i = 0; i < 2; i++) {
 180		if (!exec[i])
 181			continue;
 182
 183		unref_list_count = 0;
 184		list_for_each_entry(bo, &exec[i]->unref_list, unref_head)
 185			unref_list_count++;
 186		state->bo_count += exec[i]->bo_count + unref_list_count;
 187	}
 188
 189	kernel_state->bo = kcalloc(state->bo_count,
 190				   sizeof(*kernel_state->bo), GFP_ATOMIC);
 191
 192	if (!kernel_state->bo)
 193		goto err_free_state;
 194
 195	k = 0;
 196	for (i = 0; i < 2; i++) {
 197		if (!exec[i])
 198			continue;
 199
 200		for (j = 0; j < exec[i]->bo_count; j++) {
 201			bo = to_vc4_bo(exec[i]->bo[j]);
 202
 203			/* Retain BOs just in case they were marked purgeable.
 204			 * This prevents the BO from being purged before
 205			 * someone had a chance to dump the hang state.
 206			 */
 207			WARN_ON(!refcount_read(&bo->usecnt));
 208			refcount_inc(&bo->usecnt);
 209			drm_gem_object_get(exec[i]->bo[j]);
 210			kernel_state->bo[k++] = exec[i]->bo[j];
 211		}
 212
 213		list_for_each_entry(bo, &exec[i]->unref_list, unref_head) {
 214			/* No need to retain BOs coming from the ->unref_list
 215			 * because they are naturally unpurgeable.
 216			 */
 217			drm_gem_object_get(&bo->base.base);
 218			kernel_state->bo[k++] = &bo->base.base;
 219		}
 220	}
 221
 222	WARN_ON_ONCE(k != state->bo_count);
 223
 224	if (exec[0])
 225		state->start_bin = exec[0]->ct0ca;
 226	if (exec[1])
 227		state->start_render = exec[1]->ct1ca;
 228
 229	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 230
 231	state->ct0ca = V3D_READ(V3D_CTNCA(0));
 232	state->ct0ea = V3D_READ(V3D_CTNEA(0));
 233
 234	state->ct1ca = V3D_READ(V3D_CTNCA(1));
 235	state->ct1ea = V3D_READ(V3D_CTNEA(1));
 236
 237	state->ct0cs = V3D_READ(V3D_CTNCS(0));
 238	state->ct1cs = V3D_READ(V3D_CTNCS(1));
 239
 240	state->ct0ra0 = V3D_READ(V3D_CT00RA0);
 241	state->ct1ra0 = V3D_READ(V3D_CT01RA0);
 242
 243	state->bpca = V3D_READ(V3D_BPCA);
 244	state->bpcs = V3D_READ(V3D_BPCS);
 245	state->bpoa = V3D_READ(V3D_BPOA);
 246	state->bpos = V3D_READ(V3D_BPOS);
 247
 248	state->vpmbase = V3D_READ(V3D_VPMBASE);
 249
 250	state->dbge = V3D_READ(V3D_DBGE);
 251	state->fdbgo = V3D_READ(V3D_FDBGO);
 252	state->fdbgb = V3D_READ(V3D_FDBGB);
 253	state->fdbgr = V3D_READ(V3D_FDBGR);
 254	state->fdbgs = V3D_READ(V3D_FDBGS);
 255	state->errstat = V3D_READ(V3D_ERRSTAT);
 256
 257	/* We need to turn purgeable BOs into unpurgeable ones so that
 258	 * userspace has a chance to dump the hang state before the kernel
 259	 * decides to purge those BOs.
 260	 * Note that BO consistency at dump time cannot be guaranteed. For
 261	 * example, if the owner of these BOs decides to re-use them or mark
 262	 * them purgeable again there's nothing we can do to prevent it.
 263	 */
 264	for (i = 0; i < kernel_state->user_state.bo_count; i++) {
 265		struct vc4_bo *bo = to_vc4_bo(kernel_state->bo[i]);
 266
 267		if (bo->madv == __VC4_MADV_NOTSUPP)
 268			continue;
 269
 270		mutex_lock(&bo->madv_lock);
 271		if (!WARN_ON(bo->madv == __VC4_MADV_PURGED))
 272			bo->madv = VC4_MADV_WILLNEED;
 273		refcount_dec(&bo->usecnt);
 274		mutex_unlock(&bo->madv_lock);
 275	}
 276
 277	spin_lock_irqsave(&vc4->job_lock, irqflags);
 278	if (vc4->hang_state) {
 279		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 280		vc4_free_hang_state(dev, kernel_state);
 281	} else {
 282		vc4->hang_state = kernel_state;
 283		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 284	}
 285
 286	return;
 287
 288err_free_state:
 289	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 290	kfree(kernel_state);
 291}
 292
 293static void
 294vc4_reset(struct drm_device *dev)
 295{
 296	struct vc4_dev *vc4 = to_vc4_dev(dev);
 297
 298	DRM_INFO("Resetting GPU.\n");
 299
 300	mutex_lock(&vc4->power_lock);
 301	if (vc4->power_refcount) {
 302		/* Power the device off and back on the by dropping the
 303		 * reference on runtime PM.
 304		 */
 305		pm_runtime_put_sync_suspend(&vc4->v3d->pdev->dev);
 306		pm_runtime_get_sync(&vc4->v3d->pdev->dev);
 307	}
 308	mutex_unlock(&vc4->power_lock);
 309
 310	vc4_irq_reset(dev);
 311
 312	/* Rearm the hangcheck -- another job might have been waiting
 313	 * for our hung one to get kicked off, and vc4_irq_reset()
 314	 * would have started it.
 315	 */
 316	vc4_queue_hangcheck(dev);
 317}
 318
 319static void
 320vc4_reset_work(struct work_struct *work)
 321{
 322	struct vc4_dev *vc4 =
 323		container_of(work, struct vc4_dev, hangcheck.reset_work);
 324
 325	vc4_save_hang_state(&vc4->base);
 326
 327	vc4_reset(&vc4->base);
 328}
 329
 330static void
 331vc4_hangcheck_elapsed(struct timer_list *t)
 332{
 333	struct vc4_dev *vc4 = timer_container_of(vc4, t, hangcheck.timer);
 334	struct drm_device *dev = &vc4->base;
 335	uint32_t ct0ca, ct1ca;
 336	unsigned long irqflags;
 337	struct vc4_exec_info *bin_exec, *render_exec;
 338
 339	spin_lock_irqsave(&vc4->job_lock, irqflags);
 340
 341	bin_exec = vc4_first_bin_job(vc4);
 342	render_exec = vc4_first_render_job(vc4);
 343
 344	/* If idle, we can stop watching for hangs. */
 345	if (!bin_exec && !render_exec) {
 346		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 347		return;
 348	}
 349
 350	ct0ca = V3D_READ(V3D_CTNCA(0));
 351	ct1ca = V3D_READ(V3D_CTNCA(1));
 352
 353	/* If we've made any progress in execution, rearm the timer
 354	 * and wait.
 355	 */
 356	if ((bin_exec && ct0ca != bin_exec->last_ct0ca) ||
 357	    (render_exec && ct1ca != render_exec->last_ct1ca)) {
 358		if (bin_exec)
 359			bin_exec->last_ct0ca = ct0ca;
 360		if (render_exec)
 361			render_exec->last_ct1ca = ct1ca;
 362		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 363		vc4_queue_hangcheck(dev);
 364		return;
 365	}
 366
 367	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 368
 369	/* We've gone too long with no progress, reset.  This has to
 370	 * be done from a work struct, since resetting can sleep and
 371	 * this timer hook isn't allowed to.
 372	 */
 373	schedule_work(&vc4->hangcheck.reset_work);
 374}
 375
 376static void
 377submit_cl(struct drm_device *dev, uint32_t thread, uint32_t start, uint32_t end)
 378{
 379	struct vc4_dev *vc4 = to_vc4_dev(dev);
 380
 381	/* Set the current and end address of the control list.
 382	 * Writing the end register is what starts the job.
 383	 */
 384	V3D_WRITE(V3D_CTNCA(thread), start);
 385	V3D_WRITE(V3D_CTNEA(thread), end);
 386}
 387
 388int
 389vc4_wait_for_seqno(struct drm_device *dev, uint64_t seqno, uint64_t timeout_ns,
 390		   bool interruptible)
 391{
 392	struct vc4_dev *vc4 = to_vc4_dev(dev);
 393	int ret = 0;
 394	unsigned long timeout_expire;
 395	DEFINE_WAIT(wait);
 396
 397	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 398		return -ENODEV;
 399
 400	if (vc4->finished_seqno >= seqno)
 401		return 0;
 402
 403	if (timeout_ns == 0)
 404		return -ETIME;
 405
 406	timeout_expire = jiffies + nsecs_to_jiffies(timeout_ns);
 407
 408	trace_vc4_wait_for_seqno_begin(dev, seqno, timeout_ns);
 409	for (;;) {
 410		prepare_to_wait(&vc4->job_wait_queue, &wait,
 411				interruptible ? TASK_INTERRUPTIBLE :
 412				TASK_UNINTERRUPTIBLE);
 413
 414		if (interruptible && signal_pending(current)) {
 415			ret = -ERESTARTSYS;
 416			break;
 417		}
 418
 419		if (vc4->finished_seqno >= seqno)
 420			break;
 421
 422		if (timeout_ns != ~0ull) {
 423			if (time_after_eq(jiffies, timeout_expire)) {
 424				ret = -ETIME;
 425				break;
 426			}
 427			schedule_timeout(timeout_expire - jiffies);
 428		} else {
 429			schedule();
 430		}
 431	}
 432
 433	finish_wait(&vc4->job_wait_queue, &wait);
 434	trace_vc4_wait_for_seqno_end(dev, seqno);
 435
 436	return ret;
 437}
 438
 439static void
 440vc4_flush_caches(struct drm_device *dev)
 441{
 442	struct vc4_dev *vc4 = to_vc4_dev(dev);
 443
 444	/* Flush the GPU L2 caches.  These caches sit on top of system
 445	 * L3 (the 128kb or so shared with the CPU), and are
 446	 * non-allocating in the L3.
 447	 */
 448	V3D_WRITE(V3D_L2CACTL,
 449		  V3D_L2CACTL_L2CCLR);
 450
 451	V3D_WRITE(V3D_SLCACTL,
 452		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |
 453		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC) |
 454		  VC4_SET_FIELD(0xf, V3D_SLCACTL_UCC) |
 455		  VC4_SET_FIELD(0xf, V3D_SLCACTL_ICC));
 456}
 457
 458static void
 459vc4_flush_texture_caches(struct drm_device *dev)
 460{
 461	struct vc4_dev *vc4 = to_vc4_dev(dev);
 462
 463	V3D_WRITE(V3D_L2CACTL,
 464		  V3D_L2CACTL_L2CCLR);
 465
 466	V3D_WRITE(V3D_SLCACTL,
 467		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T1CC) |
 468		  VC4_SET_FIELD(0xf, V3D_SLCACTL_T0CC));
 469}
 470
 471/* Sets the registers for the next job to be actually be executed in
 472 * the hardware.
 473 *
 474 * The job_lock should be held during this.
 475 */
 476void
 477vc4_submit_next_bin_job(struct drm_device *dev)
 478{
 479	struct vc4_dev *vc4 = to_vc4_dev(dev);
 480	struct vc4_exec_info *exec;
 481
 482	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 483		return;
 484
 485again:
 486	exec = vc4_first_bin_job(vc4);
 487	if (!exec)
 488		return;
 489
 490	vc4_flush_caches(dev);
 491
 492	/* Only start the perfmon if it was not already started by a previous
 493	 * job.
 494	 */
 495	if (exec->perfmon && vc4->active_perfmon != exec->perfmon)
 496		vc4_perfmon_start(vc4, exec->perfmon);
 497
 498	/* Either put the job in the binner if it uses the binner, or
 499	 * immediately move it to the to-be-rendered queue.
 500	 */
 501	if (exec->ct0ca != exec->ct0ea) {
 502		trace_vc4_submit_cl(dev, false, exec->seqno, exec->ct0ca,
 503				    exec->ct0ea);
 504		submit_cl(dev, 0, exec->ct0ca, exec->ct0ea);
 505	} else {
 506		struct vc4_exec_info *next;
 507
 508		vc4_move_job_to_render(dev, exec);
 509		next = vc4_first_bin_job(vc4);
 510
 511		/* We can't start the next bin job if the previous job had a
 512		 * different perfmon instance attached to it. The same goes
 513		 * if one of them had a perfmon attached to it and the other
 514		 * one doesn't.
 515		 */
 516		if (next && next->perfmon == exec->perfmon)
 517			goto again;
 518	}
 519}
 520
 521void
 522vc4_submit_next_render_job(struct drm_device *dev)
 523{
 524	struct vc4_dev *vc4 = to_vc4_dev(dev);
 525	struct vc4_exec_info *exec = vc4_first_render_job(vc4);
 526
 527	if (!exec)
 528		return;
 529
 530	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 531		return;
 532
 533	/* A previous RCL may have written to one of our textures, and
 534	 * our full cache flush at bin time may have occurred before
 535	 * that RCL completed.  Flush the texture cache now, but not
 536	 * the instructions or uniforms (since we don't write those
 537	 * from an RCL).
 538	 */
 539	vc4_flush_texture_caches(dev);
 540
 541	trace_vc4_submit_cl(dev, true, exec->seqno, exec->ct1ca, exec->ct1ea);
 542	submit_cl(dev, 1, exec->ct1ca, exec->ct1ea);
 543}
 544
 545void
 546vc4_move_job_to_render(struct drm_device *dev, struct vc4_exec_info *exec)
 547{
 548	struct vc4_dev *vc4 = to_vc4_dev(dev);
 549	bool was_empty = list_empty(&vc4->render_job_list);
 550
 551	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 552		return;
 553
 554	list_move_tail(&exec->head, &vc4->render_job_list);
 555	if (was_empty)
 556		vc4_submit_next_render_job(dev);
 557}
 558
 559static void
 560vc4_attach_fences(struct vc4_exec_info *exec)
 561{
 562	struct vc4_bo *bo;
 563	unsigned i;
 564
 565	for (i = 0; i < exec->bo_count; i++) {
 566		bo = to_vc4_bo(exec->bo[i]);
 567		dma_resv_add_fence(bo->base.base.resv, exec->fence,
 568				   DMA_RESV_USAGE_READ);
 569	}
 570
 571	for (i = 0; i < exec->rcl_write_bo_count; i++) {
 572		bo = to_vc4_bo(&exec->rcl_write_bo[i]->base);
 573		dma_resv_add_fence(bo->base.base.resv, exec->fence,
 574				   DMA_RESV_USAGE_WRITE);
 575	}
 576}
 577
 578/* Takes the reservation lock on all the BOs being referenced, so that
 579 * at queue submit time we can update the reservations.
 580 *
 581 * We don't lock the RCL the tile alloc/state BOs, or overflow memory
 582 * (all of which are on exec->unref_list).  They're entirely private
 583 * to vc4, so we don't attach dma-buf fences to them.
 584 */
 585static int
 586vc4_lock_bo_reservations(struct vc4_exec_info *exec,
 587			 struct drm_exec *exec_ctx)
 588{
 589	int ret;
 590
 591	/* Reserve space for our shared (read-only) fence references,
 592	 * before we commit the CL to the hardware.
 593	 */
 594	drm_exec_init(exec_ctx, DRM_EXEC_INTERRUPTIBLE_WAIT, exec->bo_count);
 595	drm_exec_until_all_locked(exec_ctx) {
 596		ret = drm_exec_prepare_array(exec_ctx, exec->bo,
 597					     exec->bo_count, 1);
 598	}
 599
 600	if (ret) {
 601		drm_exec_fini(exec_ctx);
 602		return ret;
 603	}
 604
 605	return 0;
 606}
 607
 608/* Queues a struct vc4_exec_info for execution.  If no job is
 609 * currently executing, then submits it.
 610 *
 611 * Unlike most GPUs, our hardware only handles one command list at a
 612 * time.  To queue multiple jobs at once, we'd need to edit the
 613 * previous command list to have a jump to the new one at the end, and
 614 * then bump the end address.  That's a change for a later date,
 615 * though.
 616 */
 617static int
 618vc4_queue_submit(struct drm_device *dev, struct vc4_exec_info *exec,
 619		 struct drm_exec *exec_ctx,
 620		 struct drm_syncobj *out_sync)
 621{
 622	struct vc4_dev *vc4 = to_vc4_dev(dev);
 623	struct vc4_exec_info *renderjob;
 624	uint64_t seqno;
 625	unsigned long irqflags;
 626	struct vc4_fence *fence;
 627
 628	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 629	if (!fence)
 630		return -ENOMEM;
 631	fence->dev = dev;
 632
 633	spin_lock_irqsave(&vc4->job_lock, irqflags);
 634
 635	seqno = ++vc4->emit_seqno;
 636	exec->seqno = seqno;
 637
 638	dma_fence_init(&fence->base, &vc4_fence_ops, &vc4->job_lock,
 639		       vc4->dma_fence_context, exec->seqno);
 640	fence->seqno = exec->seqno;
 641	exec->fence = &fence->base;
 642
 643	if (out_sync)
 644		drm_syncobj_replace_fence(out_sync, exec->fence);
 645
 646	vc4_attach_fences(exec);
 647
 648	drm_exec_fini(exec_ctx);
 649
 650	list_add_tail(&exec->head, &vc4->bin_job_list);
 651
 652	/* If no bin job was executing and if the render job (if any) has the
 653	 * same perfmon as our job attached to it (or if both jobs don't have
 654	 * perfmon activated), then kick ours off.  Otherwise, it'll get
 655	 * started when the previous job's flush/render done interrupt occurs.
 656	 */
 657	renderjob = vc4_first_render_job(vc4);
 658	if (vc4_first_bin_job(vc4) == exec &&
 659	    (!renderjob || renderjob->perfmon == exec->perfmon)) {
 660		vc4_submit_next_bin_job(dev);
 661		vc4_queue_hangcheck(dev);
 662	}
 663
 664	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 665
 666	return 0;
 667}
 668
 669/**
 670 * vc4_cl_lookup_bos() - Sets up exec->bo[] with the GEM objects
 671 * referenced by the job.
 672 * @dev: DRM device
 673 * @file_priv: DRM file for this fd
 674 * @exec: V3D job being set up
 675 *
 676 * The command validator needs to reference BOs by their index within
 677 * the submitted job's BO list.  This does the validation of the job's
 678 * BO list and reference counting for the lifetime of the job.
 679 */
 680static int
 681vc4_cl_lookup_bos(struct drm_device *dev,
 682		  struct drm_file *file_priv,
 683		  struct vc4_exec_info *exec)
 684{
 685	struct drm_vc4_submit_cl *args = exec->args;
 686	int ret = 0;
 687	int i;
 688
 689	exec->bo_count = args->bo_handle_count;
 690
 691	if (!exec->bo_count) {
 692		/* See comment on bo_index for why we have to check
 693		 * this.
 694		 */
 695		DRM_DEBUG("Rendering requires BOs to validate\n");
 696		return -EINVAL;
 697	}
 698
 699	ret = drm_gem_objects_lookup(file_priv, u64_to_user_ptr(args->bo_handles),
 700				     exec->bo_count, &exec->bo);
 701
 702	if (ret)
 703		goto fail_put_bo;
 704
 705	for (i = 0; i < exec->bo_count; i++) {
 706		ret = vc4_bo_inc_usecnt(to_vc4_bo(exec->bo[i]));
 707		if (ret)
 708			goto fail_dec_usecnt;
 709	}
 710
 711	return 0;
 712
 713fail_dec_usecnt:
 714	/* Decrease usecnt on acquired objects.
 715	 * We cannot rely on  vc4_complete_exec() to release resources here,
 716	 * because vc4_complete_exec() has no information about which BO has
 717	 * had its ->usecnt incremented.
 718	 * To make things easier we just free everything explicitly and set
 719	 * exec->bo to NULL so that vc4_complete_exec() skips the 'BO release'
 720	 * step.
 721	 */
 722	for (i-- ; i >= 0; i--)
 723		vc4_bo_dec_usecnt(to_vc4_bo(exec->bo[i]));
 724
 725fail_put_bo:
 726	/* Release any reference to acquired objects. */
 727	for (i = 0; i < exec->bo_count && exec->bo[i]; i++)
 728		drm_gem_object_put(exec->bo[i]);
 729
 730	kvfree(exec->bo);
 731	exec->bo = NULL;
 732	return ret;
 733}
 734
 735static int
 736vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 737{
 738	struct drm_vc4_submit_cl *args = exec->args;
 739	struct vc4_dev *vc4 = to_vc4_dev(dev);
 740	void *temp = NULL;
 741	void *bin;
 742	int ret = 0;
 743	uint32_t bin_offset = 0;
 744	uint32_t shader_rec_offset = roundup(bin_offset + args->bin_cl_size,
 745					     16);
 746	uint32_t uniforms_offset = shader_rec_offset + args->shader_rec_size;
 747	uint32_t exec_size = uniforms_offset + args->uniforms_size;
 748	uint32_t temp_size = exec_size + (sizeof(struct vc4_shader_state) *
 749					  args->shader_rec_count);
 750	struct vc4_bo *bo;
 751
 752	if (shader_rec_offset < args->bin_cl_size ||
 753	    uniforms_offset < shader_rec_offset ||
 754	    exec_size < uniforms_offset ||
 755	    args->shader_rec_count >= (UINT_MAX /
 756					  sizeof(struct vc4_shader_state)) ||
 757	    temp_size < exec_size) {
 758		DRM_DEBUG("overflow in exec arguments\n");
 759		ret = -EINVAL;
 760		goto fail;
 761	}
 762
 763	/* Allocate space where we'll store the copied in user command lists
 764	 * and shader records.
 765	 *
 766	 * We don't just copy directly into the BOs because we need to
 767	 * read the contents back for validation, and I think the
 768	 * bo->vaddr is uncached access.
 769	 */
 770	temp = kvmalloc_array(temp_size, 1, GFP_KERNEL);
 771	if (!temp) {
 772		drm_err(dev, "Failed to allocate storage for copying "
 773			"in bin/render CLs.\n");
 774		ret = -ENOMEM;
 775		goto fail;
 776	}
 777	bin = temp + bin_offset;
 778	exec->shader_rec_u = temp + shader_rec_offset;
 779	exec->uniforms_u = temp + uniforms_offset;
 780	exec->shader_state = temp + exec_size;
 781	exec->shader_state_size = args->shader_rec_count;
 782
 783	if (copy_from_user(bin,
 784			   u64_to_user_ptr(args->bin_cl),
 785			   args->bin_cl_size)) {
 786		ret = -EFAULT;
 787		goto fail;
 788	}
 789
 790	if (copy_from_user(exec->shader_rec_u,
 791			   u64_to_user_ptr(args->shader_rec),
 792			   args->shader_rec_size)) {
 793		ret = -EFAULT;
 794		goto fail;
 795	}
 796
 797	if (copy_from_user(exec->uniforms_u,
 798			   u64_to_user_ptr(args->uniforms),
 799			   args->uniforms_size)) {
 800		ret = -EFAULT;
 801		goto fail;
 802	}
 803
 804	bo = vc4_bo_create(dev, exec_size, true, VC4_BO_TYPE_BCL);
 805	if (IS_ERR(bo)) {
 806		drm_err(dev, "Couldn't allocate BO for binning\n");
 807		ret = PTR_ERR(bo);
 808		goto fail;
 809	}
 810	exec->exec_bo = &bo->base;
 811
 812	list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,
 813		      &exec->unref_list);
 814
 815	exec->ct0ca = exec->exec_bo->dma_addr + bin_offset;
 816
 817	exec->bin_u = bin;
 818
 819	exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;
 820	exec->shader_rec_p = exec->exec_bo->dma_addr + shader_rec_offset;
 821	exec->shader_rec_size = args->shader_rec_size;
 822
 823	exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;
 824	exec->uniforms_p = exec->exec_bo->dma_addr + uniforms_offset;
 825	exec->uniforms_size = args->uniforms_size;
 826
 827	ret = vc4_validate_bin_cl(dev,
 828				  exec->exec_bo->vaddr + bin_offset,
 829				  bin,
 830				  exec);
 831	if (ret)
 832		goto fail;
 833
 834	ret = vc4_validate_shader_recs(dev, exec);
 835	if (ret)
 836		goto fail;
 837
 838	if (exec->found_tile_binning_mode_config_packet) {
 839		ret = vc4_v3d_bin_bo_get(vc4, &exec->bin_bo_used);
 840		if (ret)
 841			goto fail;
 842	}
 843
 844fail:
 845	kvfree(temp);
 846	return ret;
 847}
 848
 849static void
 850vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 851{
 852	struct vc4_dev *vc4 = to_vc4_dev(dev);
 853	unsigned long irqflags;
 854	unsigned i;
 855
 856	/* If we got force-completed because of GPU reset rather than
 857	 * through our IRQ handler, signal the fence now.
 858	 */
 859	if (exec->fence) {
 860		dma_fence_signal(exec->fence);
 861		dma_fence_put(exec->fence);
 862	}
 863
 864	if (exec->bo) {
 865		for (i = 0; i < exec->bo_count; i++) {
 866			struct vc4_bo *bo = to_vc4_bo(exec->bo[i]);
 867
 868			vc4_bo_dec_usecnt(bo);
 869			drm_gem_object_put(exec->bo[i]);
 870		}
 871		kvfree(exec->bo);
 872	}
 873
 874	while (!list_empty(&exec->unref_list)) {
 875		struct vc4_bo *bo = list_first_entry(&exec->unref_list,
 876						     struct vc4_bo, unref_head);
 877		list_del(&bo->unref_head);
 878		drm_gem_object_put(&bo->base.base);
 879	}
 880
 881	/* Free up the allocation of any bin slots we used. */
 882	spin_lock_irqsave(&vc4->job_lock, irqflags);
 883	vc4->bin_alloc_used &= ~exec->bin_slots;
 884	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 885
 886	/* Release the reference on the binner BO if needed. */
 887	if (exec->bin_bo_used)
 888		vc4_v3d_bin_bo_put(vc4);
 889
 890	/* Release the reference we had on the perf monitor. */
 891	vc4_perfmon_put(exec->perfmon);
 892
 893	vc4_v3d_pm_put(vc4);
 894
 895	kfree(exec);
 896}
 897
 898void
 899vc4_job_handle_completed(struct vc4_dev *vc4)
 900{
 901	unsigned long irqflags;
 902
 903	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 904		return;
 905
 906	spin_lock_irqsave(&vc4->job_lock, irqflags);
 907	while (!list_empty(&vc4->job_done_list)) {
 908		struct vc4_exec_info *exec =
 909			list_first_entry(&vc4->job_done_list,
 910					 struct vc4_exec_info, head);
 911		list_del(&exec->head);
 912
 913		spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 914		vc4_complete_exec(&vc4->base, exec);
 915		spin_lock_irqsave(&vc4->job_lock, irqflags);
 916	}
 917
 918	spin_unlock_irqrestore(&vc4->job_lock, irqflags);
 919}
 920
 921/* Scheduled when any job has been completed, this walks the list of
 922 * jobs that had completed and unrefs their BOs and frees their exec
 923 * structs.
 924 */
 925static void
 926vc4_job_done_work(struct work_struct *work)
 927{
 928	struct vc4_dev *vc4 =
 929		container_of(work, struct vc4_dev, job_done_work);
 930
 931	vc4_job_handle_completed(vc4);
 932}
 933
 934static int
 935vc4_wait_for_seqno_ioctl_helper(struct drm_device *dev,
 936				uint64_t seqno,
 937				uint64_t *timeout_ns)
 938{
 939	unsigned long start = jiffies;
 940	int ret = vc4_wait_for_seqno(dev, seqno, *timeout_ns, true);
 941
 942	if ((ret == -EINTR || ret == -ERESTARTSYS) && *timeout_ns != ~0ull) {
 943		uint64_t delta = jiffies_to_nsecs(jiffies - start);
 944
 945		if (*timeout_ns >= delta)
 946			*timeout_ns -= delta;
 947	}
 948
 949	return ret;
 950}
 951
 952int
 953vc4_wait_seqno_ioctl(struct drm_device *dev, void *data,
 954		     struct drm_file *file_priv)
 955{
 956	struct vc4_dev *vc4 = to_vc4_dev(dev);
 957	struct drm_vc4_wait_seqno *args = data;
 958
 959	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 960		return -ENODEV;
 961
 962	return vc4_wait_for_seqno_ioctl_helper(dev, args->seqno,
 963					       &args->timeout_ns);
 964}
 965
 966int
 967vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
 968		  struct drm_file *file_priv)
 969{
 970	struct vc4_dev *vc4 = to_vc4_dev(dev);
 971	int ret;
 972	struct drm_vc4_wait_bo *args = data;
 973	unsigned long timeout_jiffies =
 974		usecs_to_jiffies(div_u64(args->timeout_ns, 1000));
 975	ktime_t start = ktime_get();
 976	u64 delta_ns;
 977
 978	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 979		return -ENODEV;
 980
 981	if (args->pad != 0)
 982		return -EINVAL;
 983
 984	ret = drm_gem_dma_resv_wait(file_priv, args->handle,
 985				    true, timeout_jiffies);
 986
 987	/* Decrement the user's timeout, in case we got interrupted
 988	 * such that the ioctl will be restarted.
 989	 */
 990	delta_ns = ktime_to_ns(ktime_sub(ktime_get(), start));
 991	if (delta_ns < args->timeout_ns)
 992		args->timeout_ns -= delta_ns;
 993	else
 994		args->timeout_ns = 0;
 995
 996	return ret;
 997}
 998
 999/**
1000 * vc4_submit_cl_ioctl() - Submits a job (frame) to the VC4.
1001 * @dev: DRM device
1002 * @data: ioctl argument
1003 * @file_priv: DRM file for this fd
1004 *
1005 * This is the main entrypoint for userspace to submit a 3D frame to
1006 * the GPU.  Userspace provides the binner command list (if
1007 * applicable), and the kernel sets up the render command list to draw
1008 * to the framebuffer described in the ioctl, using the command lists
1009 * that the 3D engine's binner will produce.
1010 */
1011int
1012vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
1013		    struct drm_file *file_priv)
1014{
1015	struct vc4_dev *vc4 = to_vc4_dev(dev);
1016	struct vc4_file *vc4file = file_priv->driver_priv;
1017	struct drm_vc4_submit_cl *args = data;
1018	struct drm_syncobj *out_sync = NULL;
1019	struct vc4_exec_info *exec;
1020	struct drm_exec exec_ctx;
1021	struct dma_fence *in_fence;
1022	int ret = 0;
1023
1024	trace_vc4_submit_cl_ioctl(dev, args->bin_cl_size,
1025				  args->shader_rec_size,
1026				  args->bo_handle_count);
1027
1028	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
1029		return -ENODEV;
1030
1031	if (!vc4->v3d) {
1032		DRM_DEBUG("VC4_SUBMIT_CL with no VC4 V3D probed\n");
1033		return -ENODEV;
1034	}
1035
1036	if ((args->flags & ~(VC4_SUBMIT_CL_USE_CLEAR_COLOR |
1037			     VC4_SUBMIT_CL_FIXED_RCL_ORDER |
1038			     VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X |
1039			     VC4_SUBMIT_CL_RCL_ORDER_INCREASING_Y)) != 0) {
1040		DRM_DEBUG("Unknown flags: 0x%02x\n", args->flags);
1041		return -EINVAL;
1042	}
1043
1044	if (args->pad2 != 0) {
1045		DRM_DEBUG("Invalid pad: 0x%08x\n", args->pad2);
1046		return -EINVAL;
1047	}
1048
1049	exec = kcalloc(1, sizeof(*exec), GFP_KERNEL);
1050	if (!exec)
1051		return -ENOMEM;
1052
1053	exec->dev = vc4;
1054
1055	ret = vc4_v3d_pm_get(vc4);
1056	if (ret) {
1057		kfree(exec);
1058		return ret;
1059	}
1060
1061	exec->args = args;
1062	INIT_LIST_HEAD(&exec->unref_list);
1063
1064	ret = vc4_cl_lookup_bos(dev, file_priv, exec);
1065	if (ret)
1066		goto fail;
1067
1068	if (args->perfmonid) {
1069		exec->perfmon = vc4_perfmon_find(vc4file,
1070						 args->perfmonid);
1071		if (!exec->perfmon) {
1072			ret = -ENOENT;
1073			goto fail;
1074		}
1075	}
1076
1077	if (args->in_sync) {
1078		ret = drm_syncobj_find_fence(file_priv, args->in_sync,
1079					     0, 0, &in_fence);
1080		if (ret)
1081			goto fail;
1082
1083		/* When the fence (or fence array) is exclusively from our
1084		 * context we can skip the wait since jobs are executed in
1085		 * order of their submission through this ioctl and this can
1086		 * only have fences from a prior job.
1087		 */
1088		if (!dma_fence_match_context(in_fence,
1089					     vc4->dma_fence_context)) {
1090			ret = dma_fence_wait(in_fence, true);
1091			if (ret) {
1092				dma_fence_put(in_fence);
1093				goto fail;
1094			}
1095		}
1096
1097		dma_fence_put(in_fence);
1098	}
1099
1100	if (exec->args->bin_cl_size != 0) {
1101		ret = vc4_get_bcl(dev, exec);
1102		if (ret)
1103			goto fail;
1104	} else {
1105		exec->ct0ca = 0;
1106		exec->ct0ea = 0;
1107	}
1108
1109	ret = vc4_get_rcl(dev, exec);
1110	if (ret)
1111		goto fail;
1112
1113	ret = vc4_lock_bo_reservations(exec, &exec_ctx);
1114	if (ret)
1115		goto fail;
1116
1117	if (args->out_sync) {
1118		out_sync = drm_syncobj_find(file_priv, args->out_sync);
1119		if (!out_sync) {
1120			ret = -EINVAL;
1121			goto fail_unreserve;
1122		}
1123
1124		/* We replace the fence in out_sync in vc4_queue_submit since
1125		 * the render job could execute immediately after that call.
1126		 * If it finishes before our ioctl processing resumes the
1127		 * render job fence could already have been freed.
1128		 */
1129	}
1130
1131	/* Clear this out of the struct we'll be putting in the queue,
1132	 * since it's part of our stack.
1133	 */
1134	exec->args = NULL;
1135
1136	ret = vc4_queue_submit(dev, exec, &exec_ctx, out_sync);
1137
1138	/* The syncobj isn't part of the exec data and we need to free our
1139	 * reference even if job submission failed.
1140	 */
1141	if (out_sync)
1142		drm_syncobj_put(out_sync);
1143
1144	if (ret)
1145		goto fail_unreserve;
1146
1147	/* Return the seqno for our job. */
1148	args->seqno = vc4->emit_seqno;
1149
1150	return 0;
1151
1152fail_unreserve:
1153	drm_exec_fini(&exec_ctx);
1154fail:
1155	vc4_complete_exec(&vc4->base, exec);
1156
1157	return ret;
1158}
1159
1160static void vc4_gem_destroy(struct drm_device *dev, void *unused);
1161int vc4_gem_init(struct drm_device *dev)
1162{
1163	struct vc4_dev *vc4 = to_vc4_dev(dev);
1164	int ret;
1165
1166	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
1167		return -ENODEV;
1168
1169	vc4->dma_fence_context = dma_fence_context_alloc(1);
1170
1171	INIT_LIST_HEAD(&vc4->bin_job_list);
1172	INIT_LIST_HEAD(&vc4->render_job_list);
1173	INIT_LIST_HEAD(&vc4->job_done_list);
1174	spin_lock_init(&vc4->job_lock);
1175
1176	INIT_WORK(&vc4->hangcheck.reset_work, vc4_reset_work);
1177	timer_setup(&vc4->hangcheck.timer, vc4_hangcheck_elapsed, 0);
1178
1179	INIT_WORK(&vc4->job_done_work, vc4_job_done_work);
1180
1181	ret = drmm_mutex_init(dev, &vc4->power_lock);
1182	if (ret)
1183		return ret;
1184
1185	INIT_LIST_HEAD(&vc4->purgeable.list);
1186
1187	ret = drmm_mutex_init(dev, &vc4->purgeable.lock);
1188	if (ret)
1189		return ret;
1190
1191	return drmm_add_action_or_reset(dev, vc4_gem_destroy, NULL);
1192}
1193
1194static void vc4_gem_destroy(struct drm_device *dev, void *unused)
1195{
1196	struct vc4_dev *vc4 = to_vc4_dev(dev);
1197
1198	/* Waiting for exec to finish would need to be done before
1199	 * unregistering V3D.
1200	 */
1201	WARN_ON(vc4->emit_seqno != vc4->finished_seqno);
1202
1203	/* V3D should already have disabled its interrupt and cleared
1204	 * the overflow allocation registers.  Now free the object.
1205	 */
1206	if (vc4->bin_bo) {
1207		drm_gem_object_put(&vc4->bin_bo->base.base);
1208		vc4->bin_bo = NULL;
1209	}
1210
1211	if (vc4->hang_state)
1212		vc4_free_hang_state(dev, vc4->hang_state);
1213}
1214
1215int vc4_gem_madvise_ioctl(struct drm_device *dev, void *data,
1216			  struct drm_file *file_priv)
1217{
1218	struct vc4_dev *vc4 = to_vc4_dev(dev);
1219	struct drm_vc4_gem_madvise *args = data;
1220	struct drm_gem_object *gem_obj;
1221	struct vc4_bo *bo;
1222	int ret;
1223
1224	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
1225		return -ENODEV;
1226
1227	switch (args->madv) {
1228	case VC4_MADV_DONTNEED:
1229	case VC4_MADV_WILLNEED:
1230		break;
1231	default:
1232		return -EINVAL;
1233	}
1234
1235	if (args->pad != 0)
1236		return -EINVAL;
1237
1238	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
1239	if (!gem_obj) {
1240		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
1241		return -ENOENT;
1242	}
1243
1244	bo = to_vc4_bo(gem_obj);
1245
1246	/* Only BOs exposed to userspace can be purged. */
1247	if (bo->madv == __VC4_MADV_NOTSUPP) {
1248		DRM_DEBUG("madvise not supported on this BO\n");
1249		ret = -EINVAL;
1250		goto out_put_gem;
1251	}
1252
1253	/* Not sure it's safe to purge imported BOs. Let's just assume it's
1254	 * not until proven otherwise.
1255	 */
1256	if (gem_obj->import_attach) {
1257		DRM_DEBUG("madvise not supported on imported BOs\n");
1258		ret = -EINVAL;
1259		goto out_put_gem;
1260	}
1261
1262	mutex_lock(&bo->madv_lock);
1263
1264	if (args->madv == VC4_MADV_DONTNEED && bo->madv == VC4_MADV_WILLNEED &&
1265	    !refcount_read(&bo->usecnt)) {
1266		/* If the BO is about to be marked as purgeable, is not used
1267		 * and is not already purgeable or purged, add it to the
1268		 * purgeable list.
1269		 */
1270		vc4_bo_add_to_purgeable_pool(bo);
1271	} else if (args->madv == VC4_MADV_WILLNEED &&
1272		   bo->madv == VC4_MADV_DONTNEED &&
1273		   !refcount_read(&bo->usecnt)) {
1274		/* The BO has not been purged yet, just remove it from
1275		 * the purgeable list.
1276		 */
1277		vc4_bo_remove_from_purgeable_pool(bo);
1278	}
1279
1280	/* Save the purged state. */
1281	args->retained = bo->madv != __VC4_MADV_PURGED;
1282
1283	/* Update internal madv state only if the bo was not purged. */
1284	if (bo->madv != __VC4_MADV_PURGED)
1285		bo->madv = args->madv;
1286
1287	mutex_unlock(&bo->madv_lock);
1288
1289	ret = 0;
1290
1291out_put_gem:
1292	drm_gem_object_put(gem_obj);
1293
1294	return ret;
1295}