i915_vma.c

   1/*
   2 * Copyright © 2016 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/sched/mm.h>
  26#include <linux/dma-fence-array.h>
  27
  28#include <drm/drm_gem.h>
  29#include <drm/drm_print.h>
  30
  31#include "display/intel_fb.h"
  32#include "display/intel_frontbuffer.h"
  33#include "gem/i915_gem_lmem.h"
  34#include "gem/i915_gem_object_frontbuffer.h"
  35#include "gem/i915_gem_tiling.h"
  36#include "gt/intel_engine.h"
  37#include "gt/intel_engine_heartbeat.h"
  38#include "gt/intel_gt.h"
  39#include "gt/intel_gt_pm.h"
  40#include "gt/intel_gt_requests.h"
  41#include "gt/intel_tlb.h"
  42
  43#include "i915_drv.h"
  44#include "i915_gem_evict.h"
  45#include "i915_sw_fence_work.h"
  46#include "i915_trace.h"
  47#include "i915_vma.h"
  48#include "i915_vma_resource.h"
  49
  50static inline void assert_vma_held_evict(const struct i915_vma *vma)
  51{
  52	/*
  53	 * We may be forced to unbind when the vm is dead, to clean it up.
  54	 * This is the only exception to the requirement of the object lock
  55	 * being held.
  56	 */
  57	if (kref_read(&vma->vm->ref))
  58		assert_object_held_shared(vma->obj);
  59}
  60
  61static struct kmem_cache *slab_vmas;
  62
  63static struct i915_vma *i915_vma_alloc(void)
  64{
  65	return kmem_cache_zalloc(slab_vmas, GFP_KERNEL);
  66}
  67
  68static void i915_vma_free(struct i915_vma *vma)
  69{
  70	return kmem_cache_free(slab_vmas, vma);
  71}
  72
  73#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
  74
  75#include <linux/stackdepot.h>
  76
  77static void vma_print_allocator(struct i915_vma *vma, const char *reason)
  78{
  79	char buf[512];
  80
  81	if (!vma->node.stack) {
  82		drm_dbg(vma->obj->base.dev,
  83			"vma.node [%08llx + %08llx] %s: unknown owner\n",
  84			vma->node.start, vma->node.size, reason);
  85		return;
  86	}
  87
  88	stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
  89	drm_dbg(vma->obj->base.dev,
  90		"vma.node [%08llx + %08llx] %s: inserted at %s\n",
  91		vma->node.start, vma->node.size, reason, buf);
  92}
  93
  94#else
  95
  96static void vma_print_allocator(struct i915_vma *vma, const char *reason)
  97{
  98}
  99
 100#endif
 101
 102static inline struct i915_vma *active_to_vma(struct i915_active *ref)
 103{
 104	return container_of(ref, typeof(struct i915_vma), active);
 105}
 106
 107static int __i915_vma_active(struct i915_active *ref)
 108{
 109	struct i915_vma *vma = active_to_vma(ref);
 110
 111	if (!i915_vma_tryget(vma))
 112		return -ENOENT;
 113
 114	/*
 115	 * Exclude global GTT VMA from holding a GT wakeref
 116	 * while active, otherwise GPU never goes idle.
 117	 */
 118	if (!i915_vma_is_ggtt(vma)) {
 119		/*
 120		 * Since we and our _retire() counterpart can be
 121		 * called asynchronously, storing a wakeref tracking
 122		 * handle inside struct i915_vma is not safe, and
 123		 * there is no other good place for that.  Hence,
 124		 * use untracked variants of intel_gt_pm_get/put().
 125		 */
 126		intel_gt_pm_get_untracked(vma->vm->gt);
 127	}
 128
 129	return 0;
 130}
 131
 132static void __i915_vma_retire(struct i915_active *ref)
 133{
 134	struct i915_vma *vma = active_to_vma(ref);
 135
 136	if (!i915_vma_is_ggtt(vma)) {
 137		/*
 138		 * Since we can be called from atomic contexts,
 139		 * use an async variant of intel_gt_pm_put().
 140		 */
 141		intel_gt_pm_put_async_untracked(vma->vm->gt);
 142	}
 143
 144	i915_vma_put(vma);
 145}
 146
 147static struct i915_vma *
 148vma_create(struct drm_i915_gem_object *obj,
 149	   struct i915_address_space *vm,
 150	   const struct i915_gtt_view *view)
 151{
 152	struct i915_vma *pos = ERR_PTR(-E2BIG);
 153	struct i915_vma *vma;
 154	struct rb_node *rb, **p;
 155	int err;
 156
 157	/* The aliasing_ppgtt should never be used directly! */
 158	GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm);
 159
 160	vma = i915_vma_alloc();
 161	if (vma == NULL)
 162		return ERR_PTR(-ENOMEM);
 163
 164	vma->ops = &vm->vma_ops;
 165	vma->obj = obj;
 166	vma->size = obj->base.size;
 167	vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
 168
 169	i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0);
 170
 171	/* Declare ourselves safe for use inside shrinkers */
 172	if (IS_ENABLED(CONFIG_LOCKDEP)) {
 173		fs_reclaim_acquire(GFP_KERNEL);
 174		might_lock(&vma->active.mutex);
 175		fs_reclaim_release(GFP_KERNEL);
 176	}
 177
 178	INIT_LIST_HEAD(&vma->closed_link);
 179	INIT_LIST_HEAD(&vma->obj_link);
 180	RB_CLEAR_NODE(&vma->obj_node);
 181
 182	if (view && view->type != I915_GTT_VIEW_NORMAL) {
 183		vma->gtt_view = *view;
 184		if (view->type == I915_GTT_VIEW_PARTIAL) {
 185			GEM_BUG_ON(range_overflows_t(u64,
 186						     view->partial.offset,
 187						     view->partial.size,
 188						     obj->base.size >> PAGE_SHIFT));
 189			vma->size = view->partial.size;
 190			vma->size <<= PAGE_SHIFT;
 191			GEM_BUG_ON(vma->size > obj->base.size);
 192		} else if (view->type == I915_GTT_VIEW_ROTATED) {
 193			vma->size = intel_rotation_info_size(&view->rotated);
 194			vma->size <<= PAGE_SHIFT;
 195		} else if (view->type == I915_GTT_VIEW_REMAPPED) {
 196			vma->size = intel_remapped_info_size(&view->remapped);
 197			vma->size <<= PAGE_SHIFT;
 198		}
 199	}
 200
 201	if (unlikely(vma->size > vm->total))
 202		goto err_vma;
 203
 204	GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
 205
 206	err = mutex_lock_interruptible(&vm->mutex);
 207	if (err) {
 208		pos = ERR_PTR(err);
 209		goto err_vma;
 210	}
 211
 212	vma->vm = vm;
 213	list_add_tail(&vma->vm_link, &vm->unbound_list);
 214
 215	spin_lock(&obj->vma.lock);
 216	if (i915_is_ggtt(vm)) {
 217		if (unlikely(overflows_type(vma->size, u32)))
 218			goto err_unlock;
 219
 220		vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
 221						      i915_gem_object_get_tiling(obj),
 222						      i915_gem_object_get_stride(obj));
 223		if (unlikely(vma->fence_size < vma->size || /* overflow */
 224			     vma->fence_size > vm->total))
 225			goto err_unlock;
 226
 227		GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
 228
 229		vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
 230								i915_gem_object_get_tiling(obj),
 231								i915_gem_object_get_stride(obj));
 232		GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
 233
 234		__set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
 235	}
 236
 237	rb = NULL;
 238	p = &obj->vma.tree.rb_node;
 239	while (*p) {
 240		long cmp;
 241
 242		rb = *p;
 243		pos = rb_entry(rb, struct i915_vma, obj_node);
 244
 245		/*
 246		 * If the view already exists in the tree, another thread
 247		 * already created a matching vma, so return the older instance
 248		 * and dispose of ours.
 249		 */
 250		cmp = i915_vma_compare(pos, vm, view);
 251		if (cmp < 0)
 252			p = &rb->rb_right;
 253		else if (cmp > 0)
 254			p = &rb->rb_left;
 255		else
 256			goto err_unlock;
 257	}
 258	rb_link_node(&vma->obj_node, rb, p);
 259	rb_insert_color(&vma->obj_node, &obj->vma.tree);
 260
 261	if (i915_vma_is_ggtt(vma))
 262		/*
 263		 * We put the GGTT vma at the start of the vma-list, followed
 264		 * by the ppGGTT vma. This allows us to break early when
 265		 * iterating over only the GGTT vma for an object, see
 266		 * for_each_ggtt_vma()
 267		 */
 268		list_add(&vma->obj_link, &obj->vma.list);
 269	else
 270		list_add_tail(&vma->obj_link, &obj->vma.list);
 271
 272	spin_unlock(&obj->vma.lock);
 273	mutex_unlock(&vm->mutex);
 274
 275	return vma;
 276
 277err_unlock:
 278	spin_unlock(&obj->vma.lock);
 279	list_del_init(&vma->vm_link);
 280	mutex_unlock(&vm->mutex);
 281err_vma:
 282	i915_vma_free(vma);
 283	return pos;
 284}
 285
 286static struct i915_vma *
 287i915_vma_lookup(struct drm_i915_gem_object *obj,
 288	   struct i915_address_space *vm,
 289	   const struct i915_gtt_view *view)
 290{
 291	struct rb_node *rb;
 292
 293	rb = obj->vma.tree.rb_node;
 294	while (rb) {
 295		struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
 296		long cmp;
 297
 298		cmp = i915_vma_compare(vma, vm, view);
 299		if (cmp == 0)
 300			return vma;
 301
 302		if (cmp < 0)
 303			rb = rb->rb_right;
 304		else
 305			rb = rb->rb_left;
 306	}
 307
 308	return NULL;
 309}
 310
 311/**
 312 * i915_vma_instance - return the singleton instance of the VMA
 313 * @obj: parent &struct drm_i915_gem_object to be mapped
 314 * @vm: address space in which the mapping is located
 315 * @view: additional mapping requirements
 316 *
 317 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
 318 * the same @view characteristics. If a match is not found, one is created.
 319 * Once created, the VMA is kept until either the object is freed, or the
 320 * address space is closed.
 321 *
 322 * Returns the vma, or an error pointer.
 323 */
 324struct i915_vma *
 325i915_vma_instance(struct drm_i915_gem_object *obj,
 326		  struct i915_address_space *vm,
 327		  const struct i915_gtt_view *view)
 328{
 329	struct i915_vma *vma;
 330
 331	GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
 332	GEM_BUG_ON(!kref_read(&vm->ref));
 333
 334	spin_lock(&obj->vma.lock);
 335	vma = i915_vma_lookup(obj, vm, view);
 336	spin_unlock(&obj->vma.lock);
 337
 338	/* vma_create() will resolve the race if another creates the vma */
 339	if (unlikely(!vma))
 340		vma = vma_create(obj, vm, view);
 341
 342	GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
 343	return vma;
 344}
 345
 346struct i915_vma_work {
 347	struct dma_fence_work base;
 348	struct i915_address_space *vm;
 349	struct i915_vm_pt_stash stash;
 350	struct i915_vma_resource *vma_res;
 351	struct drm_i915_gem_object *obj;
 352	struct i915_sw_dma_fence_cb cb;
 353	unsigned int pat_index;
 354	unsigned int flags;
 355};
 356
 357static void __vma_bind(struct dma_fence_work *work)
 358{
 359	struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
 360	struct i915_vma_resource *vma_res = vw->vma_res;
 361
 362	/*
 363	 * We are about the bind the object, which must mean we have already
 364	 * signaled the work to potentially clear/move the pages underneath. If
 365	 * something went wrong at that stage then the object should have
 366	 * unknown_state set, in which case we need to skip the bind.
 367	 */
 368	if (i915_gem_object_has_unknown_state(vw->obj))
 369		return;
 370
 371	vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
 372			       vma_res, vw->pat_index, vw->flags);
 373}
 374
 375static void __vma_release(struct dma_fence_work *work)
 376{
 377	struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
 378
 379	if (vw->obj)
 380		i915_gem_object_put(vw->obj);
 381
 382	i915_vm_free_pt_stash(vw->vm, &vw->stash);
 383	if (vw->vma_res)
 384		i915_vma_resource_put(vw->vma_res);
 385}
 386
 387static const struct dma_fence_work_ops bind_ops = {
 388	.name = "bind",
 389	.work = __vma_bind,
 390	.release = __vma_release,
 391};
 392
 393struct i915_vma_work *i915_vma_work(void)
 394{
 395	struct i915_vma_work *vw;
 396
 397	vw = kzalloc(sizeof(*vw), GFP_KERNEL);
 398	if (!vw)
 399		return NULL;
 400
 401	dma_fence_work_init(&vw->base, &bind_ops);
 402	vw->base.dma.error = -EAGAIN; /* disable the worker by default */
 403
 404	return vw;
 405}
 406
 407int i915_vma_wait_for_bind(struct i915_vma *vma)
 408{
 409	int err = 0;
 410
 411	if (rcu_access_pointer(vma->active.excl.fence)) {
 412		struct dma_fence *fence;
 413
 414		rcu_read_lock();
 415		fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
 416		rcu_read_unlock();
 417		if (fence) {
 418			err = dma_fence_wait(fence, true);
 419			dma_fence_put(fence);
 420		}
 421	}
 422
 423	return err;
 424}
 425
 426#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 427static int i915_vma_verify_bind_complete(struct i915_vma *vma)
 428{
 429	struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
 430	int err;
 431
 432	if (!fence)
 433		return 0;
 434
 435	if (dma_fence_is_signaled(fence))
 436		err = fence->error;
 437	else
 438		err = -EBUSY;
 439
 440	dma_fence_put(fence);
 441
 442	return err;
 443}
 444#else
 445#define i915_vma_verify_bind_complete(_vma) 0
 446#endif
 447
 448I915_SELFTEST_EXPORT void
 449i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
 450				struct i915_vma *vma)
 451{
 452	struct drm_i915_gem_object *obj = vma->obj;
 453
 454	i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes,
 455			       obj->mm.rsgt, i915_gem_object_is_readonly(obj),
 456			       i915_gem_object_is_lmem(obj), obj->mm.region,
 457			       vma->ops, vma->private, __i915_vma_offset(vma),
 458			       __i915_vma_size(vma), vma->size, vma->guard);
 459}
 460
 461/**
 462 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
 463 * @vma: VMA to map
 464 * @pat_index: PAT index to set in PTE
 465 * @flags: flags like global or local mapping
 466 * @work: preallocated worker for allocating and binding the PTE
 467 * @vma_res: pointer to a preallocated vma resource. The resource is either
 468 * consumed or freed.
 469 *
 470 * DMA addresses are taken from the scatter-gather table of this object (or of
 471 * this VMA in case of non-default GGTT views) and PTE entries set up.
 472 * Note that DMA addresses are also the only part of the SG table we care about.
 473 */
 474int i915_vma_bind(struct i915_vma *vma,
 475		  unsigned int pat_index,
 476		  u32 flags,
 477		  struct i915_vma_work *work,
 478		  struct i915_vma_resource *vma_res)
 479{
 480	u32 bind_flags;
 481	u32 vma_flags;
 482	int ret;
 483
 484	lockdep_assert_held(&vma->vm->mutex);
 485	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 486	GEM_BUG_ON(vma->size > i915_vma_size(vma));
 487
 488	if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,
 489					      vma->node.size,
 490					      vma->vm->total))) {
 491		i915_vma_resource_free(vma_res);
 492		return -ENODEV;
 493	}
 494
 495	if (GEM_DEBUG_WARN_ON(!flags)) {
 496		i915_vma_resource_free(vma_res);
 497		return -EINVAL;
 498	}
 499
 500	bind_flags = flags;
 501	bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 502
 503	vma_flags = atomic_read(&vma->flags);
 504	vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 505
 506	bind_flags &= ~vma_flags;
 507	if (bind_flags == 0) {
 508		i915_vma_resource_free(vma_res);
 509		return 0;
 510	}
 511
 512	GEM_BUG_ON(!atomic_read(&vma->pages_count));
 513
 514	/* Wait for or await async unbinds touching our range */
 515	if (work && bind_flags & vma->vm->bind_async_flags)
 516		ret = i915_vma_resource_bind_dep_await(vma->vm,
 517						       &work->base.chain,
 518						       vma->node.start,
 519						       vma->node.size,
 520						       true,
 521						       GFP_NOWAIT |
 522						       __GFP_RETRY_MAYFAIL |
 523						       __GFP_NOWARN);
 524	else
 525		ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start,
 526						      vma->node.size, true);
 527	if (ret) {
 528		i915_vma_resource_free(vma_res);
 529		return ret;
 530	}
 531
 532	if (vma->resource || !vma_res) {
 533		/* Rebinding with an additional I915_VMA_*_BIND */
 534		GEM_WARN_ON(!vma_flags);
 535		i915_vma_resource_free(vma_res);
 536	} else {
 537		i915_vma_resource_init_from_vma(vma_res, vma);
 538		vma->resource = vma_res;
 539	}
 540	trace_i915_vma_bind(vma, bind_flags);
 541	if (work && bind_flags & vma->vm->bind_async_flags) {
 542		struct dma_fence *prev;
 543
 544		work->vma_res = i915_vma_resource_get(vma->resource);
 545		work->pat_index = pat_index;
 546		work->flags = bind_flags;
 547
 548		/*
 549		 * Note we only want to chain up to the migration fence on
 550		 * the pages (not the object itself). As we don't track that,
 551		 * yet, we have to use the exclusive fence instead.
 552		 *
 553		 * Also note that we do not want to track the async vma as
 554		 * part of the obj->resv->excl_fence as it only affects
 555		 * execution and not content or object's backing store lifetime.
 556		 */
 557		prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
 558		if (prev) {
 559			__i915_sw_fence_await_dma_fence(&work->base.chain,
 560							prev,
 561							&work->cb);
 562			dma_fence_put(prev);
 563		}
 564
 565		work->base.dma.error = 0; /* enable the queue_work() */
 566		work->obj = i915_gem_object_get(vma->obj);
 567	} else {
 568		ret = i915_gem_object_wait_moving_fence(vma->obj, true);
 569		if (ret) {
 570			i915_vma_resource_free(vma->resource);
 571			vma->resource = NULL;
 572
 573			return ret;
 574		}
 575		vma->ops->bind_vma(vma->vm, NULL, vma->resource, pat_index,
 576				   bind_flags);
 577	}
 578
 579	atomic_or(bind_flags, &vma->flags);
 580	return 0;
 581}
 582
 583void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 584{
 585	void __iomem *ptr;
 586	int err;
 587
 588	if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY))
 589		return IOMEM_ERR_PTR(-EINVAL);
 590
 591	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 592	GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
 593	GEM_BUG_ON(i915_vma_verify_bind_complete(vma));
 594
 595	ptr = READ_ONCE(vma->iomap);
 596	if (ptr == NULL) {
 597		/*
 598		 * TODO: consider just using i915_gem_object_pin_map() for lmem
 599		 * instead, which already supports mapping non-contiguous chunks
 600		 * of pages, that way we can also drop the
 601		 * I915_BO_ALLOC_CONTIGUOUS when allocating the object.
 602		 */
 603		if (i915_gem_object_is_lmem(vma->obj)) {
 604			ptr = i915_gem_object_lmem_io_map(vma->obj, 0,
 605							  vma->obj->base.size);
 606		} else if (i915_vma_is_map_and_fenceable(vma)) {
 607			ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
 608						i915_vma_offset(vma),
 609						i915_vma_size(vma));
 610		} else {
 611			ptr = (void __iomem *)
 612				i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
 613			if (IS_ERR(ptr)) {
 614				err = PTR_ERR(ptr);
 615				goto err;
 616			}
 617			ptr = page_pack_bits(ptr, 1);
 618		}
 619
 620		if (ptr == NULL) {
 621			err = -ENOMEM;
 622			goto err;
 623		}
 624
 625		if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) {
 626			if (page_unmask_bits(ptr))
 627				__i915_gem_object_release_map(vma->obj);
 628			else
 629				io_mapping_unmap(ptr);
 630			ptr = vma->iomap;
 631		}
 632	}
 633
 634	__i915_vma_pin(vma);
 635
 636	err = i915_vma_pin_fence(vma);
 637	if (err)
 638		goto err_unpin;
 639
 640	i915_vma_set_ggtt_write(vma);
 641
 642	/* NB Access through the GTT requires the device to be awake. */
 643	return page_mask_bits(ptr);
 644
 645err_unpin:
 646	__i915_vma_unpin(vma);
 647err:
 648	return IOMEM_ERR_PTR(err);
 649}
 650
 651void i915_vma_flush_writes(struct i915_vma *vma)
 652{
 653	if (i915_vma_unset_ggtt_write(vma))
 654		intel_gt_flush_ggtt_writes(vma->vm->gt);
 655}
 656
 657void i915_vma_unpin_iomap(struct i915_vma *vma)
 658{
 659	GEM_BUG_ON(vma->iomap == NULL);
 660
 661	/* XXX We keep the mapping until __i915_vma_unbind()/evict() */
 662
 663	i915_vma_flush_writes(vma);
 664
 665	i915_vma_unpin_fence(vma);
 666	i915_vma_unpin(vma);
 667}
 668
 669void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
 670{
 671	struct i915_vma *vma;
 672	struct drm_i915_gem_object *obj;
 673
 674	vma = fetch_and_zero(p_vma);
 675	if (!vma)
 676		return;
 677
 678	obj = vma->obj;
 679	GEM_BUG_ON(!obj);
 680
 681	i915_vma_unpin(vma);
 682
 683	if (flags & I915_VMA_RELEASE_MAP)
 684		i915_gem_object_unpin_map(obj);
 685
 686	i915_gem_object_put(obj);
 687}
 688
 689bool i915_vma_misplaced(const struct i915_vma *vma,
 690			u64 size, u64 alignment, u64 flags)
 691{
 692	if (!drm_mm_node_allocated(&vma->node))
 693		return false;
 694
 695	if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma)))
 696		return true;
 697
 698	if (i915_vma_size(vma) < size)
 699		return true;
 700
 701	GEM_BUG_ON(alignment && !is_power_of_2(alignment));
 702	if (alignment && !IS_ALIGNED(i915_vma_offset(vma), alignment))
 703		return true;
 704
 705	if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
 706		return true;
 707
 708	if (flags & PIN_OFFSET_BIAS &&
 709	    i915_vma_offset(vma) < (flags & PIN_OFFSET_MASK))
 710		return true;
 711
 712	if (flags & PIN_OFFSET_FIXED &&
 713	    i915_vma_offset(vma) != (flags & PIN_OFFSET_MASK))
 714		return true;
 715
 716	if (flags & PIN_OFFSET_GUARD &&
 717	    vma->guard < (flags & PIN_OFFSET_MASK))
 718		return true;
 719
 720	return false;
 721}
 722
 723void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
 724{
 725	bool mappable, fenceable;
 726
 727	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 728	GEM_BUG_ON(!vma->fence_size);
 729
 730	fenceable = (i915_vma_size(vma) >= vma->fence_size &&
 731		     IS_ALIGNED(i915_vma_offset(vma), vma->fence_alignment));
 732
 733	mappable = i915_ggtt_offset(vma) + vma->fence_size <=
 734		   i915_vm_to_ggtt(vma->vm)->mappable_end;
 735
 736	if (mappable && fenceable)
 737		set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
 738	else
 739		clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
 740}
 741
 742bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
 743{
 744	struct drm_mm_node *node = &vma->node;
 745	struct drm_mm_node *other;
 746
 747	/*
 748	 * On some machines we have to be careful when putting differing types
 749	 * of snoopable memory together to avoid the prefetcher crossing memory
 750	 * domains and dying. During vm initialisation, we decide whether or not
 751	 * these constraints apply and set the drm_mm.color_adjust
 752	 * appropriately.
 753	 */
 754	if (!i915_vm_has_cache_coloring(vma->vm))
 755		return true;
 756
 757	/* Only valid to be called on an already inserted vma */
 758	GEM_BUG_ON(!drm_mm_node_allocated(node));
 759	GEM_BUG_ON(list_empty(&node->node_list));
 760
 761	other = list_prev_entry(node, node_list);
 762	if (i915_node_color_differs(other, color) &&
 763	    !drm_mm_hole_follows(other))
 764		return false;
 765
 766	other = list_next_entry(node, node_list);
 767	if (i915_node_color_differs(other, color) &&
 768	    !drm_mm_hole_follows(node))
 769		return false;
 770
 771	return true;
 772}
 773
 774/**
 775 * i915_vma_insert - finds a slot for the vma in its address space
 776 * @vma: the vma
 777 * @ww: An optional struct i915_gem_ww_ctx
 778 * @size: requested size in bytes (can be larger than the VMA)
 779 * @alignment: required alignment
 780 * @flags: mask of PIN_* flags to use
 781 *
 782 * First we try to allocate some free space that meets the requirements for
 783 * the VMA. Failing that, if the flags permit, it will evict an old VMA,
 784 * preferably the oldest idle entry to make room for the new VMA.
 785 *
 786 * Returns:
 787 * 0 on success, negative error code otherwise.
 788 */
 789static int
 790i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
 791		u64 size, u64 alignment, u64 flags)
 792{
 793	unsigned long color, guard;
 794	u64 start, end;
 795	int ret;
 796
 797	GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 798	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 799	GEM_BUG_ON(hweight64(flags & (PIN_OFFSET_GUARD | PIN_OFFSET_FIXED | PIN_OFFSET_BIAS)) > 1);
 800
 801	size = max(size, vma->size);
 802	alignment = max_t(typeof(alignment), alignment, vma->display_alignment);
 803	if (flags & PIN_MAPPABLE) {
 804		size = max_t(typeof(size), size, vma->fence_size);
 805		alignment = max_t(typeof(alignment),
 806				  alignment, vma->fence_alignment);
 807	}
 808
 809	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 810	GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
 811	GEM_BUG_ON(!is_power_of_2(alignment));
 812
 813	guard = vma->guard; /* retain guard across rebinds */
 814	if (flags & PIN_OFFSET_GUARD) {
 815		GEM_BUG_ON(overflows_type(flags & PIN_OFFSET_MASK, u32));
 816		guard = max_t(u32, guard, flags & PIN_OFFSET_MASK);
 817	}
 818	/*
 819	 * As we align the node upon insertion, but the hardware gets
 820	 * node.start + guard, the easiest way to make that work is
 821	 * to make the guard a multiple of the alignment size.
 822	 */
 823	guard = ALIGN(guard, alignment);
 824
 825	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
 826	GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
 827
 828	end = vma->vm->total;
 829	if (flags & PIN_MAPPABLE)
 830		end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
 831	if (flags & PIN_ZONE_4G)
 832		end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
 833	GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
 834
 835	alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj));
 836
 837	/*
 838	 * If binding the object/GGTT view requires more space than the entire
 839	 * aperture has, reject it early before evicting everything in a vain
 840	 * attempt to find space.
 841	 */
 842	if (size > end - 2 * guard) {
 843		drm_dbg(vma->obj->base.dev,
 844			"Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
 845			size, flags & PIN_MAPPABLE ? "mappable" : "total", end);
 846		return -ENOSPC;
 847	}
 848
 849	color = 0;
 850
 851	if (i915_vm_has_cache_coloring(vma->vm))
 852		color = vma->obj->pat_index;
 853
 854	if (flags & PIN_OFFSET_FIXED) {
 855		u64 offset = flags & PIN_OFFSET_MASK;
 856		if (!IS_ALIGNED(offset, alignment) ||
 857		    range_overflows(offset, size, end))
 858			return -EINVAL;
 859		/*
 860		 * The caller knows not of the guard added by others and
 861		 * requests for the offset of the start of its buffer
 862		 * to be fixed, which may not be the same as the position
 863		 * of the vma->node due to the guard pages.
 864		 */
 865		if (offset < guard || offset + size > end - guard)
 866			return -ENOSPC;
 867
 868		ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node,
 869					   size + 2 * guard,
 870					   offset - guard,
 871					   color, flags);
 872		if (ret)
 873			return ret;
 874	} else {
 875		size += 2 * guard;
 876		/*
 877		 * We only support huge gtt pages through the 48b PPGTT,
 878		 * however we also don't want to force any alignment for
 879		 * objects which need to be tightly packed into the low 32bits.
 880		 *
 881		 * Note that we assume that GGTT are limited to 4GiB for the
 882		 * foreseeable future. See also i915_ggtt_offset().
 883		 */
 884		if (upper_32_bits(end - 1) &&
 885		    vma->page_sizes.sg > I915_GTT_PAGE_SIZE &&
 886		    !HAS_64K_PAGES(vma->vm->i915)) {
 887			/*
 888			 * We can't mix 64K and 4K PTEs in the same page-table
 889			 * (2M block), and so to avoid the ugliness and
 890			 * complexity of coloring we opt for just aligning 64K
 891			 * objects to 2M.
 892			 */
 893			u64 page_alignment =
 894				rounddown_pow_of_two(vma->page_sizes.sg |
 895						     I915_GTT_PAGE_SIZE_2M);
 896
 897			/*
 898			 * Check we don't expand for the limited Global GTT
 899			 * (mappable aperture is even more precious!). This
 900			 * also checks that we exclude the aliasing-ppgtt.
 901			 */
 902			GEM_BUG_ON(i915_vma_is_ggtt(vma));
 903
 904			alignment = max(alignment, page_alignment);
 905
 906			if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
 907				size = round_up(size, I915_GTT_PAGE_SIZE_2M);
 908		}
 909
 910		ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node,
 911					  size, alignment, color,
 912					  start, end, flags);
 913		if (ret)
 914			return ret;
 915
 916		GEM_BUG_ON(vma->node.start < start);
 917		GEM_BUG_ON(vma->node.start + vma->node.size > end);
 918	}
 919	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 920	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
 921
 922	list_move_tail(&vma->vm_link, &vma->vm->bound_list);
 923	vma->guard = guard;
 924
 925	return 0;
 926}
 927
 928static void
 929i915_vma_detach(struct i915_vma *vma)
 930{
 931	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 932	GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 933
 934	/*
 935	 * And finally now the object is completely decoupled from this
 936	 * vma, we can drop its hold on the backing storage and allow
 937	 * it to be reaped by the shrinker.
 938	 */
 939	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
 940}
 941
 942static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
 943{
 944	unsigned int bound;
 945
 946	bound = atomic_read(&vma->flags);
 947
 948	if (flags & PIN_VALIDATE) {
 949		flags &= I915_VMA_BIND_MASK;
 950
 951		return (flags & bound) == flags;
 952	}
 953
 954	/* with the lock mandatory for unbind, we don't race here */
 955	flags &= I915_VMA_BIND_MASK;
 956	do {
 957		if (unlikely(flags & ~bound))
 958			return false;
 959
 960		if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR)))
 961			return false;
 962
 963		GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0);
 964	} while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
 965
 966	return true;
 967}
 968
 969static struct scatterlist *
 970rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
 971	     unsigned int width, unsigned int height,
 972	     unsigned int src_stride, unsigned int dst_stride,
 973	     struct sg_table *st, struct scatterlist *sg)
 974{
 975	unsigned int column, row;
 976	pgoff_t src_idx;
 977
 978	for (column = 0; column < width; column++) {
 979		unsigned int left;
 980
 981		src_idx = src_stride * (height - 1) + column + offset;
 982		for (row = 0; row < height; row++) {
 983			st->nents++;
 984			/*
 985			 * We don't need the pages, but need to initialize
 986			 * the entries so the sg list can be happily traversed.
 987			 * The only thing we need are DMA addresses.
 988			 */
 989			sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
 990			sg_dma_address(sg) =
 991				i915_gem_object_get_dma_address(obj, src_idx);
 992			sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
 993			sg = sg_next(sg);
 994			src_idx -= src_stride;
 995		}
 996
 997		left = (dst_stride - height) * I915_GTT_PAGE_SIZE;
 998
 999		if (!left)
1000			continue;
1001
1002		st->nents++;
1003
1004		/*
1005		 * The DE ignores the PTEs for the padding tiles, the sg entry
1006		 * here is just a convenience to indicate how many padding PTEs
1007		 * to insert at this spot.
1008		 */
1009		sg_set_page(sg, NULL, left, 0);
1010		sg_dma_address(sg) = 0;
1011		sg_dma_len(sg) = left;
1012		sg = sg_next(sg);
1013	}
1014
1015	return sg;
1016}
1017
1018static noinline struct sg_table *
1019intel_rotate_pages(struct intel_rotation_info *rot_info,
1020		   struct drm_i915_gem_object *obj)
1021{
1022	unsigned int size = intel_rotation_info_size(rot_info);
1023	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1024	struct sg_table *st;
1025	struct scatterlist *sg;
1026	int ret = -ENOMEM;
1027	int i;
1028
1029	/* Allocate target SG list. */
1030	st = kmalloc(sizeof(*st), GFP_KERNEL);
1031	if (!st)
1032		goto err_st_alloc;
1033
1034	ret = sg_alloc_table(st, size, GFP_KERNEL);
1035	if (ret)
1036		goto err_sg_alloc;
1037
1038	st->nents = 0;
1039	sg = st->sgl;
1040
1041	for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1042		sg = rotate_pages(obj, rot_info->plane[i].offset,
1043				  rot_info->plane[i].width, rot_info->plane[i].height,
1044				  rot_info->plane[i].src_stride,
1045				  rot_info->plane[i].dst_stride,
1046				  st, sg);
1047
1048	return st;
1049
1050err_sg_alloc:
1051	kfree(st);
1052err_st_alloc:
1053
1054	drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1055		obj->base.size, rot_info->plane[0].width,
1056		rot_info->plane[0].height, size);
1057
1058	return ERR_PTR(ret);
1059}
1060
1061static struct scatterlist *
1062add_padding_pages(unsigned int count,
1063		  struct sg_table *st, struct scatterlist *sg)
1064{
1065	st->nents++;
1066
1067	/*
1068	 * The DE ignores the PTEs for the padding tiles, the sg entry
1069	 * here is just a convenience to indicate how many padding PTEs
1070	 * to insert at this spot.
1071	 */
1072	sg_set_page(sg, NULL, count * I915_GTT_PAGE_SIZE, 0);
1073	sg_dma_address(sg) = 0;
1074	sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE;
1075	sg = sg_next(sg);
1076
1077	return sg;
1078}
1079
1080static struct scatterlist *
1081remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj,
1082			      unsigned long offset, unsigned int alignment_pad,
1083			      unsigned int width, unsigned int height,
1084			      unsigned int src_stride, unsigned int dst_stride,
1085			      struct sg_table *st, struct scatterlist *sg,
1086			      unsigned int *gtt_offset)
1087{
1088	unsigned int row;
1089
1090	if (!width || !height)
1091		return sg;
1092
1093	if (alignment_pad)
1094		sg = add_padding_pages(alignment_pad, st, sg);
1095
1096	for (row = 0; row < height; row++) {
1097		unsigned int left = width * I915_GTT_PAGE_SIZE;
1098
1099		while (left) {
1100			dma_addr_t addr;
1101			unsigned int length;
1102
1103			/*
1104			 * We don't need the pages, but need to initialize
1105			 * the entries so the sg list can be happily traversed.
1106			 * The only thing we need are DMA addresses.
1107			 */
1108
1109			addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1110
1111			length = min(left, length);
1112
1113			st->nents++;
1114
1115			sg_set_page(sg, NULL, length, 0);
1116			sg_dma_address(sg) = addr;
1117			sg_dma_len(sg) = length;
1118			sg = sg_next(sg);
1119
1120			offset += length / I915_GTT_PAGE_SIZE;
1121			left -= length;
1122		}
1123
1124		offset += src_stride - width;
1125
1126		left = (dst_stride - width) * I915_GTT_PAGE_SIZE;
1127
1128		if (!left)
1129			continue;
1130
1131		sg = add_padding_pages(left >> PAGE_SHIFT, st, sg);
1132	}
1133
1134	*gtt_offset += alignment_pad + dst_stride * height;
1135
1136	return sg;
1137}
1138
1139static struct scatterlist *
1140remap_contiguous_pages(struct drm_i915_gem_object *obj,
1141		       pgoff_t obj_offset,
1142		       unsigned int count,
1143		       struct sg_table *st, struct scatterlist *sg)
1144{
1145	struct scatterlist *iter;
1146	unsigned int offset;
1147
1148	iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset);
1149	GEM_BUG_ON(!iter);
1150
1151	do {
1152		unsigned int len;
1153
1154		len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1155			  count << PAGE_SHIFT);
1156		sg_set_page(sg, NULL, len, 0);
1157		sg_dma_address(sg) =
1158			sg_dma_address(iter) + (offset << PAGE_SHIFT);
1159		sg_dma_len(sg) = len;
1160
1161		st->nents++;
1162		count -= len >> PAGE_SHIFT;
1163		if (count == 0)
1164			return sg;
1165
1166		sg = __sg_next(sg);
1167		iter = __sg_next(iter);
1168		offset = 0;
1169	} while (1);
1170}
1171
1172static struct scatterlist *
1173remap_linear_color_plane_pages(struct drm_i915_gem_object *obj,
1174			       pgoff_t obj_offset, unsigned int alignment_pad,
1175			       unsigned int size,
1176			       struct sg_table *st, struct scatterlist *sg,
1177			       unsigned int *gtt_offset)
1178{
1179	if (!size)
1180		return sg;
1181
1182	if (alignment_pad)
1183		sg = add_padding_pages(alignment_pad, st, sg);
1184
1185	sg = remap_contiguous_pages(obj, obj_offset, size, st, sg);
1186	sg = sg_next(sg);
1187
1188	*gtt_offset += alignment_pad + size;
1189
1190	return sg;
1191}
1192
1193static struct scatterlist *
1194remap_color_plane_pages(const struct intel_remapped_info *rem_info,
1195			struct drm_i915_gem_object *obj,
1196			int color_plane,
1197			struct sg_table *st, struct scatterlist *sg,
1198			unsigned int *gtt_offset)
1199{
1200	unsigned int alignment_pad = 0;
1201
1202	if (rem_info->plane_alignment)
1203		alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset;
1204
1205	if (rem_info->plane[color_plane].linear)
1206		sg = remap_linear_color_plane_pages(obj,
1207						    rem_info->plane[color_plane].offset,
1208						    alignment_pad,
1209						    rem_info->plane[color_plane].size,
1210						    st, sg,
1211						    gtt_offset);
1212
1213	else
1214		sg = remap_tiled_color_plane_pages(obj,
1215						   rem_info->plane[color_plane].offset,
1216						   alignment_pad,
1217						   rem_info->plane[color_plane].width,
1218						   rem_info->plane[color_plane].height,
1219						   rem_info->plane[color_plane].src_stride,
1220						   rem_info->plane[color_plane].dst_stride,
1221						   st, sg,
1222						   gtt_offset);
1223
1224	return sg;
1225}
1226
1227static noinline struct sg_table *
1228intel_remap_pages(struct intel_remapped_info *rem_info,
1229		  struct drm_i915_gem_object *obj)
1230{
1231	unsigned int size = intel_remapped_info_size(rem_info);
1232	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1233	struct sg_table *st;
1234	struct scatterlist *sg;
1235	unsigned int gtt_offset = 0;
1236	int ret = -ENOMEM;
1237	int i;
1238
1239	/* Allocate target SG list. */
1240	st = kmalloc(sizeof(*st), GFP_KERNEL);
1241	if (!st)
1242		goto err_st_alloc;
1243
1244	ret = sg_alloc_table(st, size, GFP_KERNEL);
1245	if (ret)
1246		goto err_sg_alloc;
1247
1248	st->nents = 0;
1249	sg = st->sgl;
1250
1251	for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1252		sg = remap_color_plane_pages(rem_info, obj, i, st, sg, &gtt_offset);
1253
1254	i915_sg_trim(st);
1255
1256	return st;
1257
1258err_sg_alloc:
1259	kfree(st);
1260err_st_alloc:
1261
1262	drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1263		obj->base.size, rem_info->plane[0].width,
1264		rem_info->plane[0].height, size);
1265
1266	return ERR_PTR(ret);
1267}
1268
1269static noinline struct sg_table *
1270intel_partial_pages(const struct i915_gtt_view *view,
1271		    struct drm_i915_gem_object *obj)
1272{
1273	struct sg_table *st;
1274	struct scatterlist *sg;
1275	unsigned int count = view->partial.size;
1276	int ret = -ENOMEM;
1277
1278	st = kmalloc(sizeof(*st), GFP_KERNEL);
1279	if (!st)
1280		goto err_st_alloc;
1281
1282	ret = sg_alloc_table(st, count, GFP_KERNEL);
1283	if (ret)
1284		goto err_sg_alloc;
1285
1286	st->nents = 0;
1287
1288	sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl);
1289
1290	sg_mark_end(sg);
1291	i915_sg_trim(st); /* Drop any unused tail entries. */
1292
1293	return st;
1294
1295err_sg_alloc:
1296	kfree(st);
1297err_st_alloc:
1298	return ERR_PTR(ret);
1299}
1300
1301static int
1302__i915_vma_get_pages(struct i915_vma *vma)
1303{
1304	struct sg_table *pages;
1305
1306	/*
1307	 * The vma->pages are only valid within the lifespan of the borrowed
1308	 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1309	 * must be the vma->pages. A simple rule is that vma->pages must only
1310	 * be accessed when the obj->mm.pages are pinned.
1311	 */
1312	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1313
1314	switch (vma->gtt_view.type) {
1315	default:
1316		GEM_BUG_ON(vma->gtt_view.type);
1317		fallthrough;
1318	case I915_GTT_VIEW_NORMAL:
1319		pages = vma->obj->mm.pages;
1320		break;
1321
1322	case I915_GTT_VIEW_ROTATED:
1323		pages =
1324			intel_rotate_pages(&vma->gtt_view.rotated, vma->obj);
1325		break;
1326
1327	case I915_GTT_VIEW_REMAPPED:
1328		pages =
1329			intel_remap_pages(&vma->gtt_view.remapped, vma->obj);
1330		break;
1331
1332	case I915_GTT_VIEW_PARTIAL:
1333		pages = intel_partial_pages(&vma->gtt_view, vma->obj);
1334		break;
1335	}
1336
1337	if (IS_ERR(pages)) {
1338		drm_err(&vma->vm->i915->drm,
1339			"Failed to get pages for VMA view type %u (%ld)!\n",
1340			vma->gtt_view.type, PTR_ERR(pages));
1341		return PTR_ERR(pages);
1342	}
1343
1344	vma->pages = pages;
1345
1346	return 0;
1347}
1348
1349I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
1350{
1351	int err;
1352
1353	if (atomic_add_unless(&vma->pages_count, 1, 0))
1354		return 0;
1355
1356	err = i915_gem_object_pin_pages(vma->obj);
1357	if (err)
1358		return err;
1359
1360	err = __i915_vma_get_pages(vma);
1361	if (err)
1362		goto err_unpin;
1363
1364	vma->page_sizes = vma->obj->mm.page_sizes;
1365	atomic_inc(&vma->pages_count);
1366
1367	return 0;
1368
1369err_unpin:
1370	__i915_gem_object_unpin_pages(vma->obj);
1371
1372	return err;
1373}
1374
1375void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
1376{
1377	struct intel_gt *gt;
1378	int id;
1379
1380	if (!tlb)
1381		return;
1382
1383	/*
1384	 * Before we release the pages that were bound by this vma, we
1385	 * must invalidate all the TLBs that may still have a reference
1386	 * back to our physical address. It only needs to be done once,
1387	 * so after updating the PTE to point away from the pages, record
1388	 * the most recent TLB invalidation seqno, and if we have not yet
1389	 * flushed the TLBs upon release, perform a full invalidation.
1390	 */
1391	for_each_gt(gt, vm->i915, id)
1392		WRITE_ONCE(tlb[id],
1393			   intel_gt_next_invalidate_tlb_full(gt));
1394}
1395
1396static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1397{
1398	/* We allocate under vma_get_pages, so beware the shrinker */
1399	GEM_BUG_ON(atomic_read(&vma->pages_count) < count);
1400
1401	if (atomic_sub_return(count, &vma->pages_count) == 0) {
1402		if (vma->pages != vma->obj->mm.pages) {
1403			sg_free_table(vma->pages);
1404			kfree(vma->pages);
1405		}
1406		vma->pages = NULL;
1407
1408		i915_gem_object_unpin_pages(vma->obj);
1409	}
1410}
1411
1412I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma)
1413{
1414	if (atomic_add_unless(&vma->pages_count, -1, 1))
1415		return;
1416
1417	__vma_put_pages(vma, 1);
1418}
1419
1420static void vma_unbind_pages(struct i915_vma *vma)
1421{
1422	unsigned int count;
1423
1424	lockdep_assert_held(&vma->vm->mutex);
1425
1426	/* The upper portion of pages_count is the number of bindings */
1427	count = atomic_read(&vma->pages_count);
1428	count >>= I915_VMA_PAGES_BIAS;
1429	GEM_BUG_ON(!count);
1430
1431	__vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS);
1432}
1433
1434int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1435		    u64 size, u64 alignment, u64 flags)
1436{
1437	struct i915_vma_work *work = NULL;
1438	struct dma_fence *moving = NULL;
1439	struct i915_vma_resource *vma_res = NULL;
1440	intel_wakeref_t wakeref;
1441	unsigned int bound;
1442	int err;
1443
1444	assert_vma_held(vma);
1445	GEM_BUG_ON(!ww);
1446
1447	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
1448	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
1449
1450	GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)));
1451
1452	/* First try and grab the pin without rebinding the vma */
1453	if (try_qad_pin(vma, flags))
1454		return 0;
1455
1456	err = i915_vma_get_pages(vma);
1457	if (err)
1458		return err;
1459
1460	/*
1461	 * In case of a global GTT, we must hold a runtime-pm wakeref
1462	 * while global PTEs are updated.  In other cases, we hold
1463	 * the rpm reference while the VMA is active.  Since runtime
1464	 * resume may require allocations, which are forbidden inside
1465	 * vm->mutex, get the first rpm wakeref outside of the mutex.
1466	 */
1467	wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
1468
1469	if (flags & vma->vm->bind_async_flags) {
1470		/* lock VM */
1471		err = i915_vm_lock_objects(vma->vm, ww);
1472		if (err)
1473			goto err_rpm;
1474
1475		work = i915_vma_work();
1476		if (!work) {
1477			err = -ENOMEM;
1478			goto err_rpm;
1479		}
1480
1481		work->vm = vma->vm;
1482
1483		err = i915_gem_object_get_moving_fence(vma->obj, &moving);
1484		if (err)
1485			goto err_rpm;
1486
1487		dma_fence_work_chain(&work->base, moving);
1488
1489		/* Allocate enough page directories to used PTE */
1490		if (vma->vm->allocate_va_range) {
1491			err = i915_vm_alloc_pt_stash(vma->vm,
1492						     &work->stash,
1493						     vma->size);
1494			if (err)
1495				goto err_fence;
1496
1497			err = i915_vm_map_pt_stash(vma->vm, &work->stash);
1498			if (err)
1499				goto err_fence;
1500		}
1501	}
1502
1503	vma_res = i915_vma_resource_alloc();
1504	if (IS_ERR(vma_res)) {
1505		err = PTR_ERR(vma_res);
1506		goto err_fence;
1507	}
1508
1509	/*
1510	 * Differentiate between user/kernel vma inside the aliasing-ppgtt.
1511	 *
1512	 * We conflate the Global GTT with the user's vma when using the
1513	 * aliasing-ppgtt, but it is still vitally important to try and
1514	 * keep the use cases distinct. For example, userptr objects are
1515	 * not allowed inside the Global GTT as that will cause lock
1516	 * inversions when we have to evict them the mmu_notifier callbacks -
1517	 * but they are allowed to be part of the user ppGTT which can never
1518	 * be mapped. As such we try to give the distinct users of the same
1519	 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1520	 * and i915_ppgtt separate].
1521	 *
1522	 * NB this may cause us to mask real lock inversions -- while the
1523	 * code is safe today, lockdep may not be able to spot future
1524	 * transgressions.
1525	 */
1526	err = mutex_lock_interruptible_nested(&vma->vm->mutex,
1527					      !(flags & PIN_GLOBAL));
1528	if (err)
1529		goto err_vma_res;
1530
1531	/* No more allocations allowed now we hold vm->mutex */
1532
1533	if (unlikely(i915_vma_is_closed(vma))) {
1534		err = -ENOENT;
1535		goto err_unlock;
1536	}
1537
1538	bound = atomic_read(&vma->flags);
1539	if (unlikely(bound & I915_VMA_ERROR)) {
1540		err = -ENOMEM;
1541		goto err_unlock;
1542	}
1543
1544	if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) {
1545		err = -EAGAIN; /* pins are meant to be fairly temporary */
1546		goto err_unlock;
1547	}
1548
1549	if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) {
1550		if (!(flags & PIN_VALIDATE))
1551			__i915_vma_pin(vma);
1552		goto err_unlock;
1553	}
1554
1555	err = i915_active_acquire(&vma->active);
1556	if (err)
1557		goto err_unlock;
1558
1559	if (!(bound & I915_VMA_BIND_MASK)) {
1560		err = i915_vma_insert(vma, ww, size, alignment, flags);
1561		if (err)
1562			goto err_active;
1563
1564		if (i915_is_ggtt(vma->vm))
1565			__i915_vma_set_map_and_fenceable(vma);
1566	}
1567
1568	GEM_BUG_ON(!vma->pages);
1569	err = i915_vma_bind(vma,
1570			    vma->obj->pat_index,
1571			    flags, work, vma_res);
1572	vma_res = NULL;
1573	if (err)
1574		goto err_remove;
1575
1576	/* There should only be at most 2 active bindings (user, global) */
1577	GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound);
1578	atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count);
1579	list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1580
1581	if (!(flags & PIN_VALIDATE)) {
1582		__i915_vma_pin(vma);
1583		GEM_BUG_ON(!i915_vma_is_pinned(vma));
1584	}
1585	GEM_BUG_ON(!i915_vma_is_bound(vma, flags));
1586	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
1587
1588err_remove:
1589	if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) {
1590		i915_vma_detach(vma);
1591		drm_mm_remove_node(&vma->node);
1592	}
1593err_active:
1594	i915_active_release(&vma->active);
1595err_unlock:
1596	mutex_unlock(&vma->vm->mutex);
1597err_vma_res:
1598	i915_vma_resource_free(vma_res);
1599err_fence:
1600	if (work) {
1601		/*
1602		 * When pinning VMA to GGTT on CHV or BXT with VTD enabled,
1603		 * commit VMA binding asynchronously to avoid risk of lock
1604		 * inversion among reservation_ww locks held here and
1605		 * cpu_hotplug_lock acquired from stop_machine(), which we
1606		 * wrap around GGTT updates when running in those environments.
1607		 */
1608		if (i915_vma_is_ggtt(vma) &&
1609		    intel_vm_no_concurrent_access_wa(vma->vm->i915))
1610			dma_fence_work_commit(&work->base);
1611		else
1612			dma_fence_work_commit_imm(&work->base);
1613	}
1614err_rpm:
1615	intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
1616
1617	if (moving)
1618		dma_fence_put(moving);
1619
1620	i915_vma_put_pages(vma);
1621	return err;
1622}
1623
1624int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
1625{
1626	struct i915_gem_ww_ctx ww;
1627	int err;
1628
1629	i915_gem_ww_ctx_init(&ww, true);
1630retry:
1631	err = i915_gem_object_lock(vma->obj, &ww);
1632	if (!err)
1633		err = i915_vma_pin_ww(vma, &ww, size, alignment, flags);
1634	if (err == -EDEADLK) {
1635		err = i915_gem_ww_ctx_backoff(&ww);
1636		if (!err)
1637			goto retry;
1638	}
1639	i915_gem_ww_ctx_fini(&ww);
1640
1641	return err;
1642}
1643
1644static void flush_idle_contexts(struct intel_gt *gt)
1645{
1646	struct intel_engine_cs *engine;
1647	enum intel_engine_id id;
1648
1649	for_each_engine(engine, gt, id)
1650		intel_engine_flush_barriers(engine);
1651
1652	intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
1653}
1654
1655static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1656			   u32 align, unsigned int flags)
1657{
1658	struct i915_address_space *vm = vma->vm;
1659	struct intel_gt *gt;
1660	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
1661	int err;
1662
1663	do {
1664		err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
1665
1666		if (err != -ENOSPC) {
1667			if (!err) {
1668				err = i915_vma_wait_for_bind(vma);
1669				if (err)
1670					i915_vma_unpin(vma);
1671			}
1672			return err;
1673		}
1674
1675		/* Unlike i915_vma_pin, we don't take no for an answer! */
1676		list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1677			flush_idle_contexts(gt);
1678		if (mutex_lock_interruptible(&vm->mutex) == 0) {
1679			/*
1680			 * We pass NULL ww here, as we don't want to unbind
1681			 * locked objects when called from execbuf when pinning
1682			 * is removed. This would probably regress badly.
1683			 */
1684			i915_gem_evict_vm(vm, NULL, NULL);
1685			mutex_unlock(&vm->mutex);
1686		}
1687	} while (1);
1688}
1689
1690int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1691		  u32 align, unsigned int flags)
1692{
1693	struct i915_gem_ww_ctx _ww;
1694	int err;
1695
1696	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
1697
1698	if (ww)
1699		return __i915_ggtt_pin(vma, ww, align, flags);
1700
1701	lockdep_assert_not_held(&vma->obj->base.resv->lock.base);
1702
1703	for_i915_gem_ww(&_ww, err, true) {
1704		err = i915_gem_object_lock(vma->obj, &_ww);
1705		if (!err)
1706			err = __i915_ggtt_pin(vma, &_ww, align, flags);
1707	}
1708
1709	return err;
1710}
1711
1712/**
1713 * i915_ggtt_clear_scanout - Clear scanout flag for all objects ggtt vmas
1714 * @obj: i915 GEM object
1715 * This function clears scanout flags for objects ggtt vmas. These flags are set
1716 * when object is pinned for display use and this function to clear them all is
1717 * targeted to be called by frontbuffer tracking code when the frontbuffer is
1718 * about to be released.
1719 */
1720void i915_ggtt_clear_scanout(struct drm_i915_gem_object *obj)
1721{
1722	struct i915_vma *vma;
1723
1724	spin_lock(&obj->vma.lock);
1725	for_each_ggtt_vma(vma, obj) {
1726		i915_vma_clear_scanout(vma);
1727		vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
1728	}
1729	spin_unlock(&obj->vma.lock);
1730}
1731
1732static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1733{
1734	/*
1735	 * We defer actually closing, unbinding and destroying the VMA until
1736	 * the next idle point, or if the object is freed in the meantime. By
1737	 * postponing the unbind, we allow for it to be resurrected by the
1738	 * client, avoiding the work required to rebind the VMA. This is
1739	 * advantageous for DRI, where the client/server pass objects
1740	 * between themselves, temporarily opening a local VMA to the
1741	 * object, and then closing it again. The same object is then reused
1742	 * on the next frame (or two, depending on the depth of the swap queue)
1743	 * causing us to rebind the VMA once more. This ends up being a lot
1744	 * of wasted work for the steady state.
1745	 */
1746	GEM_BUG_ON(i915_vma_is_closed(vma));
1747	list_add(&vma->closed_link, &gt->closed_vma);
1748}
1749
1750void i915_vma_close(struct i915_vma *vma)
1751{
1752	struct intel_gt *gt = vma->vm->gt;
1753	unsigned long flags;
1754
1755	if (i915_vma_is_ggtt(vma))
1756		return;
1757
1758	GEM_BUG_ON(!atomic_read(&vma->open_count));
1759	if (atomic_dec_and_lock_irqsave(&vma->open_count,
1760					&gt->closed_lock,
1761					flags)) {
1762		__vma_close(vma, gt);
1763		spin_unlock_irqrestore(&gt->closed_lock, flags);
1764	}
1765}
1766
1767static void __i915_vma_remove_closed(struct i915_vma *vma)
1768{
1769	list_del_init(&vma->closed_link);
1770}
1771
1772void i915_vma_reopen(struct i915_vma *vma)
1773{
1774	struct intel_gt *gt = vma->vm->gt;
1775
1776	spin_lock_irq(&gt->closed_lock);
1777	if (i915_vma_is_closed(vma))
1778		__i915_vma_remove_closed(vma);
1779	spin_unlock_irq(&gt->closed_lock);
1780}
1781
1782static void force_unbind(struct i915_vma *vma)
1783{
1784	if (!drm_mm_node_allocated(&vma->node))
1785		return;
1786
1787	atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
1788	WARN_ON(__i915_vma_unbind(vma));
1789	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1790}
1791
1792static void release_references(struct i915_vma *vma, struct intel_gt *gt,
1793			       bool vm_ddestroy)
1794{
1795	struct drm_i915_gem_object *obj = vma->obj;
1796
1797	GEM_BUG_ON(i915_vma_is_active(vma));
1798
1799	spin_lock(&obj->vma.lock);
1800	list_del(&vma->obj_link);
1801	if (!RB_EMPTY_NODE(&vma->obj_node))
1802		rb_erase(&vma->obj_node, &obj->vma.tree);
1803
1804	spin_unlock(&obj->vma.lock);
1805
1806	spin_lock_irq(&gt->closed_lock);
1807	__i915_vma_remove_closed(vma);
1808	spin_unlock_irq(&gt->closed_lock);
1809
1810	if (vm_ddestroy)
1811		i915_vm_resv_put(vma->vm);
1812
1813	i915_active_fini(&vma->active);
1814	GEM_WARN_ON(vma->resource);
1815	i915_vma_free(vma);
1816}
1817
1818/*
1819 * i915_vma_destroy_locked - Remove all weak reference to the vma and put
1820 * the initial reference.
1821 *
1822 * This function should be called when it's decided the vma isn't needed
1823 * anymore. The caller must assure that it doesn't race with another lookup
1824 * plus destroy, typically by taking an appropriate reference.
1825 *
1826 * Current callsites are
1827 * - __i915_gem_object_pages_fini()
1828 * - __i915_vm_close() - Blocks the above function by taking a reference on
1829 * the object.
1830 * - __i915_vma_parked() - Blocks the above functions by taking a reference
1831 * on the vm and a reference on the object. Also takes the object lock so
1832 * destruction from __i915_vma_parked() can be blocked by holding the
1833 * object lock. Since the object lock is only allowed from within i915 with
1834 * an object refcount, holding the object lock also implicitly blocks the
1835 * vma freeing from __i915_gem_object_pages_fini().
1836 *
1837 * Because of locks taken during destruction, a vma is also guaranteed to
1838 * stay alive while the following locks are held if it was looked up while
1839 * holding one of the locks:
1840 * - vm->mutex
1841 * - obj->vma.lock
1842 * - gt->closed_lock
1843 */
1844void i915_vma_destroy_locked(struct i915_vma *vma)
1845{
1846	lockdep_assert_held(&vma->vm->mutex);
1847
1848	force_unbind(vma);
1849	list_del_init(&vma->vm_link);
1850	release_references(vma, vma->vm->gt, false);
1851}
1852
1853void i915_vma_destroy(struct i915_vma *vma)
1854{
1855	struct intel_gt *gt;
1856	bool vm_ddestroy;
1857
1858	mutex_lock(&vma->vm->mutex);
1859	force_unbind(vma);
1860	list_del_init(&vma->vm_link);
1861	vm_ddestroy = vma->vm_ddestroy;
1862	vma->vm_ddestroy = false;
1863
1864	/* vma->vm may be freed when releasing vma->vm->mutex. */
1865	gt = vma->vm->gt;
1866	mutex_unlock(&vma->vm->mutex);
1867	release_references(vma, gt, vm_ddestroy);
1868}
1869
1870void i915_vma_parked(struct intel_gt *gt)
1871{
1872	struct i915_vma *vma, *next;
1873	LIST_HEAD(closed);
1874
1875	spin_lock_irq(&gt->closed_lock);
1876	list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link) {
1877		struct drm_i915_gem_object *obj = vma->obj;
1878		struct i915_address_space *vm = vma->vm;
1879
1880		/* XXX All to avoid keeping a reference on i915_vma itself */
1881
1882		if (!kref_get_unless_zero(&obj->base.refcount))
1883			continue;
1884
1885		if (!i915_vm_tryget(vm)) {
1886			i915_gem_object_put(obj);
1887			continue;
1888		}
1889
1890		list_move(&vma->closed_link, &closed);
1891	}
1892	spin_unlock_irq(&gt->closed_lock);
1893
1894	/* As the GT is held idle, no vma can be reopened as we destroy them */
1895	list_for_each_entry_safe(vma, next, &closed, closed_link) {
1896		struct drm_i915_gem_object *obj = vma->obj;
1897		struct i915_address_space *vm = vma->vm;
1898
1899		if (i915_gem_object_trylock(obj, NULL)) {
1900			INIT_LIST_HEAD(&vma->closed_link);
1901			i915_vma_destroy(vma);
1902			i915_gem_object_unlock(obj);
1903		} else {
1904			/* back you go.. */
1905			spin_lock_irq(&gt->closed_lock);
1906			list_add(&vma->closed_link, &gt->closed_vma);
1907			spin_unlock_irq(&gt->closed_lock);
1908		}
1909
1910		i915_gem_object_put(obj);
1911		i915_vm_put(vm);
1912	}
1913}
1914
1915static void __i915_vma_iounmap(struct i915_vma *vma)
1916{
1917	GEM_BUG_ON(i915_vma_is_pinned(vma));
1918
1919	if (vma->iomap == NULL)
1920		return;
1921
1922	if (page_unmask_bits(vma->iomap))
1923		__i915_gem_object_release_map(vma->obj);
1924	else
1925		io_mapping_unmap(vma->iomap);
1926	vma->iomap = NULL;
1927}
1928
1929void i915_vma_revoke_mmap(struct i915_vma *vma)
1930{
1931	struct drm_vma_offset_node *node;
1932	u64 vma_offset;
1933
1934	if (!i915_vma_has_userfault(vma))
1935		return;
1936
1937	GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
1938	GEM_BUG_ON(!vma->obj->userfault_count);
1939
1940	node = &vma->mmo->vma_node;
1941	vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT;
1942	unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
1943			    drm_vma_node_offset_addr(node) + vma_offset,
1944			    vma->size,
1945			    1);
1946
1947	i915_vma_unset_userfault(vma);
1948	if (!--vma->obj->userfault_count)
1949		list_del(&vma->obj->userfault_link);
1950}
1951
1952static int
1953__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1954{
1955	return __i915_request_await_exclusive(rq, &vma->active);
1956}
1957
1958static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1959{
1960	int err;
1961
1962	/* Wait for the vma to be bound before we start! */
1963	err = __i915_request_await_bind(rq, vma);
1964	if (err)
1965		return err;
1966
1967	return i915_active_add_request(&vma->active, rq);
1968}
1969
1970int _i915_vma_move_to_active(struct i915_vma *vma,
1971			     struct i915_request *rq,
1972			     struct dma_fence *fence,
1973			     unsigned int flags)
1974{
1975	struct drm_i915_gem_object *obj = vma->obj;
1976	int err;
1977
1978	assert_object_held(obj);
1979
1980	GEM_BUG_ON(!vma->pages);
1981
1982	if (!(flags & __EXEC_OBJECT_NO_REQUEST_AWAIT)) {
1983		err = i915_request_await_object(rq, vma->obj, flags & EXEC_OBJECT_WRITE);
1984		if (unlikely(err))
1985			return err;
1986	}
1987	err = __i915_vma_move_to_active(vma, rq);
1988	if (unlikely(err))
1989		return err;
1990
1991	/*
1992	 * Reserve fences slot early to prevent an allocation after preparing
1993	 * the workload and associating fences with dma_resv.
1994	 */
1995	if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE)) {
1996		struct dma_fence *curr;
1997		int idx;
1998
1999		dma_fence_array_for_each(curr, idx, fence)
2000			;
2001		err = dma_resv_reserve_fences(vma->obj->base.resv, idx);
2002		if (unlikely(err))
2003			return err;
2004	}
2005
2006	if (flags & EXEC_OBJECT_WRITE) {
2007		struct i915_frontbuffer *front;
2008
2009		front = i915_gem_object_frontbuffer_lookup(obj);
2010		if (unlikely(front)) {
2011			if (intel_frontbuffer_invalidate(&front->base, ORIGIN_CS))
2012				i915_active_add_request(&front->write, rq);
2013			i915_gem_object_frontbuffer_put(front);
2014		}
2015	}
2016
2017	if (fence) {
2018		struct dma_fence *curr;
2019		enum dma_resv_usage usage;
2020		int idx;
2021
2022		if (flags & EXEC_OBJECT_WRITE) {
2023			usage = DMA_RESV_USAGE_WRITE;
2024			obj->write_domain = I915_GEM_DOMAIN_RENDER;
2025			obj->read_domains = 0;
2026		} else {
2027			usage = DMA_RESV_USAGE_READ;
2028			obj->write_domain = 0;
2029		}
2030
2031		dma_fence_array_for_each(curr, idx, fence)
2032			dma_resv_add_fence(vma->obj->base.resv, curr, usage);
2033	}
2034
2035	if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence)
2036		i915_active_add_request(&vma->fence->active, rq);
2037
2038	obj->read_domains |= I915_GEM_GPU_DOMAINS;
2039	obj->mm.dirty = true;
2040
2041	GEM_BUG_ON(!i915_vma_is_active(vma));
2042	return 0;
2043}
2044
2045struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
2046{
2047	struct i915_vma_resource *vma_res = vma->resource;
2048	struct dma_fence *unbind_fence;
2049
2050	GEM_BUG_ON(i915_vma_is_pinned(vma));
2051	assert_vma_held_evict(vma);
2052
2053	if (i915_vma_is_map_and_fenceable(vma)) {
2054		/* Force a pagefault for domain tracking on next user access */
2055		i915_vma_revoke_mmap(vma);
2056
2057		/*
2058		 * Check that we have flushed all writes through the GGTT
2059		 * before the unbind, other due to non-strict nature of those
2060		 * indirect writes they may end up referencing the GGTT PTE
2061		 * after the unbind.
2062		 *
2063		 * Note that we may be concurrently poking at the GGTT_WRITE
2064		 * bit from set-domain, as we mark all GGTT vma associated
2065		 * with an object. We know this is for another vma, as we
2066		 * are currently unbinding this one -- so if this vma will be
2067		 * reused, it will be refaulted and have its dirty bit set
2068		 * before the next write.
2069		 */
2070		i915_vma_flush_writes(vma);
2071
2072		/* release the fence reg _after_ flushing */
2073		i915_vma_revoke_fence(vma);
2074
2075		clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
2076	}
2077
2078	__i915_vma_iounmap(vma);
2079
2080	GEM_BUG_ON(vma->fence);
2081	GEM_BUG_ON(i915_vma_has_userfault(vma));
2082
2083	/* Object backend must be async capable. */
2084	GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt);
2085
2086	/* If vm is not open, unbind is a nop. */
2087	vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) &&
2088		kref_read(&vma->vm->ref);
2089	vma_res->skip_pte_rewrite = !kref_read(&vma->vm->ref) ||
2090		vma->vm->skip_pte_rewrite;
2091	trace_i915_vma_unbind(vma);
2092
2093	if (async)
2094		unbind_fence = i915_vma_resource_unbind(vma_res,
2095							vma->obj->mm.tlb);
2096	else
2097		unbind_fence = i915_vma_resource_unbind(vma_res, NULL);
2098
2099	vma->resource = NULL;
2100
2101	atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),
2102		   &vma->flags);
2103
2104	i915_vma_detach(vma);
2105
2106	if (!async) {
2107		if (unbind_fence) {
2108			dma_fence_wait(unbind_fence, false);
2109			dma_fence_put(unbind_fence);
2110			unbind_fence = NULL;
2111		}
2112		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb);
2113	}
2114
2115	/*
2116	 * Binding itself may not have completed until the unbind fence signals,
2117	 * so don't drop the pages until that happens, unless the resource is
2118	 * async_capable.
2119	 */
2120
2121	vma_unbind_pages(vma);
2122	return unbind_fence;
2123}
2124
2125int __i915_vma_unbind(struct i915_vma *vma)
2126{
2127	int ret;
2128
2129	lockdep_assert_held(&vma->vm->mutex);
2130	assert_vma_held_evict(vma);
2131
2132	if (!drm_mm_node_allocated(&vma->node))
2133		return 0;
2134
2135	if (i915_vma_is_pinned(vma)) {
2136		vma_print_allocator(vma, "is pinned");
2137		return -EAGAIN;
2138	}
2139
2140	/*
2141	 * After confirming that no one else is pinning this vma, wait for
2142	 * any laggards who may have crept in during the wait (through
2143	 * a residual pin skipping the vm->mutex) to complete.
2144	 */
2145	ret = i915_vma_sync(vma);
2146	if (ret)
2147		return ret;
2148
2149	GEM_BUG_ON(i915_vma_is_active(vma));
2150	__i915_vma_evict(vma, false);
2151
2152	drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
2153	return 0;
2154}
2155
2156static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
2157{
2158	struct dma_fence *fence;
2159
2160	lockdep_assert_held(&vma->vm->mutex);
2161
2162	if (!drm_mm_node_allocated(&vma->node))
2163		return NULL;
2164
2165	if (i915_vma_is_pinned(vma) ||
2166	    &vma->obj->mm.rsgt->table != vma->resource->bi.pages)
2167		return ERR_PTR(-EAGAIN);
2168
2169	/*
2170	 * We probably need to replace this with awaiting the fences of the
2171	 * object's dma_resv when the vma active goes away. When doing that
2172	 * we need to be careful to not add the vma_resource unbind fence
2173	 * immediately to the object's dma_resv, because then unbinding
2174	 * the next vma from the object, in case there are many, will
2175	 * actually await the unbinding of the previous vmas, which is
2176	 * undesirable.
2177	 */
2178	if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active,
2179				       I915_ACTIVE_AWAIT_EXCL |
2180				       I915_ACTIVE_AWAIT_ACTIVE) < 0) {
2181		return ERR_PTR(-EBUSY);
2182	}
2183
2184	fence = __i915_vma_evict(vma, true);
2185
2186	drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
2187
2188	return fence;
2189}
2190
2191int i915_vma_unbind(struct i915_vma *vma)
2192{
2193	struct i915_address_space *vm = vma->vm;
2194	intel_wakeref_t wakeref = NULL;
2195	int err;
2196
2197	assert_object_held_shared(vma->obj);
2198
2199	/* Optimistic wait before taking the mutex */
2200	err = i915_vma_sync(vma);
2201	if (err)
2202		return err;
2203
2204	if (!drm_mm_node_allocated(&vma->node))
2205		return 0;
2206
2207	if (i915_vma_is_pinned(vma)) {
2208		vma_print_allocator(vma, "is pinned");
2209		return -EAGAIN;
2210	}
2211
2212	if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2213		/* XXX not always required: nop_clear_range */
2214		wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
2215
2216	err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref);
2217	if (err)
2218		goto out_rpm;
2219
2220	err = __i915_vma_unbind(vma);
2221	mutex_unlock(&vm->mutex);
2222
2223out_rpm:
2224	if (wakeref)
2225		intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
2226	return err;
2227}
2228
2229int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm)
2230{
2231	struct drm_i915_gem_object *obj = vma->obj;
2232	struct i915_address_space *vm = vma->vm;
2233	intel_wakeref_t wakeref = NULL;
2234	struct dma_fence *fence;
2235	int err;
2236
2237	/*
2238	 * We need the dma-resv lock since we add the
2239	 * unbind fence to the dma-resv object.
2240	 */
2241	assert_object_held(obj);
2242
2243	if (!drm_mm_node_allocated(&vma->node))
2244		return 0;
2245
2246	if (i915_vma_is_pinned(vma)) {
2247		vma_print_allocator(vma, "is pinned");
2248		return -EAGAIN;
2249	}
2250
2251	if (!obj->mm.rsgt)
2252		return -EBUSY;
2253
2254	err = dma_resv_reserve_fences(obj->base.resv, 2);
2255	if (err)
2256		return -EBUSY;
2257
2258	/*
2259	 * It would be great if we could grab this wakeref from the
2260	 * async unbind work if needed, but we can't because it uses
2261	 * kmalloc and it's in the dma-fence signalling critical path.
2262	 */
2263	if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2264		wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
2265
2266	if (trylock_vm && !mutex_trylock(&vm->mutex)) {
2267		err = -EBUSY;
2268		goto out_rpm;
2269	} else if (!trylock_vm) {
2270		err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref);
2271		if (err)
2272			goto out_rpm;
2273	}
2274
2275	fence = __i915_vma_unbind_async(vma);
2276	mutex_unlock(&vm->mutex);
2277	if (IS_ERR_OR_NULL(fence)) {
2278		err = PTR_ERR_OR_ZERO(fence);
2279		goto out_rpm;
2280	}
2281
2282	dma_resv_add_fence(obj->base.resv, fence, DMA_RESV_USAGE_READ);
2283	dma_fence_put(fence);
2284
2285out_rpm:
2286	if (wakeref)
2287		intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
2288	return err;
2289}
2290
2291int i915_vma_unbind_unlocked(struct i915_vma *vma)
2292{
2293	int err;
2294
2295	i915_gem_object_lock(vma->obj, NULL);
2296	err = i915_vma_unbind(vma);
2297	i915_gem_object_unlock(vma->obj);
2298
2299	return err;
2300}
2301
2302struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
2303{
2304	i915_gem_object_make_unshrinkable(vma->obj);
2305	return vma;
2306}
2307
2308void i915_vma_make_shrinkable(struct i915_vma *vma)
2309{
2310	i915_gem_object_make_shrinkable(vma->obj);
2311}
2312
2313void i915_vma_make_purgeable(struct i915_vma *vma)
2314{
2315	i915_gem_object_make_purgeable(vma->obj);
2316}
2317
2318#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2319#include "selftests/i915_vma.c"
2320#endif
2321
2322void i915_vma_module_exit(void)
2323{
2324	kmem_cache_destroy(slab_vmas);
2325}
2326
2327int __init i915_vma_module_init(void)
2328{
2329	slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
2330	if (!slab_vmas)
2331		return -ENOMEM;
2332
2333	return 0;
2334}