i915_gem.c

   1/*
   2 * Copyright © 2008-2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * Authors:
  24 *    Eric Anholt <eric@anholt.net>
  25 *
  26 */
  27
  28#include <linux/dma-fence-array.h>
  29#include <linux/kthread.h>
  30#include <linux/dma-resv.h>
  31#include <linux/shmem_fs.h>
  32#include <linux/slab.h>
  33#include <linux/stop_machine.h>
  34#include <linux/swap.h>
  35#include <linux/pci.h>
  36#include <linux/dma-buf.h>
  37#include <linux/mman.h>
  38
  39#include <drm/drm_cache.h>
  40#include <drm/drm_print.h>
  41#include <drm/drm_vma_manager.h>
  42
  43#include "gem/i915_gem_clflush.h"
  44#include "gem/i915_gem_context.h"
  45#include "gem/i915_gem_ioctls.h"
  46#include "gem/i915_gem_mman.h"
  47#include "gem/i915_gem_object_frontbuffer.h"
  48#include "gem/i915_gem_pm.h"
  49#include "gem/i915_gem_region.h"
  50#include "gt/intel_engine_user.h"
  51#include "gt/intel_gt.h"
  52#include "gt/intel_gt_pm.h"
  53#include "gt/intel_workarounds.h"
  54
  55#include "i915_drv.h"
  56#include "i915_file_private.h"
  57#include "i915_trace.h"
  58#include "i915_vgpu.h"
  59#include "intel_clock_gating.h"
  60
  61static int
  62insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  63{
  64	int err;
  65
  66	err = mutex_lock_interruptible(&ggtt->vm.mutex);
  67	if (err)
  68		return err;
  69
  70	memset(node, 0, sizeof(*node));
  71	err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  72					  size, 0, I915_COLOR_UNEVICTABLE,
  73					  0, ggtt->mappable_end,
  74					  DRM_MM_INSERT_LOW);
  75
  76	mutex_unlock(&ggtt->vm.mutex);
  77
  78	return err;
  79}
  80
  81static void
  82remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  83{
  84	mutex_lock(&ggtt->vm.mutex);
  85	drm_mm_remove_node(node);
  86	mutex_unlock(&ggtt->vm.mutex);
  87}
  88
  89int
  90i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  91			    struct drm_file *file)
  92{
  93	struct drm_i915_private *i915 = to_i915(dev);
  94	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
  95	struct drm_i915_gem_get_aperture *args = data;
  96	struct i915_vma *vma;
  97	u64 pinned;
  98
  99	if (mutex_lock_interruptible(&ggtt->vm.mutex))
 100		return -EINTR;
 101
 102	pinned = ggtt->vm.reserved;
 103	list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 104		if (i915_vma_is_pinned(vma))
 105			pinned += vma->node.size;
 106
 107	mutex_unlock(&ggtt->vm.mutex);
 108
 109	args->aper_size = ggtt->vm.total;
 110	args->aper_available_size = args->aper_size - pinned;
 111
 112	return 0;
 113}
 114
 115int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 116			   unsigned long flags)
 117{
 118	struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 119	bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK);
 120	LIST_HEAD(still_in_list);
 121	intel_wakeref_t wakeref;
 122	struct i915_vma *vma;
 123	int ret;
 124
 125	assert_object_held(obj);
 126
 127	if (list_empty(&obj->vma.list))
 128		return 0;
 129
 130	/*
 131	 * As some machines use ACPI to handle runtime-resume callbacks, and
 132	 * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 133	 * as they are required by the shrinker. Ergo, we wake the device up
 134	 * first just in case.
 135	 */
 136	wakeref = intel_runtime_pm_get(rpm);
 137
 138try_again:
 139	ret = 0;
 140	spin_lock(&obj->vma.lock);
 141	while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 142						       struct i915_vma,
 143						       obj_link))) {
 144		list_move_tail(&vma->obj_link, &still_in_list);
 145		if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 146			continue;
 147
 148		if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 149			ret = -EBUSY;
 150			break;
 151		}
 152
 153		/*
 154		 * Requiring the vm destructor to take the object lock
 155		 * before destroying a vma would help us eliminate the
 156		 * i915_vm_tryget() here, AND thus also the barrier stuff
 157		 * at the end. That's an easy fix, but sleeping locks in
 158		 * a kthread should generally be avoided.
 159		 */
 160		ret = -EAGAIN;
 161		if (!i915_vm_tryget(vma->vm))
 162			break;
 163
 164		spin_unlock(&obj->vma.lock);
 165
 166		/*
 167		 * Since i915_vma_parked() takes the object lock
 168		 * before vma destruction, it won't race us here,
 169		 * and destroy the vma from under us.
 170		 */
 171
 172		ret = -EBUSY;
 173		if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) {
 174			assert_object_held(vma->obj);
 175			ret = i915_vma_unbind_async(vma, vm_trylock);
 176		}
 177
 178		if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 179				      !i915_vma_is_active(vma))) {
 180			if (vm_trylock) {
 181				if (mutex_trylock(&vma->vm->mutex)) {
 182					ret = __i915_vma_unbind(vma);
 183					mutex_unlock(&vma->vm->mutex);
 184				}
 185			} else {
 186				ret = i915_vma_unbind(vma);
 187			}
 188		}
 189
 190		i915_vm_put(vma->vm);
 191		spin_lock(&obj->vma.lock);
 192	}
 193	list_splice_init(&still_in_list, &obj->vma.list);
 194	spin_unlock(&obj->vma.lock);
 195
 196	if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 197		rcu_barrier(); /* flush the i915_vm_release() */
 198		goto try_again;
 199	}
 200
 201	intel_runtime_pm_put(rpm, wakeref);
 202
 203	return ret;
 204}
 205
 206static int
 207shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 208	    bool needs_clflush)
 209{
 210	char *vaddr;
 211	int ret;
 212
 213	vaddr = kmap(page);
 214
 215	if (needs_clflush)
 216		drm_clflush_virt_range(vaddr + offset, len);
 217
 218	ret = __copy_to_user(user_data, vaddr + offset, len);
 219
 220	kunmap(page);
 221
 222	return ret ? -EFAULT : 0;
 223}
 224
 225static int
 226i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 227		     struct drm_i915_gem_pread *args)
 228{
 229	unsigned int needs_clflush;
 230	char __user *user_data;
 231	unsigned long offset;
 232	pgoff_t idx;
 233	u64 remain;
 234	int ret;
 235
 236	ret = i915_gem_object_lock_interruptible(obj, NULL);
 237	if (ret)
 238		return ret;
 239
 240	ret = i915_gem_object_pin_pages(obj);
 241	if (ret)
 242		goto err_unlock;
 243
 244	ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 245	if (ret)
 246		goto err_unpin;
 247
 248	i915_gem_object_finish_access(obj);
 249	i915_gem_object_unlock(obj);
 250
 251	remain = args->size;
 252	user_data = u64_to_user_ptr(args->data_ptr);
 253	offset = offset_in_page(args->offset);
 254	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 255		struct page *page = i915_gem_object_get_page(obj, idx);
 256		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 257
 258		ret = shmem_pread(page, offset, length, user_data,
 259				  needs_clflush);
 260		if (ret)
 261			break;
 262
 263		remain -= length;
 264		user_data += length;
 265		offset = 0;
 266	}
 267
 268	i915_gem_object_unpin_pages(obj);
 269	return ret;
 270
 271err_unpin:
 272	i915_gem_object_unpin_pages(obj);
 273err_unlock:
 274	i915_gem_object_unlock(obj);
 275	return ret;
 276}
 277
 278static inline bool
 279gtt_user_read(struct io_mapping *mapping,
 280	      loff_t base, int offset,
 281	      char __user *user_data, int length)
 282{
 283	void __iomem *vaddr;
 284	unsigned long unwritten;
 285
 286	/* We can use the cpu mem copy function because this is X86. */
 287	vaddr = io_mapping_map_atomic_wc(mapping, base);
 288	unwritten = __copy_to_user_inatomic(user_data,
 289					    (void __force *)vaddr + offset,
 290					    length);
 291	io_mapping_unmap_atomic(vaddr);
 292	if (unwritten) {
 293		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 294		unwritten = copy_to_user(user_data,
 295					 (void __force *)vaddr + offset,
 296					 length);
 297		io_mapping_unmap(vaddr);
 298	}
 299	return unwritten;
 300}
 301
 302static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 303					     struct drm_mm_node *node,
 304					     bool write)
 305{
 306	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 307	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 308	struct i915_vma *vma;
 309	struct i915_gem_ww_ctx ww;
 310	int ret;
 311
 312	i915_gem_ww_ctx_init(&ww, true);
 313retry:
 314	vma = ERR_PTR(-ENODEV);
 315	ret = i915_gem_object_lock(obj, &ww);
 316	if (ret)
 317		goto err_ww;
 318
 319	ret = i915_gem_object_set_to_gtt_domain(obj, write);
 320	if (ret)
 321		goto err_ww;
 322
 323	if (!i915_gem_object_is_tiled(obj))
 324		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 325						  PIN_MAPPABLE |
 326						  PIN_NONBLOCK /* NOWARN */ |
 327						  PIN_NOEVICT);
 328	if (vma == ERR_PTR(-EDEADLK)) {
 329		ret = -EDEADLK;
 330		goto err_ww;
 331	} else if (!IS_ERR(vma)) {
 332		node->start = i915_ggtt_offset(vma);
 333		node->flags = 0;
 334	} else {
 335		ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 336		if (ret)
 337			goto err_ww;
 338		GEM_BUG_ON(!drm_mm_node_allocated(node));
 339		vma = NULL;
 340	}
 341
 342	ret = i915_gem_object_pin_pages(obj);
 343	if (ret) {
 344		if (drm_mm_node_allocated(node)) {
 345			ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 346			remove_mappable_node(ggtt, node);
 347		} else {
 348			i915_vma_unpin(vma);
 349		}
 350	}
 351
 352err_ww:
 353	if (ret == -EDEADLK) {
 354		ret = i915_gem_ww_ctx_backoff(&ww);
 355		if (!ret)
 356			goto retry;
 357	}
 358	i915_gem_ww_ctx_fini(&ww);
 359
 360	return ret ? ERR_PTR(ret) : vma;
 361}
 362
 363static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 364				 struct drm_mm_node *node,
 365				 struct i915_vma *vma)
 366{
 367	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 368	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 369
 370	i915_gem_object_unpin_pages(obj);
 371	if (drm_mm_node_allocated(node)) {
 372		ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 373		remove_mappable_node(ggtt, node);
 374	} else {
 375		i915_vma_unpin(vma);
 376	}
 377}
 378
 379static int
 380i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 381		   const struct drm_i915_gem_pread *args)
 382{
 383	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 384	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 385	unsigned long remain, offset;
 386	intel_wakeref_t wakeref;
 387	struct drm_mm_node node;
 388	void __user *user_data;
 389	struct i915_vma *vma;
 390	int ret = 0;
 391
 392	if (overflows_type(args->size, remain) ||
 393	    overflows_type(args->offset, offset))
 394		return -EINVAL;
 395
 396	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 397
 398	vma = i915_gem_gtt_prepare(obj, &node, false);
 399	if (IS_ERR(vma)) {
 400		ret = PTR_ERR(vma);
 401		goto out_rpm;
 402	}
 403
 404	user_data = u64_to_user_ptr(args->data_ptr);
 405	remain = args->size;
 406	offset = args->offset;
 407
 408	while (remain > 0) {
 409		/* Operation in this page
 410		 *
 411		 * page_base = page offset within aperture
 412		 * page_offset = offset within page
 413		 * page_length = bytes to copy for this page
 414		 */
 415		u32 page_base = node.start;
 416		unsigned page_offset = offset_in_page(offset);
 417		unsigned page_length = PAGE_SIZE - page_offset;
 418		page_length = remain < page_length ? remain : page_length;
 419		if (drm_mm_node_allocated(&node)) {
 420			ggtt->vm.insert_page(&ggtt->vm,
 421					     i915_gem_object_get_dma_address(obj,
 422									     offset >> PAGE_SHIFT),
 423					     node.start,
 424					     i915_gem_get_pat_index(i915,
 425								    I915_CACHE_NONE), 0);
 426		} else {
 427			page_base += offset & PAGE_MASK;
 428		}
 429
 430		if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 431				  user_data, page_length)) {
 432			ret = -EFAULT;
 433			break;
 434		}
 435
 436		remain -= page_length;
 437		user_data += page_length;
 438		offset += page_length;
 439	}
 440
 441	i915_gem_gtt_cleanup(obj, &node, vma);
 442out_rpm:
 443	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 444	return ret;
 445}
 446
 447/**
 448 * i915_gem_pread_ioctl - Reads data from the object referenced by handle.
 449 * @dev: drm device pointer
 450 * @data: ioctl data blob
 451 * @file: drm file pointer
 452 *
 453 * On error, the contents of *data are undefined.
 454 */
 455int
 456i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 457		     struct drm_file *file)
 458{
 459	struct drm_i915_private *i915 = to_i915(dev);
 460	struct drm_i915_gem_pread *args = data;
 461	struct drm_i915_gem_object *obj;
 462	int ret;
 463
 464	/* PREAD is disallowed for all platforms after TGL-LP.  This also
 465	 * covers all platforms with local memory.
 466	 */
 467	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 468		return -EOPNOTSUPP;
 469
 470	if (args->size == 0)
 471		return 0;
 472
 473	if (!access_ok(u64_to_user_ptr(args->data_ptr),
 474		       args->size))
 475		return -EFAULT;
 476
 477	obj = i915_gem_object_lookup(file, args->handle);
 478	if (!obj)
 479		return -ENOENT;
 480
 481	/* Bounds check source.  */
 482	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 483		ret = -EINVAL;
 484		goto out;
 485	}
 486
 487	trace_i915_gem_object_pread(obj, args->offset, args->size);
 488	ret = -ENODEV;
 489	if (obj->ops->pread)
 490		ret = obj->ops->pread(obj, args);
 491	if (ret != -ENODEV)
 492		goto out;
 493
 494	ret = i915_gem_object_wait(obj,
 495				   I915_WAIT_INTERRUPTIBLE,
 496				   MAX_SCHEDULE_TIMEOUT);
 497	if (ret)
 498		goto out;
 499
 500	ret = i915_gem_shmem_pread(obj, args);
 501	if (ret == -EFAULT || ret == -ENODEV)
 502		ret = i915_gem_gtt_pread(obj, args);
 503
 504out:
 505	i915_gem_object_put(obj);
 506	return ret;
 507}
 508
 509/* This is the fast write path which cannot handle
 510 * page faults in the source data
 511 */
 512
 513static inline bool
 514ggtt_write(struct io_mapping *mapping,
 515	   loff_t base, int offset,
 516	   char __user *user_data, int length)
 517{
 518	void __iomem *vaddr;
 519	unsigned long unwritten;
 520
 521	/* We can use the cpu mem copy function because this is X86. */
 522	vaddr = io_mapping_map_atomic_wc(mapping, base);
 523	unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset,
 524						      user_data, length);
 525	io_mapping_unmap_atomic(vaddr);
 526	if (unwritten) {
 527		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 528		unwritten = copy_from_user((void __force *)vaddr + offset,
 529					   user_data, length);
 530		io_mapping_unmap(vaddr);
 531	}
 532
 533	return unwritten;
 534}
 535
 536/**
 537 * i915_gem_gtt_pwrite_fast - This is the fast pwrite path, where we copy the data directly from the
 538 * user into the GTT, uncached.
 539 * @obj: i915 GEM object
 540 * @args: pwrite arguments structure
 541 */
 542static int
 543i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 544			 const struct drm_i915_gem_pwrite *args)
 545{
 546	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 547	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 548	struct intel_runtime_pm *rpm = &i915->runtime_pm;
 549	unsigned long remain, offset;
 550	intel_wakeref_t wakeref;
 551	struct drm_mm_node node;
 552	struct i915_vma *vma;
 553	void __user *user_data;
 554	int ret = 0;
 555
 556	if (overflows_type(args->size, remain) ||
 557	    overflows_type(args->offset, offset))
 558		return -EINVAL;
 559
 560	if (i915_gem_object_has_struct_page(obj)) {
 561		/*
 562		 * Avoid waking the device up if we can fallback, as
 563		 * waking/resuming is very slow (worst-case 10-100 ms
 564		 * depending on PCI sleeps and our own resume time).
 565		 * This easily dwarfs any performance advantage from
 566		 * using the cache bypass of indirect GGTT access.
 567		 */
 568		wakeref = intel_runtime_pm_get_if_in_use(rpm);
 569		if (!wakeref)
 570			return -EFAULT;
 571	} else {
 572		/* No backing pages, no fallback, we must force GGTT access */
 573		wakeref = intel_runtime_pm_get(rpm);
 574	}
 575
 576	vma = i915_gem_gtt_prepare(obj, &node, true);
 577	if (IS_ERR(vma)) {
 578		ret = PTR_ERR(vma);
 579		goto out_rpm;
 580	}
 581
 582	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 583
 584	user_data = u64_to_user_ptr(args->data_ptr);
 585	offset = args->offset;
 586	remain = args->size;
 587	while (remain) {
 588		/* Operation in this page
 589		 *
 590		 * page_base = page offset within aperture
 591		 * page_offset = offset within page
 592		 * page_length = bytes to copy for this page
 593		 */
 594		u32 page_base = node.start;
 595		unsigned int page_offset = offset_in_page(offset);
 596		unsigned int page_length = PAGE_SIZE - page_offset;
 597		page_length = remain < page_length ? remain : page_length;
 598		if (drm_mm_node_allocated(&node)) {
 599			/* flush the write before we modify the GGTT */
 600			intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 601			ggtt->vm.insert_page(&ggtt->vm,
 602					     i915_gem_object_get_dma_address(obj,
 603									     offset >> PAGE_SHIFT),
 604					     node.start,
 605					     i915_gem_get_pat_index(i915,
 606								    I915_CACHE_NONE), 0);
 607			wmb(); /* flush modifications to the GGTT (insert_page) */
 608		} else {
 609			page_base += offset & PAGE_MASK;
 610		}
 611		/* If we get a fault while copying data, then (presumably) our
 612		 * source page isn't available.  Return the error and we'll
 613		 * retry in the slow path.
 614		 * If the object is non-shmem backed, we retry again with the
 615		 * path that handles page fault.
 616		 */
 617		if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 618			       user_data, page_length)) {
 619			ret = -EFAULT;
 620			break;
 621		}
 622
 623		remain -= page_length;
 624		user_data += page_length;
 625		offset += page_length;
 626	}
 627
 628	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 629	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 630
 631	i915_gem_gtt_cleanup(obj, &node, vma);
 632out_rpm:
 633	intel_runtime_pm_put(rpm, wakeref);
 634	return ret;
 635}
 636
 637/* Per-page copy function for the shmem pwrite fastpath.
 638 * Flushes invalid cachelines before writing to the target if
 639 * needs_clflush_before is set and flushes out any written cachelines after
 640 * writing if needs_clflush is set.
 641 */
 642static int
 643shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 644	     bool needs_clflush_before,
 645	     bool needs_clflush_after)
 646{
 647	char *vaddr;
 648	int ret;
 649
 650	vaddr = kmap(page);
 651
 652	if (needs_clflush_before)
 653		drm_clflush_virt_range(vaddr + offset, len);
 654
 655	ret = __copy_from_user(vaddr + offset, user_data, len);
 656	if (!ret && needs_clflush_after)
 657		drm_clflush_virt_range(vaddr + offset, len);
 658
 659	kunmap(page);
 660
 661	return ret ? -EFAULT : 0;
 662}
 663
 664static int
 665i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 666		      const struct drm_i915_gem_pwrite *args)
 667{
 668	unsigned int partial_cacheline_write;
 669	unsigned int needs_clflush;
 670	void __user *user_data;
 671	unsigned long offset;
 672	pgoff_t idx;
 673	u64 remain;
 674	int ret;
 675
 676	ret = i915_gem_object_lock_interruptible(obj, NULL);
 677	if (ret)
 678		return ret;
 679
 680	ret = i915_gem_object_pin_pages(obj);
 681	if (ret)
 682		goto err_unlock;
 683
 684	ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 685	if (ret)
 686		goto err_unpin;
 687
 688	i915_gem_object_finish_access(obj);
 689	i915_gem_object_unlock(obj);
 690
 691	/* If we don't overwrite a cacheline completely we need to be
 692	 * careful to have up-to-date data by first clflushing. Don't
 693	 * overcomplicate things and flush the entire patch.
 694	 */
 695	partial_cacheline_write = 0;
 696	if (needs_clflush & CLFLUSH_BEFORE)
 697		partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 698
 699	user_data = u64_to_user_ptr(args->data_ptr);
 700	remain = args->size;
 701	offset = offset_in_page(args->offset);
 702	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 703		struct page *page = i915_gem_object_get_page(obj, idx);
 704		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 705
 706		ret = shmem_pwrite(page, offset, length, user_data,
 707				   (offset | length) & partial_cacheline_write,
 708				   needs_clflush & CLFLUSH_AFTER);
 709		if (ret)
 710			break;
 711
 712		remain -= length;
 713		user_data += length;
 714		offset = 0;
 715	}
 716
 717	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 718
 719	i915_gem_object_unpin_pages(obj);
 720	return ret;
 721
 722err_unpin:
 723	i915_gem_object_unpin_pages(obj);
 724err_unlock:
 725	i915_gem_object_unlock(obj);
 726	return ret;
 727}
 728
 729/**
 730 * i915_gem_pwrite_ioctl - Writes data to the object referenced by handle.
 731 * @dev: drm device
 732 * @data: ioctl data blob
 733 * @file: drm file
 734 *
 735 * On error, the contents of the buffer that were to be modified are undefined.
 736 */
 737int
 738i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 739		      struct drm_file *file)
 740{
 741	struct drm_i915_private *i915 = to_i915(dev);
 742	struct drm_i915_gem_pwrite *args = data;
 743	struct drm_i915_gem_object *obj;
 744	int ret;
 745
 746	/* PWRITE is disallowed for all platforms after TGL-LP.  This also
 747	 * covers all platforms with local memory.
 748	 */
 749	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 750		return -EOPNOTSUPP;
 751
 752	if (args->size == 0)
 753		return 0;
 754
 755	if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 756		return -EFAULT;
 757
 758	obj = i915_gem_object_lookup(file, args->handle);
 759	if (!obj)
 760		return -ENOENT;
 761
 762	/* Bounds check destination. */
 763	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 764		ret = -EINVAL;
 765		goto err;
 766	}
 767
 768	/* Writes not allowed into this read-only object */
 769	if (i915_gem_object_is_readonly(obj)) {
 770		ret = -EINVAL;
 771		goto err;
 772	}
 773
 774	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 775
 776	ret = -ENODEV;
 777	if (obj->ops->pwrite)
 778		ret = obj->ops->pwrite(obj, args);
 779	if (ret != -ENODEV)
 780		goto err;
 781
 782	ret = i915_gem_object_wait(obj,
 783				   I915_WAIT_INTERRUPTIBLE |
 784				   I915_WAIT_ALL,
 785				   MAX_SCHEDULE_TIMEOUT);
 786	if (ret)
 787		goto err;
 788
 789	ret = -EFAULT;
 790	/* We can only do the GTT pwrite on untiled buffers, as otherwise
 791	 * it would end up going through the fenced access, and we'll get
 792	 * different detiling behavior between reading and writing.
 793	 * pread/pwrite currently are reading and writing from the CPU
 794	 * perspective, requiring manual detiling by the client.
 795	 */
 796	if (!i915_gem_object_has_struct_page(obj) ||
 797	    i915_gem_cpu_write_needs_clflush(obj))
 798		/* Note that the gtt paths might fail with non-page-backed user
 799		 * pointers (e.g. gtt mappings when moving data between
 800		 * textures). Fallback to the shmem path in that case.
 801		 */
 802		ret = i915_gem_gtt_pwrite_fast(obj, args);
 803
 804	if (ret == -EFAULT || ret == -ENOSPC) {
 805		if (i915_gem_object_has_struct_page(obj))
 806			ret = i915_gem_shmem_pwrite(obj, args);
 807	}
 808
 809err:
 810	i915_gem_object_put(obj);
 811	return ret;
 812}
 813
 814/**
 815 * i915_gem_sw_finish_ioctl - Called when user space has done writes to this buffer
 816 * @dev: drm device
 817 * @data: ioctl data blob
 818 * @file: drm file
 819 */
 820int
 821i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 822			 struct drm_file *file)
 823{
 824	struct drm_i915_gem_sw_finish *args = data;
 825	struct drm_i915_gem_object *obj;
 826
 827	obj = i915_gem_object_lookup(file, args->handle);
 828	if (!obj)
 829		return -ENOENT;
 830
 831	/*
 832	 * Proxy objects are barred from CPU access, so there is no
 833	 * need to ban sw_finish as it is a nop.
 834	 */
 835
 836	/* Pinned buffers may be scanout, so flush the cache */
 837	i915_gem_object_flush_if_display(obj);
 838	i915_gem_object_put(obj);
 839
 840	return 0;
 841}
 842
 843void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 844{
 845	struct drm_i915_gem_object *obj, *on;
 846	int i;
 847
 848	/*
 849	 * Only called during RPM suspend. All users of the userfault_list
 850	 * must be holding an RPM wakeref to ensure that this can not
 851	 * run concurrently with themselves.
 852	 */
 853
 854	list_for_each_entry_safe(obj, on,
 855				 &to_gt(i915)->ggtt->userfault_list, userfault_link)
 856		__i915_gem_object_release_mmap_gtt(obj);
 857
 858	list_for_each_entry_safe(obj, on,
 859				 &i915->runtime_pm.lmem_userfault_list, userfault_link)
 860		i915_gem_object_runtime_pm_release_mmap_offset(obj);
 861
 862	/*
 863	 * The fence will be lost when the device powers down. If any were
 864	 * in use by hardware (i.e. they are pinned), we should not be powering
 865	 * down! All other fences will be reacquired by the user upon waking.
 866	 */
 867	for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {
 868		struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];
 869
 870		/*
 871		 * Ideally we want to assert that the fence register is not
 872		 * live at this point (i.e. that no piece of code will be
 873		 * trying to write through fence + GTT, as that both violates
 874		 * our tracking of activity and associated locking/barriers,
 875		 * but also is illegal given that the hw is powered down).
 876		 *
 877		 * Previously we used reg->pin_count as a "liveness" indicator.
 878		 * That is not sufficient, and we need a more fine-grained
 879		 * tool if we want to have a sanity check here.
 880		 */
 881
 882		if (!reg->vma)
 883			continue;
 884
 885		GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 886		reg->dirty = true;
 887	}
 888}
 889
 890static void discard_ggtt_vma(struct i915_vma *vma)
 891{
 892	struct drm_i915_gem_object *obj = vma->obj;
 893
 894	spin_lock(&obj->vma.lock);
 895	if (!RB_EMPTY_NODE(&vma->obj_node)) {
 896		rb_erase(&vma->obj_node, &obj->vma.tree);
 897		RB_CLEAR_NODE(&vma->obj_node);
 898	}
 899	spin_unlock(&obj->vma.lock);
 900}
 901
 902struct i915_vma *
 903i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 904			    struct i915_gem_ww_ctx *ww,
 905			    const struct i915_gtt_view *view,
 906			    u64 size, u64 alignment, u64 flags)
 907{
 908	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 909	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 910	struct i915_vma *vma;
 911	int ret;
 912
 913	GEM_WARN_ON(!ww);
 914
 915	if (flags & PIN_MAPPABLE &&
 916	    (!view || view->type == I915_GTT_VIEW_NORMAL)) {
 917		/*
 918		 * If the required space is larger than the available
 919		 * aperture, we will not able to find a slot for the
 920		 * object and unbinding the object now will be in
 921		 * vain. Worse, doing so may cause us to ping-pong
 922		 * the object in and out of the Global GTT and
 923		 * waste a lot of cycles under the mutex.
 924		 */
 925		if (obj->base.size > ggtt->mappable_end)
 926			return ERR_PTR(-E2BIG);
 927
 928		/*
 929		 * If NONBLOCK is set the caller is optimistically
 930		 * trying to cache the full object within the mappable
 931		 * aperture, and *must* have a fallback in place for
 932		 * situations where we cannot bind the object. We
 933		 * can be a little more lax here and use the fallback
 934		 * more often to avoid costly migrations of ourselves
 935		 * and other objects within the aperture.
 936		 *
 937		 * Half-the-aperture is used as a simple heuristic.
 938		 * More interesting would to do search for a free
 939		 * block prior to making the commitment to unbind.
 940		 * That caters for the self-harm case, and with a
 941		 * little more heuristics (e.g. NOFAULT, NOEVICT)
 942		 * we could try to minimise harm to others.
 943		 */
 944		if (flags & PIN_NONBLOCK &&
 945		    obj->base.size > ggtt->mappable_end / 2)
 946			return ERR_PTR(-ENOSPC);
 947	}
 948
 949new_vma:
 950	vma = i915_vma_instance(obj, &ggtt->vm, view);
 951	if (IS_ERR(vma))
 952		return vma;
 953
 954	if (i915_vma_misplaced(vma, size, alignment, flags)) {
 955		if (flags & PIN_NONBLOCK) {
 956			if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 957				return ERR_PTR(-ENOSPC);
 958
 959			/*
 960			 * If this misplaced vma is too big (i.e, at-least
 961			 * half the size of aperture) or hasn't been pinned
 962			 * mappable before, we ignore the misplacement when
 963			 * PIN_NONBLOCK is set in order to avoid the ping-pong
 964			 * issue described above. In other words, we try to
 965			 * avoid the costly operation of unbinding this vma
 966			 * from the GGTT and rebinding it back because there
 967			 * may not be enough space for this vma in the aperture.
 968			 */
 969			if (flags & PIN_MAPPABLE &&
 970			    (vma->fence_size > ggtt->mappable_end / 2 ||
 971			    !i915_vma_is_map_and_fenceable(vma)))
 972				return ERR_PTR(-ENOSPC);
 973		}
 974
 975		if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 976			discard_ggtt_vma(vma);
 977			goto new_vma;
 978		}
 979
 980		ret = i915_vma_unbind(vma);
 981		if (ret)
 982			return ERR_PTR(ret);
 983	}
 984
 985	ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 986
 987	if (ret)
 988		return ERR_PTR(ret);
 989
 990	if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 991		mutex_lock(&ggtt->vm.mutex);
 992		i915_vma_revoke_fence(vma);
 993		mutex_unlock(&ggtt->vm.mutex);
 994	}
 995
 996	ret = i915_vma_wait_for_bind(vma);
 997	if (ret) {
 998		i915_vma_unpin(vma);
 999		return ERR_PTR(ret);
1000	}
1001
1002	return vma;
1003}
1004
1005struct i915_vma * __must_check
1006i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
1007			 const struct i915_gtt_view *view,
1008			 u64 size, u64 alignment, u64 flags)
1009{
1010	struct i915_gem_ww_ctx ww;
1011	struct i915_vma *ret;
1012	int err;
1013
1014	for_i915_gem_ww(&ww, err, true) {
1015		err = i915_gem_object_lock(obj, &ww);
1016		if (err)
1017			continue;
1018
1019		ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,
1020						  alignment, flags);
1021		if (IS_ERR(ret))
1022			err = PTR_ERR(ret);
1023	}
1024
1025	return err ? ERR_PTR(err) : ret;
1026}
1027
1028int
1029i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
1030		       struct drm_file *file_priv)
1031{
1032	struct drm_i915_private *i915 = to_i915(dev);
1033	struct drm_i915_gem_madvise *args = data;
1034	struct drm_i915_gem_object *obj;
1035	int err;
1036
1037	switch (args->madv) {
1038	case I915_MADV_DONTNEED:
1039	case I915_MADV_WILLNEED:
1040	    break;
1041	default:
1042	    return -EINVAL;
1043	}
1044
1045	obj = i915_gem_object_lookup(file_priv, args->handle);
1046	if (!obj)
1047		return -ENOENT;
1048
1049	err = i915_gem_object_lock_interruptible(obj, NULL);
1050	if (err)
1051		goto out;
1052
1053	if (i915_gem_object_has_pages(obj) &&
1054	    i915_gem_object_is_tiled(obj) &&
1055	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
1056		if (obj->mm.madv == I915_MADV_WILLNEED) {
1057			GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
1058			i915_gem_object_clear_tiling_quirk(obj);
1059			i915_gem_object_make_shrinkable(obj);
1060		}
1061		if (args->madv == I915_MADV_WILLNEED) {
1062			GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1063			i915_gem_object_make_unshrinkable(obj);
1064			i915_gem_object_set_tiling_quirk(obj);
1065		}
1066	}
1067
1068	if (obj->mm.madv != __I915_MADV_PURGED) {
1069		obj->mm.madv = args->madv;
1070		if (obj->ops->adjust_lru)
1071			obj->ops->adjust_lru(obj);
1072	}
1073
1074	if (i915_gem_object_has_pages(obj) ||
1075	    i915_gem_object_has_self_managed_shrink_list(obj)) {
1076		unsigned long flags;
1077
1078		spin_lock_irqsave(&i915->mm.obj_lock, flags);
1079		if (!list_empty(&obj->mm.link)) {
1080			struct list_head *list;
1081
1082			if (obj->mm.madv != I915_MADV_WILLNEED)
1083				list = &i915->mm.purge_list;
1084			else
1085				list = &i915->mm.shrink_list;
1086			list_move_tail(&obj->mm.link, list);
1087
1088		}
1089		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1090	}
1091
1092	/* if the object is no longer attached, discard its backing storage */
1093	if (obj->mm.madv == I915_MADV_DONTNEED &&
1094	    !i915_gem_object_has_pages(obj))
1095		i915_gem_object_truncate(obj);
1096
1097	args->retained = obj->mm.madv != __I915_MADV_PURGED;
1098
1099	i915_gem_object_unlock(obj);
1100out:
1101	i915_gem_object_put(obj);
1102	return err;
1103}
1104
1105/*
1106 * A single pass should suffice to release all the freed objects (along most
1107 * call paths), but be a little more paranoid in that freeing the objects does
1108 * take a little amount of time, during which the rcu callbacks could have added
1109 * new objects into the freed list, and armed the work again.
1110 */
1111void i915_gem_drain_freed_objects(struct drm_i915_private *i915)
1112{
1113	while (atomic_read(&i915->mm.free_count)) {
1114		flush_work(&i915->mm.free_work);
1115		drain_workqueue(i915->bdev.wq);
1116		rcu_barrier();
1117	}
1118}
1119
1120/*
1121 * Similar to objects above (see i915_gem_drain_freed-objects), in general we
1122 * have workers that are armed by RCU and then rearm themselves in their
1123 * callbacks. To be paranoid, we need to drain the workqueue a second time after
1124 * waiting for the RCU grace period so that we catch work queued via RCU from
1125 * the first pass. As neither drain_workqueue() nor flush_workqueue() report a
1126 * result, we make an assumption that we only don't require more than 3 passes
1127 * to catch all _recursive_ RCU delayed work.
1128 */
1129void i915_gem_drain_workqueue(struct drm_i915_private *i915)
1130{
1131	int i;
1132
1133	for (i = 0; i < 3; i++) {
1134		flush_workqueue(i915->wq);
1135		rcu_barrier();
1136		i915_gem_drain_freed_objects(i915);
1137	}
1138
1139	drain_workqueue(i915->wq);
1140}
1141
1142int i915_gem_init(struct drm_i915_private *dev_priv)
1143{
1144	struct intel_gt *gt;
1145	unsigned int i;
1146	int ret;
1147
1148	/*
1149	 * In the process of replacing cache_level with pat_index a tricky
1150	 * dependency is created on the definition of the enum i915_cache_level.
1151	 * In case this enum is changed, PTE encode would be broken.
1152	 * Add a WARNING here. And remove when we completely quit using this
1153	 * enum.
1154	 */
1155	BUILD_BUG_ON(I915_CACHE_NONE != 0 ||
1156		     I915_CACHE_LLC != 1 ||
1157		     I915_CACHE_L3_LLC != 2 ||
1158		     I915_CACHE_WT != 3 ||
1159		     I915_MAX_CACHE_LEVEL != 4);
1160
1161	/* We need to fallback to 4K pages if host doesn't support huge gtt. */
1162	if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1163		RUNTIME_INFO(dev_priv)->page_sizes = I915_GTT_PAGE_SIZE_4K;
1164
1165	for_each_gt(gt, dev_priv, i) {
1166		intel_uc_fetch_firmwares(&gt->uc);
1167		intel_wopcm_init(&gt->wopcm);
1168		if (GRAPHICS_VER(dev_priv) >= 8)
1169			setup_private_pat(gt);
1170	}
1171
1172	ret = i915_init_ggtt(dev_priv);
1173	if (ret) {
1174		GEM_BUG_ON(ret == -EIO);
1175		goto err_unlock;
1176	}
1177
1178	/*
1179	 * Despite its name intel_clock_gating_init applies both display
1180	 * clock gating workarounds; GT mmio workarounds and the occasional
1181	 * GT power context workaround. Worse, sometimes it includes a context
1182	 * register workaround which we need to apply before we record the
1183	 * default HW state for all contexts.
1184	 *
1185	 * FIXME: break up the workarounds and apply them at the right time!
1186	 */
1187	intel_clock_gating_init(dev_priv);
1188
1189	for_each_gt(gt, dev_priv, i) {
1190		ret = intel_gt_init(gt);
1191		if (ret)
1192			goto err_unlock;
1193	}
1194
1195	/*
1196	 * Register engines early to ensure the engine list is in its final
1197	 * rb-tree form, lowering the amount of code that has to deal with
1198	 * the intermediate llist state.
1199	 */
1200	intel_engines_driver_register(dev_priv);
1201
1202	return 0;
1203
1204	/*
1205	 * Unwinding is complicated by that we want to handle -EIO to mean
1206	 * disable GPU submission but keep KMS alive. We want to mark the
1207	 * HW as irrevisibly wedged, but keep enough state around that the
1208	 * driver doesn't explode during runtime.
1209	 */
1210err_unlock:
1211	i915_gem_drain_workqueue(dev_priv);
1212
1213	if (ret != -EIO) {
1214		for_each_gt(gt, dev_priv, i) {
1215			intel_gt_driver_remove(gt);
1216			intel_gt_driver_release(gt);
1217			intel_uc_cleanup_firmwares(&gt->uc);
1218		}
1219	}
1220
1221	if (ret == -EIO) {
1222		/*
1223		 * Allow engines or uC initialisation to fail by marking the GPU
1224		 * as wedged. But we only want to do this when the GPU is angry,
1225		 * for all other failure, such as an allocation failure, bail.
1226		 */
1227		for_each_gt(gt, dev_priv, i) {
1228			if (!intel_gt_is_wedged(gt)) {
1229				i915_probe_error(dev_priv,
1230						 "Failed to initialize GPU, declaring it wedged!\n");
1231				intel_gt_set_wedged(gt);
1232			}
1233		}
1234
1235		/* Minimal basic recovery for KMS */
1236		ret = i915_ggtt_enable_hw(dev_priv);
1237		i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1238		intel_clock_gating_init(dev_priv);
1239	}
1240
1241	i915_gem_drain_freed_objects(dev_priv);
1242
1243	return ret;
1244}
1245
1246void i915_gem_driver_register(struct drm_i915_private *i915)
1247{
1248	i915_gem_driver_register__shrinker(i915);
1249}
1250
1251void i915_gem_driver_unregister(struct drm_i915_private *i915)
1252{
1253	i915_gem_driver_unregister__shrinker(i915);
1254}
1255
1256void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1257{
1258	struct intel_gt *gt;
1259	unsigned int i;
1260
1261	i915_gem_suspend_late(dev_priv);
1262	for_each_gt(gt, dev_priv, i)
1263		intel_gt_driver_remove(gt);
1264	dev_priv->uabi_engines = RB_ROOT;
1265
1266	/* Flush any outstanding unpin_work. */
1267	i915_gem_drain_workqueue(dev_priv);
1268}
1269
1270void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1271{
1272	struct intel_gt *gt;
1273	unsigned int i;
1274
1275	for_each_gt(gt, dev_priv, i) {
1276		intel_gt_driver_release(gt);
1277		intel_uc_cleanup_firmwares(&gt->uc);
1278	}
1279
1280	/* Flush any outstanding work, including i915_gem_context.release_work. */
1281	i915_gem_drain_workqueue(dev_priv);
1282
1283	drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1284}
1285
1286static void i915_gem_init__mm(struct drm_i915_private *i915)
1287{
1288	spin_lock_init(&i915->mm.obj_lock);
1289
1290	init_llist_head(&i915->mm.free_list);
1291
1292	INIT_LIST_HEAD(&i915->mm.purge_list);
1293	INIT_LIST_HEAD(&i915->mm.shrink_list);
1294
1295	i915_gem_init__objects(i915);
1296}
1297
1298void i915_gem_init_early(struct drm_i915_private *dev_priv)
1299{
1300	i915_gem_init__mm(dev_priv);
1301	i915_gem_init__contexts(dev_priv);
1302
1303	spin_lock_init(&dev_priv->frontbuffer_lock);
1304}
1305
1306void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1307{
1308	i915_gem_drain_workqueue(dev_priv);
1309	GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1310	GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1311	drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1312}
1313
1314int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1315{
1316	struct drm_i915_file_private *file_priv;
1317	struct i915_drm_client *client;
1318	int ret = -ENOMEM;
1319
1320	drm_dbg(&i915->drm, "\n");
1321
1322	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1323	if (!file_priv)
1324		goto err_alloc;
1325
1326	client = i915_drm_client_alloc();
1327	if (!client)
1328		goto err_client;
1329
1330	file->driver_priv = file_priv;
1331	file_priv->i915 = i915;
1332	file_priv->file = file;
1333	file_priv->client = client;
1334
1335	file_priv->bsd_engine = -1;
1336	file_priv->hang_timestamp = jiffies;
1337
1338	ret = i915_gem_context_open(i915, file);
1339	if (ret)
1340		goto err_context;
1341
1342	return 0;
1343
1344err_context:
1345	i915_drm_client_put(client);
1346err_client:
1347	kfree(file_priv);
1348err_alloc:
1349	return ret;
1350}
1351
1352#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1353#include "selftests/mock_gem_device.c"
1354#include "selftests/i915_gem.c"
1355#endif