i915_driver.c

   1/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
   2 */
   3/*
   4 *
   5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   6 * All Rights Reserved.
   7 *
   8 * Permission is hereby granted, free of charge, to any person obtaining a
   9 * copy of this software and associated documentation files (the
  10 * "Software"), to deal in the Software without restriction, including
  11 * without limitation the rights to use, copy, modify, merge, publish,
  12 * distribute, sub license, and/or sell copies of the Software, and to
  13 * permit persons to whom the Software is furnished to do so, subject to
  14 * the following conditions:
  15 *
  16 * The above copyright notice and this permission notice (including the
  17 * next paragraph) shall be included in all copies or substantial portions
  18 * of the Software.
  19 *
  20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 *
  28 */
  29
  30#include <linux/aperture.h>
  31#include <linux/acpi.h>
  32#include <linux/device.h>
  33#include <linux/module.h>
  34#include <linux/oom.h>
  35#include <linux/pci.h>
  36#include <linux/pm.h>
  37#include <linux/pm_runtime.h>
  38#include <linux/slab.h>
  39#include <linux/string_helpers.h>
  40#include <linux/vga_switcheroo.h>
  41#include <linux/vt.h>
  42
  43#include <drm/drm_atomic_helper.h>
  44#include <drm/drm_client.h>
  45#include <drm/drm_client_event.h>
  46#include <drm/drm_ioctl.h>
  47#include <drm/drm_managed.h>
  48#include <drm/drm_probe_helper.h>
  49#include <drm/intel/display_member.h>
  50#include <drm/intel/display_parent_interface.h>
  51
  52#include "display/i9xx_display_sr.h"
  53#include "display/intel_bw.h"
  54#include "display/intel_cdclk.h"
  55#include "display/intel_crtc.h"
  56#include "display/intel_display_device.h"
  57#include "display/intel_display_driver.h"
  58#include "display/intel_display_power.h"
  59#include "display/intel_dmc.h"
  60#include "display/intel_dp.h"
  61#include "display/intel_dpt.h"
  62#include "display/intel_encoder.h"
  63#include "display/intel_fbdev.h"
  64#include "display/intel_gmbus.h"
  65#include "display/intel_hotplug.h"
  66#include "display/intel_opregion.h"
  67#include "display/intel_overlay.h"
  68#include "display/intel_pch_refclk.h"
  69#include "display/intel_pps.h"
  70#include "display/intel_sbi.h"
  71#include "display/intel_sprite_uapi.h"
  72#include "display/skl_watermark.h"
  73
  74#include "gem/i915_gem_context.h"
  75#include "gem/i915_gem_create.h"
  76#include "gem/i915_gem_dmabuf.h"
  77#include "gem/i915_gem_ioctls.h"
  78#include "gem/i915_gem_mman.h"
  79#include "gem/i915_gem_pm.h"
  80#include "gt/intel_gt.h"
  81#include "gt/intel_gt_pm.h"
  82#include "gt/intel_gt_print.h"
  83#include "gt/intel_rc6.h"
  84
  85#include "pxp/intel_pxp.h"
  86#include "pxp/intel_pxp_debugfs.h"
  87#include "pxp/intel_pxp_pm.h"
  88
  89#include "soc/intel_dram.h"
  90#include "soc/intel_gmch.h"
  91
  92#include "i915_debugfs.h"
  93#include "i915_driver.h"
  94#include "i915_drm_client.h"
  95#include "i915_drv.h"
  96#include "i915_file_private.h"
  97#include "i915_getparam.h"
  98#include "i915_hwmon.h"
  99#include "i915_ioc32.h"
 100#include "i915_ioctl.h"
 101#include "i915_irq.h"
 102#include "i915_memcpy.h"
 103#include "i915_perf.h"
 104#include "i915_query.h"
 105#include "i915_reg.h"
 106#include "i915_switcheroo.h"
 107#include "i915_sysfs.h"
 108#include "i915_utils.h"
 109#include "i915_vgpu.h"
 110#include "intel_clock_gating.h"
 111#include "intel_cpu_info.h"
 112#include "intel_gvt.h"
 113#include "intel_memory_region.h"
 114#include "intel_pci_config.h"
 115#include "intel_pcode.h"
 116#include "intel_region_ttm.h"
 117#include "vlv_iosf_sb.h"
 118#include "vlv_suspend.h"
 119
 120static const struct drm_driver i915_drm_driver;
 121
 122static int i915_workqueues_init(struct drm_i915_private *dev_priv)
 123{
 124	/*
 125	 * The i915 workqueue is primarily used for batched retirement of
 126	 * requests (and thus managing bo) once the task has been completed
 127	 * by the GPU. i915_retire_requests() is called directly when we
 128	 * need high-priority retirement, such as waiting for an explicit
 129	 * bo.
 130	 *
 131	 * It is also used for periodic low-priority events, such as
 132	 * idle-timers and recording error state.
 133	 *
 134	 * All tasks on the workqueue are expected to acquire the dev mutex
 135	 * so there is no point in running more than one instance of the
 136	 * workqueue at any time.  Use an ordered one.
 137	 */
 138	dev_priv->wq = alloc_ordered_workqueue("i915", 0);
 139	if (dev_priv->wq == NULL)
 140		goto out_err;
 141
 142	/*
 143	 * The unordered i915 workqueue should be used for all work
 144	 * scheduling that do not require running in order, which used
 145	 * to be scheduled on the system_wq before moving to a driver
 146	 * instance due deprecation of flush_scheduled_work().
 147	 */
 148	dev_priv->unordered_wq = alloc_workqueue("i915-unordered", 0, 0);
 149	if (dev_priv->unordered_wq == NULL)
 150		goto out_free_wq;
 151
 152	return 0;
 153
 154out_free_wq:
 155	destroy_workqueue(dev_priv->wq);
 156out_err:
 157	drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n");
 158
 159	return -ENOMEM;
 160}
 161
 162static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
 163{
 164	destroy_workqueue(dev_priv->unordered_wq);
 165	destroy_workqueue(dev_priv->wq);
 166}
 167
 168/*
 169 * We don't keep the workarounds for pre-production hardware, so we expect our
 170 * driver to fail on these machines in one way or another. A little warning on
 171 * dmesg may help both the user and the bug triagers.
 172 *
 173 * Our policy for removing pre-production workarounds is to keep the
 174 * current gen workarounds as a guide to the bring-up of the next gen
 175 * (workarounds have a habit of persisting!). Anything older than that
 176 * should be removed along with the complications they introduce.
 177 */
 178static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
 179{
 180	bool pre = false;
 181
 182	pre |= IS_HASWELL_EARLY_SDV(dev_priv);
 183	pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6;
 184	pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA;
 185	pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
 186	pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3;
 187	pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7;
 188	pre |= IS_TIGERLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
 189	pre |= IS_DG1(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
 190	pre |= IS_DG2_G10(dev_priv) && INTEL_REVID(dev_priv) < 0x8;
 191	pre |= IS_DG2_G11(dev_priv) && INTEL_REVID(dev_priv) < 0x5;
 192	pre |= IS_DG2_G12(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
 193
 194	if (pre) {
 195		drm_err(&dev_priv->drm, "This is a pre-production stepping. "
 196			  "It may not be fully functional.\n");
 197		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
 198	}
 199}
 200
 201static void sanitize_gpu(struct drm_i915_private *i915)
 202{
 203	if (!intel_gt_gpu_reset_clobbers_display(to_gt(i915))) {
 204		struct intel_gt *gt;
 205		unsigned int i;
 206
 207		for_each_gt(gt, i915, i)
 208			intel_gt_reset_all_engines(gt);
 209	}
 210}
 211
 212/**
 213 * i915_driver_early_probe - setup state not requiring device access
 214 * @dev_priv: device private
 215 *
 216 * Initialize everything that is a "SW-only" state, that is state not
 217 * requiring accessing the device or exposing the driver via kernel internal
 218 * or userspace interfaces. Example steps belonging here: lock initialization,
 219 * system memory allocation, setting up device specific attributes and
 220 * function hooks not requiring accessing the device.
 221 */
 222static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 223{
 224	struct intel_display *display = dev_priv->display;
 225	int ret = 0;
 226
 227	if (i915_inject_probe_failure(dev_priv))
 228		return -ENODEV;
 229
 230	intel_device_info_runtime_init_early(dev_priv);
 231
 232	intel_step_init(dev_priv);
 233
 234	intel_uncore_mmio_debug_init_early(dev_priv);
 235
 236	spin_lock_init(&dev_priv->gpu_error.lock);
 237
 238	intel_sbi_init(display);
 239	vlv_iosf_sb_init(dev_priv);
 240	mutex_init(&dev_priv->sb_lock);
 241
 242	i915_memcpy_init_early(dev_priv);
 243	intel_runtime_pm_init_early(&dev_priv->runtime_pm);
 244
 245	ret = i915_workqueues_init(dev_priv);
 246	if (ret < 0)
 247		return ret;
 248
 249	ret = vlv_suspend_init(dev_priv);
 250	if (ret < 0)
 251		goto err_workqueues;
 252
 253	ret = intel_region_ttm_device_init(dev_priv);
 254	if (ret)
 255		goto err_ttm;
 256
 257	ret = intel_root_gt_init_early(dev_priv);
 258	if (ret < 0)
 259		goto err_rootgt;
 260
 261	i915_gem_init_early(dev_priv);
 262
 263	intel_irq_init(dev_priv);
 264	intel_display_driver_early_probe(display);
 265	intel_clock_gating_hooks_init(dev_priv);
 266
 267	intel_detect_preproduction_hw(dev_priv);
 268
 269	return 0;
 270
 271err_rootgt:
 272	intel_region_ttm_device_fini(dev_priv);
 273err_ttm:
 274	vlv_suspend_cleanup(dev_priv);
 275err_workqueues:
 276	i915_workqueues_cleanup(dev_priv);
 277	return ret;
 278}
 279
 280/**
 281 * i915_driver_late_release - cleanup the setup done in
 282 *			       i915_driver_early_probe()
 283 * @dev_priv: device private
 284 */
 285static void i915_driver_late_release(struct drm_i915_private *dev_priv)
 286{
 287	struct intel_display *display = dev_priv->display;
 288
 289	intel_irq_fini(dev_priv);
 290	intel_power_domains_cleanup(display);
 291	i915_gem_cleanup_early(dev_priv);
 292	intel_gt_driver_late_release_all(dev_priv);
 293	intel_region_ttm_device_fini(dev_priv);
 294	vlv_suspend_cleanup(dev_priv);
 295	i915_workqueues_cleanup(dev_priv);
 296
 297	mutex_destroy(&dev_priv->sb_lock);
 298	vlv_iosf_sb_fini(dev_priv);
 299	intel_sbi_fini(display);
 300
 301	i915_params_free(&dev_priv->params);
 302
 303	intel_display_device_remove(display);
 304}
 305
 306/**
 307 * i915_driver_mmio_probe - setup device MMIO
 308 * @dev_priv: device private
 309 *
 310 * Setup minimal device state necessary for MMIO accesses later in the
 311 * initialization sequence. The setup here should avoid any other device-wide
 312 * side effects or exposing the driver via kernel internal or user space
 313 * interfaces.
 314 */
 315static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 316{
 317	struct intel_display *display = dev_priv->display;
 318	struct intel_gt *gt;
 319	int ret, i;
 320
 321	if (i915_inject_probe_failure(dev_priv))
 322		return -ENODEV;
 323
 324	ret = intel_gmch_bridge_setup(dev_priv);
 325	if (ret < 0)
 326		return ret;
 327
 328	for_each_gt(gt, dev_priv, i) {
 329		ret = intel_uncore_init_mmio(gt->uncore);
 330		if (ret)
 331			return ret;
 332
 333		ret = drmm_add_action_or_reset(&dev_priv->drm,
 334					       intel_uncore_fini_mmio,
 335					       gt->uncore);
 336		if (ret)
 337			return ret;
 338	}
 339
 340	/* Try to make sure MCHBAR is enabled before poking at it */
 341	intel_gmch_bar_setup(dev_priv);
 342	intel_device_info_runtime_init(dev_priv);
 343	intel_display_device_info_runtime_init(display);
 344
 345	for_each_gt(gt, dev_priv, i) {
 346		ret = intel_gt_init_mmio(gt);
 347		if (ret)
 348			goto err_uncore;
 349	}
 350
 351	/* As early as possible, scrub existing GPU state before clobbering */
 352	sanitize_gpu(dev_priv);
 353
 354	return 0;
 355
 356err_uncore:
 357	intel_gmch_bar_teardown(dev_priv);
 358
 359	return ret;
 360}
 361
 362/**
 363 * i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe()
 364 * @dev_priv: device private
 365 */
 366static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
 367{
 368	intel_gmch_bar_teardown(dev_priv);
 369}
 370
 371/**
 372 * i915_set_dma_info - set all relevant PCI dma info as configured for the
 373 * platform
 374 * @i915: valid i915 instance
 375 *
 376 * Set the dma max segment size, device and coherent masks.  The dma mask set
 377 * needs to occur before i915_ggtt_probe_hw.
 378 *
 379 * A couple of platforms have special needs.  Address them as well.
 380 *
 381 */
 382static int i915_set_dma_info(struct drm_i915_private *i915)
 383{
 384	unsigned int mask_size = INTEL_INFO(i915)->dma_mask_size;
 385	int ret;
 386
 387	GEM_BUG_ON(!mask_size);
 388
 389	/*
 390	 * We don't have a max segment size, so set it to the max so sg's
 391	 * debugging layer doesn't complain
 392	 */
 393	dma_set_max_seg_size(i915->drm.dev, UINT_MAX);
 394
 395	ret = dma_set_mask(i915->drm.dev, DMA_BIT_MASK(mask_size));
 396	if (ret)
 397		goto mask_err;
 398
 399	/* overlay on gen2 is broken and can't address above 1G */
 400	if (GRAPHICS_VER(i915) == 2)
 401		mask_size = 30;
 402
 403	/*
 404	 * 965GM sometimes incorrectly writes to hardware status page (HWS)
 405	 * using 32bit addressing, overwriting memory if HWS is located
 406	 * above 4GB.
 407	 *
 408	 * The documentation also mentions an issue with undefined
 409	 * behaviour if any general state is accessed within a page above 4GB,
 410	 * which also needs to be handled carefully.
 411	 */
 412	if (IS_I965G(i915) || IS_I965GM(i915))
 413		mask_size = 32;
 414
 415	ret = dma_set_coherent_mask(i915->drm.dev, DMA_BIT_MASK(mask_size));
 416	if (ret)
 417		goto mask_err;
 418
 419	return 0;
 420
 421mask_err:
 422	drm_err(&i915->drm, "Can't set DMA mask/consistent mask (%d)\n", ret);
 423	return ret;
 424}
 425
 426/* Wa_14022698537:dg2 */
 427static void i915_enable_g8(struct drm_i915_private *i915)
 428{
 429	if (IS_DG2(i915)) {
 430		if (IS_DG2_D(i915) && !intel_match_g8_cpu())
 431			return;
 432
 433		snb_pcode_write_p(&i915->uncore, PCODE_POWER_SETUP,
 434				  POWER_SETUP_SUBCOMMAND_G8_ENABLE, 0, 0);
 435	}
 436}
 437
 438static int i915_pcode_init(struct drm_i915_private *i915)
 439{
 440	struct intel_gt *gt;
 441	int id, ret;
 442
 443	for_each_gt(gt, i915, id) {
 444		ret = intel_pcode_init(gt->uncore);
 445		if (ret) {
 446			gt_err(gt, "intel_pcode_init failed %d\n", ret);
 447			return ret;
 448		}
 449	}
 450
 451	i915_enable_g8(i915);
 452	return 0;
 453}
 454
 455/**
 456 * i915_driver_hw_probe - setup state requiring device access
 457 * @dev_priv: device private
 458 *
 459 * Setup state that requires accessing the device, but doesn't require
 460 * exposing the driver via kernel internal or userspace interfaces.
 461 */
 462static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 463{
 464	struct intel_display *display = dev_priv->display;
 465	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 466	int ret;
 467
 468	if (i915_inject_probe_failure(dev_priv))
 469		return -ENODEV;
 470
 471	if (HAS_PPGTT(dev_priv)) {
 472		if (intel_vgpu_active(dev_priv) &&
 473		    !intel_vgpu_has_full_ppgtt(dev_priv)) {
 474			drm_err(&dev_priv->drm,
 475				"incompatible vGPU found, support for isolated ppGTT required\n");
 476			return -ENXIO;
 477		}
 478	}
 479
 480	if (HAS_EXECLISTS(dev_priv)) {
 481		/*
 482		 * Older GVT emulation depends upon intercepting CSB mmio,
 483		 * which we no longer use, preferring to use the HWSP cache
 484		 * instead.
 485		 */
 486		if (intel_vgpu_active(dev_priv) &&
 487		    !intel_vgpu_has_hwsp_emulation(dev_priv)) {
 488			drm_err(&dev_priv->drm,
 489				"old vGPU host found, support for HWSP emulation required\n");
 490			return -ENXIO;
 491		}
 492	}
 493
 494	/* needs to be done before ggtt probe */
 495	intel_dram_edram_detect(dev_priv);
 496
 497	ret = i915_set_dma_info(dev_priv);
 498	if (ret)
 499		return ret;
 500
 501	ret = i915_perf_init(dev_priv);
 502	if (ret)
 503		return ret;
 504
 505	ret = i915_ggtt_probe_hw(dev_priv);
 506	if (ret)
 507		goto err_perf;
 508
 509	ret = aperture_remove_conflicting_pci_devices(pdev, dev_priv->drm.driver->name);
 510	if (ret)
 511		goto err_ggtt;
 512
 513	ret = i915_ggtt_init_hw(dev_priv);
 514	if (ret)
 515		goto err_ggtt;
 516
 517	/*
 518	 * Make sure we probe lmem before we probe stolen-lmem. The BAR size
 519	 * might be different due to bar resizing.
 520	 */
 521	ret = intel_gt_tiles_init(dev_priv);
 522	if (ret)
 523		goto err_ggtt;
 524
 525	ret = intel_memory_regions_hw_probe(dev_priv);
 526	if (ret)
 527		goto err_ggtt;
 528
 529	ret = i915_ggtt_enable_hw(dev_priv);
 530	if (ret) {
 531		drm_err(&dev_priv->drm, "failed to enable GGTT\n");
 532		goto err_mem_regions;
 533	}
 534
 535	pci_set_master(pdev);
 536
 537	/* On the 945G/GM, the chipset reports the MSI capability on the
 538	 * integrated graphics even though the support isn't actually there
 539	 * according to the published specs.  It doesn't appear to function
 540	 * correctly in testing on 945G.
 541	 * This may be a side effect of MSI having been made available for PEG
 542	 * and the registers being closely associated.
 543	 *
 544	 * According to chipset errata, on the 965GM, MSI interrupts may
 545	 * be lost or delayed, and was defeatured. MSI interrupts seem to
 546	 * get lost on g4x as well, and interrupt delivery seems to stay
 547	 * properly dead afterwards. So we'll just disable them for all
 548	 * pre-gen5 chipsets.
 549	 *
 550	 * dp aux and gmbus irq on gen4 seems to be able to generate legacy
 551	 * interrupts even when in MSI mode. This results in spurious
 552	 * interrupt warnings if the legacy irq no. is shared with another
 553	 * device. The kernel then disables that interrupt source and so
 554	 * prevents the other device from working properly.
 555	 */
 556	if (GRAPHICS_VER(dev_priv) >= 5) {
 557		if (pci_enable_msi(pdev) < 0)
 558			drm_dbg(&dev_priv->drm, "can't enable MSI");
 559	}
 560
 561	ret = intel_gvt_init(dev_priv);
 562	if (ret)
 563		goto err_msi;
 564
 565	intel_opregion_setup(display);
 566
 567	ret = i915_pcode_init(dev_priv);
 568	if (ret)
 569		goto err_opregion;
 570
 571	/*
 572	 * Fill the dram structure to get the system dram info. This will be
 573	 * used for memory latency calculation.
 574	 */
 575	ret = intel_dram_detect(dev_priv);
 576	if (ret)
 577		goto err_opregion;
 578
 579	intel_bw_init_hw(display);
 580
 581	return 0;
 582
 583err_opregion:
 584	intel_opregion_cleanup(display);
 585err_msi:
 586	if (pdev->msi_enabled)
 587		pci_disable_msi(pdev);
 588err_mem_regions:
 589	intel_memory_regions_driver_release(dev_priv);
 590err_ggtt:
 591	i915_ggtt_driver_release(dev_priv);
 592	i915_gem_drain_freed_objects(dev_priv);
 593	i915_ggtt_driver_late_release(dev_priv);
 594err_perf:
 595	i915_perf_fini(dev_priv);
 596	return ret;
 597}
 598
 599/**
 600 * i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe()
 601 * @dev_priv: device private
 602 */
 603static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
 604{
 605	struct intel_display *display = dev_priv->display;
 606	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 607
 608	i915_perf_fini(dev_priv);
 609
 610	intel_opregion_cleanup(display);
 611
 612	if (pdev->msi_enabled)
 613		pci_disable_msi(pdev);
 614}
 615
 616/**
 617 * i915_driver_register - register the driver with the rest of the system
 618 * @dev_priv: device private
 619 *
 620 * Perform any steps necessary to make the driver available via kernel
 621 * internal or userspace interfaces.
 622 */
 623static int i915_driver_register(struct drm_i915_private *dev_priv)
 624{
 625	struct intel_display *display = dev_priv->display;
 626	struct intel_gt *gt;
 627	unsigned int i;
 628	int ret;
 629
 630	i915_gem_driver_register(dev_priv);
 631	i915_pmu_register(dev_priv);
 632
 633	intel_vgpu_register(dev_priv);
 634
 635	/* Reveal our presence to userspace */
 636	ret = drm_dev_register(&dev_priv->drm, 0);
 637	if (ret) {
 638		i915_probe_error(dev_priv,
 639				 "Failed to register driver for userspace access!\n");
 640		drm_dev_unregister(&dev_priv->drm);
 641		i915_pmu_unregister(dev_priv);
 642		i915_gem_driver_unregister(dev_priv);
 643		return ret;
 644	}
 645
 646	i915_debugfs_register(dev_priv);
 647	i915_setup_sysfs(dev_priv);
 648
 649	/* Depends on sysfs having been initialized */
 650	i915_perf_register(dev_priv);
 651
 652	for_each_gt(gt, dev_priv, i)
 653		intel_gt_driver_register(gt);
 654
 655	intel_pxp_debugfs_register(dev_priv->pxp);
 656
 657	i915_hwmon_register(dev_priv);
 658
 659	intel_display_driver_register(display);
 660
 661	intel_power_domains_enable(display);
 662	intel_runtime_pm_enable(&dev_priv->runtime_pm);
 663
 664	if (i915_switcheroo_register(dev_priv))
 665		drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n");
 666
 667	return 0;
 668}
 669
 670/**
 671 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
 672 * @dev_priv: device private
 673 */
 674static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 675{
 676	struct intel_display *display = dev_priv->display;
 677	struct intel_gt *gt;
 678	unsigned int i;
 679
 680	i915_switcheroo_unregister(dev_priv);
 681
 682	intel_runtime_pm_disable(&dev_priv->runtime_pm);
 683	intel_power_domains_disable(display);
 684
 685	intel_display_driver_unregister(display);
 686
 687	intel_pxp_fini(dev_priv);
 688
 689	for_each_gt(gt, dev_priv, i)
 690		intel_gt_driver_unregister(gt);
 691
 692	i915_hwmon_unregister(dev_priv);
 693
 694	i915_perf_unregister(dev_priv);
 695	i915_pmu_unregister(dev_priv);
 696
 697	i915_teardown_sysfs(dev_priv);
 698	drm_dev_unplug(&dev_priv->drm);
 699
 700	i915_gem_driver_unregister(dev_priv);
 701}
 702
 703void
 704i915_print_iommu_status(struct drm_i915_private *i915, struct drm_printer *p)
 705{
 706	drm_printf(p, "iommu: %s\n",
 707		   str_enabled_disabled(i915_vtd_active(i915)));
 708}
 709
 710static void i915_welcome_messages(struct drm_i915_private *dev_priv)
 711{
 712	if (drm_debug_enabled(DRM_UT_DRIVER)) {
 713		struct drm_printer p = drm_dbg_printer(&dev_priv->drm, DRM_UT_DRIVER,
 714						       "device info:");
 715		struct intel_gt *gt;
 716		unsigned int i;
 717
 718		drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n",
 719			   INTEL_DEVID(dev_priv),
 720			   INTEL_REVID(dev_priv),
 721			   intel_platform_name(INTEL_INFO(dev_priv)->platform),
 722			   intel_subplatform(RUNTIME_INFO(dev_priv),
 723					     INTEL_INFO(dev_priv)->platform),
 724			   GRAPHICS_VER(dev_priv));
 725
 726		intel_device_info_print(INTEL_INFO(dev_priv),
 727					RUNTIME_INFO(dev_priv), &p);
 728		i915_print_iommu_status(dev_priv, &p);
 729		for_each_gt(gt, dev_priv, i)
 730			intel_gt_info_print(&gt->info, &p);
 731	}
 732
 733	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
 734		drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n");
 735	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 736		drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n");
 737	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
 738		drm_info(&dev_priv->drm,
 739			 "DRM_I915_DEBUG_RUNTIME_PM enabled\n");
 740}
 741
 742static const struct intel_display_parent_interface parent = {
 743	.rpm = &i915_display_rpm_interface,
 744};
 745
 746const struct intel_display_parent_interface *i915_driver_parent_interface(void)
 747{
 748	return &parent;
 749}
 750
 751/* Ensure drm and display members are placed properly. */
 752INTEL_DISPLAY_MEMBER_STATIC_ASSERT(struct drm_i915_private, drm, display);
 753
 754static struct drm_i915_private *
 755i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
 756{
 757	const struct intel_device_info *match_info =
 758		(struct intel_device_info *)ent->driver_data;
 759	struct drm_i915_private *i915;
 760	struct intel_display *display;
 761
 762	i915 = devm_drm_dev_alloc(&pdev->dev, &i915_drm_driver,
 763				  struct drm_i915_private, drm);
 764	if (IS_ERR(i915))
 765		return i915;
 766
 767	pci_set_drvdata(pdev, &i915->drm);
 768
 769	/* Device parameters start as a copy of module parameters. */
 770	i915_params_copy(&i915->params, &i915_modparams);
 771
 772	/* Set up device info and initial runtime info. */
 773	intel_device_info_driver_create(i915, pdev->device, match_info);
 774
 775	display = intel_display_device_probe(pdev, &parent);
 776	if (IS_ERR(display))
 777		return ERR_CAST(display);
 778
 779	i915->display = display;
 780
 781	return i915;
 782}
 783
 784/**
 785 * i915_driver_probe - setup chip and create an initial config
 786 * @pdev: PCI device
 787 * @ent: matching PCI ID entry
 788 *
 789 * The driver probe routine has to do several things:
 790 *   - drive output discovery via intel_display_driver_probe()
 791 *   - initialize the memory manager
 792 *   - allocate initial config memory
 793 *   - setup the DRM framebuffer with the allocated memory
 794 */
 795int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 796{
 797	struct drm_i915_private *i915;
 798	struct intel_display *display;
 799	int ret;
 800
 801	ret = pci_enable_device(pdev);
 802	if (ret) {
 803		pr_err("Failed to enable graphics device: %pe\n", ERR_PTR(ret));
 804		return ret;
 805	}
 806
 807	i915 = i915_driver_create(pdev, ent);
 808	if (IS_ERR(i915)) {
 809		pci_disable_device(pdev);
 810		return PTR_ERR(i915);
 811	}
 812
 813	display = i915->display;
 814
 815	ret = i915_driver_early_probe(i915);
 816	if (ret < 0)
 817		goto out_pci_disable;
 818
 819	disable_rpm_wakeref_asserts(&i915->runtime_pm);
 820
 821	intel_vgpu_detect(i915);
 822
 823	ret = intel_gt_probe_all(i915);
 824	if (ret < 0)
 825		goto out_runtime_pm_put;
 826
 827	ret = i915_driver_mmio_probe(i915);
 828	if (ret < 0)
 829		goto out_runtime_pm_put;
 830
 831	ret = i915_driver_hw_probe(i915);
 832	if (ret < 0)
 833		goto out_cleanup_mmio;
 834
 835	ret = intel_display_driver_probe_noirq(display);
 836	if (ret < 0)
 837		goto out_cleanup_hw;
 838
 839	ret = intel_irq_install(i915);
 840	if (ret)
 841		goto out_cleanup_modeset;
 842
 843	ret = intel_display_driver_probe_nogem(display);
 844	if (ret)
 845		goto out_cleanup_irq;
 846
 847	ret = i915_gem_init(i915);
 848	if (ret)
 849		goto out_cleanup_modeset2;
 850
 851	ret = intel_pxp_init(i915);
 852	if (ret && ret != -ENODEV)
 853		drm_dbg(&i915->drm, "pxp init failed with %d\n", ret);
 854
 855	ret = intel_display_driver_probe(display);
 856	if (ret)
 857		goto out_cleanup_gem;
 858
 859	ret = i915_driver_register(i915);
 860	if (ret)
 861		goto out_cleanup_gem;
 862
 863	enable_rpm_wakeref_asserts(&i915->runtime_pm);
 864
 865	i915_welcome_messages(i915);
 866
 867	i915->do_release = true;
 868
 869	return 0;
 870
 871out_cleanup_gem:
 872	intel_pxp_fini(i915);
 873	i915_gem_suspend(i915);
 874	i915_gem_driver_remove(i915);
 875	i915_gem_driver_release(i915);
 876out_cleanup_modeset2:
 877	/* FIXME clean up the error path */
 878	intel_display_driver_remove(display);
 879	intel_irq_uninstall(i915);
 880	intel_display_driver_remove_noirq(display);
 881	goto out_cleanup_modeset;
 882out_cleanup_irq:
 883	intel_irq_uninstall(i915);
 884out_cleanup_modeset:
 885	intel_display_driver_remove_nogem(display);
 886out_cleanup_hw:
 887	i915_driver_hw_remove(i915);
 888	intel_memory_regions_driver_release(i915);
 889	i915_ggtt_driver_release(i915);
 890	i915_gem_drain_freed_objects(i915);
 891	i915_ggtt_driver_late_release(i915);
 892out_cleanup_mmio:
 893	i915_driver_mmio_release(i915);
 894out_runtime_pm_put:
 895	enable_rpm_wakeref_asserts(&i915->runtime_pm);
 896	i915_driver_late_release(i915);
 897out_pci_disable:
 898	pci_disable_device(pdev);
 899	i915_probe_error(i915, "Device initialization failed (%d)\n", ret);
 900	return ret;
 901}
 902
 903void i915_driver_remove(struct drm_i915_private *i915)
 904{
 905	struct intel_display *display = i915->display;
 906	intel_wakeref_t wakeref;
 907
 908	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 909
 910	i915_driver_unregister(i915);
 911
 912	/* Flush any external code that still may be under the RCU lock */
 913	synchronize_rcu();
 914
 915	i915_gem_suspend(i915);
 916
 917	intel_gvt_driver_remove(i915);
 918
 919	intel_display_driver_remove(display);
 920
 921	intel_irq_uninstall(i915);
 922
 923	intel_display_driver_remove_noirq(display);
 924
 925	i915_reset_error_state(i915);
 926	i915_gem_driver_remove(i915);
 927
 928	intel_display_driver_remove_nogem(display);
 929
 930	i915_driver_hw_remove(i915);
 931
 932	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 933}
 934
 935static void i915_driver_release(struct drm_device *dev)
 936{
 937	struct drm_i915_private *dev_priv = to_i915(dev);
 938	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
 939	intel_wakeref_t wakeref;
 940
 941	if (!dev_priv->do_release)
 942		return;
 943
 944	wakeref = intel_runtime_pm_get(rpm);
 945
 946	i915_gem_driver_release(dev_priv);
 947
 948	intel_memory_regions_driver_release(dev_priv);
 949	i915_ggtt_driver_release(dev_priv);
 950	i915_gem_drain_freed_objects(dev_priv);
 951	i915_ggtt_driver_late_release(dev_priv);
 952
 953	i915_driver_mmio_release(dev_priv);
 954
 955	intel_runtime_pm_put(rpm, wakeref);
 956
 957	intel_runtime_pm_driver_release(rpm);
 958
 959	i915_driver_late_release(dev_priv);
 960}
 961
 962static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
 963{
 964	struct drm_i915_private *i915 = to_i915(dev);
 965	int ret;
 966
 967	ret = i915_gem_open(i915, file);
 968	if (ret)
 969		return ret;
 970
 971	return 0;
 972}
 973
 974static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
 975{
 976	struct drm_i915_file_private *file_priv = file->driver_priv;
 977
 978	i915_gem_context_close(file);
 979	i915_drm_client_put(file_priv->client);
 980
 981	kfree_rcu(file_priv, rcu);
 982
 983	/* Catch up with all the deferred frees from "this" client */
 984	i915_gem_flush_free_objects(to_i915(dev));
 985}
 986
 987void i915_driver_shutdown(struct drm_i915_private *i915)
 988{
 989	struct intel_display *display = i915->display;
 990
 991	disable_rpm_wakeref_asserts(&i915->runtime_pm);
 992	intel_runtime_pm_disable(&i915->runtime_pm);
 993	intel_power_domains_disable(display);
 994
 995	drm_client_dev_suspend(&i915->drm);
 996	if (intel_display_device_present(display)) {
 997		drm_kms_helper_poll_disable(&i915->drm);
 998		intel_display_driver_disable_user_access(display);
 999
1000		drm_atomic_helper_shutdown(&i915->drm);
1001	}
1002
1003	intel_dp_mst_suspend(display);
1004
1005	intel_irq_suspend(i915);
1006	intel_hpd_cancel_work(display);
1007
1008	if (intel_display_device_present(display))
1009		intel_display_driver_suspend_access(display);
1010
1011	intel_encoder_suspend_all(display);
1012	intel_encoder_shutdown_all(display);
1013
1014	intel_dmc_suspend(display);
1015
1016	i915_gem_suspend(i915);
1017
1018	/*
1019	 * The only requirement is to reboot with display DC states disabled,
1020	 * for now leaving all display power wells in the INIT power domain
1021	 * enabled.
1022	 *
1023	 * TODO:
1024	 * - unify the pci_driver::shutdown sequence here with the
1025	 *   pci_driver.driver.pm.poweroff,poweroff_late sequence.
1026	 * - unify the driver remove and system/runtime suspend sequences with
1027	 *   the above unified shutdown/poweroff sequence.
1028	 */
1029	intel_power_domains_driver_remove(display);
1030	enable_rpm_wakeref_asserts(&i915->runtime_pm);
1031
1032	intel_runtime_pm_driver_last_release(&i915->runtime_pm);
1033}
1034
1035static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1036{
1037#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1038	if (acpi_target_system_state() < ACPI_STATE_S3)
1039		return true;
1040#endif
1041	return false;
1042}
1043
1044static void i915_drm_complete(struct drm_device *dev)
1045{
1046	struct drm_i915_private *i915 = to_i915(dev);
1047
1048	intel_pxp_resume_complete(i915->pxp);
1049}
1050
1051static int i915_drm_prepare(struct drm_device *dev)
1052{
1053	struct drm_i915_private *i915 = to_i915(dev);
1054
1055	intel_pxp_suspend_prepare(i915->pxp);
1056
1057	/*
1058	 * NB intel_display_driver_suspend() may issue new requests after we've
1059	 * ostensibly marked the GPU as ready-to-sleep here. We need to
1060	 * split out that work and pull it forward so that after point,
1061	 * the GPU is not woken again.
1062	 */
1063	return i915_gem_backup_suspend(i915);
1064}
1065
1066static int i915_drm_suspend(struct drm_device *dev)
1067{
1068	struct drm_i915_private *dev_priv = to_i915(dev);
1069	struct intel_display *display = dev_priv->display;
1070	pci_power_t opregion_target_state;
1071
1072	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1073
1074	/* We do a lot of poking in a lot of registers, make sure they work
1075	 * properly. */
1076	intel_power_domains_disable(display);
1077	drm_client_dev_suspend(dev);
1078	if (intel_display_device_present(display)) {
1079		drm_kms_helper_poll_disable(dev);
1080		intel_display_driver_disable_user_access(display);
1081	}
1082
1083	intel_display_driver_suspend(display);
1084
1085	intel_irq_suspend(dev_priv);
1086	intel_hpd_cancel_work(display);
1087
1088	if (intel_display_device_present(display))
1089		intel_display_driver_suspend_access(display);
1090
1091	intel_encoder_suspend_all(display);
1092
1093	/* Must be called before GGTT is suspended. */
1094	intel_dpt_suspend(display);
1095	i915_ggtt_suspend(to_gt(dev_priv)->ggtt);
1096
1097	i9xx_display_sr_save(display);
1098
1099	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
1100	intel_opregion_suspend(display, opregion_target_state);
1101
1102	dev_priv->suspend_count++;
1103
1104	intel_dmc_suspend(display);
1105
1106	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1107
1108	i915_gem_drain_freed_objects(dev_priv);
1109
1110	return 0;
1111}
1112
1113static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1114{
1115	struct drm_i915_private *dev_priv = to_i915(dev);
1116	struct intel_display *display = dev_priv->display;
1117	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1118	struct intel_gt *gt;
1119	int ret, i;
1120	bool s2idle = !hibernation && suspend_to_idle(dev_priv);
1121
1122	disable_rpm_wakeref_asserts(rpm);
1123
1124	intel_pxp_suspend(dev_priv->pxp);
1125
1126	i915_gem_suspend_late(dev_priv);
1127
1128	for_each_gt(gt, dev_priv, i)
1129		intel_uncore_suspend(gt->uncore);
1130
1131	intel_display_power_suspend_late(display, s2idle);
1132
1133	ret = vlv_suspend_complete(dev_priv);
1134	if (ret) {
1135		drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);
1136		intel_display_power_resume_early(display);
1137	}
1138
1139	enable_rpm_wakeref_asserts(rpm);
1140
1141	if (!dev_priv->uncore.user_forcewake_count)
1142		intel_runtime_pm_driver_release(rpm);
1143
1144	return ret;
1145}
1146
1147static int i915_drm_suspend_noirq(struct drm_device *dev, bool hibernation)
1148{
1149	struct drm_i915_private *dev_priv = to_i915(dev);
1150	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1151
1152	/*
1153	 * During hibernation on some platforms the BIOS may try to access
1154	 * the device even though it's already in D3 and hang the machine. So
1155	 * leave the device in D0 on those platforms and hope the BIOS will
1156	 * power down the device properly. The issue was seen on multiple old
1157	 * GENs with different BIOS vendors, so having an explicit blacklist
1158	 * is impractical; apply the workaround on everything pre GEN6. The
1159	 * platforms where the issue was seen:
1160	 * Lenovo Thinkpad X301, X61s, X60, T60, X41
1161	 * Fujitsu FSC S7110
1162	 * Acer Aspire 1830T
1163	 *
1164	 * pci_save_state() prevents drivers/pci from
1165	 * automagically putting the device into D3.
1166	 */
1167	if (hibernation && GRAPHICS_VER(dev_priv) < 6)
1168		pci_save_state(pdev);
1169
1170	return 0;
1171}
1172
1173int i915_driver_suspend_switcheroo(struct drm_i915_private *i915,
1174				   pm_message_t state)
1175{
1176	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1177	int error;
1178
1179	if (drm_WARN_ON_ONCE(&i915->drm, state.event != PM_EVENT_SUSPEND &&
1180			     state.event != PM_EVENT_FREEZE))
1181		return -EINVAL;
1182
1183	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1184		return 0;
1185
1186	error = i915_drm_suspend(&i915->drm);
1187	if (error)
1188		return error;
1189
1190	error = i915_drm_suspend_late(&i915->drm, false);
1191	if (error)
1192		return error;
1193
1194	pci_save_state(pdev);
1195	pci_set_power_state(pdev, PCI_D3hot);
1196
1197	return 0;
1198}
1199
1200static int i915_drm_resume(struct drm_device *dev)
1201{
1202	struct drm_i915_private *dev_priv = to_i915(dev);
1203	struct intel_display *display = dev_priv->display;
1204	struct intel_gt *gt;
1205	int ret, i;
1206
1207	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1208
1209	ret = i915_pcode_init(dev_priv);
1210	if (ret)
1211		return ret;
1212
1213	sanitize_gpu(dev_priv);
1214
1215	ret = i915_ggtt_enable_hw(dev_priv);
1216	if (ret)
1217		drm_err(&dev_priv->drm, "failed to re-enable GGTT\n");
1218
1219	i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1220
1221	for_each_gt(gt, dev_priv, i)
1222		if (GRAPHICS_VER(gt->i915) >= 8)
1223			setup_private_pat(gt);
1224
1225	/* Must be called after GGTT is resumed. */
1226	intel_dpt_resume(display);
1227
1228	intel_dmc_resume(display);
1229
1230	i9xx_display_sr_restore(display);
1231
1232	intel_gmbus_reset(display);
1233
1234	intel_pps_unlock_regs_wa(display);
1235
1236	intel_init_pch_refclk(display);
1237
1238	/*
1239	 * Interrupts have to be enabled before any batches are run. If not the
1240	 * GPU will hang. i915_gem_init_hw() will initiate batches to
1241	 * update/restore the context.
1242	 *
1243	 * drm_mode_config_reset() needs AUX interrupts.
1244	 *
1245	 * Modeset enabling in intel_display_driver_init_hw() also needs working
1246	 * interrupts.
1247	 */
1248	intel_irq_resume(dev_priv);
1249
1250	if (intel_display_device_present(display))
1251		drm_mode_config_reset(dev);
1252
1253	i915_gem_resume(dev_priv);
1254
1255	intel_display_driver_init_hw(display);
1256
1257	intel_clock_gating_init(dev_priv);
1258
1259	if (intel_display_device_present(display))
1260		intel_display_driver_resume_access(display);
1261
1262	intel_hpd_init(display);
1263
1264	intel_display_driver_resume(display);
1265
1266	if (intel_display_device_present(display)) {
1267		intel_display_driver_enable_user_access(display);
1268		drm_kms_helper_poll_enable(dev);
1269	}
1270	intel_hpd_poll_disable(display);
1271
1272	intel_opregion_resume(display);
1273
1274	drm_client_dev_resume(dev);
1275
1276	intel_power_domains_enable(display);
1277
1278	intel_gvt_resume(dev_priv);
1279
1280	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1281
1282	return 0;
1283}
1284
1285static int i915_drm_resume_early(struct drm_device *dev)
1286{
1287	struct drm_i915_private *dev_priv = to_i915(dev);
1288	struct intel_display *display = dev_priv->display;
1289	struct intel_gt *gt;
1290	int ret, i;
1291
1292	/*
1293	 * We have a resume ordering issue with the snd-hda driver also
1294	 * requiring our device to be power up. Due to the lack of a
1295	 * parent/child relationship we currently solve this with an early
1296	 * resume hook.
1297	 *
1298	 * FIXME: This should be solved with a special hdmi sink device or
1299	 * similar so that power domains can be employed.
1300	 */
1301
1302	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1303
1304	ret = vlv_resume_prepare(dev_priv, false);
1305	if (ret)
1306		drm_err(&dev_priv->drm,
1307			"Resume prepare failed: %d, continuing anyway\n", ret);
1308
1309	for_each_gt(gt, dev_priv, i)
1310		intel_gt_resume_early(gt);
1311
1312	intel_display_power_resume_early(display);
1313
1314	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1315
1316	return ret;
1317}
1318
1319int i915_driver_resume_switcheroo(struct drm_i915_private *i915)
1320{
1321	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1322	int ret;
1323
1324	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1325		return 0;
1326
1327	ret = pci_set_power_state(pdev, PCI_D0);
1328	if (ret)
1329		return ret;
1330
1331	pci_restore_state(pdev);
1332
1333	ret = i915_drm_resume_early(&i915->drm);
1334	if (ret)
1335		return ret;
1336
1337	return i915_drm_resume(&i915->drm);
1338}
1339
1340static int i915_pm_prepare(struct device *kdev)
1341{
1342	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1343
1344	if (!i915) {
1345		dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1346		return -ENODEV;
1347	}
1348
1349	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1350		return 0;
1351
1352	return i915_drm_prepare(&i915->drm);
1353}
1354
1355static int i915_pm_suspend(struct device *kdev)
1356{
1357	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1358
1359	if (!i915) {
1360		dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1361		return -ENODEV;
1362	}
1363
1364	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1365		return 0;
1366
1367	return i915_drm_suspend(&i915->drm);
1368}
1369
1370static int i915_pm_suspend_late(struct device *kdev)
1371{
1372	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1373
1374	/*
1375	 * We have a suspend ordering issue with the snd-hda driver also
1376	 * requiring our device to be power up. Due to the lack of a
1377	 * parent/child relationship we currently solve this with an late
1378	 * suspend hook.
1379	 *
1380	 * FIXME: This should be solved with a special hdmi sink device or
1381	 * similar so that power domains can be employed.
1382	 */
1383	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1384		return 0;
1385
1386	return i915_drm_suspend_late(&i915->drm, false);
1387}
1388
1389static int i915_pm_suspend_noirq(struct device *kdev)
1390{
1391	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1392
1393	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1394		return 0;
1395
1396	return i915_drm_suspend_noirq(&i915->drm, false);
1397}
1398
1399static int i915_pm_poweroff_late(struct device *kdev)
1400{
1401	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1402
1403	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1404		return 0;
1405
1406	return i915_drm_suspend_late(&i915->drm, true);
1407}
1408
1409static int i915_pm_poweroff_noirq(struct device *kdev)
1410{
1411	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1412
1413	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1414		return 0;
1415
1416	return i915_drm_suspend_noirq(&i915->drm, true);
1417}
1418
1419static int i915_pm_resume_early(struct device *kdev)
1420{
1421	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1422
1423	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1424		return 0;
1425
1426	return i915_drm_resume_early(&i915->drm);
1427}
1428
1429static int i915_pm_resume(struct device *kdev)
1430{
1431	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1432
1433	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1434		return 0;
1435
1436	return i915_drm_resume(&i915->drm);
1437}
1438
1439static void i915_pm_complete(struct device *kdev)
1440{
1441	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1442
1443	if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1444		return;
1445
1446	i915_drm_complete(&i915->drm);
1447}
1448
1449/* freeze: before creating the hibernation_image */
1450static int i915_pm_freeze(struct device *kdev)
1451{
1452	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1453	int ret;
1454
1455	if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
1456		ret = i915_drm_suspend(&i915->drm);
1457		if (ret)
1458			return ret;
1459	}
1460
1461	ret = i915_gem_freeze(i915);
1462	if (ret)
1463		return ret;
1464
1465	return 0;
1466}
1467
1468static int i915_pm_freeze_late(struct device *kdev)
1469{
1470	struct drm_i915_private *i915 = kdev_to_i915(kdev);
1471	int ret;
1472
1473	if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
1474		ret = i915_drm_suspend_late(&i915->drm, true);
1475		if (ret)
1476			return ret;
1477	}
1478
1479	ret = i915_gem_freeze_late(i915);
1480	if (ret)
1481		return ret;
1482
1483	return 0;
1484}
1485
1486/* thaw: called after creating the hibernation image, but before turning off. */
1487static int i915_pm_thaw_early(struct device *kdev)
1488{
1489	return i915_pm_resume_early(kdev);
1490}
1491
1492static int i915_pm_thaw(struct device *kdev)
1493{
1494	return i915_pm_resume(kdev);
1495}
1496
1497/* restore: called after loading the hibernation image. */
1498static int i915_pm_restore_early(struct device *kdev)
1499{
1500	return i915_pm_resume_early(kdev);
1501}
1502
1503static int i915_pm_restore(struct device *kdev)
1504{
1505	return i915_pm_resume(kdev);
1506}
1507
1508static int intel_runtime_suspend(struct device *kdev)
1509{
1510	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
1511	struct intel_display *display = dev_priv->display;
1512	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1513	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1514	struct pci_dev *root_pdev;
1515	struct intel_gt *gt;
1516	int ret, i;
1517
1518	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
1519		return -ENODEV;
1520
1521	drm_dbg(&dev_priv->drm, "Suspending device\n");
1522
1523	disable_rpm_wakeref_asserts(rpm);
1524
1525	/*
1526	 * We are safe here against re-faults, since the fault handler takes
1527	 * an RPM reference.
1528	 */
1529	i915_gem_runtime_suspend(dev_priv);
1530
1531	intel_pxp_runtime_suspend(dev_priv->pxp);
1532
1533	for_each_gt(gt, dev_priv, i)
1534		intel_gt_runtime_suspend(gt);
1535
1536	intel_irq_suspend(dev_priv);
1537
1538	for_each_gt(gt, dev_priv, i)
1539		intel_uncore_suspend(gt->uncore);
1540
1541	intel_display_power_suspend(display);
1542
1543	ret = vlv_suspend_complete(dev_priv);
1544	if (ret) {
1545		drm_err(&dev_priv->drm,
1546			"Runtime suspend failed, disabling it (%d)\n", ret);
1547		intel_uncore_runtime_resume(&dev_priv->uncore);
1548
1549		intel_irq_resume(dev_priv);
1550
1551		for_each_gt(gt, dev_priv, i)
1552			intel_gt_runtime_resume(gt);
1553
1554		enable_rpm_wakeref_asserts(rpm);
1555
1556		return ret;
1557	}
1558
1559	enable_rpm_wakeref_asserts(rpm);
1560	intel_runtime_pm_driver_release(rpm);
1561
1562	if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))
1563		drm_err(&dev_priv->drm,
1564			"Unclaimed access detected prior to suspending\n");
1565
1566	/*
1567	 * FIXME: Temporary hammer to avoid freezing the machine on our DGFX
1568	 * This should be totally removed when we handle the pci states properly
1569	 * on runtime PM.
1570	 */
1571	root_pdev = pcie_find_root_port(pdev);
1572	if (root_pdev)
1573		pci_d3cold_disable(root_pdev);
1574
1575	/*
1576	 * FIXME: We really should find a document that references the arguments
1577	 * used below!
1578	 */
1579	if (IS_BROADWELL(dev_priv)) {
1580		/*
1581		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
1582		 * being detected, and the call we do at intel_runtime_resume()
1583		 * won't be able to restore them. Since PCI_D3hot matches the
1584		 * actual specification and appears to be working, use it.
1585		 */
1586		intel_opregion_notify_adapter(display, PCI_D3hot);
1587	} else {
1588		/*
1589		 * current versions of firmware which depend on this opregion
1590		 * notification have repurposed the D1 definition to mean
1591		 * "runtime suspended" vs. what you would normally expect (D3)
1592		 * to distinguish it from notifications that might be sent via
1593		 * the suspend path.
1594		 */
1595		intel_opregion_notify_adapter(display, PCI_D1);
1596	}
1597
1598	assert_forcewakes_inactive(&dev_priv->uncore);
1599
1600	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
1601		intel_hpd_poll_enable(display);
1602
1603	drm_dbg(&dev_priv->drm, "Device suspended\n");
1604	return 0;
1605}
1606
1607static int intel_runtime_resume(struct device *kdev)
1608{
1609	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
1610	struct intel_display *display = dev_priv->display;
1611	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1612	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1613	struct pci_dev *root_pdev;
1614	struct intel_gt *gt;
1615	int ret, i;
1616
1617	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
1618		return -ENODEV;
1619
1620	drm_dbg(&dev_priv->drm, "Resuming device\n");
1621
1622	drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count));
1623	disable_rpm_wakeref_asserts(rpm);
1624
1625	intel_opregion_notify_adapter(display, PCI_D0);
1626
1627	root_pdev = pcie_find_root_port(pdev);
1628	if (root_pdev)
1629		pci_d3cold_enable(root_pdev);
1630
1631	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))
1632		drm_dbg(&dev_priv->drm,
1633			"Unclaimed access during suspend, bios?\n");
1634
1635	intel_display_power_resume(display);
1636
1637	ret = vlv_resume_prepare(dev_priv, true);
1638
1639	for_each_gt(gt, dev_priv, i)
1640		intel_uncore_runtime_resume(gt->uncore);
1641
1642	intel_irq_resume(dev_priv);
1643
1644	/*
1645	 * No point of rolling back things in case of an error, as the best
1646	 * we can do is to hope that things will still work (and disable RPM).
1647	 */
1648	for_each_gt(gt, dev_priv, i)
1649		intel_gt_runtime_resume(gt);
1650
1651	intel_pxp_runtime_resume(dev_priv->pxp);
1652
1653	/*
1654	 * On VLV/CHV display interrupts are part of the display
1655	 * power well, so hpd is reinitialized from there. For
1656	 * everyone else do it here.
1657	 */
1658	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
1659		intel_hpd_init(display);
1660		intel_hpd_poll_disable(display);
1661	}
1662
1663	skl_watermark_ipc_update(display);
1664
1665	enable_rpm_wakeref_asserts(rpm);
1666
1667	if (ret)
1668		drm_err(&dev_priv->drm,
1669			"Runtime resume failed, disabling it (%d)\n", ret);
1670	else
1671		drm_dbg(&dev_priv->drm, "Device resumed\n");
1672
1673	return ret;
1674}
1675
1676const struct dev_pm_ops i915_pm_ops = {
1677	/*
1678	 * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
1679	 * PMSG_RESUME]
1680	 */
1681	.prepare = i915_pm_prepare,
1682	.suspend = i915_pm_suspend,
1683	.suspend_late = i915_pm_suspend_late,
1684	.suspend_noirq = i915_pm_suspend_noirq,
1685	.resume_early = i915_pm_resume_early,
1686	.resume = i915_pm_resume,
1687	.complete = i915_pm_complete,
1688
1689	/*
1690	 * S4 event handlers
1691	 * @freeze*   : called (1) before creating the
1692	 *              hibernation image [PMSG_FREEZE] and
1693	 *              (2) after rebooting, before restoring
1694	 *              the image [PMSG_QUIESCE]
1695	 * @thaw*     : called (1) after creating the hibernation
1696	 *              image, before writing it [PMSG_THAW]
1697	 *              and (2) after failing to create or
1698	 *              restore the image [PMSG_RECOVER]
1699	 * @poweroff* : called after writing the hibernation
1700	 *              image, before rebooting [PMSG_HIBERNATE]
1701	 * @restore*  : called after rebooting and restoring the
1702	 *              hibernation image [PMSG_RESTORE]
1703	 */
1704	.freeze = i915_pm_freeze,
1705	.freeze_late = i915_pm_freeze_late,
1706	.thaw_early = i915_pm_thaw_early,
1707	.thaw = i915_pm_thaw,
1708	.poweroff = i915_pm_suspend,
1709	.poweroff_late = i915_pm_poweroff_late,
1710	.poweroff_noirq = i915_pm_poweroff_noirq,
1711	.restore_early = i915_pm_restore_early,
1712	.restore = i915_pm_restore,
1713
1714	/* S0ix (via runtime suspend) event handlers */
1715	.runtime_suspend = intel_runtime_suspend,
1716	.runtime_resume = intel_runtime_resume,
1717};
1718
1719static const struct file_operations i915_driver_fops = {
1720	.owner = THIS_MODULE,
1721	.open = drm_open,
1722	.release = drm_release_noglobal,
1723	.unlocked_ioctl = drm_ioctl,
1724	.mmap = i915_gem_mmap,
1725	.poll = drm_poll,
1726	.read = drm_read,
1727	.compat_ioctl = i915_ioc32_compat_ioctl,
1728	.llseek = noop_llseek,
1729#ifdef CONFIG_PROC_FS
1730	.show_fdinfo = drm_show_fdinfo,
1731#endif
1732	.fop_flags = FOP_UNSIGNED_OFFSET,
1733};
1734
1735static int
1736i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
1737			  struct drm_file *file)
1738{
1739	return -ENODEV;
1740}
1741
1742static const struct drm_ioctl_desc i915_ioctls[] = {
1743	DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1744	DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
1745	DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
1746	DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
1747	DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
1748	DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
1749	DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_RENDER_ALLOW),
1750	DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1751	DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
1752	DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
1753	DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1754	DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
1755	DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1756	DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1757	DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH),
1758	DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
1759	DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1760	DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1761	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, drm_invalid_op, DRM_AUTH),
1762	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),
1763	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
1764	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
1765	DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_RENDER_ALLOW),
1766	DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
1767	DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
1768	DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_RENDER_ALLOW),
1769	DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1770	DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1771	DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
1772	DRM_IOCTL_DEF_DRV(I915_GEM_CREATE_EXT, i915_gem_create_ext_ioctl, DRM_RENDER_ALLOW),
1773	DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
1774	DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
1775	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
1776	DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_OFFSET, i915_gem_mmap_offset_ioctl, DRM_RENDER_ALLOW),
1777	DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
1778	DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
1779	DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW),
1780	DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW),
1781	DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
1782	DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_crtc_get_pipe_from_crtc_id_ioctl, 0),
1783	DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
1784	DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER),
1785	DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER),
1786	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER),
1787	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER),
1788	DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_RENDER_ALLOW),
1789	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
1790	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
1791	DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
1792	DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
1793	DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
1794	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
1795	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
1796	DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW),
1797	DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_RENDER_ALLOW),
1798	DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_RENDER_ALLOW),
1799	DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_RENDER_ALLOW),
1800	DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW),
1801	DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW),
1802};
1803
1804/*
1805 * Interface history:
1806 *
1807 * 1.1: Original.
1808 * 1.2: Add Power Management
1809 * 1.3: Add vblank support
1810 * 1.4: Fix cmdbuffer path, add heap destroy
1811 * 1.5: Add vblank pipe configuration
1812 * 1.6: - New ioctl for scheduling buffer swaps on vertical blank
1813 *      - Support vertical blank on secondary display pipe
1814 */
1815#define DRIVER_MAJOR		1
1816#define DRIVER_MINOR		6
1817#define DRIVER_PATCHLEVEL	0
1818
1819static const struct drm_driver i915_drm_driver = {
1820	/* Don't use MTRRs here; the Xserver or userspace app should
1821	 * deal with them for Intel hardware.
1822	 */
1823	.driver_features =
1824	    DRIVER_GEM |
1825	    DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ |
1826	    DRIVER_SYNCOBJ_TIMELINE,
1827	.release = i915_driver_release,
1828	.open = i915_driver_open,
1829	.postclose = i915_driver_postclose,
1830	.show_fdinfo = PTR_IF(IS_ENABLED(CONFIG_PROC_FS), i915_drm_client_fdinfo),
1831
1832	.gem_prime_import = i915_gem_prime_import,
1833
1834	.dumb_create = i915_gem_dumb_create,
1835	.dumb_map_offset = i915_gem_dumb_mmap_offset,
1836
1837	INTEL_FBDEV_DRIVER_OPS,
1838
1839	.ioctls = i915_ioctls,
1840	.num_ioctls = ARRAY_SIZE(i915_ioctls),
1841	.fops = &i915_driver_fops,
1842	.name = DRIVER_NAME,
1843	.desc = DRIVER_DESC,
1844	.major = DRIVER_MAJOR,
1845	.minor = DRIVER_MINOR,
1846	.patchlevel = DRIVER_PATCHLEVEL,
1847};