i915_pci.c

   1/*
   2 * Copyright © 2016 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <drm/drm_color_mgmt.h>
  26#include <drm/drm_drv.h>
  27#include <drm/intel/pciids.h>
  28
  29#include "display/intel_display_driver.h"
  30#include "gt/intel_gt_regs.h"
  31#include "gt/intel_sa_media.h"
  32#include "gem/i915_gem_object_types.h"
  33
  34#include "i915_driver.h"
  35#include "i915_drv.h"
  36#include "i915_pci.h"
  37#include "i915_reg.h"
  38#include "intel_pci_config.h"
  39
  40__diag_push();
  41__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for device info");
  42
  43#define PLATFORM(x) .platform = (x)
  44#define GEN(x) \
  45	.__runtime.graphics.ip.ver = (x), \
  46	.__runtime.media.ip.ver = (x)
  47
  48#define LEGACY_CACHELEVEL \
  49	.cachelevel_to_pat = { \
  50		[I915_CACHE_NONE]   = 0, \
  51		[I915_CACHE_LLC]    = 1, \
  52		[I915_CACHE_L3_LLC] = 2, \
  53		[I915_CACHE_WT]     = 3, \
  54	}
  55
  56#define TGL_CACHELEVEL \
  57	.cachelevel_to_pat = { \
  58		[I915_CACHE_NONE]   = 3, \
  59		[I915_CACHE_LLC]    = 0, \
  60		[I915_CACHE_L3_LLC] = 0, \
  61		[I915_CACHE_WT]     = 2, \
  62	}
  63
  64#define MTL_CACHELEVEL \
  65	.cachelevel_to_pat = { \
  66		[I915_CACHE_NONE]   = 2, \
  67		[I915_CACHE_LLC]    = 3, \
  68		[I915_CACHE_L3_LLC] = 3, \
  69		[I915_CACHE_WT]     = 1, \
  70	}
  71
  72/* Keep in gen based order, and chronological order within a gen */
  73
  74#define GEN_DEFAULT_PAGE_SIZES \
  75	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K
  76
  77#define GEN_DEFAULT_REGIONS \
  78	.memory_regions = BIT(INTEL_REGION_SMEM) | BIT(INTEL_REGION_STOLEN_SMEM)
  79
  80#define I830_FEATURES \
  81	GEN(2), \
  82	.is_mobile = 1, \
  83	.gpu_reset_clobbers_display = true, \
  84	.has_3d_pipeline = 1, \
  85	.hws_needs_physical = 1, \
  86	.unfenced_needs_alignment = 1, \
  87	.platform_engine_mask = BIT(RCS0), \
  88	.has_snoop = true, \
  89	.has_coherent_ggtt = false, \
  90	.dma_mask_size = 32, \
  91	.max_pat_index = 3, \
  92	GEN_DEFAULT_PAGE_SIZES, \
  93	GEN_DEFAULT_REGIONS, \
  94	LEGACY_CACHELEVEL
  95
  96#define I845_FEATURES \
  97	GEN(2), \
  98	.has_3d_pipeline = 1, \
  99	.gpu_reset_clobbers_display = true, \
 100	.hws_needs_physical = 1, \
 101	.unfenced_needs_alignment = 1, \
 102	.platform_engine_mask = BIT(RCS0), \
 103	.has_snoop = true, \
 104	.has_coherent_ggtt = false, \
 105	.dma_mask_size = 32, \
 106	.max_pat_index = 3, \
 107	GEN_DEFAULT_PAGE_SIZES, \
 108	GEN_DEFAULT_REGIONS, \
 109	LEGACY_CACHELEVEL
 110
 111static const struct intel_device_info i830_info = {
 112	I830_FEATURES,
 113	PLATFORM(INTEL_I830),
 114};
 115
 116static const struct intel_device_info i845g_info = {
 117	I845_FEATURES,
 118	PLATFORM(INTEL_I845G),
 119};
 120
 121static const struct intel_device_info i85x_info = {
 122	I830_FEATURES,
 123	PLATFORM(INTEL_I85X),
 124};
 125
 126static const struct intel_device_info i865g_info = {
 127	I845_FEATURES,
 128	PLATFORM(INTEL_I865G),
 129};
 130
 131#define GEN3_FEATURES \
 132	GEN(3), \
 133	.gpu_reset_clobbers_display = true, \
 134	.platform_engine_mask = BIT(RCS0), \
 135	.has_3d_pipeline = 1, \
 136	.has_snoop = true, \
 137	.has_coherent_ggtt = true, \
 138	.dma_mask_size = 32, \
 139	.max_pat_index = 3, \
 140	GEN_DEFAULT_PAGE_SIZES, \
 141	GEN_DEFAULT_REGIONS, \
 142	LEGACY_CACHELEVEL
 143
 144static const struct intel_device_info i915g_info = {
 145	GEN3_FEATURES,
 146	PLATFORM(INTEL_I915G),
 147	.has_coherent_ggtt = false,
 148	.hws_needs_physical = 1,
 149	.unfenced_needs_alignment = 1,
 150};
 151
 152static const struct intel_device_info i915gm_info = {
 153	GEN3_FEATURES,
 154	PLATFORM(INTEL_I915GM),
 155	.is_mobile = 1,
 156	.hws_needs_physical = 1,
 157	.unfenced_needs_alignment = 1,
 158};
 159
 160static const struct intel_device_info i945g_info = {
 161	GEN3_FEATURES,
 162	PLATFORM(INTEL_I945G),
 163	.hws_needs_physical = 1,
 164	.unfenced_needs_alignment = 1,
 165};
 166
 167static const struct intel_device_info i945gm_info = {
 168	GEN3_FEATURES,
 169	PLATFORM(INTEL_I945GM),
 170	.is_mobile = 1,
 171	.hws_needs_physical = 1,
 172	.unfenced_needs_alignment = 1,
 173};
 174
 175static const struct intel_device_info g33_info = {
 176	GEN3_FEATURES,
 177	PLATFORM(INTEL_G33),
 178	.dma_mask_size = 36,
 179};
 180
 181static const struct intel_device_info pnv_g_info = {
 182	GEN3_FEATURES,
 183	PLATFORM(INTEL_PINEVIEW),
 184	.dma_mask_size = 36,
 185};
 186
 187static const struct intel_device_info pnv_m_info = {
 188	GEN3_FEATURES,
 189	PLATFORM(INTEL_PINEVIEW),
 190	.is_mobile = 1,
 191	.dma_mask_size = 36,
 192};
 193
 194#define GEN4_FEATURES \
 195	GEN(4), \
 196	.gpu_reset_clobbers_display = true, \
 197	.platform_engine_mask = BIT(RCS0), \
 198	.has_3d_pipeline = 1, \
 199	.has_snoop = true, \
 200	.has_coherent_ggtt = true, \
 201	.dma_mask_size = 36, \
 202	.max_pat_index = 3, \
 203	GEN_DEFAULT_PAGE_SIZES, \
 204	GEN_DEFAULT_REGIONS, \
 205	LEGACY_CACHELEVEL
 206
 207static const struct intel_device_info i965g_info = {
 208	GEN4_FEATURES,
 209	PLATFORM(INTEL_I965G),
 210	.hws_needs_physical = 1,
 211	.has_snoop = false,
 212};
 213
 214static const struct intel_device_info i965gm_info = {
 215	GEN4_FEATURES,
 216	PLATFORM(INTEL_I965GM),
 217	.is_mobile = 1,
 218	.hws_needs_physical = 1,
 219	.has_snoop = false,
 220};
 221
 222static const struct intel_device_info g45_info = {
 223	GEN4_FEATURES,
 224	PLATFORM(INTEL_G45),
 225	.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
 226	.gpu_reset_clobbers_display = false,
 227};
 228
 229static const struct intel_device_info gm45_info = {
 230	GEN4_FEATURES,
 231	PLATFORM(INTEL_GM45),
 232	.is_mobile = 1,
 233	.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
 234	.gpu_reset_clobbers_display = false,
 235};
 236
 237#define GEN5_FEATURES \
 238	GEN(5), \
 239	.platform_engine_mask = BIT(RCS0) | BIT(VCS0), \
 240	.has_3d_pipeline = 1, \
 241	.has_snoop = true, \
 242	.has_coherent_ggtt = true, \
 243	/* ilk does support rc6, but we do not implement [power] contexts */ \
 244	.has_rc6 = 0, \
 245	.dma_mask_size = 36, \
 246	.max_pat_index = 3, \
 247	GEN_DEFAULT_PAGE_SIZES, \
 248	GEN_DEFAULT_REGIONS, \
 249	LEGACY_CACHELEVEL
 250
 251static const struct intel_device_info ilk_d_info = {
 252	GEN5_FEATURES,
 253	PLATFORM(INTEL_IRONLAKE),
 254};
 255
 256static const struct intel_device_info ilk_m_info = {
 257	GEN5_FEATURES,
 258	PLATFORM(INTEL_IRONLAKE),
 259	.is_mobile = 1,
 260	.has_rps = true,
 261};
 262
 263#define GEN6_FEATURES \
 264	GEN(6), \
 265	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 266	.has_3d_pipeline = 1, \
 267	.has_coherent_ggtt = true, \
 268	.has_llc = 1, \
 269	.has_rc6 = 1, \
 270	/* snb does support rc6p, but enabling it causes various issues */ \
 271	.has_rc6p = 0, \
 272	.has_rps = true, \
 273	.dma_mask_size = 40, \
 274	.max_pat_index = 3, \
 275	.__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \
 276	.__runtime.ppgtt_size = 31, \
 277	GEN_DEFAULT_PAGE_SIZES, \
 278	GEN_DEFAULT_REGIONS, \
 279	LEGACY_CACHELEVEL
 280
 281#define SNB_D_PLATFORM \
 282	GEN6_FEATURES, \
 283	PLATFORM(INTEL_SANDYBRIDGE)
 284
 285static const struct intel_device_info snb_d_gt1_info = {
 286	SNB_D_PLATFORM,
 287	.gt = 1,
 288};
 289
 290static const struct intel_device_info snb_d_gt2_info = {
 291	SNB_D_PLATFORM,
 292	.gt = 2,
 293};
 294
 295#define SNB_M_PLATFORM \
 296	GEN6_FEATURES, \
 297	PLATFORM(INTEL_SANDYBRIDGE), \
 298	.is_mobile = 1
 299
 300
 301static const struct intel_device_info snb_m_gt1_info = {
 302	SNB_M_PLATFORM,
 303	.gt = 1,
 304};
 305
 306static const struct intel_device_info snb_m_gt2_info = {
 307	SNB_M_PLATFORM,
 308	.gt = 2,
 309};
 310
 311#define GEN7_FEATURES  \
 312	GEN(7), \
 313	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 314	.has_3d_pipeline = 1, \
 315	.has_coherent_ggtt = true, \
 316	.has_llc = 1, \
 317	.has_rc6 = 1, \
 318	.has_rc6p = 1, \
 319	.has_reset_engine = true, \
 320	.has_rps = true, \
 321	.dma_mask_size = 40, \
 322	.max_pat_index = 3, \
 323	.__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \
 324	.__runtime.ppgtt_size = 31, \
 325	GEN_DEFAULT_PAGE_SIZES, \
 326	GEN_DEFAULT_REGIONS, \
 327	LEGACY_CACHELEVEL
 328
 329#define IVB_D_PLATFORM \
 330	GEN7_FEATURES, \
 331	PLATFORM(INTEL_IVYBRIDGE), \
 332	.has_l3_dpf = 1
 333
 334static const struct intel_device_info ivb_d_gt1_info = {
 335	IVB_D_PLATFORM,
 336	.gt = 1,
 337};
 338
 339static const struct intel_device_info ivb_d_gt2_info = {
 340	IVB_D_PLATFORM,
 341	.gt = 2,
 342};
 343
 344#define IVB_M_PLATFORM \
 345	GEN7_FEATURES, \
 346	PLATFORM(INTEL_IVYBRIDGE), \
 347	.is_mobile = 1, \
 348	.has_l3_dpf = 1
 349
 350static const struct intel_device_info ivb_m_gt1_info = {
 351	IVB_M_PLATFORM,
 352	.gt = 1,
 353};
 354
 355static const struct intel_device_info ivb_m_gt2_info = {
 356	IVB_M_PLATFORM,
 357	.gt = 2,
 358};
 359
 360static const struct intel_device_info ivb_q_info = {
 361	GEN7_FEATURES,
 362	PLATFORM(INTEL_IVYBRIDGE),
 363	.gt = 2,
 364	.has_l3_dpf = 1,
 365};
 366
 367static const struct intel_device_info vlv_info = {
 368	PLATFORM(INTEL_VALLEYVIEW),
 369	GEN(7),
 370	.has_runtime_pm = 1,
 371	.has_rc6 = 1,
 372	.has_reset_engine = true,
 373	.has_rps = true,
 374	.dma_mask_size = 40,
 375	.max_pat_index = 3,
 376	.__runtime.ppgtt_type = INTEL_PPGTT_ALIASING,
 377	.__runtime.ppgtt_size = 31,
 378	.has_snoop = true,
 379	.has_coherent_ggtt = false,
 380	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0),
 381	GEN_DEFAULT_PAGE_SIZES,
 382	GEN_DEFAULT_REGIONS,
 383	LEGACY_CACHELEVEL,
 384};
 385
 386#define G75_FEATURES  \
 387	GEN7_FEATURES, \
 388	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
 389	.has_rc6p = 0 /* RC6p removed-by HSW */, \
 390	.has_runtime_pm = 1
 391
 392#define HSW_PLATFORM \
 393	G75_FEATURES, \
 394	PLATFORM(INTEL_HASWELL), \
 395	.has_l3_dpf = 1
 396
 397static const struct intel_device_info hsw_gt1_info = {
 398	HSW_PLATFORM,
 399	.gt = 1,
 400};
 401
 402static const struct intel_device_info hsw_gt2_info = {
 403	HSW_PLATFORM,
 404	.gt = 2,
 405};
 406
 407static const struct intel_device_info hsw_gt3_info = {
 408	HSW_PLATFORM,
 409	.gt = 3,
 410};
 411
 412#define GEN8_FEATURES \
 413	G75_FEATURES, \
 414	GEN(8), \
 415	.has_logical_ring_contexts = 1, \
 416	.dma_mask_size = 39, \
 417	.__runtime.ppgtt_type = INTEL_PPGTT_FULL, \
 418	.__runtime.ppgtt_size = 48, \
 419	.has_64bit_reloc = 1
 420
 421#define BDW_PLATFORM \
 422	GEN8_FEATURES, \
 423	PLATFORM(INTEL_BROADWELL)
 424
 425static const struct intel_device_info bdw_gt1_info = {
 426	BDW_PLATFORM,
 427	.gt = 1,
 428};
 429
 430static const struct intel_device_info bdw_gt2_info = {
 431	BDW_PLATFORM,
 432	.gt = 2,
 433};
 434
 435static const struct intel_device_info bdw_rsvd_info = {
 436	BDW_PLATFORM,
 437	.gt = 3,
 438	/* According to the device ID those devices are GT3, they were
 439	 * previously treated as not GT3, keep it like that.
 440	 */
 441};
 442
 443static const struct intel_device_info bdw_gt3_info = {
 444	BDW_PLATFORM,
 445	.gt = 3,
 446	.platform_engine_mask =
 447		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 448};
 449
 450static const struct intel_device_info chv_info = {
 451	PLATFORM(INTEL_CHERRYVIEW),
 452	GEN(8),
 453	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0),
 454	.has_64bit_reloc = 1,
 455	.has_runtime_pm = 1,
 456	.has_rc6 = 1,
 457	.has_rps = true,
 458	.has_logical_ring_contexts = 1,
 459	.dma_mask_size = 39,
 460	.max_pat_index = 3,
 461	.__runtime.ppgtt_type = INTEL_PPGTT_FULL,
 462	.__runtime.ppgtt_size = 32,
 463	.has_reset_engine = 1,
 464	.has_snoop = true,
 465	.has_coherent_ggtt = false,
 466	GEN_DEFAULT_PAGE_SIZES,
 467	GEN_DEFAULT_REGIONS,
 468	LEGACY_CACHELEVEL,
 469};
 470
 471#define GEN9_DEFAULT_PAGE_SIZES \
 472	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 473		I915_GTT_PAGE_SIZE_64K
 474
 475#define GEN9_FEATURES \
 476	GEN8_FEATURES, \
 477	GEN(9), \
 478	GEN9_DEFAULT_PAGE_SIZES, \
 479	.has_gt_uc = 1
 480
 481#define SKL_PLATFORM \
 482	GEN9_FEATURES, \
 483	PLATFORM(INTEL_SKYLAKE)
 484
 485static const struct intel_device_info skl_gt1_info = {
 486	SKL_PLATFORM,
 487	.gt = 1,
 488};
 489
 490static const struct intel_device_info skl_gt2_info = {
 491	SKL_PLATFORM,
 492	.gt = 2,
 493};
 494
 495#define SKL_GT3_PLUS_PLATFORM \
 496	SKL_PLATFORM, \
 497	.platform_engine_mask = \
 498		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1)
 499
 500
 501static const struct intel_device_info skl_gt3_info = {
 502	SKL_GT3_PLUS_PLATFORM,
 503	.gt = 3,
 504};
 505
 506static const struct intel_device_info skl_gt4_info = {
 507	SKL_GT3_PLUS_PLATFORM,
 508	.gt = 4,
 509};
 510
 511#define GEN9_LP_FEATURES \
 512	GEN(9), \
 513	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
 514	.has_3d_pipeline = 1, \
 515	.has_64bit_reloc = 1, \
 516	.has_runtime_pm = 1, \
 517	.has_rc6 = 1, \
 518	.has_rps = true, \
 519	.has_logical_ring_contexts = 1, \
 520	.has_gt_uc = 1, \
 521	.dma_mask_size = 39, \
 522	.__runtime.ppgtt_type = INTEL_PPGTT_FULL, \
 523	.__runtime.ppgtt_size = 48, \
 524	.has_reset_engine = 1, \
 525	.has_snoop = true, \
 526	.has_coherent_ggtt = false, \
 527	.max_pat_index = 3, \
 528	GEN9_DEFAULT_PAGE_SIZES, \
 529	GEN_DEFAULT_REGIONS, \
 530	LEGACY_CACHELEVEL
 531
 532static const struct intel_device_info bxt_info = {
 533	GEN9_LP_FEATURES,
 534	PLATFORM(INTEL_BROXTON),
 535};
 536
 537static const struct intel_device_info glk_info = {
 538	GEN9_LP_FEATURES,
 539	PLATFORM(INTEL_GEMINILAKE),
 540};
 541
 542#define KBL_PLATFORM \
 543	GEN9_FEATURES, \
 544	PLATFORM(INTEL_KABYLAKE)
 545
 546static const struct intel_device_info kbl_gt1_info = {
 547	KBL_PLATFORM,
 548	.gt = 1,
 549};
 550
 551static const struct intel_device_info kbl_gt2_info = {
 552	KBL_PLATFORM,
 553	.gt = 2,
 554};
 555
 556static const struct intel_device_info kbl_gt3_info = {
 557	KBL_PLATFORM,
 558	.gt = 3,
 559	.platform_engine_mask =
 560		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 561};
 562
 563#define CFL_PLATFORM \
 564	GEN9_FEATURES, \
 565	PLATFORM(INTEL_COFFEELAKE)
 566
 567static const struct intel_device_info cfl_gt1_info = {
 568	CFL_PLATFORM,
 569	.gt = 1,
 570};
 571
 572static const struct intel_device_info cfl_gt2_info = {
 573	CFL_PLATFORM,
 574	.gt = 2,
 575};
 576
 577static const struct intel_device_info cfl_gt3_info = {
 578	CFL_PLATFORM,
 579	.gt = 3,
 580	.platform_engine_mask =
 581		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 582};
 583
 584#define CML_PLATFORM \
 585	GEN9_FEATURES, \
 586	PLATFORM(INTEL_COMETLAKE)
 587
 588static const struct intel_device_info cml_gt1_info = {
 589	CML_PLATFORM,
 590	.gt = 1,
 591};
 592
 593static const struct intel_device_info cml_gt2_info = {
 594	CML_PLATFORM,
 595	.gt = 2,
 596};
 597
 598#define GEN11_DEFAULT_PAGE_SIZES \
 599	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 600		I915_GTT_PAGE_SIZE_64K |		\
 601		I915_GTT_PAGE_SIZE_2M
 602
 603#define GEN11_FEATURES \
 604	GEN9_FEATURES, \
 605	GEN11_DEFAULT_PAGE_SIZES, \
 606	GEN(11), \
 607	.has_coherent_ggtt = false, \
 608	.has_logical_ring_elsq = 1
 609
 610static const struct intel_device_info icl_info = {
 611	GEN11_FEATURES,
 612	PLATFORM(INTEL_ICELAKE),
 613	.platform_engine_mask =
 614		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 615};
 616
 617static const struct intel_device_info ehl_info = {
 618	GEN11_FEATURES,
 619	PLATFORM(INTEL_ELKHARTLAKE),
 620	.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
 621	.__runtime.ppgtt_size = 36,
 622};
 623
 624static const struct intel_device_info jsl_info = {
 625	GEN11_FEATURES,
 626	PLATFORM(INTEL_JASPERLAKE),
 627	.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
 628	.__runtime.ppgtt_size = 36,
 629};
 630
 631#define GEN12_FEATURES \
 632	GEN11_FEATURES, \
 633	GEN(12), \
 634	TGL_CACHELEVEL, \
 635	.has_global_mocs = 1, \
 636	.has_pxp = 1, \
 637	.max_pat_index = 3
 638
 639static const struct intel_device_info tgl_info = {
 640	GEN12_FEATURES,
 641	PLATFORM(INTEL_TIGERLAKE),
 642	.platform_engine_mask =
 643		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 644};
 645
 646static const struct intel_device_info rkl_info = {
 647	GEN12_FEATURES,
 648	PLATFORM(INTEL_ROCKETLAKE),
 649	.platform_engine_mask =
 650		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0),
 651};
 652
 653#define DGFX_FEATURES \
 654	.memory_regions = BIT(INTEL_REGION_SMEM) | BIT(INTEL_REGION_LMEM_0) | BIT(INTEL_REGION_STOLEN_LMEM), \
 655	.has_llc = 0, \
 656	.has_pxp = 0, \
 657	.has_snoop = 1, \
 658	.is_dgfx = 1, \
 659	.has_heci_gscfi = 1
 660
 661static const struct intel_device_info dg1_info = {
 662	GEN12_FEATURES,
 663	DGFX_FEATURES,
 664	.__runtime.graphics.ip.rel = 10,
 665	PLATFORM(INTEL_DG1),
 666	.platform_engine_mask =
 667		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) |
 668		BIT(VCS0) | BIT(VCS2),
 669	/* Wa_16011227922 */
 670	.__runtime.ppgtt_size = 47,
 671};
 672
 673static const struct intel_device_info adl_s_info = {
 674	GEN12_FEATURES,
 675	PLATFORM(INTEL_ALDERLAKE_S),
 676	.platform_engine_mask =
 677		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 678	.dma_mask_size = 39,
 679};
 680
 681static const struct intel_device_info adl_p_info = {
 682	GEN12_FEATURES,
 683	PLATFORM(INTEL_ALDERLAKE_P),
 684	.platform_engine_mask =
 685		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 686	.__runtime.ppgtt_size = 48,
 687	.dma_mask_size = 39,
 688};
 689
 690#undef GEN
 691
 692#define XE_HP_PAGE_SIZES \
 693	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 694		I915_GTT_PAGE_SIZE_64K |		\
 695		I915_GTT_PAGE_SIZE_2M
 696
 697#define XE_HP_FEATURES \
 698	XE_HP_PAGE_SIZES, \
 699	TGL_CACHELEVEL, \
 700	.dma_mask_size = 46, \
 701	.has_3d_pipeline = 1, \
 702	.has_64bit_reloc = 1, \
 703	.has_flat_ccs = 1, \
 704	.has_global_mocs = 1, \
 705	.has_gt_uc = 1, \
 706	.has_llc = 1, \
 707	.has_logical_ring_contexts = 1, \
 708	.has_logical_ring_elsq = 1, \
 709	.has_mslice_steering = 1, \
 710	.has_oa_bpc_reporting = 1, \
 711	.has_oa_slice_contrib_limits = 1, \
 712	.has_oam = 1, \
 713	.has_rc6 = 1, \
 714	.has_reset_engine = 1, \
 715	.has_rps = 1, \
 716	.has_runtime_pm = 1, \
 717	.max_pat_index = 3, \
 718	.__runtime.ppgtt_size = 48, \
 719	.__runtime.ppgtt_type = INTEL_PPGTT_FULL
 720
 721#define DG2_FEATURES \
 722	XE_HP_FEATURES, \
 723	DGFX_FEATURES, \
 724	.__runtime.graphics.ip.ver = 12, \
 725	.__runtime.graphics.ip.rel = 55, \
 726	.__runtime.media.ip.ver = 12, \
 727	.__runtime.media.ip.rel = 55, \
 728	PLATFORM(INTEL_DG2), \
 729	.has_64k_pages = 1, \
 730	.has_guc_deprivilege = 1, \
 731	.has_heci_pxp = 1, \
 732	.has_media_ratio_mode = 1, \
 733	.platform_engine_mask = \
 734		BIT(RCS0) | BIT(BCS0) | \
 735		BIT(VECS0) | BIT(VECS1) | \
 736		BIT(VCS0) | BIT(VCS2) | \
 737		BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3)
 738
 739static const struct intel_device_info dg2_info = {
 740	DG2_FEATURES,
 741};
 742
 743static const struct intel_device_info ats_m_info = {
 744	DG2_FEATURES,
 745	.require_force_probe = 1,
 746	.tuning_thread_rr_after_dep = 1,
 747};
 748
 749static const struct intel_gt_definition xelpmp_extra_gt[] = {
 750	{
 751		.type = GT_MEDIA,
 752		.name = "Standalone Media GT",
 753		.gsi_offset = MTL_MEDIA_GSI_BASE,
 754		.engine_mask = BIT(VECS0) | BIT(VCS0) | BIT(VCS2) | BIT(GSC0),
 755	},
 756	{}
 757};
 758
 759static const struct intel_device_info mtl_info = {
 760	XE_HP_FEATURES,
 761	/*
 762	 * Real graphics IP version will be obtained from hardware GMD_ID
 763	 * register.  Value provided here is just for sanity checking.
 764	 */
 765	.__runtime.graphics.ip.ver = 12,
 766	.__runtime.graphics.ip.rel = 70,
 767	.__runtime.media.ip.ver = 13,
 768	PLATFORM(INTEL_METEORLAKE),
 769	.extra_gt_list = xelpmp_extra_gt,
 770	.has_flat_ccs = 0,
 771	.has_gmd_id = 1,
 772	.has_guc_deprivilege = 1,
 773	.has_guc_tlb_invalidation = 1,
 774	.has_llc = 0,
 775	.has_mslice_steering = 0,
 776	.has_snoop = 1,
 777	.max_pat_index = 4,
 778	.has_pxp = 1,
 779	.memory_regions = BIT(INTEL_REGION_SMEM) | BIT(INTEL_REGION_STOLEN_LMEM),
 780	.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(CCS0),
 781	MTL_CACHELEVEL,
 782};
 783
 784#undef PLATFORM
 785
 786__diag_pop();
 787
 788/*
 789 * Make sure any device matches here are from most specific to most
 790 * general.  For example, since the Quanta match is based on the subsystem
 791 * and subvendor IDs, we need it to come before the more general IVB
 792 * PCI ID matches, otherwise we'll use the wrong info struct above.
 793 */
 794static const struct pci_device_id pciidlist[] = {
 795	INTEL_I830_IDS(INTEL_VGA_DEVICE, &i830_info),
 796	INTEL_I845G_IDS(INTEL_VGA_DEVICE, &i845g_info),
 797	INTEL_I85X_IDS(INTEL_VGA_DEVICE, &i85x_info),
 798	INTEL_I865G_IDS(INTEL_VGA_DEVICE, &i865g_info),
 799	INTEL_I915G_IDS(INTEL_VGA_DEVICE, &i915g_info),
 800	INTEL_I915GM_IDS(INTEL_VGA_DEVICE, &i915gm_info),
 801	INTEL_I945G_IDS(INTEL_VGA_DEVICE, &i945g_info),
 802	INTEL_I945GM_IDS(INTEL_VGA_DEVICE, &i945gm_info),
 803	INTEL_I965G_IDS(INTEL_VGA_DEVICE, &i965g_info),
 804	INTEL_G33_IDS(INTEL_VGA_DEVICE, &g33_info),
 805	INTEL_I965GM_IDS(INTEL_VGA_DEVICE, &i965gm_info),
 806	INTEL_GM45_IDS(INTEL_VGA_DEVICE, &gm45_info),
 807	INTEL_G45_IDS(INTEL_VGA_DEVICE, &g45_info),
 808	INTEL_PNV_G_IDS(INTEL_VGA_DEVICE, &pnv_g_info),
 809	INTEL_PNV_M_IDS(INTEL_VGA_DEVICE, &pnv_m_info),
 810	INTEL_ILK_D_IDS(INTEL_VGA_DEVICE, &ilk_d_info),
 811	INTEL_ILK_M_IDS(INTEL_VGA_DEVICE, &ilk_m_info),
 812	INTEL_SNB_D_GT1_IDS(INTEL_VGA_DEVICE, &snb_d_gt1_info),
 813	INTEL_SNB_D_GT2_IDS(INTEL_VGA_DEVICE, &snb_d_gt2_info),
 814	INTEL_SNB_M_GT1_IDS(INTEL_VGA_DEVICE, &snb_m_gt1_info),
 815	INTEL_SNB_M_GT2_IDS(INTEL_VGA_DEVICE, &snb_m_gt2_info),
 816	INTEL_IVB_Q_IDS(INTEL_VGA_DEVICE, &ivb_q_info), /* must be first IVB */
 817	INTEL_IVB_M_GT1_IDS(INTEL_VGA_DEVICE, &ivb_m_gt1_info),
 818	INTEL_IVB_M_GT2_IDS(INTEL_VGA_DEVICE, &ivb_m_gt2_info),
 819	INTEL_IVB_D_GT1_IDS(INTEL_VGA_DEVICE, &ivb_d_gt1_info),
 820	INTEL_IVB_D_GT2_IDS(INTEL_VGA_DEVICE, &ivb_d_gt2_info),
 821	INTEL_HSW_GT1_IDS(INTEL_VGA_DEVICE, &hsw_gt1_info),
 822	INTEL_HSW_GT2_IDS(INTEL_VGA_DEVICE, &hsw_gt2_info),
 823	INTEL_HSW_GT3_IDS(INTEL_VGA_DEVICE, &hsw_gt3_info),
 824	INTEL_VLV_IDS(INTEL_VGA_DEVICE, &vlv_info),
 825	INTEL_BDW_GT1_IDS(INTEL_VGA_DEVICE, &bdw_gt1_info),
 826	INTEL_BDW_GT2_IDS(INTEL_VGA_DEVICE, &bdw_gt2_info),
 827	INTEL_BDW_GT3_IDS(INTEL_VGA_DEVICE, &bdw_gt3_info),
 828	INTEL_BDW_RSVD_IDS(INTEL_VGA_DEVICE, &bdw_rsvd_info),
 829	INTEL_CHV_IDS(INTEL_VGA_DEVICE, &chv_info),
 830	INTEL_SKL_GT1_IDS(INTEL_VGA_DEVICE, &skl_gt1_info),
 831	INTEL_SKL_GT2_IDS(INTEL_VGA_DEVICE, &skl_gt2_info),
 832	INTEL_SKL_GT3_IDS(INTEL_VGA_DEVICE, &skl_gt3_info),
 833	INTEL_SKL_GT4_IDS(INTEL_VGA_DEVICE, &skl_gt4_info),
 834	INTEL_BXT_IDS(INTEL_VGA_DEVICE, &bxt_info),
 835	INTEL_GLK_IDS(INTEL_VGA_DEVICE, &glk_info),
 836	INTEL_KBL_GT1_IDS(INTEL_VGA_DEVICE, &kbl_gt1_info),
 837	INTEL_KBL_GT2_IDS(INTEL_VGA_DEVICE, &kbl_gt2_info),
 838	INTEL_KBL_GT3_IDS(INTEL_VGA_DEVICE, &kbl_gt3_info),
 839	INTEL_KBL_GT4_IDS(INTEL_VGA_DEVICE, &kbl_gt3_info),
 840	INTEL_AML_KBL_GT2_IDS(INTEL_VGA_DEVICE, &kbl_gt2_info),
 841	INTEL_CFL_S_GT1_IDS(INTEL_VGA_DEVICE, &cfl_gt1_info),
 842	INTEL_CFL_S_GT2_IDS(INTEL_VGA_DEVICE, &cfl_gt2_info),
 843	INTEL_CFL_H_GT1_IDS(INTEL_VGA_DEVICE, &cfl_gt1_info),
 844	INTEL_CFL_H_GT2_IDS(INTEL_VGA_DEVICE, &cfl_gt2_info),
 845	INTEL_CFL_U_GT2_IDS(INTEL_VGA_DEVICE, &cfl_gt2_info),
 846	INTEL_CFL_U_GT3_IDS(INTEL_VGA_DEVICE, &cfl_gt3_info),
 847	INTEL_WHL_U_GT1_IDS(INTEL_VGA_DEVICE, &cfl_gt1_info),
 848	INTEL_WHL_U_GT2_IDS(INTEL_VGA_DEVICE, &cfl_gt2_info),
 849	INTEL_AML_CFL_GT2_IDS(INTEL_VGA_DEVICE, &cfl_gt2_info),
 850	INTEL_WHL_U_GT3_IDS(INTEL_VGA_DEVICE, &cfl_gt3_info),
 851	INTEL_CML_GT1_IDS(INTEL_VGA_DEVICE, &cml_gt1_info),
 852	INTEL_CML_GT2_IDS(INTEL_VGA_DEVICE, &cml_gt2_info),
 853	INTEL_CML_U_GT1_IDS(INTEL_VGA_DEVICE, &cml_gt1_info),
 854	INTEL_CML_U_GT2_IDS(INTEL_VGA_DEVICE, &cml_gt2_info),
 855	INTEL_ICL_IDS(INTEL_VGA_DEVICE, &icl_info),
 856	INTEL_EHL_IDS(INTEL_VGA_DEVICE, &ehl_info),
 857	INTEL_JSL_IDS(INTEL_VGA_DEVICE, &jsl_info),
 858	INTEL_TGL_IDS(INTEL_VGA_DEVICE, &tgl_info),
 859	INTEL_RKL_IDS(INTEL_VGA_DEVICE, &rkl_info),
 860	INTEL_ADLS_IDS(INTEL_VGA_DEVICE, &adl_s_info),
 861	INTEL_ADLP_IDS(INTEL_VGA_DEVICE, &adl_p_info),
 862	INTEL_ADLN_IDS(INTEL_VGA_DEVICE, &adl_p_info),
 863	INTEL_DG1_IDS(INTEL_VGA_DEVICE, &dg1_info),
 864	INTEL_RPLS_IDS(INTEL_VGA_DEVICE, &adl_s_info),
 865	INTEL_RPLU_IDS(INTEL_VGA_DEVICE, &adl_p_info),
 866	INTEL_RPLP_IDS(INTEL_VGA_DEVICE, &adl_p_info),
 867	INTEL_DG2_IDS(INTEL_VGA_DEVICE, &dg2_info),
 868	INTEL_ATS_M_IDS(INTEL_VGA_DEVICE, &ats_m_info),
 869	INTEL_ARL_IDS(INTEL_VGA_DEVICE, &mtl_info),
 870	INTEL_MTL_IDS(INTEL_VGA_DEVICE, &mtl_info),
 871	{}
 872};
 873MODULE_DEVICE_TABLE(pci, pciidlist);
 874
 875static void i915_pci_remove(struct pci_dev *pdev)
 876{
 877	struct drm_i915_private *i915;
 878
 879	i915 = pdev_to_i915(pdev);
 880	if (!i915) /* driver load aborted, nothing to cleanup */
 881		return;
 882
 883	i915_driver_remove(i915);
 884	pci_set_drvdata(pdev, NULL);
 885}
 886
 887/* is device_id present in comma separated list of ids */
 888static bool device_id_in_list(u16 device_id, const char *devices, bool negative)
 889{
 890	char *s, *p, *tok;
 891	bool ret;
 892
 893	if (!devices || !*devices)
 894		return false;
 895
 896	/* match everything */
 897	if (negative && strcmp(devices, "!*") == 0)
 898		return true;
 899	if (!negative && strcmp(devices, "*") == 0)
 900		return true;
 901
 902	s = kstrdup(devices, GFP_KERNEL);
 903	if (!s)
 904		return false;
 905
 906	for (p = s, ret = false; (tok = strsep(&p, ",")) != NULL; ) {
 907		u16 val;
 908
 909		if (negative && tok[0] == '!')
 910			tok++;
 911		else if ((negative && tok[0] != '!') ||
 912			 (!negative && tok[0] == '!'))
 913			continue;
 914
 915		if (kstrtou16(tok, 16, &val) == 0 && val == device_id) {
 916			ret = true;
 917			break;
 918		}
 919	}
 920
 921	kfree(s);
 922
 923	return ret;
 924}
 925
 926static bool id_forced(u16 device_id)
 927{
 928	return device_id_in_list(device_id, i915_modparams.force_probe, false);
 929}
 930
 931static bool id_blocked(u16 device_id)
 932{
 933	return device_id_in_list(device_id, i915_modparams.force_probe, true);
 934}
 935
 936bool i915_pci_resource_valid(struct pci_dev *pdev, int bar)
 937{
 938	if (!pci_resource_flags(pdev, bar))
 939		return false;
 940
 941	if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)
 942		return false;
 943
 944	if (!pci_resource_len(pdev, bar))
 945		return false;
 946
 947	return true;
 948}
 949
 950static bool intel_mmio_bar_valid(struct pci_dev *pdev, struct intel_device_info *intel_info)
 951{
 952	return i915_pci_resource_valid(pdev, intel_mmio_bar(intel_info->__runtime.graphics.ip.ver));
 953}
 954
 955static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 956{
 957	struct intel_device_info *intel_info =
 958		(struct intel_device_info *) ent->driver_data;
 959	int err;
 960
 961	if (intel_info->require_force_probe && !id_forced(pdev->device)) {
 962		dev_info(&pdev->dev,
 963			 "Your graphics device %04x is not properly supported by i915 in this\n"
 964			 "kernel version. To force driver probe anyway, use i915.force_probe=%04x\n"
 965			 "module parameter or CONFIG_DRM_I915_FORCE_PROBE=%04x configuration option,\n"
 966			 "or (recommended) check for kernel updates.\n",
 967			 pdev->device, pdev->device, pdev->device);
 968		return -ENODEV;
 969	}
 970
 971	if (id_blocked(pdev->device)) {
 972		dev_info(&pdev->dev, "I915 probe blocked for Device ID %04x.\n",
 973			 pdev->device);
 974		return -ENODEV;
 975	}
 976
 977	if (intel_info->require_force_probe) {
 978		dev_info(&pdev->dev, "Force probing unsupported Device ID %04x, tainting kernel\n",
 979			 pdev->device);
 980		add_taint(TAINT_USER, LOCKDEP_STILL_OK);
 981	}
 982
 983	/* Only bind to function 0 of the device. Early generations
 984	 * used function 1 as a placeholder for multi-head. This causes
 985	 * us confusion instead, especially on the systems where both
 986	 * functions have the same PCI-ID!
 987	 */
 988	if (PCI_FUNC(pdev->devfn))
 989		return -ENODEV;
 990
 991	if (!intel_mmio_bar_valid(pdev, intel_info))
 992		return -ENXIO;
 993
 994	/* Detect if we need to wait for other drivers early on */
 995	if (intel_display_driver_probe_defer(pdev))
 996		return -EPROBE_DEFER;
 997
 998	err = i915_driver_probe(pdev, ent);
 999	if (err)
1000		return err;
1001
1002	if (i915_inject_probe_failure(pdev_to_i915(pdev))) {
1003		i915_pci_remove(pdev);
1004		return -ENODEV;
1005	}
1006
1007	err = i915_live_selftests(pdev);
1008	if (err) {
1009		i915_pci_remove(pdev);
1010		return err > 0 ? -ENOTTY : err;
1011	}
1012
1013	err = i915_perf_selftests(pdev);
1014	if (err) {
1015		i915_pci_remove(pdev);
1016		return err > 0 ? -ENOTTY : err;
1017	}
1018
1019	return 0;
1020}
1021
1022static void i915_pci_shutdown(struct pci_dev *pdev)
1023{
1024	struct drm_i915_private *i915 = pdev_to_i915(pdev);
1025
1026	i915_driver_shutdown(i915);
1027}
1028
1029static struct pci_driver i915_pci_driver = {
1030	.name = DRIVER_NAME,
1031	.id_table = pciidlist,
1032	.probe = i915_pci_probe,
1033	.remove = i915_pci_remove,
1034	.shutdown = i915_pci_shutdown,
1035	.driver.pm = &i915_pm_ops,
1036};
1037
1038int i915_pci_register_driver(void)
1039{
1040	return pci_register_driver(&i915_pci_driver);
1041}
1042
1043void i915_pci_unregister_driver(void)
1044{
1045	pci_unregister_driver(&i915_pci_driver);
1046}