i915_gem_stolen.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2008-2012 Intel Corporation
   4 */
   5
   6#include <linux/errno.h>
   7#include <linux/mutex.h>
   8
   9#include <drm/drm_mm.h>
  10#include <drm/drm_print.h>
  11#include <drm/intel/i915_drm.h>
  12
  13#include "gem/i915_gem_lmem.h"
  14#include "gem/i915_gem_region.h"
  15#include "gt/intel_gt.h"
  16#include "gt/intel_gt_mcr.h"
  17#include "gt/intel_gt_regs.h"
  18#include "gt/intel_region_lmem.h"
  19#include "i915_drv.h"
  20#include "i915_gem_stolen.h"
  21#include "i915_pci.h"
  22#include "i915_reg.h"
  23#include "i915_utils.h"
  24#include "i915_vgpu.h"
  25#include "intel_mchbar_regs.h"
  26#include "intel_pci_config.h"
  27
  28struct intel_stolen_node {
  29	struct drm_i915_private *i915;
  30	struct drm_mm_node node;
  31};
  32
  33/*
  34 * The BIOS typically reserves some of the system's memory for the exclusive
  35 * use of the integrated graphics. This memory is no longer available for
  36 * use by the OS and so the user finds that his system has less memory
  37 * available than he put in. We refer to this memory as stolen.
  38 *
  39 * The BIOS will allocate its framebuffer from the stolen memory. Our
  40 * goal is try to reuse that object for our own fbcon which must always
  41 * be available for panics. Anything else we can reuse the stolen memory
  42 * for is a boon.
  43 */
  44
  45static int __i915_gem_stolen_insert_node_in_range(struct drm_i915_private *i915,
  46						  struct drm_mm_node *node, u64 size,
  47						  unsigned int alignment, u64 start, u64 end)
  48{
  49	int ret;
  50
  51	if (!drm_mm_initialized(&i915->mm.stolen))
  52		return -ENODEV;
  53
  54	/* WaSkipStolenMemoryFirstPage:bdw+ */
  55	if (GRAPHICS_VER(i915) >= 8 && start < 4096)
  56		start = 4096;
  57
  58	mutex_lock(&i915->mm.stolen_lock);
  59	ret = drm_mm_insert_node_in_range(&i915->mm.stolen, node,
  60					  size, alignment, 0,
  61					  start, end, DRM_MM_INSERT_BEST);
  62	mutex_unlock(&i915->mm.stolen_lock);
  63
  64	return ret;
  65}
  66
  67int i915_gem_stolen_insert_node_in_range(struct intel_stolen_node *node, u64 size,
  68					 unsigned int alignment, u64 start, u64 end)
  69{
  70	return __i915_gem_stolen_insert_node_in_range(node->i915, &node->node,
  71						      size, alignment,
  72						      start, end);
  73}
  74
  75static int __i915_gem_stolen_insert_node(struct drm_i915_private *i915,
  76					 struct drm_mm_node *node, u64 size,
  77					 unsigned int alignment)
  78{
  79	return __i915_gem_stolen_insert_node_in_range(i915, node,
  80						      size, alignment,
  81						      I915_GEM_STOLEN_BIAS,
  82						      U64_MAX);
  83}
  84
  85int i915_gem_stolen_insert_node(struct intel_stolen_node *node, u64 size,
  86				unsigned int alignment)
  87{
  88	return __i915_gem_stolen_insert_node(node->i915, &node->node, size, alignment);
  89}
  90
  91static void __i915_gem_stolen_remove_node(struct drm_i915_private *i915,
  92					  struct drm_mm_node *node)
  93{
  94	mutex_lock(&i915->mm.stolen_lock);
  95	drm_mm_remove_node(node);
  96	mutex_unlock(&i915->mm.stolen_lock);
  97}
  98
  99void i915_gem_stolen_remove_node(struct intel_stolen_node *node)
 100{
 101	__i915_gem_stolen_remove_node(node->i915, &node->node);
 102}
 103
 104static bool valid_stolen_size(struct drm_i915_private *i915, struct resource *dsm)
 105{
 106	return (dsm->start != 0 || HAS_LMEMBAR_SMEM_STOLEN(i915)) && dsm->end > dsm->start;
 107}
 108
 109static int adjust_stolen(struct drm_i915_private *i915,
 110			 struct resource *dsm)
 111{
 112	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 113	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 114
 115	if (!valid_stolen_size(i915, dsm))
 116		return -EINVAL;
 117
 118	/*
 119	 * Make sure we don't clobber the GTT if it's within stolen memory
 120	 *
 121	 * TODO: We have yet too encounter the case where the GTT wasn't at the
 122	 * end of stolen. With that assumption we could simplify this.
 123	 */
 124	if (GRAPHICS_VER(i915) <= 4 &&
 125	    !IS_G33(i915) && !IS_PINEVIEW(i915) && !IS_G4X(i915)) {
 126		struct resource stolen[2] = {*dsm, *dsm};
 127		struct resource ggtt_res;
 128		resource_size_t ggtt_start;
 129
 130		ggtt_start = intel_uncore_read(uncore, PGTBL_CTL);
 131		if (GRAPHICS_VER(i915) == 4)
 132			ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
 133				     (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
 134		else
 135			ggtt_start &= PGTBL_ADDRESS_LO_MASK;
 136
 137		ggtt_res = DEFINE_RES_MEM(ggtt_start, ggtt_total_entries(ggtt) * 4);
 138
 139		if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
 140			stolen[0].end = ggtt_res.start;
 141		if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
 142			stolen[1].start = ggtt_res.end;
 143
 144		/* Pick the larger of the two chunks */
 145		if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
 146			*dsm = stolen[0];
 147		else
 148			*dsm = stolen[1];
 149
 150		if (stolen[0].start != stolen[1].start ||
 151		    stolen[0].end != stolen[1].end) {
 152			drm_dbg(&i915->drm,
 153				"GTT within stolen memory at %pR\n",
 154				&ggtt_res);
 155			drm_dbg(&i915->drm, "Stolen memory adjusted to %pR\n",
 156				dsm);
 157		}
 158	}
 159
 160	if (!valid_stolen_size(i915, dsm))
 161		return -EINVAL;
 162
 163	return 0;
 164}
 165
 166static int request_smem_stolen(struct drm_i915_private *i915,
 167			       struct resource *dsm)
 168{
 169	struct resource *r;
 170
 171	/*
 172	 * With stolen lmem, we don't need to request system memory for the
 173	 * address range since it's local to the gpu.
 174	 *
 175	 * Starting MTL, in IGFX devices the stolen memory is exposed via
 176	 * LMEMBAR and shall be considered similar to stolen lmem.
 177	 */
 178	if (HAS_LMEM(i915) || HAS_LMEMBAR_SMEM_STOLEN(i915))
 179		return 0;
 180
 181	/*
 182	 * Verify that nothing else uses this physical address. Stolen
 183	 * memory should be reserved by the BIOS and hidden from the
 184	 * kernel. So if the region is already marked as busy, something
 185	 * is seriously wrong.
 186	 */
 187	r = devm_request_mem_region(i915->drm.dev, dsm->start,
 188				    resource_size(dsm),
 189				    "Graphics Stolen Memory");
 190	if (r == NULL) {
 191		/*
 192		 * One more attempt but this time requesting region from
 193		 * start + 1, as we have seen that this resolves the region
 194		 * conflict with the PCI Bus.
 195		 * This is a BIOS w/a: Some BIOS wrap stolen in the root
 196		 * PCI bus, but have an off-by-one error. Hence retry the
 197		 * reservation starting from 1 instead of 0.
 198		 * There's also BIOS with off-by-one on the other end.
 199		 */
 200		r = devm_request_mem_region(i915->drm.dev, dsm->start + 1,
 201					    resource_size(dsm) - 2,
 202					    "Graphics Stolen Memory");
 203		/*
 204		 * GEN3 firmware likes to smash pci bridges into the stolen
 205		 * range. Apparently this works.
 206		 */
 207		if (!r && GRAPHICS_VER(i915) != 3) {
 208			drm_err(&i915->drm,
 209				"conflict detected with stolen region: %pR\n",
 210				dsm);
 211
 212			return -EBUSY;
 213		}
 214	}
 215
 216	return 0;
 217}
 218
 219static void i915_gem_cleanup_stolen(struct drm_i915_private *i915)
 220{
 221	if (!drm_mm_initialized(&i915->mm.stolen))
 222		return;
 223
 224	drm_mm_takedown(&i915->mm.stolen);
 225}
 226
 227static void g4x_get_stolen_reserved(struct drm_i915_private *i915,
 228				    struct intel_uncore *uncore,
 229				    resource_size_t *base,
 230				    resource_size_t *size)
 231{
 232	u32 reg_val = intel_uncore_read(uncore,
 233					IS_GM45(i915) ?
 234					CTG_STOLEN_RESERVED :
 235					ELK_STOLEN_RESERVED);
 236	resource_size_t stolen_top = i915->dsm.stolen.end + 1;
 237
 238	drm_dbg(&i915->drm, "%s_STOLEN_RESERVED = %08x\n",
 239		IS_GM45(i915) ? "CTG" : "ELK", reg_val);
 240
 241	if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
 242		return;
 243
 244	/*
 245	 * Whether ILK really reuses the ELK register for this is unclear.
 246	 * Let's see if we catch anyone with this supposedly enabled on ILK.
 247	 */
 248	drm_WARN(&i915->drm, GRAPHICS_VER(i915) == 5,
 249		 "ILK stolen reserved found? 0x%08x\n",
 250		 reg_val);
 251
 252	if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
 253		return;
 254
 255	*base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 256	drm_WARN_ON(&i915->drm,
 257		    (reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 258
 259	*size = stolen_top - *base;
 260}
 261
 262static void gen6_get_stolen_reserved(struct drm_i915_private *i915,
 263				     struct intel_uncore *uncore,
 264				     resource_size_t *base,
 265				     resource_size_t *size)
 266{
 267	u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 268
 269	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 270
 271	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 272		return;
 273
 274	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 275
 276	switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
 277	case GEN6_STOLEN_RESERVED_1M:
 278		*size = 1024 * 1024;
 279		break;
 280	case GEN6_STOLEN_RESERVED_512K:
 281		*size = 512 * 1024;
 282		break;
 283	case GEN6_STOLEN_RESERVED_256K:
 284		*size = 256 * 1024;
 285		break;
 286	case GEN6_STOLEN_RESERVED_128K:
 287		*size = 128 * 1024;
 288		break;
 289	default:
 290		*size = 1024 * 1024;
 291		MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK);
 292	}
 293}
 294
 295static void vlv_get_stolen_reserved(struct drm_i915_private *i915,
 296				    struct intel_uncore *uncore,
 297				    resource_size_t *base,
 298				    resource_size_t *size)
 299{
 300	u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 301	resource_size_t stolen_top = i915->dsm.stolen.end + 1;
 302
 303	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 304
 305	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 306		return;
 307
 308	switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 309	default:
 310		MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 311		fallthrough;
 312	case GEN7_STOLEN_RESERVED_1M:
 313		*size = 1024 * 1024;
 314		break;
 315	}
 316
 317	/*
 318	 * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
 319	 * reserved location as (top - size).
 320	 */
 321	*base = stolen_top - *size;
 322}
 323
 324static void gen7_get_stolen_reserved(struct drm_i915_private *i915,
 325				     struct intel_uncore *uncore,
 326				     resource_size_t *base,
 327				     resource_size_t *size)
 328{
 329	u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 330
 331	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 332
 333	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 334		return;
 335
 336	*base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 337
 338	switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 339	case GEN7_STOLEN_RESERVED_1M:
 340		*size = 1024 * 1024;
 341		break;
 342	case GEN7_STOLEN_RESERVED_256K:
 343		*size = 256 * 1024;
 344		break;
 345	default:
 346		*size = 1024 * 1024;
 347		MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 348	}
 349}
 350
 351static void chv_get_stolen_reserved(struct drm_i915_private *i915,
 352				    struct intel_uncore *uncore,
 353				    resource_size_t *base,
 354				    resource_size_t *size)
 355{
 356	u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 357
 358	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 359
 360	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 361		return;
 362
 363	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 364
 365	switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 366	case GEN8_STOLEN_RESERVED_1M:
 367		*size = 1024 * 1024;
 368		break;
 369	case GEN8_STOLEN_RESERVED_2M:
 370		*size = 2 * 1024 * 1024;
 371		break;
 372	case GEN8_STOLEN_RESERVED_4M:
 373		*size = 4 * 1024 * 1024;
 374		break;
 375	case GEN8_STOLEN_RESERVED_8M:
 376		*size = 8 * 1024 * 1024;
 377		break;
 378	default:
 379		*size = 8 * 1024 * 1024;
 380		MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 381	}
 382}
 383
 384static void bdw_get_stolen_reserved(struct drm_i915_private *i915,
 385				    struct intel_uncore *uncore,
 386				    resource_size_t *base,
 387				    resource_size_t *size)
 388{
 389	u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 390	resource_size_t stolen_top = i915->dsm.stolen.end + 1;
 391
 392	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 393
 394	if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 395		return;
 396
 397	if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
 398		return;
 399
 400	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 401	*size = stolen_top - *base;
 402}
 403
 404static void icl_get_stolen_reserved(struct drm_i915_private *i915,
 405				    struct intel_uncore *uncore,
 406				    resource_size_t *base,
 407				    resource_size_t *size)
 408{
 409	u64 reg_val = intel_uncore_read64(uncore, GEN6_STOLEN_RESERVED);
 410
 411	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);
 412
 413	/* Wa_14019821291 */
 414	if (MEDIA_VER_FULL(i915) == IP_VER(13, 0)) {
 415		/*
 416		 * This workaround is primarily implemented by the BIOS.  We
 417		 * just need to figure out whether the BIOS has applied the
 418		 * workaround (meaning the programmed address falls within
 419		 * the DSM) and, if so, reserve that part of the DSM to
 420		 * prevent accidental reuse.  The DSM location should be just
 421		 * below the WOPCM.
 422		 */
 423		u64 gscpsmi_base = intel_uncore_read64_2x32(uncore,
 424							    MTL_GSCPSMI_BASEADDR_LSB,
 425							    MTL_GSCPSMI_BASEADDR_MSB);
 426		if (gscpsmi_base >= i915->dsm.stolen.start &&
 427		    gscpsmi_base < i915->dsm.stolen.end) {
 428			*base = gscpsmi_base;
 429			*size = i915->dsm.stolen.end - gscpsmi_base;
 430			return;
 431		}
 432	}
 433
 434	switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 435	case GEN8_STOLEN_RESERVED_1M:
 436		*size = 1024 * 1024;
 437		break;
 438	case GEN8_STOLEN_RESERVED_2M:
 439		*size = 2 * 1024 * 1024;
 440		break;
 441	case GEN8_STOLEN_RESERVED_4M:
 442		*size = 4 * 1024 * 1024;
 443		break;
 444	case GEN8_STOLEN_RESERVED_8M:
 445		*size = 8 * 1024 * 1024;
 446		break;
 447	default:
 448		*size = 8 * 1024 * 1024;
 449		MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 450	}
 451
 452	if (HAS_LMEMBAR_SMEM_STOLEN(i915))
 453		/* the base is initialized to stolen top so subtract size to get base */
 454		*base -= *size;
 455	else
 456		*base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;
 457}
 458
 459/*
 460 * Initialize i915->dsm.reserved to contain the reserved space within the Data
 461 * Stolen Memory. This is a range on the top of DSM that is reserved, not to
 462 * be used by driver, so must be excluded from the region passed to the
 463 * allocator later. In the spec this is also called as WOPCM.
 464 *
 465 * Our expectation is that the reserved space is at the top of the stolen
 466 * region, as it has been the case for every platform, and *never* at the
 467 * bottom, so the calculation here can be simplified.
 468 */
 469static int init_reserved_stolen(struct drm_i915_private *i915)
 470{
 471	struct intel_uncore *uncore = &i915->uncore;
 472	resource_size_t reserved_base, stolen_top;
 473	resource_size_t reserved_size;
 474	int ret = 0;
 475
 476	stolen_top = i915->dsm.stolen.end + 1;
 477	reserved_base = stolen_top;
 478	reserved_size = 0;
 479
 480	if (GRAPHICS_VER(i915) >= 11) {
 481		icl_get_stolen_reserved(i915, uncore,
 482					&reserved_base, &reserved_size);
 483	} else if (GRAPHICS_VER(i915) >= 8) {
 484		if (IS_CHERRYVIEW(i915) || IS_BROXTON(i915) || IS_GEMINILAKE(i915))
 485			chv_get_stolen_reserved(i915, uncore,
 486						&reserved_base, &reserved_size);
 487		else
 488			bdw_get_stolen_reserved(i915, uncore,
 489						&reserved_base, &reserved_size);
 490	} else if (GRAPHICS_VER(i915) >= 7) {
 491		if (IS_VALLEYVIEW(i915))
 492			vlv_get_stolen_reserved(i915, uncore,
 493						&reserved_base, &reserved_size);
 494		else
 495			gen7_get_stolen_reserved(i915, uncore,
 496						 &reserved_base, &reserved_size);
 497	} else if (GRAPHICS_VER(i915) >= 6) {
 498		gen6_get_stolen_reserved(i915, uncore,
 499					 &reserved_base, &reserved_size);
 500	} else if (GRAPHICS_VER(i915) >= 5 || IS_G4X(i915)) {
 501		g4x_get_stolen_reserved(i915, uncore,
 502					&reserved_base, &reserved_size);
 503	}
 504
 505	/* No reserved stolen */
 506	if (reserved_base == stolen_top)
 507		goto bail_out;
 508
 509	if (!reserved_base) {
 510		drm_err(&i915->drm,
 511			"inconsistent reservation %pa + %pa; ignoring\n",
 512			&reserved_base, &reserved_size);
 513		ret = -EINVAL;
 514		goto bail_out;
 515	}
 516
 517	i915->dsm.reserved = DEFINE_RES_MEM(reserved_base, reserved_size);
 518
 519	if (!resource_contains(&i915->dsm.stolen, &i915->dsm.reserved)) {
 520		drm_err(&i915->drm,
 521			"Stolen reserved area %pR outside stolen memory %pR\n",
 522			&i915->dsm.reserved, &i915->dsm.stolen);
 523		ret = -EINVAL;
 524		goto bail_out;
 525	}
 526
 527	return 0;
 528
 529bail_out:
 530	i915->dsm.reserved = DEFINE_RES_MEM(reserved_base, 0);
 531
 532	return ret;
 533}
 534
 535static int i915_gem_init_stolen(struct intel_memory_region *mem)
 536{
 537	struct drm_i915_private *i915 = mem->i915;
 538
 539	mutex_init(&i915->mm.stolen_lock);
 540
 541	if (intel_vgpu_active(i915)) {
 542		drm_notice(&i915->drm,
 543			   "%s, disabling use of stolen memory\n",
 544			   "iGVT-g active");
 545		return -ENOSPC;
 546	}
 547
 548	if (i915_vtd_active(i915) && GRAPHICS_VER(i915) < 8) {
 549		drm_notice(&i915->drm,
 550			   "%s, disabling use of stolen memory\n",
 551			   "DMAR active");
 552		return -ENOSPC;
 553	}
 554
 555	if (adjust_stolen(i915, &mem->region))
 556		return -ENOSPC;
 557
 558	if (request_smem_stolen(i915, &mem->region))
 559		return -ENOSPC;
 560
 561	i915->dsm.stolen = mem->region;
 562
 563	if (init_reserved_stolen(i915))
 564		return -ENOSPC;
 565
 566	/* Exclude the reserved region from driver use */
 567	mem->region.end = i915->dsm.reserved.start - 1;
 568	mem->io = DEFINE_RES_MEM(mem->io.start,
 569				 min(resource_size(&mem->io),
 570				     resource_size(&mem->region)));
 571
 572	i915->dsm.usable_size = resource_size(&mem->region);
 573
 574	drm_dbg(&i915->drm,
 575		"Memory reserved for graphics device: %lluK, usable: %lluK\n",
 576		(u64)resource_size(&i915->dsm.stolen) >> 10,
 577		(u64)i915->dsm.usable_size >> 10);
 578
 579	if (i915->dsm.usable_size == 0)
 580		return -ENOSPC;
 581
 582	/* Basic memrange allocator for stolen space. */
 583	drm_mm_init(&i915->mm.stolen, 0, i915->dsm.usable_size);
 584
 585	/*
 586	 * Access to stolen lmem beyond certain size for MTL A0 stepping
 587	 * would crash the machine. Disable stolen lmem for userspace access
 588	 * by setting usable_size to zero.
 589	 */
 590	if (IS_METEORLAKE(i915) && INTEL_REVID(i915) == 0x0)
 591		i915->dsm.usable_size = 0;
 592
 593	return 0;
 594}
 595
 596static void dbg_poison(struct i915_ggtt *ggtt,
 597		       dma_addr_t addr, resource_size_t size,
 598		       u8 x)
 599{
 600#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 601	if (!drm_mm_node_allocated(&ggtt->error_capture))
 602		return;
 603
 604	if (ggtt->vm.bind_async_flags & I915_VMA_GLOBAL_BIND)
 605		return; /* beware stop_machine() inversion */
 606
 607	GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
 608
 609	mutex_lock(&ggtt->error_mutex);
 610	while (size) {
 611		void __iomem *s;
 612
 613		ggtt->vm.insert_page(&ggtt->vm, addr,
 614				     ggtt->error_capture.start,
 615				     i915_gem_get_pat_index(ggtt->vm.i915,
 616							    I915_CACHE_NONE),
 617				     0);
 618		mb();
 619
 620		s = io_mapping_map_wc(&ggtt->iomap,
 621				      ggtt->error_capture.start,
 622				      PAGE_SIZE);
 623		memset_io(s, x, PAGE_SIZE);
 624		io_mapping_unmap(s);
 625
 626		addr += PAGE_SIZE;
 627		size -= PAGE_SIZE;
 628	}
 629	mb();
 630	ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE);
 631	mutex_unlock(&ggtt->error_mutex);
 632#endif
 633}
 634
 635static struct sg_table *
 636i915_pages_create_for_stolen(struct drm_device *dev,
 637			     resource_size_t offset, resource_size_t size)
 638{
 639	struct drm_i915_private *i915 = to_i915(dev);
 640	struct sg_table *st;
 641	struct scatterlist *sg;
 642
 643	GEM_BUG_ON(range_overflows(offset, size, resource_size(&i915->dsm.stolen)));
 644
 645	/* We hide that we have no struct page backing our stolen object
 646	 * by wrapping the contiguous physical allocation with a fake
 647	 * dma mapping in a single scatterlist.
 648	 */
 649
 650	st = kmalloc(sizeof(*st), GFP_KERNEL);
 651	if (st == NULL)
 652		return ERR_PTR(-ENOMEM);
 653
 654	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 655		kfree(st);
 656		return ERR_PTR(-ENOMEM);
 657	}
 658
 659	sg = st->sgl;
 660	sg->offset = 0;
 661	sg->length = size;
 662
 663	sg_dma_address(sg) = (dma_addr_t)i915->dsm.stolen.start + offset;
 664	sg_dma_len(sg) = size;
 665
 666	return st;
 667}
 668
 669static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 670{
 671	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 672	struct sg_table *pages =
 673		i915_pages_create_for_stolen(obj->base.dev,
 674					     obj->stolen->start,
 675					     obj->stolen->size);
 676	if (IS_ERR(pages))
 677		return PTR_ERR(pages);
 678
 679	dbg_poison(to_gt(i915)->ggtt,
 680		   sg_dma_address(pages->sgl),
 681		   sg_dma_len(pages->sgl),
 682		   POISON_INUSE);
 683
 684	__i915_gem_object_set_pages(obj, pages);
 685
 686	return 0;
 687}
 688
 689static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
 690					     struct sg_table *pages)
 691{
 692	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 693	/* Should only be called from i915_gem_object_release_stolen() */
 694
 695	dbg_poison(to_gt(i915)->ggtt,
 696		   sg_dma_address(pages->sgl),
 697		   sg_dma_len(pages->sgl),
 698		   POISON_FREE);
 699
 700	sg_free_table(pages);
 701	kfree(pages);
 702}
 703
 704static void
 705i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 706{
 707	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 708	struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
 709
 710	GEM_BUG_ON(!stolen);
 711	__i915_gem_stolen_remove_node(i915, stolen);
 712	kfree(stolen);
 713
 714	i915_gem_object_release_memory_region(obj);
 715}
 716
 717static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
 718	.name = "i915_gem_object_stolen",
 719	.get_pages = i915_gem_object_get_pages_stolen,
 720	.put_pages = i915_gem_object_put_pages_stolen,
 721	.release = i915_gem_object_release_stolen,
 722};
 723
 724static int __i915_gem_object_create_stolen(struct intel_memory_region *mem,
 725					   struct drm_i915_gem_object *obj,
 726					   struct drm_mm_node *stolen)
 727{
 728	static struct lock_class_key lock_class;
 729	unsigned int cache_level;
 730	unsigned int flags;
 731	int err;
 732
 733	/*
 734	 * Stolen objects are always physically contiguous since we just
 735	 * allocate one big block underneath using the drm_mm range allocator.
 736	 */
 737	flags = I915_BO_ALLOC_CONTIGUOUS;
 738
 739	drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size);
 740	i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, flags);
 741
 742	obj->stolen = stolen;
 743	obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 744	cache_level = HAS_LLC(mem->i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
 745	i915_gem_object_set_cache_coherency(obj, cache_level);
 746
 747	if (WARN_ON(!i915_gem_object_trylock(obj, NULL)))
 748		return -EBUSY;
 749
 750	i915_gem_object_init_memory_region(obj, mem);
 751
 752	err = i915_gem_object_pin_pages(obj);
 753	if (err)
 754		i915_gem_object_release_memory_region(obj);
 755	i915_gem_object_unlock(obj);
 756
 757	return err;
 758}
 759
 760static int _i915_gem_object_stolen_init(struct intel_memory_region *mem,
 761					struct drm_i915_gem_object *obj,
 762					resource_size_t offset,
 763					resource_size_t size,
 764					resource_size_t page_size,
 765					unsigned int flags)
 766{
 767	struct drm_i915_private *i915 = mem->i915;
 768	struct drm_mm_node *stolen;
 769	int ret;
 770
 771	if (!drm_mm_initialized(&i915->mm.stolen))
 772		return -ENODEV;
 773
 774	if (size == 0)
 775		return -EINVAL;
 776
 777	/*
 778	 * With discrete devices, where we lack a mappable aperture there is no
 779	 * possible way to ever access this memory on the CPU side.
 780	 */
 781	if (mem->type == INTEL_MEMORY_STOLEN_LOCAL && !resource_size(&mem->io) &&
 782	    !(flags & I915_BO_ALLOC_GPU_ONLY))
 783		return -ENOSPC;
 784
 785	stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 786	if (!stolen)
 787		return -ENOMEM;
 788
 789	if (offset != I915_BO_INVALID_OFFSET) {
 790		drm_dbg(&i915->drm,
 791			"creating preallocated stolen object: stolen_offset=%pa, size=%pa\n",
 792			&offset, &size);
 793
 794		stolen->start = offset;
 795		stolen->size = size;
 796		mutex_lock(&i915->mm.stolen_lock);
 797		ret = drm_mm_reserve_node(&i915->mm.stolen, stolen);
 798		mutex_unlock(&i915->mm.stolen_lock);
 799	} else {
 800		ret = __i915_gem_stolen_insert_node(i915, stolen, size,
 801						    mem->min_page_size);
 802	}
 803	if (ret)
 804		goto err_free;
 805
 806	ret = __i915_gem_object_create_stolen(mem, obj, stolen);
 807	if (ret)
 808		goto err_remove;
 809
 810	return 0;
 811
 812err_remove:
 813	__i915_gem_stolen_remove_node(i915, stolen);
 814err_free:
 815	kfree(stolen);
 816	return ret;
 817}
 818
 819struct drm_i915_gem_object *
 820i915_gem_object_create_stolen(struct drm_i915_private *i915,
 821			      resource_size_t size)
 822{
 823	return i915_gem_object_create_region(i915->mm.stolen_region, size, 0, 0);
 824}
 825
 826static int init_stolen_smem(struct intel_memory_region *mem)
 827{
 828	int err;
 829
 830	/*
 831	 * Initialise stolen early so that we may reserve preallocated
 832	 * objects for the BIOS to KMS transition.
 833	 */
 834	err = i915_gem_init_stolen(mem);
 835	if (err)
 836		drm_dbg(&mem->i915->drm, "Skip stolen region: failed to setup\n");
 837
 838	return 0;
 839}
 840
 841static int release_stolen_smem(struct intel_memory_region *mem)
 842{
 843	i915_gem_cleanup_stolen(mem->i915);
 844	return 0;
 845}
 846
 847static const struct intel_memory_region_ops i915_region_stolen_smem_ops = {
 848	.init = init_stolen_smem,
 849	.release = release_stolen_smem,
 850	.init_object = _i915_gem_object_stolen_init,
 851};
 852
 853static int init_stolen_lmem(struct intel_memory_region *mem)
 854{
 855	int err;
 856
 857	if (GEM_WARN_ON(resource_size(&mem->region) == 0))
 858		return 0;
 859
 860	err = i915_gem_init_stolen(mem);
 861	if (err) {
 862		drm_dbg(&mem->i915->drm, "Skip stolen region: failed to setup\n");
 863		return 0;
 864	}
 865
 866	if (resource_size(&mem->io) &&
 867	    !io_mapping_init_wc(&mem->iomap, mem->io.start, resource_size(&mem->io)))
 868		goto err_cleanup;
 869
 870	return 0;
 871
 872err_cleanup:
 873	i915_gem_cleanup_stolen(mem->i915);
 874	return err;
 875}
 876
 877static int release_stolen_lmem(struct intel_memory_region *mem)
 878{
 879	if (resource_size(&mem->io))
 880		io_mapping_fini(&mem->iomap);
 881	i915_gem_cleanup_stolen(mem->i915);
 882	return 0;
 883}
 884
 885static const struct intel_memory_region_ops i915_region_stolen_lmem_ops = {
 886	.init = init_stolen_lmem,
 887	.release = release_stolen_lmem,
 888	.init_object = _i915_gem_object_stolen_init,
 889};
 890
 891static int mtl_get_gms_size(struct intel_uncore *uncore)
 892{
 893	u16 ggc, gms;
 894
 895	ggc = intel_uncore_read16(uncore, GGC);
 896
 897	/* check GGMS, should be fixed 0x3 (8MB) */
 898	if ((ggc & GGMS_MASK) != GGMS_MASK)
 899		return -EIO;
 900
 901	/* return valid GMS value, -EIO if invalid */
 902	gms = REG_FIELD_GET(GMS_MASK, ggc);
 903	switch (gms) {
 904	case 0x0 ... 0x04:
 905		return gms * 32;
 906	case 0xf0 ... 0xfe:
 907		return (gms - 0xf0 + 1) * 4;
 908	default:
 909		MISSING_CASE(gms);
 910		return -EIO;
 911	}
 912}
 913
 914struct intel_memory_region *
 915i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 916			   u16 instance)
 917{
 918	struct intel_uncore *uncore = &i915->uncore;
 919	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 920	resource_size_t dsm_size, dsm_base, lmem_size;
 921	struct intel_memory_region *mem;
 922	resource_size_t io_start, io_size;
 923	resource_size_t min_page_size;
 924	int ret;
 925
 926	if (WARN_ON_ONCE(instance))
 927		return ERR_PTR(-ENODEV);
 928
 929	if (!i915_pci_resource_valid(pdev, GEN12_LMEM_BAR))
 930		return ERR_PTR(-ENXIO);
 931
 932	if (HAS_LMEMBAR_SMEM_STOLEN(i915) || IS_DG1(i915)) {
 933		lmem_size = pci_resource_len(pdev, GEN12_LMEM_BAR);
 934	} else {
 935		resource_size_t lmem_range;
 936
 937		lmem_range = intel_gt_mcr_read_any(to_gt(i915), XEHP_TILE0_ADDR_RANGE) & 0xFFFF;
 938		lmem_size = lmem_range >> XEHP_TILE_LMEM_RANGE_SHIFT;
 939		lmem_size *= SZ_1G;
 940	}
 941
 942	if (HAS_LMEMBAR_SMEM_STOLEN(i915)) {
 943		/*
 944		 * MTL dsm size is in GGC register.
 945		 * Also MTL uses offset to GSMBASE in ptes, so i915
 946		 * uses dsm_base = 8MBs to setup stolen region, since
 947		 * DSMBASE = GSMBASE + 8MB.
 948		 */
 949		ret = mtl_get_gms_size(uncore);
 950		if (ret < 0) {
 951			drm_err(&i915->drm, "invalid MTL GGC register setting\n");
 952			return ERR_PTR(ret);
 953		}
 954
 955		dsm_base = SZ_8M;
 956		dsm_size = (resource_size_t)(ret * SZ_1M);
 957
 958		GEM_BUG_ON(pci_resource_len(pdev, GEN12_LMEM_BAR) != SZ_256M);
 959		GEM_BUG_ON((dsm_base + dsm_size) > lmem_size);
 960	} else {
 961		/* Use DSM base address instead for stolen memory */
 962		dsm_base = intel_uncore_read64(uncore, GEN6_DSMBASE) & GEN11_BDSM_MASK;
 963		if (lmem_size < dsm_base) {
 964			drm_dbg(&i915->drm,
 965				"Disabling stolen memory support due to OOB placement: lmem_size = %pa vs dsm_base = %pa\n",
 966				&lmem_size, &dsm_base);
 967			return NULL;
 968		}
 969		dsm_size = ALIGN_DOWN(lmem_size - dsm_base, SZ_1M);
 970	}
 971
 972	if (i915_direct_stolen_access(i915)) {
 973		drm_dbg(&i915->drm, "Using direct DSM access\n");
 974		io_start = intel_uncore_read64(uncore, GEN6_DSMBASE) & GEN11_BDSM_MASK;
 975		io_size = dsm_size;
 976	} else if (pci_resource_len(pdev, GEN12_LMEM_BAR) < lmem_size) {
 977		io_start = 0;
 978		io_size = 0;
 979	} else {
 980		io_start = pci_resource_start(pdev, GEN12_LMEM_BAR) + dsm_base;
 981		io_size = dsm_size;
 982	}
 983
 984	min_page_size = HAS_64K_PAGES(i915) ? I915_GTT_PAGE_SIZE_64K :
 985						I915_GTT_PAGE_SIZE_4K;
 986
 987	mem = intel_memory_region_create(i915, dsm_base, dsm_size,
 988					 min_page_size,
 989					 io_start, io_size,
 990					 type, instance,
 991					 &i915_region_stolen_lmem_ops);
 992	if (IS_ERR(mem))
 993		return mem;
 994
 995	intel_memory_region_set_name(mem, "stolen-local");
 996
 997	mem->private = true;
 998
 999	return mem;
1000}
1001
1002struct intel_memory_region*
1003i915_gem_stolen_smem_setup(struct drm_i915_private *i915, u16 type,
1004			   u16 instance)
1005{
1006	struct intel_memory_region *mem;
1007
1008	mem = intel_memory_region_create(i915,
1009					 intel_graphics_stolen_res.start,
1010					 resource_size(&intel_graphics_stolen_res),
1011					 PAGE_SIZE, 0, 0, type, instance,
1012					 &i915_region_stolen_smem_ops);
1013	if (IS_ERR(mem))
1014		return mem;
1015
1016	intel_memory_region_set_name(mem, "stolen-system");
1017
1018	mem->private = true;
1019
1020	return mem;
1021}
1022
1023bool i915_gem_object_is_stolen(const struct drm_i915_gem_object *obj)
1024{
1025	return obj->ops == &i915_gem_object_stolen_ops;
1026}
1027
1028bool i915_gem_stolen_initialized(struct drm_device *drm)
1029{
1030	struct drm_i915_private *i915 = to_i915(drm);
1031
1032	return drm_mm_initialized(&i915->mm.stolen);
1033}
1034
1035u64 i915_gem_stolen_area_address(struct drm_device *drm)
1036{
1037	struct drm_i915_private *i915 = to_i915(drm);
1038
1039	return i915->dsm.stolen.start;
1040}
1041
1042u64 i915_gem_stolen_area_size(struct drm_device *drm)
1043{
1044	struct drm_i915_private *i915 = to_i915(drm);
1045
1046	return resource_size(&i915->dsm.stolen);
1047}
1048
1049u64 i915_gem_stolen_node_address(const struct intel_stolen_node *node)
1050{
1051	struct drm_i915_private *i915 = node->i915;
1052
1053	return i915->dsm.stolen.start + i915_gem_stolen_node_offset(node);
1054}
1055
1056bool i915_gem_stolen_node_allocated(const struct intel_stolen_node *node)
1057{
1058	return drm_mm_node_allocated(&node->node);
1059}
1060
1061u64 i915_gem_stolen_node_offset(const struct intel_stolen_node *node)
1062{
1063	return node->node.start;
1064}
1065
1066u64 i915_gem_stolen_node_size(const struct intel_stolen_node *node)
1067{
1068	return node->node.size;
1069}
1070
1071struct intel_stolen_node *i915_gem_stolen_node_alloc(struct drm_device *drm)
1072{
1073	struct drm_i915_private *i915 = to_i915(drm);
1074	struct intel_stolen_node *node;
1075
1076	node = kzalloc(sizeof(*node), GFP_KERNEL);
1077	if (!node)
1078		return NULL;
1079
1080	node->i915 = i915;
1081
1082	return node;
1083}
1084
1085void i915_gem_stolen_node_free(const struct intel_stolen_node *node)
1086{
1087	kfree(node);
1088}