intel_ggtt.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2020 Intel Corporation
   4 */
   5
   6#include <asm/set_memory.h>
   7#include <asm/smp.h>
   8#include <linux/types.h>
   9#include <linux/stop_machine.h>
  10
  11#include <drm/drm_managed.h>
  12#include <drm/drm_print.h>
  13#include <drm/intel/i915_drm.h>
  14#include <drm/intel/intel-gtt.h>
  15
  16#include "gem/i915_gem_lmem.h"
  17
  18#include "intel_context.h"
  19#include "intel_ggtt_gmch.h"
  20#include "intel_gpu_commands.h"
  21#include "intel_gt.h"
  22#include "intel_gt_regs.h"
  23#include "intel_pci_config.h"
  24#include "intel_ring.h"
  25#include "i915_drv.h"
  26#include "i915_pci.h"
  27#include "i915_reg.h"
  28#include "i915_request.h"
  29#include "i915_scatterlist.h"
  30#include "i915_utils.h"
  31#include "i915_vgpu.h"
  32
  33#include "intel_gtt.h"
  34#include "gen8_ppgtt.h"
  35#include "intel_engine_pm.h"
  36
  37static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
  38				   unsigned long color,
  39				   u64 *start,
  40				   u64 *end)
  41{
  42	if (i915_node_color_differs(node, color))
  43		*start += I915_GTT_PAGE_SIZE;
  44
  45	/*
  46	 * Also leave a space between the unallocated reserved node after the
  47	 * GTT and any objects within the GTT, i.e. we use the color adjustment
  48	 * to insert a guard page to prevent prefetches crossing over the
  49	 * GTT boundary.
  50	 */
  51	node = list_next_entry(node, node_list);
  52	if (node->color != color)
  53		*end -= I915_GTT_PAGE_SIZE;
  54}
  55
  56static int ggtt_init_hw(struct i915_ggtt *ggtt)
  57{
  58	struct drm_i915_private *i915 = ggtt->vm.i915;
  59
  60	i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
  61
  62	ggtt->vm.is_ggtt = true;
  63
  64	/* Only VLV supports read-only GGTT mappings */
  65	ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
  66
  67	if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
  68		ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
  69
  70	if (ggtt->mappable_end) {
  71		if (!io_mapping_init_wc(&ggtt->iomap,
  72					ggtt->gmadr.start,
  73					ggtt->mappable_end)) {
  74			ggtt->vm.cleanup(&ggtt->vm);
  75			return -EIO;
  76		}
  77
  78		ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
  79					      ggtt->mappable_end);
  80	}
  81
  82	intel_ggtt_init_fences(ggtt);
  83
  84	return 0;
  85}
  86
  87/**
  88 * i915_ggtt_init_hw - Initialize GGTT hardware
  89 * @i915: i915 device
  90 */
  91int i915_ggtt_init_hw(struct drm_i915_private *i915)
  92{
  93	int ret;
  94
  95	/*
  96	 * Note that we use page colouring to enforce a guard page at the
  97	 * end of the address space. This is required as the CS may prefetch
  98	 * beyond the end of the batch buffer, across the page boundary,
  99	 * and beyond the end of the GTT if we do not provide a guard.
 100	 */
 101	ret = ggtt_init_hw(to_gt(i915)->ggtt);
 102	if (ret)
 103		return ret;
 104
 105	return 0;
 106}
 107
 108/**
 109 * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM
 110 * @vm: The VM to suspend the mappings for
 111 * @evict_all: Evict all VMAs
 112 *
 113 * Suspend the memory mappings for all objects mapped to HW via the GGTT or a
 114 * DPT page table.
 115 */
 116void i915_ggtt_suspend_vm(struct i915_address_space *vm, bool evict_all)
 117{
 118	struct i915_vma *vma, *vn;
 119	int save_skip_rewrite;
 120
 121	drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
 122
 123retry:
 124	i915_gem_drain_freed_objects(vm->i915);
 125
 126	mutex_lock(&vm->mutex);
 127
 128	/*
 129	 * Skip rewriting PTE on VMA unbind.
 130	 * FIXME: Use an argument to i915_vma_unbind() instead?
 131	 */
 132	save_skip_rewrite = vm->skip_pte_rewrite;
 133	vm->skip_pte_rewrite = true;
 134
 135	list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
 136		struct drm_i915_gem_object *obj = vma->obj;
 137
 138		GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 139
 140		if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
 141			continue;
 142
 143		/* unlikely to race when GPU is idle, so no worry about slowpath.. */
 144		if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {
 145			/*
 146			 * No dead objects should appear here, GPU should be
 147			 * completely idle, and userspace suspended
 148			 */
 149			i915_gem_object_get(obj);
 150
 151			mutex_unlock(&vm->mutex);
 152
 153			i915_gem_object_lock(obj, NULL);
 154			GEM_WARN_ON(i915_vma_unbind(vma));
 155			i915_gem_object_unlock(obj);
 156			i915_gem_object_put(obj);
 157
 158			vm->skip_pte_rewrite = save_skip_rewrite;
 159			goto retry;
 160		}
 161
 162		if (evict_all || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
 163			i915_vma_wait_for_bind(vma);
 164
 165			__i915_vma_evict(vma, false);
 166			drm_mm_remove_node(&vma->node);
 167		}
 168
 169		i915_gem_object_unlock(obj);
 170	}
 171
 172	vm->clear_range(vm, 0, vm->total);
 173
 174	vm->skip_pte_rewrite = save_skip_rewrite;
 175
 176	mutex_unlock(&vm->mutex);
 177
 178	drm_WARN_ON(&vm->i915->drm, evict_all && !list_empty(&vm->bound_list));
 179}
 180
 181void i915_ggtt_suspend(struct i915_ggtt *ggtt)
 182{
 183	struct intel_gt *gt;
 184
 185	i915_ggtt_suspend_vm(&ggtt->vm, false);
 186	ggtt->invalidate(ggtt);
 187
 188	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
 189		intel_gt_check_and_clear_faults(gt);
 190}
 191
 192void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
 193{
 194	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 195
 196	spin_lock_irq(&uncore->lock);
 197	intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 198	intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
 199	spin_unlock_irq(&uncore->lock);
 200}
 201
 202static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915)
 203{
 204	/*
 205	 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
 206	 * will be dropped. For WC mappings in general we have 64 byte burst
 207	 * writes when the WC buffer is flushed, so we can't use it, but have to
 208	 * resort to an uncached mapping. The WC issue is easily caught by the
 209	 * readback check when writing GTT PTE entries.
 210	 */
 211	if (!IS_GEN9_LP(i915) && GRAPHICS_VER(i915) < 11)
 212		return true;
 213
 214	return false;
 215}
 216
 217static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
 218{
 219	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 220
 221	/*
 222	 * Note that as an uncached mmio write, this will flush the
 223	 * WCB of the writes into the GGTT before it triggers the invalidate.
 224	 *
 225	 * Only perform this when GGTT is mapped as WC, see ggtt_probe_common().
 226	 */
 227	if (needs_wc_ggtt_mapping(ggtt->vm.i915))
 228		intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6,
 229				      GFX_FLSH_CNTL_EN);
 230}
 231
 232static void guc_ggtt_ct_invalidate(struct intel_gt *gt)
 233{
 234	struct intel_uncore *uncore = gt->uncore;
 235	intel_wakeref_t wakeref;
 236
 237	with_intel_runtime_pm_if_active(uncore->rpm, wakeref)
 238		intel_guc_invalidate_tlb_guc(gt_to_guc(gt));
 239}
 240
 241static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 242{
 243	struct drm_i915_private *i915 = ggtt->vm.i915;
 244	struct intel_gt *gt;
 245
 246	gen8_ggtt_invalidate(ggtt);
 247
 248	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) {
 249		if (intel_guc_tlb_invalidation_is_available(gt_to_guc(gt)))
 250			guc_ggtt_ct_invalidate(gt);
 251		else if (GRAPHICS_VER(i915) >= 12)
 252			intel_uncore_write_fw(gt->uncore,
 253					      GEN12_GUC_TLB_INV_CR,
 254					      GEN12_GUC_TLB_INV_CR_INVALIDATE);
 255		else
 256			intel_uncore_write_fw(gt->uncore,
 257					      GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 258	}
 259}
 260
 261static u64 mtl_ggtt_pte_encode(dma_addr_t addr,
 262			       unsigned int pat_index,
 263			       u32 flags)
 264{
 265	gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
 266
 267	WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK);
 268
 269	if (flags & PTE_LM)
 270		pte |= GEN12_GGTT_PTE_LM;
 271
 272	if (pat_index & BIT(0))
 273		pte |= MTL_GGTT_PTE_PAT0;
 274
 275	if (pat_index & BIT(1))
 276		pte |= MTL_GGTT_PTE_PAT1;
 277
 278	return pte;
 279}
 280
 281u64 gen8_ggtt_pte_encode(dma_addr_t addr,
 282			 unsigned int pat_index,
 283			 u32 flags)
 284{
 285	gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;
 286
 287	if (flags & PTE_LM)
 288		pte |= GEN12_GGTT_PTE_LM;
 289
 290	return pte;
 291}
 292
 293static dma_addr_t gen8_ggtt_pte_decode(u64 pte, bool *is_present, bool *is_local)
 294{
 295	*is_present = pte & GEN8_PAGE_PRESENT;
 296	*is_local = pte & GEN12_GGTT_PTE_LM;
 297
 298	return pte & GEN12_GGTT_PTE_ADDR_MASK;
 299}
 300
 301static bool should_update_ggtt_with_bind(struct i915_ggtt *ggtt)
 302{
 303	struct intel_gt *gt = ggtt->vm.gt;
 304
 305	return intel_gt_is_bind_context_ready(gt);
 306}
 307
 308static struct intel_context *gen8_ggtt_bind_get_ce(struct i915_ggtt *ggtt, intel_wakeref_t *wakeref)
 309{
 310	struct intel_context *ce;
 311	struct intel_gt *gt = ggtt->vm.gt;
 312
 313	if (intel_gt_is_wedged(gt))
 314		return NULL;
 315
 316	ce = gt->engine[BCS0]->bind_context;
 317	GEM_BUG_ON(!ce);
 318
 319	/*
 320	 * If the GT is not awake already at this stage then fallback
 321	 * to pci based GGTT update otherwise __intel_wakeref_get_first()
 322	 * would conflict with fs_reclaim trying to allocate memory while
 323	 * doing rpm_resume().
 324	 */
 325	*wakeref = intel_gt_pm_get_if_awake(gt);
 326	if (!*wakeref)
 327		return NULL;
 328
 329	intel_engine_pm_get(ce->engine);
 330
 331	return ce;
 332}
 333
 334static void gen8_ggtt_bind_put_ce(struct intel_context *ce, intel_wakeref_t wakeref)
 335{
 336	intel_engine_pm_put(ce->engine);
 337	intel_gt_pm_put(ce->engine->gt, wakeref);
 338}
 339
 340static bool gen8_ggtt_bind_ptes(struct i915_ggtt *ggtt, u32 offset,
 341				struct sg_table *pages, u32 num_entries,
 342				const gen8_pte_t pte)
 343{
 344	struct i915_sched_attr attr = {};
 345	struct intel_gt *gt = ggtt->vm.gt;
 346	const gen8_pte_t scratch_pte = ggtt->vm.scratch[0]->encode;
 347	struct sgt_iter iter;
 348	struct i915_request *rq;
 349	struct intel_context *ce;
 350	intel_wakeref_t wakeref;
 351	u32 *cs;
 352
 353	if (!num_entries)
 354		return true;
 355
 356	ce = gen8_ggtt_bind_get_ce(ggtt, &wakeref);
 357	if (!ce)
 358		return false;
 359
 360	if (pages)
 361		iter = __sgt_iter(pages->sgl, true);
 362
 363	while (num_entries) {
 364		int count = 0;
 365		dma_addr_t addr;
 366		/*
 367		 * MI_UPDATE_GTT can update 512 entries in a single command but
 368		 * that end up with engine reset, 511 works.
 369		 */
 370		u32 n_ptes = min_t(u32, 511, num_entries);
 371
 372		if (mutex_lock_interruptible(&ce->timeline->mutex))
 373			goto put_ce;
 374
 375		intel_context_enter(ce);
 376		rq = __i915_request_create(ce, GFP_NOWAIT | GFP_ATOMIC);
 377		intel_context_exit(ce);
 378		if (IS_ERR(rq)) {
 379			GT_TRACE(gt, "Failed to get bind request\n");
 380			mutex_unlock(&ce->timeline->mutex);
 381			goto put_ce;
 382		}
 383
 384		cs = intel_ring_begin(rq, 2 * n_ptes + 2);
 385		if (IS_ERR(cs)) {
 386			GT_TRACE(gt, "Failed to ring space for GGTT bind\n");
 387			i915_request_set_error_once(rq, PTR_ERR(cs));
 388			/* once a request is created, it must be queued */
 389			goto queue_err_rq;
 390		}
 391
 392		*cs++ = MI_UPDATE_GTT | (2 * n_ptes);
 393		*cs++ = offset << 12;
 394
 395		if (pages) {
 396			for_each_sgt_daddr_next(addr, iter) {
 397				if (count == n_ptes)
 398					break;
 399				*cs++ = lower_32_bits(pte | addr);
 400				*cs++ = upper_32_bits(pte | addr);
 401				count++;
 402			}
 403			/* fill remaining with scratch pte, if any */
 404			if (count < n_ptes) {
 405				memset64((u64 *)cs, scratch_pte,
 406					 n_ptes - count);
 407				cs += (n_ptes - count) * 2;
 408			}
 409		} else {
 410			memset64((u64 *)cs, pte, n_ptes);
 411			cs += n_ptes * 2;
 412		}
 413
 414		intel_ring_advance(rq, cs);
 415queue_err_rq:
 416		i915_request_get(rq);
 417		__i915_request_commit(rq);
 418		__i915_request_queue(rq, &attr);
 419
 420		mutex_unlock(&ce->timeline->mutex);
 421		/* This will break if the request is complete or after engine reset */
 422		i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
 423		if (rq->fence.error)
 424			goto err_rq;
 425
 426		i915_request_put(rq);
 427
 428		num_entries -= n_ptes;
 429		offset += n_ptes;
 430	}
 431
 432	gen8_ggtt_bind_put_ce(ce, wakeref);
 433	return true;
 434
 435err_rq:
 436	i915_request_put(rq);
 437put_ce:
 438	gen8_ggtt_bind_put_ce(ce, wakeref);
 439	return false;
 440}
 441
 442static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 443{
 444	writeq(pte, addr);
 445}
 446
 447static gen8_pte_t gen8_get_pte(void __iomem *addr)
 448{
 449	return readq(addr);
 450}
 451
 452static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 453				  dma_addr_t addr,
 454				  u64 offset,
 455				  unsigned int pat_index,
 456				  u32 flags)
 457{
 458	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 459	gen8_pte_t __iomem *pte =
 460		(gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 461
 462	gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags));
 463
 464	ggtt->invalidate(ggtt);
 465}
 466
 467static dma_addr_t gen8_ggtt_read_entry(struct i915_address_space *vm,
 468				       u64 offset, bool *is_present, bool *is_local)
 469{
 470	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 471	gen8_pte_t __iomem *pte =
 472		(gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 473
 474	return ggtt->vm.pte_decode(gen8_get_pte(pte), is_present, is_local);
 475}
 476
 477static void gen8_ggtt_insert_page_bind(struct i915_address_space *vm,
 478				       dma_addr_t addr, u64 offset,
 479				       unsigned int pat_index, u32 flags)
 480{
 481	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 482	gen8_pte_t pte;
 483
 484	pte = ggtt->vm.pte_encode(addr, pat_index, flags);
 485	if (should_update_ggtt_with_bind(i915_vm_to_ggtt(vm)) &&
 486	    gen8_ggtt_bind_ptes(ggtt, offset, NULL, 1, pte))
 487		return ggtt->invalidate(ggtt);
 488
 489	gen8_ggtt_insert_page(vm, addr, offset, pat_index, flags);
 490}
 491
 492static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 493				     struct i915_vma_resource *vma_res,
 494				     unsigned int pat_index,
 495				     u32 flags)
 496{
 497	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 498	const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);
 499	gen8_pte_t __iomem *gte;
 500	gen8_pte_t __iomem *end;
 501	struct sgt_iter iter;
 502	dma_addr_t addr;
 503
 504	/*
 505	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
 506	 * not to allow the user to override access to a read only page.
 507	 */
 508
 509	gte = (gen8_pte_t __iomem *)ggtt->gsm;
 510	gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
 511	end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
 512	while (gte < end)
 513		gen8_set_pte(gte++, vm->scratch[0]->encode);
 514	end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
 515
 516	for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
 517		gen8_set_pte(gte++, pte_encode | addr);
 518	GEM_BUG_ON(gte > end);
 519
 520	/* Fill the allocated but "unused" space beyond the end of the buffer */
 521	while (gte < end)
 522		gen8_set_pte(gte++, vm->scratch[0]->encode);
 523
 524	/*
 525	 * We want to flush the TLBs only after we're certain all the PTE
 526	 * updates have finished.
 527	 */
 528	ggtt->invalidate(ggtt);
 529}
 530
 531static bool __gen8_ggtt_insert_entries_bind(struct i915_address_space *vm,
 532					    struct i915_vma_resource *vma_res,
 533					    unsigned int pat_index, u32 flags)
 534{
 535	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 536	gen8_pte_t scratch_pte = vm->scratch[0]->encode;
 537	gen8_pte_t pte_encode;
 538	u64 start, end;
 539
 540	pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);
 541	start = (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
 542	end = start + vma_res->guard / I915_GTT_PAGE_SIZE;
 543	if (!gen8_ggtt_bind_ptes(ggtt, start, NULL, end - start, scratch_pte))
 544		goto err;
 545
 546	start = end;
 547	end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
 548	if (!gen8_ggtt_bind_ptes(ggtt, start, vma_res->bi.pages,
 549	      vma_res->node_size / I915_GTT_PAGE_SIZE, pte_encode))
 550		goto err;
 551
 552	start += vma_res->node_size / I915_GTT_PAGE_SIZE;
 553	if (!gen8_ggtt_bind_ptes(ggtt, start, NULL, end - start, scratch_pte))
 554		goto err;
 555
 556	return true;
 557
 558err:
 559	return false;
 560}
 561
 562static void gen8_ggtt_insert_entries_bind(struct i915_address_space *vm,
 563					  struct i915_vma_resource *vma_res,
 564					  unsigned int pat_index, u32 flags)
 565{
 566	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 567
 568	if (should_update_ggtt_with_bind(i915_vm_to_ggtt(vm)) &&
 569	    __gen8_ggtt_insert_entries_bind(vm, vma_res, pat_index, flags))
 570		return ggtt->invalidate(ggtt);
 571
 572	gen8_ggtt_insert_entries(vm, vma_res, pat_index, flags);
 573}
 574
 575static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 576				  u64 start, u64 length)
 577{
 578	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 579	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 580	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 581	const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
 582	gen8_pte_t __iomem *gtt_base =
 583		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
 584	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 585	int i;
 586
 587	if (WARN(num_entries > max_entries,
 588		 "First entry = %d; Num entries = %d (max=%d)\n",
 589		 first_entry, num_entries, max_entries))
 590		num_entries = max_entries;
 591
 592	for (i = 0; i < num_entries; i++)
 593		gen8_set_pte(&gtt_base[i], scratch_pte);
 594}
 595
 596static void gen8_ggtt_scratch_range_bind(struct i915_address_space *vm,
 597					 u64 start, u64 length)
 598{
 599	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 600	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 601	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 602	const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
 603	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 604
 605	if (WARN(num_entries > max_entries,
 606		 "First entry = %d; Num entries = %d (max=%d)\n",
 607		 first_entry, num_entries, max_entries))
 608		num_entries = max_entries;
 609
 610	if (should_update_ggtt_with_bind(ggtt) && gen8_ggtt_bind_ptes(ggtt, first_entry,
 611	     NULL, num_entries, scratch_pte))
 612		return ggtt->invalidate(ggtt);
 613
 614	gen8_ggtt_clear_range(vm, start, length);
 615}
 616
 617static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 618				  dma_addr_t addr,
 619				  u64 offset,
 620				  unsigned int pat_index,
 621				  u32 flags)
 622{
 623	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 624	gen6_pte_t __iomem *pte =
 625		(gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 626
 627	iowrite32(vm->pte_encode(addr, pat_index, flags), pte);
 628
 629	ggtt->invalidate(ggtt);
 630}
 631
 632static dma_addr_t gen6_ggtt_read_entry(struct i915_address_space *vm,
 633				       u64 offset,
 634				       bool *is_present, bool *is_local)
 635{
 636	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 637	gen6_pte_t __iomem *pte =
 638		(gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 639
 640	return vm->pte_decode(ioread32(pte), is_present, is_local);
 641}
 642
 643/*
 644 * Binds an object into the global gtt with the specified cache level.
 645 * The object will be accessible to the GPU via commands whose operands
 646 * reference offsets within the global GTT as well as accessible by the GPU
 647 * through the GMADR mapped BAR (i915->mm.gtt->gtt).
 648 */
 649static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 650				     struct i915_vma_resource *vma_res,
 651				     unsigned int pat_index,
 652				     u32 flags)
 653{
 654	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 655	gen6_pte_t __iomem *gte;
 656	gen6_pte_t __iomem *end;
 657	struct sgt_iter iter;
 658	dma_addr_t addr;
 659
 660	gte = (gen6_pte_t __iomem *)ggtt->gsm;
 661	gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;
 662
 663	end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;
 664	while (gte < end)
 665		iowrite32(vm->scratch[0]->encode, gte++);
 666	end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;
 667	for_each_sgt_daddr(addr, iter, vma_res->bi.pages)
 668		iowrite32(vm->pte_encode(addr, pat_index, flags), gte++);
 669	GEM_BUG_ON(gte > end);
 670
 671	/* Fill the allocated but "unused" space beyond the end of the buffer */
 672	while (gte < end)
 673		iowrite32(vm->scratch[0]->encode, gte++);
 674
 675	/*
 676	 * We want to flush the TLBs only after we're certain all the PTE
 677	 * updates have finished.
 678	 */
 679	ggtt->invalidate(ggtt);
 680}
 681
 682static void nop_clear_range(struct i915_address_space *vm,
 683			    u64 start, u64 length)
 684{
 685}
 686
 687static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
 688{
 689	/*
 690	 * Make sure the internal GAM fifo has been cleared of all GTT
 691	 * writes before exiting stop_machine(). This guarantees that
 692	 * any aperture accesses waiting to start in another process
 693	 * cannot back up behind the GTT writes causing a hang.
 694	 * The register can be any arbitrary GAM register.
 695	 */
 696	intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
 697}
 698
 699struct insert_page {
 700	struct i915_address_space *vm;
 701	dma_addr_t addr;
 702	u64 offset;
 703	unsigned int pat_index;
 704};
 705
 706static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
 707{
 708	struct insert_page *arg = _arg;
 709
 710	gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset,
 711			      arg->pat_index, 0);
 712	bxt_vtd_ggtt_wa(arg->vm);
 713
 714	return 0;
 715}
 716
 717static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
 718					  dma_addr_t addr,
 719					  u64 offset,
 720					  unsigned int pat_index,
 721					  u32 unused)
 722{
 723	struct insert_page arg = { vm, addr, offset, pat_index };
 724
 725	stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
 726}
 727
 728struct insert_entries {
 729	struct i915_address_space *vm;
 730	struct i915_vma_resource *vma_res;
 731	unsigned int pat_index;
 732	u32 flags;
 733};
 734
 735static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 736{
 737	struct insert_entries *arg = _arg;
 738
 739	gen8_ggtt_insert_entries(arg->vm, arg->vma_res,
 740				 arg->pat_index, arg->flags);
 741	bxt_vtd_ggtt_wa(arg->vm);
 742
 743	return 0;
 744}
 745
 746static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
 747					     struct i915_vma_resource *vma_res,
 748					     unsigned int pat_index,
 749					     u32 flags)
 750{
 751	struct insert_entries arg = { vm, vma_res, pat_index, flags };
 752
 753	stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
 754}
 755
 756static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 757				  u64 start, u64 length)
 758{
 759	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 760	unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 761	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 762	gen6_pte_t scratch_pte, __iomem *gtt_base =
 763		(gen6_pte_t __iomem *)ggtt->gsm + first_entry;
 764	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 765	int i;
 766
 767	if (WARN(num_entries > max_entries,
 768		 "First entry = %d; Num entries = %d (max=%d)\n",
 769		 first_entry, num_entries, max_entries))
 770		num_entries = max_entries;
 771
 772	scratch_pte = vm->scratch[0]->encode;
 773	for (i = 0; i < num_entries; i++)
 774		iowrite32(scratch_pte, &gtt_base[i]);
 775}
 776
 777void intel_ggtt_bind_vma(struct i915_address_space *vm,
 778			 struct i915_vm_pt_stash *stash,
 779			 struct i915_vma_resource *vma_res,
 780			 unsigned int pat_index,
 781			 u32 flags)
 782{
 783	u32 pte_flags;
 784
 785	if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))
 786		return;
 787
 788	vma_res->bound_flags |= flags;
 789
 790	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 791	pte_flags = 0;
 792	if (vma_res->bi.readonly)
 793		pte_flags |= PTE_READ_ONLY;
 794	if (vma_res->bi.lmem)
 795		pte_flags |= PTE_LM;
 796
 797	vm->insert_entries(vm, vma_res, pat_index, pte_flags);
 798	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
 799}
 800
 801void intel_ggtt_unbind_vma(struct i915_address_space *vm,
 802			   struct i915_vma_resource *vma_res)
 803{
 804	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 805}
 806
 807dma_addr_t intel_ggtt_read_entry(struct i915_address_space *vm,
 808				 u64 offset, bool *is_present, bool *is_local)
 809{
 810	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 811
 812	return ggtt->vm.read_entry(vm, offset, is_present, is_local);
 813}
 814
 815/*
 816 * Reserve the top of the GuC address space for firmware images. Addresses
 817 * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC,
 818 * which makes for a suitable range to hold GuC/HuC firmware images if the
 819 * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT
 820 * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk
 821 * of the same size anyway, which is far more than needed, to keep the logic
 822 * in uc_fw_ggtt_offset() simple.
 823 */
 824#define GUC_TOP_RESERVE_SIZE (SZ_4G - GUC_GGTT_TOP)
 825
 826static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
 827{
 828	u64 offset;
 829	int ret;
 830
 831	if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
 832		return 0;
 833
 834	GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE);
 835	offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE;
 836
 837	ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw,
 838				   GUC_TOP_RESERVE_SIZE, offset,
 839				   I915_COLOR_UNEVICTABLE, PIN_NOEVICT);
 840	if (ret)
 841		drm_dbg(&ggtt->vm.i915->drm,
 842			"Failed to reserve top of GGTT for GuC\n");
 843
 844	return ret;
 845}
 846
 847static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
 848{
 849	if (drm_mm_node_allocated(&ggtt->uc_fw))
 850		drm_mm_remove_node(&ggtt->uc_fw);
 851}
 852
 853static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
 854{
 855	ggtt_release_guc_top(ggtt);
 856	if (drm_mm_node_allocated(&ggtt->error_capture))
 857		drm_mm_remove_node(&ggtt->error_capture);
 858	mutex_destroy(&ggtt->error_mutex);
 859}
 860
 861static int init_ggtt(struct i915_ggtt *ggtt)
 862{
 863	/*
 864	 * Let GEM Manage all of the aperture.
 865	 *
 866	 * However, leave one page at the end still bound to the scratch page.
 867	 * There are a number of places where the hardware apparently prefetches
 868	 * past the end of the object, and we've seen multiple hangs with the
 869	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
 870	 * aperture.  One page should be enough to keep any prefetching inside
 871	 * of the aperture.
 872	 */
 873	unsigned long hole_start, hole_end;
 874	struct drm_mm_node *entry;
 875	int ret;
 876
 877	/*
 878	 * GuC requires all resources that we're sharing with it to be placed in
 879	 * non-WOPCM memory. If GuC is not present or not in use we still need a
 880	 * small bias as ring wraparound at offset 0 sometimes hangs. No idea
 881	 * why.
 882	 */
 883	ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
 884			       intel_wopcm_guc_size(&ggtt->vm.gt->wopcm));
 885
 886	ret = intel_vgt_balloon(ggtt);
 887	if (ret)
 888		return ret;
 889
 890	mutex_init(&ggtt->error_mutex);
 891	if (ggtt->mappable_end) {
 892		/*
 893		 * Reserve a mappable slot for our lockless error capture.
 894		 *
 895		 * We strongly prefer taking address 0x0 in order to protect
 896		 * other critical buffers against accidental overwrites,
 897		 * as writing to address 0 is a very common mistake.
 898		 *
 899		 * Since 0 may already be in use by the system (e.g. the BIOS
 900		 * framebuffer), we let the reservation fail quietly and hope
 901		 * 0 remains reserved always.
 902		 *
 903		 * If we fail to reserve 0, and then fail to find any space
 904		 * for an error-capture, remain silent. We can afford not
 905		 * to reserve an error_capture node as we have fallback
 906		 * paths, and we trust that 0 will remain reserved. However,
 907		 * the only likely reason for failure to insert is a driver
 908		 * bug, which we expect to cause other failures...
 909		 *
 910		 * Since CPU can perform speculative reads on error capture
 911		 * (write-combining allows it) add scratch page after error
 912		 * capture to avoid DMAR errors.
 913		 */
 914		ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE;
 915		ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;
 916		if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))
 917			drm_mm_insert_node_in_range(&ggtt->vm.mm,
 918						    &ggtt->error_capture,
 919						    ggtt->error_capture.size, 0,
 920						    ggtt->error_capture.color,
 921						    0, ggtt->mappable_end,
 922						    DRM_MM_INSERT_LOW);
 923	}
 924	if (drm_mm_node_allocated(&ggtt->error_capture)) {
 925		u64 start = ggtt->error_capture.start;
 926		u64 size = ggtt->error_capture.size;
 927
 928		ggtt->vm.scratch_range(&ggtt->vm, start, size);
 929		drm_dbg(&ggtt->vm.i915->drm,
 930			"Reserved GGTT:[%llx, %llx] for use by error capture\n",
 931			start, start + size);
 932	}
 933
 934	/*
 935	 * The upper portion of the GuC address space has a sizeable hole
 936	 * (several MB) that is inaccessible by GuC. Reserve this range within
 937	 * GGTT as it can comfortably hold GuC/HuC firmware images.
 938	 */
 939	ret = ggtt_reserve_guc_top(ggtt);
 940	if (ret)
 941		goto err;
 942
 943	/* Clear any non-preallocated blocks */
 944	drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
 945		drm_dbg(&ggtt->vm.i915->drm,
 946			"clearing unused GTT space: [%lx, %lx]\n",
 947			hole_start, hole_end);
 948		ggtt->vm.clear_range(&ggtt->vm, hole_start,
 949				     hole_end - hole_start);
 950	}
 951
 952	/* And finally clear the reserved guard page */
 953	ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
 954
 955	return 0;
 956
 957err:
 958	cleanup_init_ggtt(ggtt);
 959	return ret;
 960}
 961
 962static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
 963				  struct i915_vm_pt_stash *stash,
 964				  struct i915_vma_resource *vma_res,
 965				  unsigned int pat_index,
 966				  u32 flags)
 967{
 968	u32 pte_flags;
 969
 970	/* Currently applicable only to VLV */
 971	pte_flags = 0;
 972	if (vma_res->bi.readonly)
 973		pte_flags |= PTE_READ_ONLY;
 974
 975	if (flags & I915_VMA_LOCAL_BIND)
 976		ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
 977			       stash, vma_res, pat_index, flags);
 978
 979	if (flags & I915_VMA_GLOBAL_BIND)
 980		vm->insert_entries(vm, vma_res, pat_index, pte_flags);
 981
 982	vma_res->bound_flags |= flags;
 983}
 984
 985static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
 986				    struct i915_vma_resource *vma_res)
 987{
 988	if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)
 989		vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 990
 991	if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)
 992		ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);
 993}
 994
 995static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
 996{
 997	struct i915_vm_pt_stash stash = {};
 998	struct i915_ppgtt *ppgtt;
 999	int err;
1000
1001	ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);
1002	if (IS_ERR(ppgtt))
1003		return PTR_ERR(ppgtt);
1004
1005	if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
1006		err = -ENODEV;
1007		goto err_ppgtt;
1008	}
1009
1010	err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
1011	if (err)
1012		goto err_ppgtt;
1013
1014	i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
1015	err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
1016	i915_gem_object_unlock(ppgtt->vm.scratch[0]);
1017	if (err)
1018		goto err_stash;
1019
1020	/*
1021	 * Note we only pre-allocate as far as the end of the global
1022	 * GTT. On 48b / 4-level page-tables, the difference is very,
1023	 * very significant! We have to preallocate as GVT/vgpu does
1024	 * not like the page directory disappearing.
1025	 */
1026	ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
1027
1028	ggtt->alias = ppgtt;
1029	ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
1030
1031	GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);
1032	ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
1033
1034	GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);
1035	ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
1036
1037	i915_vm_free_pt_stash(&ppgtt->vm, &stash);
1038	return 0;
1039
1040err_stash:
1041	i915_vm_free_pt_stash(&ppgtt->vm, &stash);
1042err_ppgtt:
1043	i915_vm_put(&ppgtt->vm);
1044	return err;
1045}
1046
1047static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
1048{
1049	struct i915_ppgtt *ppgtt;
1050
1051	ppgtt = fetch_and_zero(&ggtt->alias);
1052	if (!ppgtt)
1053		return;
1054
1055	i915_vm_put(&ppgtt->vm);
1056
1057	ggtt->vm.vma_ops.bind_vma   = intel_ggtt_bind_vma;
1058	ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;
1059}
1060
1061int i915_init_ggtt(struct drm_i915_private *i915)
1062{
1063	int ret;
1064
1065	ret = init_ggtt(to_gt(i915)->ggtt);
1066	if (ret)
1067		return ret;
1068
1069	if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
1070		ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);
1071		if (ret)
1072			cleanup_init_ggtt(to_gt(i915)->ggtt);
1073	}
1074
1075	return 0;
1076}
1077
1078static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
1079{
1080	struct i915_vma *vma, *vn;
1081
1082	flush_workqueue(ggtt->vm.i915->wq);
1083	i915_gem_drain_freed_objects(ggtt->vm.i915);
1084
1085	mutex_lock(&ggtt->vm.mutex);
1086
1087	ggtt->vm.skip_pte_rewrite = true;
1088
1089	list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
1090		struct drm_i915_gem_object *obj = vma->obj;
1091		bool trylock;
1092
1093		trylock = i915_gem_object_trylock(obj, NULL);
1094		WARN_ON(!trylock);
1095
1096		WARN_ON(__i915_vma_unbind(vma));
1097		if (trylock)
1098			i915_gem_object_unlock(obj);
1099	}
1100
1101	if (drm_mm_node_allocated(&ggtt->error_capture))
1102		drm_mm_remove_node(&ggtt->error_capture);
1103	mutex_destroy(&ggtt->error_mutex);
1104
1105	ggtt_release_guc_top(ggtt);
1106	intel_vgt_deballoon(ggtt);
1107
1108	ggtt->vm.cleanup(&ggtt->vm);
1109
1110	mutex_unlock(&ggtt->vm.mutex);
1111	i915_address_space_fini(&ggtt->vm);
1112
1113	arch_phys_wc_del(ggtt->mtrr);
1114
1115	if (ggtt->iomap.size)
1116		io_mapping_fini(&ggtt->iomap);
1117}
1118
1119/**
1120 * i915_ggtt_driver_release - Clean up GGTT hardware initialization
1121 * @i915: i915 device
1122 */
1123void i915_ggtt_driver_release(struct drm_i915_private *i915)
1124{
1125	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1126
1127	fini_aliasing_ppgtt(ggtt);
1128
1129	intel_ggtt_fini_fences(ggtt);
1130	ggtt_cleanup_hw(ggtt);
1131}
1132
1133/**
1134 * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after
1135 * all free objects have been drained.
1136 * @i915: i915 device
1137 */
1138void i915_ggtt_driver_late_release(struct drm_i915_private *i915)
1139{
1140	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1141
1142	GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);
1143	dma_resv_fini(&ggtt->vm._resv);
1144}
1145
1146static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1147{
1148	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1149	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1150	return snb_gmch_ctl << 20;
1151}
1152
1153static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1154{
1155	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1156	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1157	if (bdw_gmch_ctl)
1158		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1159
1160#ifdef CONFIG_X86_32
1161	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
1162	if (bdw_gmch_ctl > 4)
1163		bdw_gmch_ctl = 4;
1164#endif
1165
1166	return bdw_gmch_ctl << 20;
1167}
1168
1169static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
1170{
1171	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
1172	gmch_ctrl &= SNB_GMCH_GGMS_MASK;
1173
1174	if (gmch_ctrl)
1175		return 1 << (20 + gmch_ctrl);
1176
1177	return 0;
1178}
1179
1180static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915)
1181{
1182	/*
1183	 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset
1184	 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset
1185	 */
1186	GEM_BUG_ON(GRAPHICS_VER(i915) < 6);
1187	return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M;
1188}
1189
1190static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915)
1191{
1192	return gen6_gttmmadr_size(i915) / 2;
1193}
1194
1195static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
1196{
1197	struct drm_i915_private *i915 = ggtt->vm.i915;
1198	struct intel_uncore *uncore = ggtt->vm.gt->uncore;
1199	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1200	phys_addr_t phys_addr;
1201	u32 pte_flags;
1202	int ret;
1203
1204	GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915));
1205
1206	if (i915_direct_stolen_access(i915)) {
1207		drm_dbg(&i915->drm, "Using direct GSM access\n");
1208		phys_addr = intel_uncore_read64(uncore, GEN6_GSMBASE) & GEN11_BDSM_MASK;
1209	} else {
1210		phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915);
1211	}
1212
1213	if (needs_wc_ggtt_mapping(i915))
1214		ggtt->gsm = ioremap_wc(phys_addr, size);
1215	else
1216		ggtt->gsm = ioremap(phys_addr, size);
1217
1218	if (!ggtt->gsm) {
1219		drm_err(&i915->drm, "Failed to map the ggtt page table\n");
1220		return -ENOMEM;
1221	}
1222
1223	kref_init(&ggtt->vm.resv_ref);
1224	ret = setup_scratch_page(&ggtt->vm);
1225	if (ret) {
1226		drm_err(&i915->drm, "Scratch setup failed\n");
1227		/* iounmap will also get called at remove, but meh */
1228		iounmap(ggtt->gsm);
1229		return ret;
1230	}
1231
1232	pte_flags = 0;
1233	if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))
1234		pte_flags |= PTE_LM;
1235
1236	ggtt->vm.scratch[0]->encode =
1237		ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),
1238				    i915_gem_get_pat_index(i915,
1239							   I915_CACHE_NONE),
1240				    pte_flags);
1241
1242	return 0;
1243}
1244
1245static void gen6_gmch_remove(struct i915_address_space *vm)
1246{
1247	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
1248
1249	iounmap(ggtt->gsm);
1250	free_scratch(vm);
1251}
1252
1253static struct resource pci_resource(struct pci_dev *pdev, int bar)
1254{
1255	return DEFINE_RES_MEM(pci_resource_start(pdev, bar),
1256			      pci_resource_len(pdev, bar));
1257}
1258
1259static int gen8_gmch_probe(struct i915_ggtt *ggtt)
1260{
1261	struct drm_i915_private *i915 = ggtt->vm.i915;
1262	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1263	unsigned int size;
1264	u16 snb_gmch_ctl;
1265
1266	if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) {
1267		if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
1268			return -ENXIO;
1269
1270		ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
1271		ggtt->mappable_end = resource_size(&ggtt->gmadr);
1272	}
1273
1274	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1275	if (IS_CHERRYVIEW(i915))
1276		size = chv_get_total_gtt_size(snb_gmch_ctl);
1277	else
1278		size = gen8_get_total_gtt_size(snb_gmch_ctl);
1279
1280	ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1281	ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1282	ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY;
1283
1284	ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
1285	ggtt->vm.cleanup = gen6_gmch_remove;
1286	ggtt->vm.insert_page = gen8_ggtt_insert_page;
1287	ggtt->vm.clear_range = nop_clear_range;
1288	ggtt->vm.scratch_range = gen8_ggtt_clear_range;
1289
1290	ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
1291	ggtt->vm.read_entry = gen8_ggtt_read_entry;
1292
1293	/*
1294	 * Serialize GTT updates with aperture access on BXT if VT-d is on,
1295	 * and always on CHV.
1296	 */
1297	if (intel_vm_no_concurrent_access_wa(i915)) {
1298		ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
1299		ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
1300
1301		/*
1302		 * Calling stop_machine() version of GGTT update function
1303		 * at error capture/reset path will raise lockdep warning.
1304		 * Allow calling gen8_ggtt_insert_* directly at reset path
1305		 * which is safe from parallel GGTT updates.
1306		 */
1307		ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
1308		ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;
1309
1310		ggtt->vm.bind_async_flags =
1311			I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
1312	}
1313
1314	if (i915_ggtt_require_binder(i915)) {
1315		ggtt->vm.scratch_range = gen8_ggtt_scratch_range_bind;
1316		ggtt->vm.insert_page = gen8_ggtt_insert_page_bind;
1317		ggtt->vm.insert_entries = gen8_ggtt_insert_entries_bind;
1318		/*
1319		 * On GPU is hung, we might bind VMAs for error capture.
1320		 * Fallback to CPU GGTT updates in that case.
1321		 */
1322		ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
1323	}
1324
1325	if (intel_uc_wants_guc_submission(&ggtt->vm.gt->uc))
1326		ggtt->invalidate = guc_ggtt_invalidate;
1327	else
1328		ggtt->invalidate = gen8_ggtt_invalidate;
1329
1330	ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1331	ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1332
1333	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1334		ggtt->vm.pte_encode = mtl_ggtt_pte_encode;
1335	else
1336		ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
1337
1338	ggtt->vm.pte_decode = gen8_ggtt_pte_decode;
1339
1340	return ggtt_probe_common(ggtt, size);
1341}
1342
1343/*
1344 * For pre-gen8 platforms pat_index is the same as enum i915_cache_level,
1345 * so the switch-case statements in these PTE encode functions are still valid.
1346 * See translation table LEGACY_CACHELEVEL.
1347 */
1348static u64 snb_pte_encode(dma_addr_t addr,
1349			  unsigned int pat_index,
1350			  u32 flags)
1351{
1352	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1353
1354	switch (pat_index) {
1355	case I915_CACHE_L3_LLC:
1356	case I915_CACHE_LLC:
1357		pte |= GEN6_PTE_CACHE_LLC;
1358		break;
1359	case I915_CACHE_NONE:
1360		pte |= GEN6_PTE_UNCACHED;
1361		break;
1362	default:
1363		MISSING_CASE(pat_index);
1364	}
1365
1366	return pte;
1367}
1368
1369static u64 ivb_pte_encode(dma_addr_t addr,
1370			  unsigned int pat_index,
1371			  u32 flags)
1372{
1373	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1374
1375	switch (pat_index) {
1376	case I915_CACHE_L3_LLC:
1377		pte |= GEN7_PTE_CACHE_L3_LLC;
1378		break;
1379	case I915_CACHE_LLC:
1380		pte |= GEN6_PTE_CACHE_LLC;
1381		break;
1382	case I915_CACHE_NONE:
1383		pte |= GEN6_PTE_UNCACHED;
1384		break;
1385	default:
1386		MISSING_CASE(pat_index);
1387	}
1388
1389	return pte;
1390}
1391
1392static u64 byt_pte_encode(dma_addr_t addr,
1393			  unsigned int pat_index,
1394			  u32 flags)
1395{
1396	gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1397
1398	if (!(flags & PTE_READ_ONLY))
1399		pte |= BYT_PTE_WRITEABLE;
1400
1401	if (pat_index != I915_CACHE_NONE)
1402		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
1403
1404	return pte;
1405}
1406
1407static u64 hsw_pte_encode(dma_addr_t addr,
1408			  unsigned int pat_index,
1409			  u32 flags)
1410{
1411	gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1412
1413	if (pat_index != I915_CACHE_NONE)
1414		pte |= HSW_WB_LLC_AGE3;
1415
1416	return pte;
1417}
1418
1419static u64 iris_pte_encode(dma_addr_t addr,
1420			   unsigned int pat_index,
1421			   u32 flags)
1422{
1423	gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
1424
1425	switch (pat_index) {
1426	case I915_CACHE_NONE:
1427		break;
1428	case I915_CACHE_WT:
1429		pte |= HSW_WT_ELLC_LLC_AGE3;
1430		break;
1431	default:
1432		pte |= HSW_WB_ELLC_LLC_AGE3;
1433		break;
1434	}
1435
1436	return pte;
1437}
1438
1439static dma_addr_t gen6_pte_decode(u64 pte, bool *is_present, bool *is_local)
1440{
1441	*is_present = pte & GEN6_PTE_VALID;
1442	*is_local = false;
1443
1444	return ((pte & 0xff0) << 28) | (pte & ~0xfff);
1445}
1446
1447static int gen6_gmch_probe(struct i915_ggtt *ggtt)
1448{
1449	struct drm_i915_private *i915 = ggtt->vm.i915;
1450	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
1451	unsigned int size;
1452	u16 snb_gmch_ctl;
1453
1454	if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))
1455		return -ENXIO;
1456
1457	ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);
1458	ggtt->mappable_end = resource_size(&ggtt->gmadr);
1459
1460	/*
1461	 * 64/512MB is the current min/max we actually know of, but this is
1462	 * just a coarse sanity check.
1463	 */
1464	if (ggtt->mappable_end < (64 << 20) ||
1465	    ggtt->mappable_end > (512 << 20)) {
1466		drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",
1467			&ggtt->mappable_end);
1468		return -ENXIO;
1469	}
1470
1471	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1472
1473	size = gen6_get_total_gtt_size(snb_gmch_ctl);
1474	ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
1475
1476	ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1477	ggtt->vm.alloc_scratch_dma = alloc_pt_dma;
1478
1479	ggtt->vm.clear_range = nop_clear_range;
1480	if (!HAS_FULL_PPGTT(i915))
1481		ggtt->vm.clear_range = gen6_ggtt_clear_range;
1482	ggtt->vm.scratch_range = gen6_ggtt_clear_range;
1483	ggtt->vm.insert_page = gen6_ggtt_insert_page;
1484	ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
1485	ggtt->vm.read_entry = gen6_ggtt_read_entry;
1486	ggtt->vm.cleanup = gen6_gmch_remove;
1487
1488	ggtt->invalidate = gen6_ggtt_invalidate;
1489
1490	if (HAS_EDRAM(i915))
1491		ggtt->vm.pte_encode = iris_pte_encode;
1492	else if (IS_HASWELL(i915))
1493		ggtt->vm.pte_encode = hsw_pte_encode;
1494	else if (IS_VALLEYVIEW(i915))
1495		ggtt->vm.pte_encode = byt_pte_encode;
1496	else if (GRAPHICS_VER(i915) >= 7)
1497		ggtt->vm.pte_encode = ivb_pte_encode;
1498	else
1499		ggtt->vm.pte_encode = snb_pte_encode;
1500
1501	ggtt->vm.pte_decode = gen6_pte_decode;
1502
1503	ggtt->vm.vma_ops.bind_vma    = intel_ggtt_bind_vma;
1504	ggtt->vm.vma_ops.unbind_vma  = intel_ggtt_unbind_vma;
1505
1506	return ggtt_probe_common(ggtt, size);
1507}
1508
1509static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1510{
1511	struct drm_i915_private *i915 = gt->i915;
1512	int ret;
1513
1514	ggtt->vm.gt = gt;
1515	ggtt->vm.i915 = i915;
1516	ggtt->vm.dma = i915->drm.dev;
1517	dma_resv_init(&ggtt->vm._resv);
1518
1519	if (GRAPHICS_VER(i915) >= 8)
1520		ret = gen8_gmch_probe(ggtt);
1521	else if (GRAPHICS_VER(i915) >= 6)
1522		ret = gen6_gmch_probe(ggtt);
1523	else
1524		ret = intel_ggtt_gmch_probe(ggtt);
1525
1526	if (ret) {
1527		dma_resv_fini(&ggtt->vm._resv);
1528		return ret;
1529	}
1530
1531	if ((ggtt->vm.total - 1) >> 32) {
1532		drm_err(&i915->drm,
1533			"We never expected a Global GTT with more than 32bits"
1534			" of address space! Found %lldM!\n",
1535			ggtt->vm.total >> 20);
1536		ggtt->vm.total = 1ULL << 32;
1537		ggtt->mappable_end =
1538			min_t(u64, ggtt->mappable_end, ggtt->vm.total);
1539	}
1540
1541	if (ggtt->mappable_end > ggtt->vm.total) {
1542		drm_err(&i915->drm,
1543			"mappable aperture extends past end of GGTT,"
1544			" aperture=%pa, total=%llx\n",
1545			&ggtt->mappable_end, ggtt->vm.total);
1546		ggtt->mappable_end = ggtt->vm.total;
1547	}
1548
1549	/* GMADR is the PCI mmio aperture into the global GTT. */
1550	drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
1551	drm_dbg(&i915->drm, "GMADR size = %lluM\n",
1552		(u64)ggtt->mappable_end >> 20);
1553	drm_dbg(&i915->drm, "DSM size = %lluM\n",
1554		(u64)resource_size(&intel_graphics_stolen_res) >> 20);
1555
1556	return 0;
1557}
1558
1559/**
1560 * i915_ggtt_probe_hw - Probe GGTT hardware location
1561 * @i915: i915 device
1562 */
1563int i915_ggtt_probe_hw(struct drm_i915_private *i915)
1564{
1565	struct intel_gt *gt;
1566	int ret, i;
1567
1568	for_each_gt(gt, i915, i) {
1569		ret = intel_gt_assign_ggtt(gt);
1570		if (ret)
1571			return ret;
1572	}
1573
1574	ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
1575	if (ret)
1576		return ret;
1577
1578	if (i915_vtd_active(i915))
1579		drm_info(&i915->drm, "VT-d active for gfx access\n");
1580
1581	return 0;
1582}
1583
1584struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915)
1585{
1586	struct i915_ggtt *ggtt;
1587
1588	ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL);
1589	if (!ggtt)
1590		return ERR_PTR(-ENOMEM);
1591
1592	INIT_LIST_HEAD(&ggtt->gt_list);
1593
1594	return ggtt;
1595}
1596
1597int i915_ggtt_enable_hw(struct drm_i915_private *i915)
1598{
1599	if (GRAPHICS_VER(i915) < 6)
1600		return intel_ggtt_gmch_enable_hw(i915);
1601
1602	return 0;
1603}
1604
1605/**
1606 * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM
1607 * @vm: The VM to restore the mappings for
1608 * @all_evicted: Were all VMAs expected to be evicted on suspend?
1609 *
1610 * Restore the memory mappings for all objects mapped to HW via the GGTT or a
1611 * DPT page table.
1612 *
1613 * Returns %true if restoring the mapping for any object that was in a write
1614 * domain before suspend.
1615 */
1616bool i915_ggtt_resume_vm(struct i915_address_space *vm, bool all_evicted)
1617{
1618	struct i915_vma *vma;
1619	bool write_domain_objs = false;
1620
1621	drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);
1622
1623	if (all_evicted) {
1624		drm_WARN_ON(&vm->i915->drm, !list_empty(&vm->bound_list));
1625		return false;
1626	}
1627
1628	/* First fill our portion of the GTT with scratch pages */
1629	vm->clear_range(vm, 0, vm->total);
1630
1631	/* clflush objects bound into the GGTT and rebind them. */
1632	list_for_each_entry(vma, &vm->bound_list, vm_link) {
1633		struct drm_i915_gem_object *obj = vma->obj;
1634		unsigned int was_bound =
1635			atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
1636
1637		GEM_BUG_ON(!was_bound);
1638
1639		/*
1640		 * Clear the bound flags of the vma resource to allow
1641		 * ptes to be repopulated.
1642		 */
1643		vma->resource->bound_flags = 0;
1644		vma->ops->bind_vma(vm, NULL, vma->resource,
1645				   obj ? obj->pat_index :
1646					 i915_gem_get_pat_index(vm->i915,
1647								I915_CACHE_NONE),
1648				   was_bound);
1649
1650		if (obj) { /* only used during resume => exclusive access */
1651			write_domain_objs |= fetch_and_zero(&obj->write_domain);
1652			obj->read_domains |= I915_GEM_DOMAIN_GTT;
1653		}
1654	}
1655
1656	return write_domain_objs;
1657}
1658
1659void i915_ggtt_resume(struct i915_ggtt *ggtt)
1660{
1661	struct intel_gt *gt;
1662	bool flush;
1663
1664	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1665		intel_gt_check_and_clear_faults(gt);
1666
1667	flush = i915_ggtt_resume_vm(&ggtt->vm, false);
1668
1669	if (drm_mm_node_allocated(&ggtt->error_capture))
1670		ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start,
1671				       ggtt->error_capture.size);
1672
1673	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1674		intel_uc_resume_mappings(&gt->uc);
1675
1676	ggtt->invalidate(ggtt);
1677
1678	if (flush)
1679		wbinvd_on_all_cpus();
1680
1681	intel_ggtt_restore_fences(ggtt);
1682}