xe_ggtt.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2021 Intel Corporation
   4 */
   5
   6#include "xe_ggtt.h"
   7
   8#include <kunit/visibility.h>
   9#include <linux/fault-inject.h>
  10#include <linux/io-64-nonatomic-lo-hi.h>
  11#include <linux/sizes.h>
  12
  13#include <drm/drm_drv.h>
  14#include <drm/drm_managed.h>
  15#include <drm/intel/i915_drm.h>
  16#include <generated/xe_wa_oob.h>
  17
  18#include "regs/xe_gt_regs.h"
  19#include "regs/xe_gtt_defs.h"
  20#include "regs/xe_regs.h"
  21#include "xe_assert.h"
  22#include "xe_bo.h"
  23#include "xe_device.h"
  24#include "xe_gt.h"
  25#include "xe_gt_printk.h"
  26#include "xe_map.h"
  27#include "xe_mmio.h"
  28#include "xe_pm.h"
  29#include "xe_res_cursor.h"
  30#include "xe_sriov.h"
  31#include "xe_tile_printk.h"
  32#include "xe_tile_sriov_vf.h"
  33#include "xe_tlb_inval.h"
  34#include "xe_wa.h"
  35#include "xe_wopcm.h"
  36
  37/**
  38 * DOC: Global Graphics Translation Table (GGTT)
  39 *
  40 * Xe GGTT implements the support for a Global Virtual Address space that is used
  41 * for resources that are accessible to privileged (i.e. kernel-mode) processes,
  42 * and not tied to a specific user-level process. For example, the Graphics
  43 * micro-Controller (GuC) and Display Engine (if present) utilize this Global
  44 * address space.
  45 *
  46 * The Global GTT (GGTT) translates from the Global virtual address to a physical
  47 * address that can be accessed by HW. The GGTT is a flat, single-level table.
  48 *
  49 * Xe implements a simplified version of the GGTT specifically managing only a
  50 * certain range of it that goes from the Write Once Protected Content Memory (WOPCM)
  51 * Layout to a predefined GUC_GGTT_TOP. This approach avoids complications related to
  52 * the GuC (Graphics Microcontroller) hardware limitations. The GuC address space
  53 * is limited on both ends of the GGTT, because the GuC shim HW redirects
  54 * accesses to those addresses to other HW areas instead of going through the
  55 * GGTT. On the bottom end, the GuC can't access offsets below the WOPCM size,
  56 * while on the top side the limit is fixed at GUC_GGTT_TOP. To keep things
  57 * simple, instead of checking each object to see if they are accessed by GuC or
  58 * not, we just exclude those areas from the allocator. Additionally, to simplify
  59 * the driver load, we use the maximum WOPCM size in this logic instead of the
  60 * programmed one, so we don't need to wait until the actual size to be
  61 * programmed is determined (which requires FW fetch) before initializing the
  62 * GGTT. These simplifications might waste space in the GGTT (about 20-25 MBs
  63 * depending on the platform) but we can live with this. Another benefit of this
  64 * is the GuC bootrom can't access anything below the WOPCM max size so anything
  65 * the bootrom needs to access (e.g. a RSA key) needs to be placed in the GGTT
  66 * above the WOPCM max size. Starting the GGTT allocations above the WOPCM max
  67 * give us the correct placement for free.
  68 */
  69
  70static u64 xelp_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
  71{
  72	u64 pte = XE_PAGE_PRESENT;
  73
  74	if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo))
  75		pte |= XE_GGTT_PTE_DM;
  76
  77	return pte;
  78}
  79
  80static u64 xelpg_ggtt_pte_flags(struct xe_bo *bo, u16 pat_index)
  81{
  82	struct xe_device *xe = xe_bo_device(bo);
  83	u64 pte;
  84
  85	pte = xelp_ggtt_pte_flags(bo, pat_index);
  86
  87	xe_assert(xe, pat_index <= 3);
  88
  89	if (pat_index & BIT(0))
  90		pte |= XELPG_GGTT_PTE_PAT0;
  91
  92	if (pat_index & BIT(1))
  93		pte |= XELPG_GGTT_PTE_PAT1;
  94
  95	return pte;
  96}
  97
  98static unsigned int probe_gsm_size(struct pci_dev *pdev)
  99{
 100	u16 gmch_ctl, ggms;
 101
 102	pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl);
 103	ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK;
 104	return ggms ? SZ_1M << ggms : 0;
 105}
 106
 107static void ggtt_update_access_counter(struct xe_ggtt *ggtt)
 108{
 109	struct xe_tile *tile = ggtt->tile;
 110	struct xe_gt *affected_gt;
 111	u32 max_gtt_writes;
 112
 113	if (tile->primary_gt && XE_GT_WA(tile->primary_gt, 22019338487)) {
 114		affected_gt = tile->primary_gt;
 115		max_gtt_writes = 1100;
 116
 117		/* Only expected to apply to primary GT on dgpu platforms */
 118		xe_tile_assert(tile, IS_DGFX(tile_to_xe(tile)));
 119	} else {
 120		affected_gt = tile->media_gt;
 121		max_gtt_writes = 63;
 122
 123		/* Only expected to apply to media GT on igpu platforms */
 124		xe_tile_assert(tile, !IS_DGFX(tile_to_xe(tile)));
 125	}
 126
 127	/*
 128	 * Wa_22019338487: GMD_ID is a RO register, a dummy write forces gunit
 129	 * to wait for completion of prior GTT writes before letting this through.
 130	 * This needs to be done for all GGTT writes originating from the CPU.
 131	 */
 132	lockdep_assert_held(&ggtt->lock);
 133
 134	if ((++ggtt->access_count % max_gtt_writes) == 0) {
 135		xe_mmio_write32(&affected_gt->mmio, GMD_ID, 0x0);
 136		ggtt->access_count = 0;
 137	}
 138}
 139
 140static void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte)
 141{
 142	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
 143	xe_tile_assert(ggtt->tile, addr < ggtt->size);
 144
 145	writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]);
 146}
 147
 148static void xe_ggtt_set_pte_and_flush(struct xe_ggtt *ggtt, u64 addr, u64 pte)
 149{
 150	xe_ggtt_set_pte(ggtt, addr, pte);
 151	ggtt_update_access_counter(ggtt);
 152}
 153
 154static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr)
 155{
 156	xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK));
 157	xe_tile_assert(ggtt->tile, addr < ggtt->size);
 158
 159	return readq(&ggtt->gsm[addr >> XE_PTE_SHIFT]);
 160}
 161
 162static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
 163{
 164	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
 165	u64 end = start + size - 1;
 166	u64 scratch_pte;
 167
 168	xe_tile_assert(ggtt->tile, start < end);
 169
 170	if (ggtt->scratch)
 171		scratch_pte = xe_bo_addr(ggtt->scratch, 0, XE_PAGE_SIZE) |
 172			      ggtt->pt_ops->pte_encode_flags(ggtt->scratch,
 173							     pat_index);
 174	else
 175		scratch_pte = 0;
 176
 177	while (start < end) {
 178		ggtt->pt_ops->ggtt_set_pte(ggtt, start, scratch_pte);
 179		start += XE_PAGE_SIZE;
 180	}
 181}
 182
 183static void primelockdep(struct xe_ggtt *ggtt)
 184{
 185	if (!IS_ENABLED(CONFIG_LOCKDEP))
 186		return;
 187
 188	fs_reclaim_acquire(GFP_KERNEL);
 189	might_lock(&ggtt->lock);
 190	fs_reclaim_release(GFP_KERNEL);
 191}
 192
 193/**
 194 * xe_ggtt_alloc - Allocate a GGTT for a given &xe_tile
 195 * @tile: &xe_tile
 196 *
 197 * Allocates a &xe_ggtt for a given tile.
 198 *
 199 * Return: &xe_ggtt on success, or NULL when out of memory.
 200 */
 201struct xe_ggtt *xe_ggtt_alloc(struct xe_tile *tile)
 202{
 203	struct xe_device *xe = tile_to_xe(tile);
 204	struct xe_ggtt *ggtt;
 205
 206	ggtt = drmm_kzalloc(&xe->drm, sizeof(*ggtt), GFP_KERNEL);
 207	if (!ggtt)
 208		return NULL;
 209
 210	if (drmm_mutex_init(&xe->drm, &ggtt->lock))
 211		return NULL;
 212
 213	primelockdep(ggtt);
 214	ggtt->tile = tile;
 215
 216	return ggtt;
 217}
 218
 219static void ggtt_fini_early(struct drm_device *drm, void *arg)
 220{
 221	struct xe_ggtt *ggtt = arg;
 222
 223	destroy_workqueue(ggtt->wq);
 224	drm_mm_takedown(&ggtt->mm);
 225}
 226
 227static void ggtt_fini(void *arg)
 228{
 229	struct xe_ggtt *ggtt = arg;
 230
 231	ggtt->scratch = NULL;
 232}
 233
 234#ifdef CONFIG_LOCKDEP
 235void xe_ggtt_might_lock(struct xe_ggtt *ggtt)
 236{
 237	might_lock(&ggtt->lock);
 238}
 239#endif
 240
 241static const struct xe_ggtt_pt_ops xelp_pt_ops = {
 242	.pte_encode_flags = xelp_ggtt_pte_flags,
 243	.ggtt_set_pte = xe_ggtt_set_pte,
 244	.ggtt_get_pte = xe_ggtt_get_pte,
 245};
 246
 247static const struct xe_ggtt_pt_ops xelpg_pt_ops = {
 248	.pte_encode_flags = xelpg_ggtt_pte_flags,
 249	.ggtt_set_pte = xe_ggtt_set_pte,
 250	.ggtt_get_pte = xe_ggtt_get_pte,
 251};
 252
 253static const struct xe_ggtt_pt_ops xelpg_pt_wa_ops = {
 254	.pte_encode_flags = xelpg_ggtt_pte_flags,
 255	.ggtt_set_pte = xe_ggtt_set_pte_and_flush,
 256	.ggtt_get_pte = xe_ggtt_get_pte,
 257};
 258
 259static void __xe_ggtt_init_early(struct xe_ggtt *ggtt, u32 reserved)
 260{
 261	drm_mm_init(&ggtt->mm, reserved,
 262		    ggtt->size - reserved);
 263}
 264
 265int xe_ggtt_init_kunit(struct xe_ggtt *ggtt, u32 reserved, u32 size)
 266{
 267	ggtt->size = size;
 268	__xe_ggtt_init_early(ggtt, reserved);
 269	return 0;
 270}
 271EXPORT_SYMBOL_IF_KUNIT(xe_ggtt_init_kunit);
 272
 273static void dev_fini_ggtt(void *arg)
 274{
 275	struct xe_ggtt *ggtt = arg;
 276
 277	scoped_guard(mutex, &ggtt->lock)
 278		ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE;
 279	drain_workqueue(ggtt->wq);
 280}
 281
 282/**
 283 * xe_ggtt_init_early - Early GGTT initialization
 284 * @ggtt: the &xe_ggtt to be initialized
 285 *
 286 * It allows to create new mappings usable by the GuC.
 287 * Mappings are not usable by the HW engines, as it doesn't have scratch nor
 288 * initial clear done to it yet. That will happen in the regular, non-early
 289 * GGTT initialization.
 290 *
 291 * Return: 0 on success or a negative error code on failure.
 292 */
 293int xe_ggtt_init_early(struct xe_ggtt *ggtt)
 294{
 295	struct xe_device *xe = tile_to_xe(ggtt->tile);
 296	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 297	unsigned int gsm_size;
 298	int err;
 299
 300	if (IS_SRIOV_VF(xe) || GRAPHICS_VERx100(xe) >= 1250)
 301		gsm_size = SZ_8M; /* GGTT is expected to be 4GiB */
 302	else
 303		gsm_size = probe_gsm_size(pdev);
 304
 305	if (gsm_size == 0) {
 306		xe_tile_err(ggtt->tile, "Hardware reported no preallocated GSM\n");
 307		return -ENOMEM;
 308	}
 309
 310	ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M;
 311	ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE;
 312
 313	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
 314		ggtt->flags |= XE_GGTT_FLAGS_64K;
 315
 316	if (ggtt->size > GUC_GGTT_TOP)
 317		ggtt->size = GUC_GGTT_TOP;
 318
 319	if (GRAPHICS_VERx100(xe) >= 1270)
 320		ggtt->pt_ops =
 321			(ggtt->tile->media_gt && XE_GT_WA(ggtt->tile->media_gt, 22019338487)) ||
 322			(ggtt->tile->primary_gt && XE_GT_WA(ggtt->tile->primary_gt, 22019338487)) ?
 323			&xelpg_pt_wa_ops : &xelpg_pt_ops;
 324	else
 325		ggtt->pt_ops = &xelp_pt_ops;
 326
 327	ggtt->wq = alloc_workqueue("xe-ggtt-wq", WQ_MEM_RECLAIM, 0);
 328	if (!ggtt->wq)
 329		return -ENOMEM;
 330
 331	__xe_ggtt_init_early(ggtt, xe_wopcm_size(xe));
 332
 333	err = drmm_add_action_or_reset(&xe->drm, ggtt_fini_early, ggtt);
 334	if (err)
 335		return err;
 336
 337	ggtt->flags |= XE_GGTT_FLAGS_ONLINE;
 338	err = devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
 339	if (err)
 340		return err;
 341
 342	if (IS_SRIOV_VF(xe)) {
 343		err = xe_tile_sriov_vf_prepare_ggtt(ggtt->tile);
 344		if (err)
 345			return err;
 346	}
 347
 348	return 0;
 349}
 350ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */
 351
 352static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
 353
 354static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
 355{
 356	struct drm_mm_node *hole;
 357	u64 start, end;
 358
 359	/* Display may have allocated inside ggtt, so be careful with clearing here */
 360	mutex_lock(&ggtt->lock);
 361	drm_mm_for_each_hole(hole, &ggtt->mm, start, end)
 362		xe_ggtt_clear(ggtt, start, end - start);
 363
 364	xe_ggtt_invalidate(ggtt);
 365	mutex_unlock(&ggtt->lock);
 366}
 367
 368static void ggtt_node_remove(struct xe_ggtt_node *node)
 369{
 370	struct xe_ggtt *ggtt = node->ggtt;
 371	bool bound;
 372
 373	mutex_lock(&ggtt->lock);
 374	bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE;
 375	if (bound)
 376		xe_ggtt_clear(ggtt, node->base.start, node->base.size);
 377	drm_mm_remove_node(&node->base);
 378	node->base.size = 0;
 379	mutex_unlock(&ggtt->lock);
 380
 381	if (!bound)
 382		goto free_node;
 383
 384	if (node->invalidate_on_remove)
 385		xe_ggtt_invalidate(ggtt);
 386
 387free_node:
 388	xe_ggtt_node_fini(node);
 389}
 390
 391static void ggtt_node_remove_work_func(struct work_struct *work)
 392{
 393	struct xe_ggtt_node *node = container_of(work, typeof(*node),
 394						 delayed_removal_work);
 395	struct xe_device *xe = tile_to_xe(node->ggtt->tile);
 396
 397	guard(xe_pm_runtime)(xe);
 398	ggtt_node_remove(node);
 399}
 400
 401/**
 402 * xe_ggtt_node_remove - Remove a &xe_ggtt_node from the GGTT
 403 * @node: the &xe_ggtt_node to be removed
 404 * @invalidate: if node needs invalidation upon removal
 405 */
 406void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate)
 407{
 408	struct xe_ggtt *ggtt;
 409	struct xe_device *xe;
 410
 411	if (!node || !node->ggtt)
 412		return;
 413
 414	ggtt = node->ggtt;
 415	xe = tile_to_xe(ggtt->tile);
 416
 417	node->invalidate_on_remove = invalidate;
 418
 419	if (xe_pm_runtime_get_if_active(xe)) {
 420		ggtt_node_remove(node);
 421		xe_pm_runtime_put(xe);
 422	} else {
 423		queue_work(ggtt->wq, &node->delayed_removal_work);
 424	}
 425}
 426
 427/**
 428 * xe_ggtt_init - Regular non-early GGTT initialization
 429 * @ggtt: the &xe_ggtt to be initialized
 430 *
 431 * Return: 0 on success or a negative error code on failure.
 432 */
 433int xe_ggtt_init(struct xe_ggtt *ggtt)
 434{
 435	struct xe_device *xe = tile_to_xe(ggtt->tile);
 436	unsigned int flags;
 437	int err;
 438
 439	/*
 440	 * So we don't need to worry about 64K GGTT layout when dealing with
 441	 * scratch entries, rather keep the scratch page in system memory on
 442	 * platforms where 64K pages are needed for VRAM.
 443	 */
 444	flags = 0;
 445	if (ggtt->flags & XE_GGTT_FLAGS_64K)
 446		flags |= XE_BO_FLAG_SYSTEM;
 447	else
 448		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
 449
 450	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
 451	if (IS_ERR(ggtt->scratch)) {
 452		err = PTR_ERR(ggtt->scratch);
 453		goto err;
 454	}
 455
 456	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, xe_bo_size(ggtt->scratch));
 457
 458	xe_ggtt_initial_clear(ggtt);
 459
 460	return devm_add_action_or_reset(xe->drm.dev, ggtt_fini, ggtt);
 461err:
 462	ggtt->scratch = NULL;
 463	return err;
 464}
 465
 466static void ggtt_invalidate_gt_tlb(struct xe_gt *gt)
 467{
 468	int err;
 469
 470	if (!gt)
 471		return;
 472
 473	err = xe_tlb_inval_ggtt(&gt->tlb_inval);
 474	xe_gt_WARN(gt, err, "Failed to invalidate GGTT (%pe)", ERR_PTR(err));
 475}
 476
 477static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
 478{
 479	struct xe_device *xe = tile_to_xe(ggtt->tile);
 480
 481	/*
 482	 * XXX: Barrier for GGTT pages. Unsure exactly why this required but
 483	 * without this LNL is having issues with the GuC reading scratch page
 484	 * vs. correct GGTT page. Not particularly a hot code path so blindly
 485	 * do a mmio read here which results in GuC reading correct GGTT page.
 486	 */
 487	xe_mmio_read32(xe_root_tile_mmio(xe), VF_CAP_REG);
 488
 489	/* Each GT in a tile has its own TLB to cache GGTT lookups */
 490	ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt);
 491	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
 492}
 493
 494static void xe_ggtt_dump_node(struct xe_ggtt *ggtt,
 495			      const struct drm_mm_node *node, const char *description)
 496{
 497	char buf[10];
 498
 499	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
 500		string_get_size(node->size, 1, STRING_UNITS_2, buf, sizeof(buf));
 501		xe_tile_dbg(ggtt->tile, "GGTT %#llx-%#llx (%s) %s\n",
 502			    node->start, node->start + node->size, buf, description);
 503	}
 504}
 505
 506/**
 507 * xe_ggtt_node_insert_balloon_locked - prevent allocation of specified GGTT addresses
 508 * @node: the &xe_ggtt_node to hold reserved GGTT node
 509 * @start: the starting GGTT address of the reserved region
 510 * @end: then end GGTT address of the reserved region
 511 *
 512 * To be used in cases where ggtt->lock is already taken.
 513 * Use xe_ggtt_node_remove_balloon_locked() to release a reserved GGTT node.
 514 *
 515 * Return: 0 on success or a negative error code on failure.
 516 */
 517int xe_ggtt_node_insert_balloon_locked(struct xe_ggtt_node *node, u64 start, u64 end)
 518{
 519	struct xe_ggtt *ggtt = node->ggtt;
 520	int err;
 521
 522	xe_tile_assert(ggtt->tile, start < end);
 523	xe_tile_assert(ggtt->tile, IS_ALIGNED(start, XE_PAGE_SIZE));
 524	xe_tile_assert(ggtt->tile, IS_ALIGNED(end, XE_PAGE_SIZE));
 525	xe_tile_assert(ggtt->tile, !drm_mm_node_allocated(&node->base));
 526	lockdep_assert_held(&ggtt->lock);
 527
 528	node->base.color = 0;
 529	node->base.start = start;
 530	node->base.size = end - start;
 531
 532	err = drm_mm_reserve_node(&ggtt->mm, &node->base);
 533
 534	if (xe_tile_WARN(ggtt->tile, err, "Failed to balloon GGTT %#llx-%#llx (%pe)\n",
 535			 node->base.start, node->base.start + node->base.size, ERR_PTR(err)))
 536		return err;
 537
 538	xe_ggtt_dump_node(ggtt, &node->base, "balloon");
 539	return 0;
 540}
 541
 542/**
 543 * xe_ggtt_node_remove_balloon_locked - release a reserved GGTT region
 544 * @node: the &xe_ggtt_node with reserved GGTT region
 545 *
 546 * To be used in cases where ggtt->lock is already taken.
 547 * See xe_ggtt_node_insert_balloon_locked() for details.
 548 */
 549void xe_ggtt_node_remove_balloon_locked(struct xe_ggtt_node *node)
 550{
 551	if (!xe_ggtt_node_allocated(node))
 552		return;
 553
 554	lockdep_assert_held(&node->ggtt->lock);
 555
 556	xe_ggtt_dump_node(node->ggtt, &node->base, "remove-balloon");
 557
 558	drm_mm_remove_node(&node->base);
 559}
 560
 561static void xe_ggtt_assert_fit(struct xe_ggtt *ggtt, u64 start, u64 size)
 562{
 563	struct xe_tile *tile = ggtt->tile;
 564	struct xe_device *xe = tile_to_xe(tile);
 565	u64 __maybe_unused wopcm = xe_wopcm_size(xe);
 566
 567	xe_tile_assert(tile, start >= wopcm);
 568	xe_tile_assert(tile, start + size < ggtt->size - wopcm);
 569}
 570
 571/**
 572 * xe_ggtt_shift_nodes_locked - Shift GGTT nodes to adjust for a change in usable address range.
 573 * @ggtt: the &xe_ggtt struct instance
 574 * @shift: change to the location of area provisioned for current VF
 575 *
 576 * This function moves all nodes from the GGTT VM, to a temp list. These nodes are expected
 577 * to represent allocations in range formerly assigned to current VF, before the range changed.
 578 * When the GGTT VM is completely clear of any nodes, they are re-added with shifted offsets.
 579 *
 580 * The function has no ability of failing - because it shifts existing nodes, without
 581 * any additional processing. If the nodes were successfully existing at the old address,
 582 * they will do the same at the new one. A fail inside this function would indicate that
 583 * the list of nodes was either already damaged, or that the shift brings the address range
 584 * outside of valid bounds. Both cases justify an assert rather than error code.
 585 */
 586void xe_ggtt_shift_nodes_locked(struct xe_ggtt *ggtt, s64 shift)
 587{
 588	struct xe_tile *tile __maybe_unused = ggtt->tile;
 589	struct drm_mm_node *node, *tmpn;
 590	LIST_HEAD(temp_list_head);
 591
 592	lockdep_assert_held(&ggtt->lock);
 593
 594	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG))
 595		drm_mm_for_each_node_safe(node, tmpn, &ggtt->mm)
 596			xe_ggtt_assert_fit(ggtt, node->start + shift, node->size);
 597
 598	drm_mm_for_each_node_safe(node, tmpn, &ggtt->mm) {
 599		drm_mm_remove_node(node);
 600		list_add(&node->node_list, &temp_list_head);
 601	}
 602
 603	list_for_each_entry_safe(node, tmpn, &temp_list_head, node_list) {
 604		list_del(&node->node_list);
 605		node->start += shift;
 606		drm_mm_reserve_node(&ggtt->mm, node);
 607		xe_tile_assert(tile, drm_mm_node_allocated(node));
 608	}
 609}
 610
 611/**
 612 * xe_ggtt_node_insert_locked - Locked version to insert a &xe_ggtt_node into the GGTT
 613 * @node: the &xe_ggtt_node to be inserted
 614 * @size: size of the node
 615 * @align: alignment constrain of the node
 616 * @mm_flags: flags to control the node behavior
 617 *
 618 * It cannot be called without first having called xe_ggtt_init() once.
 619 * To be used in cases where ggtt->lock is already taken.
 620 *
 621 * Return: 0 on success or a negative error code on failure.
 622 */
 623int xe_ggtt_node_insert_locked(struct xe_ggtt_node *node,
 624			       u32 size, u32 align, u32 mm_flags)
 625{
 626	return drm_mm_insert_node_generic(&node->ggtt->mm, &node->base, size, align, 0,
 627					  mm_flags);
 628}
 629
 630/**
 631 * xe_ggtt_node_insert - Insert a &xe_ggtt_node into the GGTT
 632 * @node: the &xe_ggtt_node to be inserted
 633 * @size: size of the node
 634 * @align: alignment constrain of the node
 635 *
 636 * It cannot be called without first having called xe_ggtt_init() once.
 637 *
 638 * Return: 0 on success or a negative error code on failure.
 639 */
 640int xe_ggtt_node_insert(struct xe_ggtt_node *node, u32 size, u32 align)
 641{
 642	int ret;
 643
 644	if (!node || !node->ggtt)
 645		return -ENOENT;
 646
 647	mutex_lock(&node->ggtt->lock);
 648	ret = xe_ggtt_node_insert_locked(node, size, align,
 649					 DRM_MM_INSERT_HIGH);
 650	mutex_unlock(&node->ggtt->lock);
 651
 652	return ret;
 653}
 654
 655/**
 656 * xe_ggtt_node_init - Initialize %xe_ggtt_node struct
 657 * @ggtt: the &xe_ggtt where the new node will later be inserted/reserved.
 658 *
 659 * This function will allocate the struct %xe_ggtt_node and return its pointer.
 660 * This struct will then be freed after the node removal upon xe_ggtt_node_remove()
 661 * or xe_ggtt_node_remove_balloon_locked().
 662 * Having %xe_ggtt_node struct allocated doesn't mean that the node is already allocated
 663 * in GGTT. Only the xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
 664 * xe_ggtt_node_insert_balloon_locked() will ensure the node is inserted or reserved in GGTT.
 665 *
 666 * Return: A pointer to %xe_ggtt_node struct on success. An ERR_PTR otherwise.
 667 **/
 668struct xe_ggtt_node *xe_ggtt_node_init(struct xe_ggtt *ggtt)
 669{
 670	struct xe_ggtt_node *node = kzalloc(sizeof(*node), GFP_NOFS);
 671
 672	if (!node)
 673		return ERR_PTR(-ENOMEM);
 674
 675	INIT_WORK(&node->delayed_removal_work, ggtt_node_remove_work_func);
 676	node->ggtt = ggtt;
 677
 678	return node;
 679}
 680
 681/**
 682 * xe_ggtt_node_fini - Forcebly finalize %xe_ggtt_node struct
 683 * @node: the &xe_ggtt_node to be freed
 684 *
 685 * If anything went wrong with either xe_ggtt_node_insert(), xe_ggtt_node_insert_locked(),
 686 * or xe_ggtt_node_insert_balloon_locked(); and this @node is not going to be reused, then,
 687 * this function needs to be called to free the %xe_ggtt_node struct
 688 **/
 689void xe_ggtt_node_fini(struct xe_ggtt_node *node)
 690{
 691	kfree(node);
 692}
 693
 694/**
 695 * xe_ggtt_node_allocated - Check if node is allocated in GGTT
 696 * @node: the &xe_ggtt_node to be inspected
 697 *
 698 * Return: True if allocated, False otherwise.
 699 */
 700bool xe_ggtt_node_allocated(const struct xe_ggtt_node *node)
 701{
 702	if (!node || !node->ggtt)
 703		return false;
 704
 705	return drm_mm_node_allocated(&node->base);
 706}
 707
 708/**
 709 * xe_ggtt_node_pt_size() - Get the size of page table entries needed to map a GGTT node.
 710 * @node: the &xe_ggtt_node
 711 *
 712 * Return: GGTT node page table entries size in bytes.
 713 */
 714size_t xe_ggtt_node_pt_size(const struct xe_ggtt_node *node)
 715{
 716	if (!node)
 717		return 0;
 718
 719	return node->base.size / XE_PAGE_SIZE * sizeof(u64);
 720}
 721
 722/**
 723 * xe_ggtt_map_bo - Map the BO into GGTT
 724 * @ggtt: the &xe_ggtt where node will be mapped
 725 * @node: the &xe_ggtt_node where this BO is mapped
 726 * @bo: the &xe_bo to be mapped
 727 * @pat_index: Which pat_index to use.
 728 */
 729void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node,
 730		    struct xe_bo *bo, u16 pat_index)
 731{
 732
 733	u64 start, pte, end;
 734	struct xe_res_cursor cur;
 735
 736	if (XE_WARN_ON(!node))
 737		return;
 738
 739	start = node->base.start;
 740	end = start + xe_bo_size(bo);
 741
 742	pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index);
 743	if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {
 744		xe_assert(xe_bo_device(bo), bo->ttm.ttm);
 745
 746		for (xe_res_first_sg(xe_bo_sg(bo), 0, xe_bo_size(bo), &cur);
 747		     cur.remaining; xe_res_next(&cur, XE_PAGE_SIZE))
 748			ggtt->pt_ops->ggtt_set_pte(ggtt, end - cur.remaining,
 749						   pte | xe_res_dma(&cur));
 750	} else {
 751		/* Prepend GPU offset */
 752		pte |= vram_region_gpu_offset(bo->ttm.resource);
 753
 754		for (xe_res_first(bo->ttm.resource, 0, xe_bo_size(bo), &cur);
 755		     cur.remaining; xe_res_next(&cur, XE_PAGE_SIZE))
 756			ggtt->pt_ops->ggtt_set_pte(ggtt, end - cur.remaining,
 757						   pte + cur.start);
 758	}
 759}
 760
 761/**
 762 * xe_ggtt_map_bo_unlocked - Restore a mapping of a BO into GGTT
 763 * @ggtt: the &xe_ggtt where node will be mapped
 764 * @bo: the &xe_bo to be mapped
 765 *
 766 * This is used to restore a GGTT mapping after suspend.
 767 */
 768void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo)
 769{
 770	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
 771	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
 772
 773	mutex_lock(&ggtt->lock);
 774	xe_ggtt_map_bo(ggtt, bo->ggtt_node[ggtt->tile->id], bo, pat_index);
 775	mutex_unlock(&ggtt->lock);
 776}
 777
 778static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
 779				  u64 start, u64 end, struct drm_exec *exec)
 780{
 781	u64 alignment = bo->min_align > 0 ? bo->min_align : XE_PAGE_SIZE;
 782	u8 tile_id = ggtt->tile->id;
 783	int err;
 784
 785	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
 786		alignment = SZ_64K;
 787
 788	if (XE_WARN_ON(bo->ggtt_node[tile_id])) {
 789		/* Someone's already inserted this BO in the GGTT */
 790		xe_tile_assert(ggtt->tile, bo->ggtt_node[tile_id]->base.size == xe_bo_size(bo));
 791		return 0;
 792	}
 793
 794	err = xe_bo_validate(bo, NULL, false, exec);
 795	if (err)
 796		return err;
 797
 798	xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
 799
 800	bo->ggtt_node[tile_id] = xe_ggtt_node_init(ggtt);
 801	if (IS_ERR(bo->ggtt_node[tile_id])) {
 802		err = PTR_ERR(bo->ggtt_node[tile_id]);
 803		bo->ggtt_node[tile_id] = NULL;
 804		goto out;
 805	}
 806
 807	mutex_lock(&ggtt->lock);
 808	err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node[tile_id]->base,
 809					  xe_bo_size(bo), alignment, 0, start, end, 0);
 810	if (err) {
 811		xe_ggtt_node_fini(bo->ggtt_node[tile_id]);
 812		bo->ggtt_node[tile_id] = NULL;
 813	} else {
 814		u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
 815		u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
 816
 817		xe_ggtt_map_bo(ggtt, bo->ggtt_node[tile_id], bo, pat_index);
 818	}
 819	mutex_unlock(&ggtt->lock);
 820
 821	if (!err && bo->flags & XE_BO_FLAG_GGTT_INVALIDATE)
 822		xe_ggtt_invalidate(ggtt);
 823
 824out:
 825	xe_pm_runtime_put(tile_to_xe(ggtt->tile));
 826
 827	return err;
 828}
 829
 830/**
 831 * xe_ggtt_insert_bo_at - Insert BO at a specific GGTT space
 832 * @ggtt: the &xe_ggtt where bo will be inserted
 833 * @bo: the &xe_bo to be inserted
 834 * @start: address where it will be inserted
 835 * @end: end of the range where it will be inserted
 836 * @exec: The drm_exec transaction to use for exhaustive eviction.
 837 *
 838 * Return: 0 on success or a negative error code on failure.
 839 */
 840int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
 841			 u64 start, u64 end, struct drm_exec *exec)
 842{
 843	return __xe_ggtt_insert_bo_at(ggtt, bo, start, end, exec);
 844}
 845
 846/**
 847 * xe_ggtt_insert_bo - Insert BO into GGTT
 848 * @ggtt: the &xe_ggtt where bo will be inserted
 849 * @bo: the &xe_bo to be inserted
 850 * @exec: The drm_exec transaction to use for exhaustive eviction.
 851 *
 852 * Return: 0 on success or a negative error code on failure.
 853 */
 854int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo,
 855		      struct drm_exec *exec)
 856{
 857	return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX, exec);
 858}
 859
 860/**
 861 * xe_ggtt_remove_bo - Remove a BO from the GGTT
 862 * @ggtt: the &xe_ggtt where node will be removed
 863 * @bo: the &xe_bo to be removed
 864 */
 865void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo)
 866{
 867	u8 tile_id = ggtt->tile->id;
 868
 869	if (XE_WARN_ON(!bo->ggtt_node[tile_id]))
 870		return;
 871
 872	/* This BO is not currently in the GGTT */
 873	xe_tile_assert(ggtt->tile, bo->ggtt_node[tile_id]->base.size == xe_bo_size(bo));
 874
 875	xe_ggtt_node_remove(bo->ggtt_node[tile_id],
 876			    bo->flags & XE_BO_FLAG_GGTT_INVALIDATE);
 877}
 878
 879/**
 880 * xe_ggtt_largest_hole - Largest GGTT hole
 881 * @ggtt: the &xe_ggtt that will be inspected
 882 * @alignment: minimum alignment
 883 * @spare: If not NULL: in: desired memory size to be spared / out: Adjusted possible spare
 884 *
 885 * Return: size of the largest continuous GGTT region
 886 */
 887u64 xe_ggtt_largest_hole(struct xe_ggtt *ggtt, u64 alignment, u64 *spare)
 888{
 889	const struct drm_mm *mm = &ggtt->mm;
 890	const struct drm_mm_node *entry;
 891	u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
 892	u64 hole_start, hole_end, hole_size;
 893	u64 max_hole = 0;
 894
 895	mutex_lock(&ggtt->lock);
 896
 897	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
 898		hole_start = max(hole_start, hole_min_start);
 899		hole_start = ALIGN(hole_start, alignment);
 900		hole_end = ALIGN_DOWN(hole_end, alignment);
 901		if (hole_start >= hole_end)
 902			continue;
 903		hole_size = hole_end - hole_start;
 904		if (spare)
 905			*spare -= min3(*spare, hole_size, max_hole);
 906		max_hole = max(max_hole, hole_size);
 907	}
 908
 909	mutex_unlock(&ggtt->lock);
 910
 911	return max_hole;
 912}
 913
 914#ifdef CONFIG_PCI_IOV
 915static u64 xe_encode_vfid_pte(u16 vfid)
 916{
 917	return FIELD_PREP(GGTT_PTE_VFID, vfid) | XE_PAGE_PRESENT;
 918}
 919
 920static void xe_ggtt_assign_locked(struct xe_ggtt *ggtt, const struct drm_mm_node *node, u16 vfid)
 921{
 922	u64 start = node->start;
 923	u64 size = node->size;
 924	u64 end = start + size - 1;
 925	u64 pte = xe_encode_vfid_pte(vfid);
 926
 927	lockdep_assert_held(&ggtt->lock);
 928
 929	if (!drm_mm_node_allocated(node))
 930		return;
 931
 932	while (start < end) {
 933		ggtt->pt_ops->ggtt_set_pte(ggtt, start, pte);
 934		start += XE_PAGE_SIZE;
 935	}
 936
 937	xe_ggtt_invalidate(ggtt);
 938}
 939
 940/**
 941 * xe_ggtt_assign - assign a GGTT region to the VF
 942 * @node: the &xe_ggtt_node to update
 943 * @vfid: the VF identifier
 944 *
 945 * This function is used by the PF driver to assign a GGTT region to the VF.
 946 * In addition to PTE's VFID bits 11:2 also PRESENT bit 0 is set as on some
 947 * platforms VFs can't modify that either.
 948 */
 949void xe_ggtt_assign(const struct xe_ggtt_node *node, u16 vfid)
 950{
 951	mutex_lock(&node->ggtt->lock);
 952	xe_ggtt_assign_locked(node->ggtt, &node->base, vfid);
 953	mutex_unlock(&node->ggtt->lock);
 954}
 955
 956/**
 957 * xe_ggtt_node_save() - Save a &xe_ggtt_node to a buffer.
 958 * @node: the &xe_ggtt_node to be saved
 959 * @dst: destination buffer
 960 * @size: destination buffer size in bytes
 961 * @vfid: VF identifier
 962 *
 963 * Return: 0 on success or a negative error code on failure.
 964 */
 965int xe_ggtt_node_save(struct xe_ggtt_node *node, void *dst, size_t size, u16 vfid)
 966{
 967	struct xe_ggtt *ggtt;
 968	u64 start, end;
 969	u64 *buf = dst;
 970	u64 pte;
 971
 972	if (!node)
 973		return -ENOENT;
 974
 975	guard(mutex)(&node->ggtt->lock);
 976
 977	if (xe_ggtt_node_pt_size(node) != size)
 978		return -EINVAL;
 979
 980	ggtt = node->ggtt;
 981	start = node->base.start;
 982	end = start + node->base.size - 1;
 983
 984	while (start < end) {
 985		pte = ggtt->pt_ops->ggtt_get_pte(ggtt, start);
 986		if (vfid != u64_get_bits(pte, GGTT_PTE_VFID))
 987			return -EPERM;
 988
 989		*buf++ = u64_replace_bits(pte, 0, GGTT_PTE_VFID);
 990		start += XE_PAGE_SIZE;
 991	}
 992
 993	return 0;
 994}
 995
 996/**
 997 * xe_ggtt_node_load() - Load a &xe_ggtt_node from a buffer.
 998 * @node: the &xe_ggtt_node to be loaded
 999 * @src: source buffer
1000 * @size: source buffer size in bytes
1001 * @vfid: VF identifier
1002 *
1003 * Return: 0 on success or a negative error code on failure.
1004 */
1005int xe_ggtt_node_load(struct xe_ggtt_node *node, const void *src, size_t size, u16 vfid)
1006{
1007	u64 vfid_pte = xe_encode_vfid_pte(vfid);
1008	const u64 *buf = src;
1009	struct xe_ggtt *ggtt;
1010	u64 start, end;
1011
1012	if (!node)
1013		return -ENOENT;
1014
1015	guard(mutex)(&node->ggtt->lock);
1016
1017	if (xe_ggtt_node_pt_size(node) != size)
1018		return -EINVAL;
1019
1020	ggtt = node->ggtt;
1021	start = node->base.start;
1022	end = start + node->base.size - 1;
1023
1024	while (start < end) {
1025		vfid_pte = u64_replace_bits(*buf++, vfid, GGTT_PTE_VFID);
1026		ggtt->pt_ops->ggtt_set_pte(ggtt, start, vfid_pte);
1027		start += XE_PAGE_SIZE;
1028	}
1029	xe_ggtt_invalidate(ggtt);
1030
1031	return 0;
1032}
1033
1034#endif
1035
1036/**
1037 * xe_ggtt_dump - Dump GGTT for debug
1038 * @ggtt: the &xe_ggtt to be dumped
1039 * @p: the &drm_mm_printer helper handle to be used to dump the information
1040 *
1041 * Return: 0 on success or a negative error code on failure.
1042 */
1043int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p)
1044{
1045	int err;
1046
1047	err = mutex_lock_interruptible(&ggtt->lock);
1048	if (err)
1049		return err;
1050
1051	drm_mm_print(&ggtt->mm, p);
1052	mutex_unlock(&ggtt->lock);
1053	return err;
1054}
1055
1056/**
1057 * xe_ggtt_print_holes - Print holes
1058 * @ggtt: the &xe_ggtt to be inspected
1059 * @alignment: min alignment
1060 * @p: the &drm_printer
1061 *
1062 * Print GGTT ranges that are available and return total size available.
1063 *
1064 * Return: Total available size.
1065 */
1066u64 xe_ggtt_print_holes(struct xe_ggtt *ggtt, u64 alignment, struct drm_printer *p)
1067{
1068	const struct drm_mm *mm = &ggtt->mm;
1069	const struct drm_mm_node *entry;
1070	u64 hole_min_start = xe_wopcm_size(tile_to_xe(ggtt->tile));
1071	u64 hole_start, hole_end, hole_size;
1072	u64 total = 0;
1073	char buf[10];
1074
1075	mutex_lock(&ggtt->lock);
1076
1077	drm_mm_for_each_hole(entry, mm, hole_start, hole_end) {
1078		hole_start = max(hole_start, hole_min_start);
1079		hole_start = ALIGN(hole_start, alignment);
1080		hole_end = ALIGN_DOWN(hole_end, alignment);
1081		if (hole_start >= hole_end)
1082			continue;
1083		hole_size = hole_end - hole_start;
1084		total += hole_size;
1085
1086		string_get_size(hole_size, 1, STRING_UNITS_2, buf, sizeof(buf));
1087		drm_printf(p, "range:\t%#llx-%#llx\t(%s)\n",
1088			   hole_start, hole_end - 1, buf);
1089	}
1090
1091	mutex_unlock(&ggtt->lock);
1092
1093	return total;
1094}
1095
1096/**
1097 * xe_ggtt_encode_pte_flags - Get PTE encoding flags for BO
1098 * @ggtt: &xe_ggtt
1099 * @bo: &xe_bo
1100 * @pat_index: The pat_index for the PTE.
1101 *
1102 * This function returns the pte_flags for a given BO, without  address.
1103 * It's used for DPT to fill a GGTT mapped BO with a linear lookup table.
1104 */
1105u64 xe_ggtt_encode_pte_flags(struct xe_ggtt *ggtt,
1106			     struct xe_bo *bo, u16 pat_index)
1107{
1108	return ggtt->pt_ops->pte_encode_flags(bo, pat_index);
1109}
1110
1111/**
1112 * xe_ggtt_read_pte - Read a PTE from the GGTT
1113 * @ggtt: &xe_ggtt
1114 * @offset: the offset for which the mapping should be read.
1115 *
1116 * Used by testcases, and by display reading out an inherited bios FB.
1117 */
1118u64 xe_ggtt_read_pte(struct xe_ggtt *ggtt, u64 offset)
1119{
1120	return ioread64(ggtt->gsm + (offset / XE_PAGE_SIZE));
1121}