xe_bo.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2021 Intel Corporation
   4 */
   5
   6#include "xe_bo.h"
   7
   8#include <linux/dma-buf.h>
   9#include <linux/nospec.h>
  10
  11#include <drm/drm_drv.h>
  12#include <drm/drm_dumb_buffers.h>
  13#include <drm/drm_gem_ttm_helper.h>
  14#include <drm/drm_managed.h>
  15#include <drm/ttm/ttm_backup.h>
  16#include <drm/ttm/ttm_device.h>
  17#include <drm/ttm/ttm_placement.h>
  18#include <drm/ttm/ttm_tt.h>
  19#include <uapi/drm/xe_drm.h>
  20
  21#include <kunit/static_stub.h>
  22
  23#include <trace/events/gpu_mem.h>
  24
  25#include "xe_device.h"
  26#include "xe_dma_buf.h"
  27#include "xe_drm_client.h"
  28#include "xe_ggtt.h"
  29#include "xe_gt.h"
  30#include "xe_map.h"
  31#include "xe_migrate.h"
  32#include "xe_pm.h"
  33#include "xe_preempt_fence.h"
  34#include "xe_pxp.h"
  35#include "xe_res_cursor.h"
  36#include "xe_shrinker.h"
  37#include "xe_sriov_vf_ccs.h"
  38#include "xe_tile.h"
  39#include "xe_trace_bo.h"
  40#include "xe_ttm_stolen_mgr.h"
  41#include "xe_vm.h"
  42#include "xe_vram_types.h"
  43
  44const char *const xe_mem_type_to_name[TTM_NUM_MEM_TYPES]  = {
  45	[XE_PL_SYSTEM] = "system",
  46	[XE_PL_TT] = "gtt",
  47	[XE_PL_VRAM0] = "vram0",
  48	[XE_PL_VRAM1] = "vram1",
  49	[XE_PL_STOLEN] = "stolen"
  50};
  51
  52static const struct ttm_place sys_placement_flags = {
  53	.fpfn = 0,
  54	.lpfn = 0,
  55	.mem_type = XE_PL_SYSTEM,
  56	.flags = 0,
  57};
  58
  59static struct ttm_placement sys_placement = {
  60	.num_placement = 1,
  61	.placement = &sys_placement_flags,
  62};
  63
  64static struct ttm_placement purge_placement;
  65
  66static const struct ttm_place tt_placement_flags[] = {
  67	{
  68		.fpfn = 0,
  69		.lpfn = 0,
  70		.mem_type = XE_PL_TT,
  71		.flags = TTM_PL_FLAG_DESIRED,
  72	},
  73	{
  74		.fpfn = 0,
  75		.lpfn = 0,
  76		.mem_type = XE_PL_SYSTEM,
  77		.flags = TTM_PL_FLAG_FALLBACK,
  78	}
  79};
  80
  81static struct ttm_placement tt_placement = {
  82	.num_placement = 2,
  83	.placement = tt_placement_flags,
  84};
  85
  86#define for_each_set_bo_vram_flag(bit__, bo_flags__) \
  87	for (unsigned int __bit_tmp = BIT(0); __bit_tmp <= XE_BO_FLAG_VRAM_MASK; __bit_tmp <<= 1) \
  88		for_each_if(((bit__) = __bit_tmp) & (bo_flags__) & XE_BO_FLAG_VRAM_MASK)
  89
  90bool mem_type_is_vram(u32 mem_type)
  91{
  92	return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;
  93}
  94
  95static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)
  96{
  97	return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);
  98}
  99
 100static bool resource_is_vram(struct ttm_resource *res)
 101{
 102	return mem_type_is_vram(res->mem_type);
 103}
 104
 105bool xe_bo_is_vram(struct xe_bo *bo)
 106{
 107	return resource_is_vram(bo->ttm.resource) ||
 108		resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);
 109}
 110
 111bool xe_bo_is_stolen(struct xe_bo *bo)
 112{
 113	return bo->ttm.resource->mem_type == XE_PL_STOLEN;
 114}
 115
 116/**
 117 * xe_bo_has_single_placement - check if BO is placed only in one memory location
 118 * @bo: The BO
 119 *
 120 * This function checks whether a given BO is placed in only one memory location.
 121 *
 122 * Returns: true if the BO is placed in a single memory location, false otherwise.
 123 *
 124 */
 125bool xe_bo_has_single_placement(struct xe_bo *bo)
 126{
 127	return bo->placement.num_placement == 1;
 128}
 129
 130/**
 131 * xe_bo_is_stolen_devmem - check if BO is of stolen type accessed via PCI BAR
 132 * @bo: The BO
 133 *
 134 * The stolen memory is accessed through the PCI BAR for both DGFX and some
 135 * integrated platforms that have a dedicated bit in the PTE for devmem (DM).
 136 *
 137 * Returns: true if it's stolen memory accessed via PCI BAR, false otherwise.
 138 */
 139bool xe_bo_is_stolen_devmem(struct xe_bo *bo)
 140{
 141	return xe_bo_is_stolen(bo) &&
 142		GRAPHICS_VERx100(xe_bo_device(bo)) >= 1270;
 143}
 144
 145/**
 146 * xe_bo_is_vm_bound - check if BO has any mappings through VM_BIND
 147 * @bo: The BO
 148 *
 149 * Check if a given bo is bound through VM_BIND. This requires the
 150 * reservation lock for the BO to be held.
 151 *
 152 * Returns: boolean
 153 */
 154bool xe_bo_is_vm_bound(struct xe_bo *bo)
 155{
 156	xe_bo_assert_held(bo);
 157
 158	return !list_empty(&bo->ttm.base.gpuva.list);
 159}
 160
 161static bool xe_bo_is_user(struct xe_bo *bo)
 162{
 163	return bo->flags & XE_BO_FLAG_USER;
 164}
 165
 166static struct xe_migrate *
 167mem_type_to_migrate(struct xe_device *xe, u32 mem_type)
 168{
 169	struct xe_tile *tile;
 170
 171	xe_assert(xe, mem_type == XE_PL_STOLEN || mem_type_is_vram(mem_type));
 172	tile = &xe->tiles[mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0)];
 173	return tile->migrate;
 174}
 175
 176static struct xe_vram_region *res_to_mem_region(struct ttm_resource *res)
 177{
 178	struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
 179	struct ttm_resource_manager *mgr;
 180	struct xe_ttm_vram_mgr *vram_mgr;
 181
 182	xe_assert(xe, resource_is_vram(res));
 183	mgr = ttm_manager_type(&xe->ttm, res->mem_type);
 184	vram_mgr = to_xe_ttm_vram_mgr(mgr);
 185
 186	return container_of(vram_mgr, struct xe_vram_region, ttm);
 187}
 188
 189static void try_add_system(struct xe_device *xe, struct xe_bo *bo,
 190			   u32 bo_flags, u32 *c)
 191{
 192	if (bo_flags & XE_BO_FLAG_SYSTEM) {
 193		xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
 194
 195		bo->placements[*c] = (struct ttm_place) {
 196			.mem_type = XE_PL_TT,
 197			.flags = (bo_flags & XE_BO_FLAG_VRAM_MASK) ?
 198			TTM_PL_FLAG_FALLBACK : 0,
 199		};
 200		*c += 1;
 201	}
 202}
 203
 204static bool force_contiguous(u32 bo_flags)
 205{
 206	if (bo_flags & XE_BO_FLAG_STOLEN)
 207		return true; /* users expect this */
 208	else if (bo_flags & XE_BO_FLAG_PINNED &&
 209		 !(bo_flags & XE_BO_FLAG_PINNED_LATE_RESTORE))
 210		return true; /* needs vmap */
 211	else if (bo_flags & XE_BO_FLAG_CPU_ADDR_MIRROR)
 212		return true;
 213
 214	/*
 215	 * For eviction / restore on suspend / resume objects pinned in VRAM
 216	 * must be contiguous, also only contiguous BOs support xe_bo_vmap.
 217	 */
 218	return bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS &&
 219	       bo_flags & XE_BO_FLAG_PINNED;
 220}
 221
 222static u8 vram_bo_flag_to_tile_id(struct xe_device *xe, u32 vram_bo_flag)
 223{
 224	xe_assert(xe, vram_bo_flag & XE_BO_FLAG_VRAM_MASK);
 225	xe_assert(xe, (vram_bo_flag & (vram_bo_flag - 1)) == 0);
 226
 227	return __ffs(vram_bo_flag >> (__ffs(XE_BO_FLAG_VRAM0) - 1)) - 1;
 228}
 229
 230static u32 bo_vram_flags_to_vram_placement(struct xe_device *xe, u32 bo_flags, u32 vram_flag,
 231					   enum ttm_bo_type type)
 232{
 233	u8 tile_id = vram_bo_flag_to_tile_id(xe, vram_flag);
 234
 235	xe_assert(xe, tile_id < xe->info.tile_count);
 236
 237	if (type == ttm_bo_type_kernel && !(bo_flags & XE_BO_FLAG_FORCE_USER_VRAM))
 238		return xe->tiles[tile_id].mem.kernel_vram->placement;
 239	else
 240		return xe->tiles[tile_id].mem.vram->placement;
 241}
 242
 243static void add_vram(struct xe_device *xe, struct xe_bo *bo,
 244		     struct ttm_place *places, u32 bo_flags, u32 mem_type, u32 *c)
 245{
 246	struct ttm_place place = { .mem_type = mem_type };
 247	struct ttm_resource_manager *mgr = ttm_manager_type(&xe->ttm, mem_type);
 248	struct xe_ttm_vram_mgr *vram_mgr = to_xe_ttm_vram_mgr(mgr);
 249
 250	struct xe_vram_region *vram;
 251	u64 io_size;
 252
 253	xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
 254
 255	vram = container_of(vram_mgr, struct xe_vram_region, ttm);
 256	xe_assert(xe, vram && vram->usable_size);
 257	io_size = vram->io_size;
 258
 259	if (force_contiguous(bo_flags))
 260		place.flags |= TTM_PL_FLAG_CONTIGUOUS;
 261
 262	if (io_size < vram->usable_size) {
 263		if (bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) {
 264			place.fpfn = 0;
 265			place.lpfn = io_size >> PAGE_SHIFT;
 266		} else {
 267			place.flags |= TTM_PL_FLAG_TOPDOWN;
 268		}
 269	}
 270	places[*c] = place;
 271	*c += 1;
 272}
 273
 274static void try_add_vram(struct xe_device *xe, struct xe_bo *bo,
 275			 u32 bo_flags, enum ttm_bo_type type, u32 *c)
 276{
 277	u32 vram_flag;
 278
 279	for_each_set_bo_vram_flag(vram_flag, bo_flags) {
 280		u32 pl = bo_vram_flags_to_vram_placement(xe, bo_flags, vram_flag, type);
 281
 282		add_vram(xe, bo, bo->placements, bo_flags, pl, c);
 283	}
 284}
 285
 286static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,
 287			   u32 bo_flags, u32 *c)
 288{
 289	if (bo_flags & XE_BO_FLAG_STOLEN) {
 290		xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
 291
 292		bo->placements[*c] = (struct ttm_place) {
 293			.mem_type = XE_PL_STOLEN,
 294			.flags = force_contiguous(bo_flags) ?
 295				TTM_PL_FLAG_CONTIGUOUS : 0,
 296		};
 297		*c += 1;
 298	}
 299}
 300
 301static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
 302				       u32 bo_flags, enum ttm_bo_type type)
 303{
 304	u32 c = 0;
 305
 306	try_add_vram(xe, bo, bo_flags, type, &c);
 307	try_add_system(xe, bo, bo_flags, &c);
 308	try_add_stolen(xe, bo, bo_flags, &c);
 309
 310	if (!c)
 311		return -EINVAL;
 312
 313	bo->placement = (struct ttm_placement) {
 314		.num_placement = c,
 315		.placement = bo->placements,
 316	};
 317
 318	return 0;
 319}
 320
 321int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
 322			      u32 bo_flags, enum ttm_bo_type type)
 323{
 324	xe_bo_assert_held(bo);
 325	return __xe_bo_placement_for_flags(xe, bo, bo_flags, type);
 326}
 327
 328static void xe_evict_flags(struct ttm_buffer_object *tbo,
 329			   struct ttm_placement *placement)
 330{
 331	struct xe_device *xe = container_of(tbo->bdev, typeof(*xe), ttm);
 332	bool device_unplugged = drm_dev_is_unplugged(&xe->drm);
 333	struct xe_bo *bo;
 334
 335	if (!xe_bo_is_xe_bo(tbo)) {
 336		/* Don't handle scatter gather BOs */
 337		if (tbo->type == ttm_bo_type_sg) {
 338			placement->num_placement = 0;
 339			return;
 340		}
 341
 342		*placement = device_unplugged ? purge_placement : sys_placement;
 343		return;
 344	}
 345
 346	bo = ttm_to_xe_bo(tbo);
 347	if (bo->flags & XE_BO_FLAG_CPU_ADDR_MIRROR) {
 348		*placement = sys_placement;
 349		return;
 350	}
 351
 352	if (device_unplugged && !tbo->base.dma_buf) {
 353		*placement = purge_placement;
 354		return;
 355	}
 356
 357	/*
 358	 * For xe, sg bos that are evicted to system just triggers a
 359	 * rebind of the sg list upon subsequent validation to XE_PL_TT.
 360	 */
 361	switch (tbo->resource->mem_type) {
 362	case XE_PL_VRAM0:
 363	case XE_PL_VRAM1:
 364	case XE_PL_STOLEN:
 365		*placement = tt_placement;
 366		break;
 367	case XE_PL_TT:
 368	default:
 369		*placement = sys_placement;
 370		break;
 371	}
 372}
 373
 374/* struct xe_ttm_tt - Subclassed ttm_tt for xe */
 375struct xe_ttm_tt {
 376	struct ttm_tt ttm;
 377	struct sg_table sgt;
 378	struct sg_table *sg;
 379	/** @purgeable: Whether the content of the pages of @ttm is purgeable. */
 380	bool purgeable;
 381};
 382
 383static int xe_tt_map_sg(struct xe_device *xe, struct ttm_tt *tt)
 384{
 385	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
 386	unsigned long num_pages = tt->num_pages;
 387	int ret;
 388
 389	XE_WARN_ON((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&
 390		   !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE));
 391
 392	if (xe_tt->sg)
 393		return 0;
 394
 395	ret = sg_alloc_table_from_pages_segment(&xe_tt->sgt, tt->pages,
 396						num_pages, 0,
 397						(u64)num_pages << PAGE_SHIFT,
 398						xe_sg_segment_size(xe->drm.dev),
 399						GFP_KERNEL);
 400	if (ret)
 401		return ret;
 402
 403	xe_tt->sg = &xe_tt->sgt;
 404	ret = dma_map_sgtable(xe->drm.dev, xe_tt->sg, DMA_BIDIRECTIONAL,
 405			      DMA_ATTR_SKIP_CPU_SYNC);
 406	if (ret) {
 407		sg_free_table(xe_tt->sg);
 408		xe_tt->sg = NULL;
 409		return ret;
 410	}
 411
 412	return 0;
 413}
 414
 415static void xe_tt_unmap_sg(struct xe_device *xe, struct ttm_tt *tt)
 416{
 417	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
 418
 419	if (xe_tt->sg) {
 420		dma_unmap_sgtable(xe->drm.dev, xe_tt->sg,
 421				  DMA_BIDIRECTIONAL, 0);
 422		sg_free_table(xe_tt->sg);
 423		xe_tt->sg = NULL;
 424	}
 425}
 426
 427struct sg_table *xe_bo_sg(struct xe_bo *bo)
 428{
 429	struct ttm_tt *tt = bo->ttm.ttm;
 430	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
 431
 432	return xe_tt->sg;
 433}
 434
 435/*
 436 * Account ttm pages against the device shrinker's shrinkable and
 437 * purgeable counts.
 438 */
 439static void xe_ttm_tt_account_add(struct xe_device *xe, struct ttm_tt *tt)
 440{
 441	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
 442
 443	if (xe_tt->purgeable)
 444		xe_shrinker_mod_pages(xe->mem.shrinker, 0, tt->num_pages);
 445	else
 446		xe_shrinker_mod_pages(xe->mem.shrinker, tt->num_pages, 0);
 447}
 448
 449static void xe_ttm_tt_account_subtract(struct xe_device *xe, struct ttm_tt *tt)
 450{
 451	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
 452
 453	if (xe_tt->purgeable)
 454		xe_shrinker_mod_pages(xe->mem.shrinker, 0, -(long)tt->num_pages);
 455	else
 456		xe_shrinker_mod_pages(xe->mem.shrinker, -(long)tt->num_pages, 0);
 457}
 458
 459static void update_global_total_pages(struct ttm_device *ttm_dev,
 460				      long num_pages)
 461{
 462#if IS_ENABLED(CONFIG_TRACE_GPU_MEM)
 463	struct xe_device *xe = ttm_to_xe_device(ttm_dev);
 464	u64 global_total_pages =
 465		atomic64_add_return(num_pages, &xe->global_total_pages);
 466
 467	trace_gpu_mem_total(xe->drm.primary->index, 0,
 468			    global_total_pages << PAGE_SHIFT);
 469#endif
 470}
 471
 472static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo,
 473				       u32 page_flags)
 474{
 475	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
 476	struct xe_device *xe = xe_bo_device(bo);
 477	struct xe_ttm_tt *xe_tt;
 478	struct ttm_tt *tt;
 479	unsigned long extra_pages;
 480	enum ttm_caching caching = ttm_cached;
 481	int err;
 482
 483	xe_tt = kzalloc(sizeof(*xe_tt), GFP_KERNEL);
 484	if (!xe_tt)
 485		return NULL;
 486
 487	tt = &xe_tt->ttm;
 488
 489	extra_pages = 0;
 490	if (xe_bo_needs_ccs_pages(bo))
 491		extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, xe_bo_size(bo)),
 492					   PAGE_SIZE);
 493
 494	/*
 495	 * DGFX system memory is always WB / ttm_cached, since
 496	 * other caching modes are only supported on x86. DGFX
 497	 * GPU system memory accesses are always coherent with the
 498	 * CPU.
 499	 */
 500	if (!IS_DGFX(xe)) {
 501		switch (bo->cpu_caching) {
 502		case DRM_XE_GEM_CPU_CACHING_WC:
 503			caching = ttm_write_combined;
 504			break;
 505		default:
 506			caching = ttm_cached;
 507			break;
 508		}
 509
 510		WARN_ON((bo->flags & XE_BO_FLAG_USER) && !bo->cpu_caching);
 511
 512		/*
 513		 * Display scanout is always non-coherent with the CPU cache.
 514		 *
 515		 * For Xe_LPG and beyond, PPGTT PTE lookups are also
 516		 * non-coherent and require a CPU:WC mapping.
 517		 */
 518		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
 519		    (xe->info.graphics_verx100 >= 1270 &&
 520		     bo->flags & XE_BO_FLAG_PAGETABLE))
 521			caching = ttm_write_combined;
 522	}
 523
 524	if (bo->flags & XE_BO_FLAG_NEEDS_UC) {
 525		/*
 526		 * Valid only for internally-created buffers only, for
 527		 * which cpu_caching is never initialized.
 528		 */
 529		xe_assert(xe, bo->cpu_caching == 0);
 530		caching = ttm_uncached;
 531	}
 532
 533	if (ttm_bo->type != ttm_bo_type_sg)
 534		page_flags |= TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE;
 535
 536	err = ttm_tt_init(tt, &bo->ttm, page_flags, caching, extra_pages);
 537	if (err) {
 538		kfree(xe_tt);
 539		return NULL;
 540	}
 541
 542	if (ttm_bo->type != ttm_bo_type_sg) {
 543		err = ttm_tt_setup_backup(tt);
 544		if (err) {
 545			ttm_tt_fini(tt);
 546			kfree(xe_tt);
 547			return NULL;
 548		}
 549	}
 550
 551	return tt;
 552}
 553
 554static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt,
 555			      struct ttm_operation_ctx *ctx)
 556{
 557	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
 558	int err;
 559
 560	/*
 561	 * dma-bufs are not populated with pages, and the dma-
 562	 * addresses are set up when moved to XE_PL_TT.
 563	 */
 564	if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&
 565	    !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))
 566		return 0;
 567
 568	if (ttm_tt_is_backed_up(tt) && !xe_tt->purgeable) {
 569		err = ttm_tt_restore(ttm_dev, tt, ctx);
 570	} else {
 571		ttm_tt_clear_backed_up(tt);
 572		err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx);
 573	}
 574	if (err)
 575		return err;
 576
 577	xe_tt->purgeable = false;
 578	xe_ttm_tt_account_add(ttm_to_xe_device(ttm_dev), tt);
 579	update_global_total_pages(ttm_dev, tt->num_pages);
 580
 581	return 0;
 582}
 583
 584static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt)
 585{
 586	struct xe_device *xe = ttm_to_xe_device(ttm_dev);
 587
 588	if ((tt->page_flags & TTM_TT_FLAG_EXTERNAL) &&
 589	    !(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE))
 590		return;
 591
 592	xe_tt_unmap_sg(xe, tt);
 593
 594	ttm_pool_free(&ttm_dev->pool, tt);
 595	xe_ttm_tt_account_subtract(xe, tt);
 596	update_global_total_pages(ttm_dev, -(long)tt->num_pages);
 597}
 598
 599static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)
 600{
 601	ttm_tt_fini(tt);
 602	kfree(tt);
 603}
 604
 605static bool xe_ttm_resource_visible(struct ttm_resource *mem)
 606{
 607	struct xe_ttm_vram_mgr_resource *vres =
 608		to_xe_ttm_vram_mgr_resource(mem);
 609
 610	return vres->used_visible_size == mem->size;
 611}
 612
 613/**
 614 * xe_bo_is_visible_vram - check if BO is placed entirely in visible VRAM.
 615 * @bo: The BO
 616 *
 617 * This function checks whether a given BO resides entirely in memory visible from the CPU
 618 *
 619 * Returns: true if the BO is entirely visible, false otherwise.
 620 *
 621 */
 622bool xe_bo_is_visible_vram(struct xe_bo *bo)
 623{
 624	if (drm_WARN_ON(bo->ttm.base.dev, !xe_bo_is_vram(bo)))
 625		return false;
 626
 627	return xe_ttm_resource_visible(bo->ttm.resource);
 628}
 629
 630static int xe_ttm_io_mem_reserve(struct ttm_device *bdev,
 631				 struct ttm_resource *mem)
 632{
 633	struct xe_device *xe = ttm_to_xe_device(bdev);
 634
 635	switch (mem->mem_type) {
 636	case XE_PL_SYSTEM:
 637	case XE_PL_TT:
 638		return 0;
 639	case XE_PL_VRAM0:
 640	case XE_PL_VRAM1: {
 641		struct xe_vram_region *vram = res_to_mem_region(mem);
 642
 643		if (!xe_ttm_resource_visible(mem))
 644			return -EINVAL;
 645
 646		mem->bus.offset = mem->start << PAGE_SHIFT;
 647
 648		if (vram->mapping &&
 649		    mem->placement & TTM_PL_FLAG_CONTIGUOUS)
 650			mem->bus.addr = (u8 __force *)vram->mapping +
 651				mem->bus.offset;
 652
 653		mem->bus.offset += vram->io_start;
 654		mem->bus.is_iomem = true;
 655
 656#if  !IS_ENABLED(CONFIG_X86)
 657		mem->bus.caching = ttm_write_combined;
 658#endif
 659		return 0;
 660	} case XE_PL_STOLEN:
 661		return xe_ttm_stolen_io_mem_reserve(xe, mem);
 662	default:
 663		return -EINVAL;
 664	}
 665}
 666
 667static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
 668				const struct ttm_operation_ctx *ctx)
 669{
 670	struct dma_resv_iter cursor;
 671	struct dma_fence *fence;
 672	struct drm_gem_object *obj = &bo->ttm.base;
 673	struct drm_gpuvm_bo *vm_bo;
 674	bool idle = false;
 675	int ret = 0;
 676
 677	dma_resv_assert_held(bo->ttm.base.resv);
 678
 679	if (!list_empty(&bo->ttm.base.gpuva.list)) {
 680		dma_resv_iter_begin(&cursor, bo->ttm.base.resv,
 681				    DMA_RESV_USAGE_BOOKKEEP);
 682		dma_resv_for_each_fence_unlocked(&cursor, fence)
 683			dma_fence_enable_sw_signaling(fence);
 684		dma_resv_iter_end(&cursor);
 685	}
 686
 687	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
 688		struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
 689		struct drm_gpuva *gpuva;
 690
 691		if (!xe_vm_in_fault_mode(vm)) {
 692			drm_gpuvm_bo_evict(vm_bo, true);
 693			continue;
 694		}
 695
 696		if (!idle) {
 697			long timeout;
 698
 699			if (ctx->no_wait_gpu &&
 700			    !dma_resv_test_signaled(bo->ttm.base.resv,
 701						    DMA_RESV_USAGE_BOOKKEEP))
 702				return -EBUSY;
 703
 704			timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
 705							DMA_RESV_USAGE_BOOKKEEP,
 706							ctx->interruptible,
 707							MAX_SCHEDULE_TIMEOUT);
 708			if (!timeout)
 709				return -ETIME;
 710			if (timeout < 0)
 711				return timeout;
 712
 713			idle = true;
 714		}
 715
 716		drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
 717			struct xe_vma *vma = gpuva_to_vma(gpuva);
 718
 719			trace_xe_vma_evict(vma);
 720			ret = xe_vm_invalidate_vma(vma);
 721			if (XE_WARN_ON(ret))
 722				return ret;
 723		}
 724	}
 725
 726	return ret;
 727}
 728
 729/*
 730 * The dma-buf map_attachment() / unmap_attachment() is hooked up here.
 731 * Note that unmapping the attachment is deferred to the next
 732 * map_attachment time, or to bo destroy (after idling) whichever comes first.
 733 * This is to avoid syncing before unmap_attachment(), assuming that the
 734 * caller relies on idling the reservation object before moving the
 735 * backing store out. Should that assumption not hold, then we will be able
 736 * to unconditionally call unmap_attachment() when moving out to system.
 737 */
 738static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo,
 739			     struct ttm_resource *new_res)
 740{
 741	struct dma_buf_attachment *attach = ttm_bo->base.import_attach;
 742	struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt,
 743					       ttm);
 744	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
 745	bool device_unplugged = drm_dev_is_unplugged(&xe->drm);
 746	struct sg_table *sg;
 747
 748	xe_assert(xe, attach);
 749	xe_assert(xe, ttm_bo->ttm);
 750
 751	if (device_unplugged && new_res->mem_type == XE_PL_SYSTEM &&
 752	    ttm_bo->sg) {
 753		dma_resv_wait_timeout(ttm_bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
 754				      false, MAX_SCHEDULE_TIMEOUT);
 755		dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
 756		ttm_bo->sg = NULL;
 757	}
 758
 759	if (new_res->mem_type == XE_PL_SYSTEM)
 760		goto out;
 761
 762	if (ttm_bo->sg) {
 763		dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL);
 764		ttm_bo->sg = NULL;
 765	}
 766
 767	sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
 768	if (IS_ERR(sg))
 769		return PTR_ERR(sg);
 770
 771	ttm_bo->sg = sg;
 772	xe_tt->sg = sg;
 773
 774out:
 775	ttm_bo_move_null(ttm_bo, new_res);
 776
 777	return 0;
 778}
 779
 780/**
 781 * xe_bo_move_notify - Notify subsystems of a pending move
 782 * @bo: The buffer object
 783 * @ctx: The struct ttm_operation_ctx controlling locking and waits.
 784 *
 785 * This function notifies subsystems of an upcoming buffer move.
 786 * Upon receiving such a notification, subsystems should schedule
 787 * halting access to the underlying pages and optionally add a fence
 788 * to the buffer object's dma_resv object, that signals when access is
 789 * stopped. The caller will wait on all dma_resv fences before
 790 * starting the move.
 791 *
 792 * A subsystem may commence access to the object after obtaining
 793 * bindings to the new backing memory under the object lock.
 794 *
 795 * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode,
 796 * negative error code on error.
 797 */
 798static int xe_bo_move_notify(struct xe_bo *bo,
 799			     const struct ttm_operation_ctx *ctx)
 800{
 801	struct ttm_buffer_object *ttm_bo = &bo->ttm;
 802	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
 803	struct ttm_resource *old_mem = ttm_bo->resource;
 804	u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;
 805	int ret;
 806
 807	/*
 808	 * If this starts to call into many components, consider
 809	 * using a notification chain here.
 810	 */
 811
 812	if (xe_bo_is_pinned(bo))
 813		return -EINVAL;
 814
 815	xe_bo_vunmap(bo);
 816	ret = xe_bo_trigger_rebind(xe, bo, ctx);
 817	if (ret)
 818		return ret;
 819
 820	/* Don't call move_notify() for imported dma-bufs. */
 821	if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach)
 822		dma_buf_move_notify(ttm_bo->base.dma_buf);
 823
 824	/*
 825	 * TTM has already nuked the mmap for us (see ttm_bo_unmap_virtual),
 826	 * so if we moved from VRAM make sure to unlink this from the userfault
 827	 * tracking.
 828	 */
 829	if (mem_type_is_vram(old_mem_type)) {
 830		mutex_lock(&xe->mem_access.vram_userfault.lock);
 831		if (!list_empty(&bo->vram_userfault_link))
 832			list_del_init(&bo->vram_userfault_link);
 833		mutex_unlock(&xe->mem_access.vram_userfault.lock);
 834	}
 835
 836	return 0;
 837}
 838
 839static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 840		      struct ttm_operation_ctx *ctx,
 841		      struct ttm_resource *new_mem,
 842		      struct ttm_place *hop)
 843{
 844	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
 845	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
 846	struct ttm_resource *old_mem = ttm_bo->resource;
 847	u32 old_mem_type = old_mem ? old_mem->mem_type : XE_PL_SYSTEM;
 848	struct ttm_tt *ttm = ttm_bo->ttm;
 849	struct xe_migrate *migrate = NULL;
 850	struct dma_fence *fence;
 851	bool move_lacks_source;
 852	bool tt_has_data;
 853	bool needs_clear;
 854	bool handle_system_ccs = (!IS_DGFX(xe) && xe_bo_needs_ccs_pages(bo) &&
 855				  ttm && ttm_tt_is_populated(ttm)) ? true : false;
 856	int ret = 0;
 857
 858	/* Bo creation path, moving to system or TT. */
 859	if ((!old_mem && ttm) && !handle_system_ccs) {
 860		if (new_mem->mem_type == XE_PL_TT)
 861			ret = xe_tt_map_sg(xe, ttm);
 862		if (!ret)
 863			ttm_bo_move_null(ttm_bo, new_mem);
 864		goto out;
 865	}
 866
 867	if (ttm_bo->type == ttm_bo_type_sg) {
 868		if (new_mem->mem_type == XE_PL_SYSTEM)
 869			ret = xe_bo_move_notify(bo, ctx);
 870		if (!ret)
 871			ret = xe_bo_move_dmabuf(ttm_bo, new_mem);
 872		return ret;
 873	}
 874
 875	tt_has_data = ttm && (ttm_tt_is_populated(ttm) || ttm_tt_is_swapped(ttm));
 876
 877	move_lacks_source = !old_mem || (handle_system_ccs ? (!bo->ccs_cleared) :
 878					 (!mem_type_is_vram(old_mem_type) && !tt_has_data));
 879
 880	needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) ||
 881		(!ttm && ttm_bo->type == ttm_bo_type_device);
 882
 883	if (new_mem->mem_type == XE_PL_TT) {
 884		ret = xe_tt_map_sg(xe, ttm);
 885		if (ret)
 886			goto out;
 887	}
 888
 889	if ((move_lacks_source && !needs_clear)) {
 890		ttm_bo_move_null(ttm_bo, new_mem);
 891		goto out;
 892	}
 893
 894	if (!move_lacks_source && (bo->flags & XE_BO_FLAG_CPU_ADDR_MIRROR) &&
 895	    new_mem->mem_type == XE_PL_SYSTEM) {
 896		ret = xe_svm_bo_evict(bo);
 897		if (!ret) {
 898			drm_dbg(&xe->drm, "Evict system allocator BO success\n");
 899			ttm_bo_move_null(ttm_bo, new_mem);
 900		} else {
 901			drm_dbg(&xe->drm, "Evict system allocator BO failed=%pe\n",
 902				ERR_PTR(ret));
 903		}
 904
 905		goto out;
 906	}
 907
 908	if (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT && !handle_system_ccs) {
 909		ttm_bo_move_null(ttm_bo, new_mem);
 910		goto out;
 911	}
 912
 913	/*
 914	 * Failed multi-hop where the old_mem is still marked as
 915	 * TTM_PL_FLAG_TEMPORARY, should just be a dummy move.
 916	 */
 917	if (old_mem_type == XE_PL_TT &&
 918	    new_mem->mem_type == XE_PL_TT) {
 919		ttm_bo_move_null(ttm_bo, new_mem);
 920		goto out;
 921	}
 922
 923	if (!move_lacks_source && !xe_bo_is_pinned(bo)) {
 924		ret = xe_bo_move_notify(bo, ctx);
 925		if (ret)
 926			goto out;
 927	}
 928
 929	if (old_mem_type == XE_PL_TT &&
 930	    new_mem->mem_type == XE_PL_SYSTEM) {
 931		long timeout = dma_resv_wait_timeout(ttm_bo->base.resv,
 932						     DMA_RESV_USAGE_BOOKKEEP,
 933						     false,
 934						     MAX_SCHEDULE_TIMEOUT);
 935		if (timeout < 0) {
 936			ret = timeout;
 937			goto out;
 938		}
 939
 940		if (!handle_system_ccs) {
 941			ttm_bo_move_null(ttm_bo, new_mem);
 942			goto out;
 943		}
 944	}
 945
 946	if (!move_lacks_source &&
 947	    ((old_mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) ||
 948	     (mem_type_is_vram(old_mem_type) &&
 949	      new_mem->mem_type == XE_PL_SYSTEM))) {
 950		hop->fpfn = 0;
 951		hop->lpfn = 0;
 952		hop->mem_type = XE_PL_TT;
 953		hop->flags = TTM_PL_FLAG_TEMPORARY;
 954		ret = -EMULTIHOP;
 955		goto out;
 956	}
 957
 958	if (bo->tile)
 959		migrate = bo->tile->migrate;
 960	else if (resource_is_vram(new_mem))
 961		migrate = mem_type_to_migrate(xe, new_mem->mem_type);
 962	else if (mem_type_is_vram(old_mem_type))
 963		migrate = mem_type_to_migrate(xe, old_mem_type);
 964	else
 965		migrate = xe->tiles[0].migrate;
 966
 967	xe_assert(xe, migrate);
 968	trace_xe_bo_move(bo, new_mem->mem_type, old_mem_type, move_lacks_source);
 969	if (xe_rpm_reclaim_safe(xe)) {
 970		/*
 971		 * We might be called through swapout in the validation path of
 972		 * another TTM device, so acquire rpm here.
 973		 */
 974		xe_pm_runtime_get(xe);
 975	} else {
 976		drm_WARN_ON(&xe->drm, handle_system_ccs);
 977		xe_pm_runtime_get_noresume(xe);
 978	}
 979
 980	if (move_lacks_source) {
 981		u32 flags = 0;
 982
 983		if (mem_type_is_vram(new_mem->mem_type))
 984			flags |= XE_MIGRATE_CLEAR_FLAG_FULL;
 985		else if (handle_system_ccs)
 986			flags |= XE_MIGRATE_CLEAR_FLAG_CCS_DATA;
 987
 988		fence = xe_migrate_clear(migrate, bo, new_mem, flags);
 989	} else {
 990		fence = xe_migrate_copy(migrate, bo, bo, old_mem, new_mem,
 991					handle_system_ccs);
 992	}
 993	if (IS_ERR(fence)) {
 994		ret = PTR_ERR(fence);
 995		xe_pm_runtime_put(xe);
 996		goto out;
 997	}
 998	if (!move_lacks_source) {
 999		ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict, true,
1000						new_mem);
1001		if (ret) {
1002			dma_fence_wait(fence, false);
1003			ttm_bo_move_null(ttm_bo, new_mem);
1004			ret = 0;
1005		}
1006	} else {
1007		/*
1008		 * ttm_bo_move_accel_cleanup() may blow up if
1009		 * bo->resource == NULL, so just attach the
1010		 * fence and set the new resource.
1011		 */
1012		dma_resv_add_fence(ttm_bo->base.resv, fence,
1013				   DMA_RESV_USAGE_KERNEL);
1014		ttm_bo_move_null(ttm_bo, new_mem);
1015	}
1016
1017	dma_fence_put(fence);
1018	xe_pm_runtime_put(xe);
1019
1020	/*
1021	 * CCS meta data is migrated from TT -> SMEM. So, let us detach the
1022	 * BBs from BO as it is no longer needed.
1023	 */
1024	if (IS_VF_CCS_READY(xe) && old_mem_type == XE_PL_TT &&
1025	    new_mem->mem_type == XE_PL_SYSTEM)
1026		xe_sriov_vf_ccs_detach_bo(bo);
1027
1028	if (IS_VF_CCS_READY(xe) &&
1029	    ((move_lacks_source && new_mem->mem_type == XE_PL_TT) ||
1030	     (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) &&
1031	    handle_system_ccs)
1032		ret = xe_sriov_vf_ccs_attach_bo(bo);
1033
1034out:
1035	if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) &&
1036	    ttm_bo->ttm) {
1037		long timeout = dma_resv_wait_timeout(ttm_bo->base.resv,
1038						     DMA_RESV_USAGE_KERNEL,
1039						     false,
1040						     MAX_SCHEDULE_TIMEOUT);
1041		if (timeout < 0)
1042			ret = timeout;
1043
1044		if (IS_VF_CCS_READY(xe))
1045			xe_sriov_vf_ccs_detach_bo(bo);
1046
1047		xe_tt_unmap_sg(xe, ttm_bo->ttm);
1048	}
1049
1050	return ret;
1051}
1052
1053static long xe_bo_shrink_purge(struct ttm_operation_ctx *ctx,
1054			       struct ttm_buffer_object *bo,
1055			       unsigned long *scanned)
1056{
1057	struct xe_device *xe = ttm_to_xe_device(bo->bdev);
1058	struct ttm_tt *tt = bo->ttm;
1059	long lret;
1060
1061	/* Fake move to system, without copying data. */
1062	if (bo->resource->mem_type != XE_PL_SYSTEM) {
1063		struct ttm_resource *new_resource;
1064
1065		lret = ttm_bo_wait_ctx(bo, ctx);
1066		if (lret)
1067			return lret;
1068
1069		lret = ttm_bo_mem_space(bo, &sys_placement, &new_resource, ctx);
1070		if (lret)
1071			return lret;
1072
1073		xe_tt_unmap_sg(xe, bo->ttm);
1074		ttm_bo_move_null(bo, new_resource);
1075	}
1076
1077	*scanned += bo->ttm->num_pages;
1078	lret = ttm_bo_shrink(ctx, bo, (struct ttm_bo_shrink_flags)
1079			     {.purge = true,
1080			      .writeback = false,
1081			      .allow_move = false});
1082
1083	if (lret > 0) {
1084		xe_ttm_tt_account_subtract(xe, bo->ttm);
1085		update_global_total_pages(bo->bdev, -(long)tt->num_pages);
1086	}
1087
1088	return lret;
1089}
1090
1091static bool
1092xe_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place)
1093{
1094	struct drm_gpuvm_bo *vm_bo;
1095
1096	if (!ttm_bo_eviction_valuable(bo, place))
1097		return false;
1098
1099	if (!xe_bo_is_xe_bo(bo))
1100		return true;
1101
1102	drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
1103		if (xe_vm_is_validating(gpuvm_to_vm(vm_bo->vm)))
1104			return false;
1105	}
1106
1107	return true;
1108}
1109
1110/**
1111 * xe_bo_shrink() - Try to shrink an xe bo.
1112 * @ctx: The struct ttm_operation_ctx used for shrinking.
1113 * @bo: The TTM buffer object whose pages to shrink.
1114 * @flags: Flags governing the shrink behaviour.
1115 * @scanned: Pointer to a counter of the number of pages
1116 * attempted to shrink.
1117 *
1118 * Try to shrink- or purge a bo, and if it succeeds, unmap dma.
1119 * Note that we need to be able to handle also non xe bos
1120 * (ghost bos), but only if the struct ttm_tt is embedded in
1121 * a struct xe_ttm_tt. When the function attempts to shrink
1122 * the pages of a buffer object, The value pointed to by @scanned
1123 * is updated.
1124 *
1125 * Return: The number of pages shrunken or purged, or negative error
1126 * code on failure.
1127 */
1128long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
1129		  const struct xe_bo_shrink_flags flags,
1130		  unsigned long *scanned)
1131{
1132	struct ttm_tt *tt = bo->ttm;
1133	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
1134	struct ttm_place place = {.mem_type = bo->resource->mem_type};
1135	struct xe_bo *xe_bo = ttm_to_xe_bo(bo);
1136	struct xe_device *xe = ttm_to_xe_device(bo->bdev);
1137	bool needs_rpm;
1138	long lret = 0L;
1139
1140	if (!(tt->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE) ||
1141	    (flags.purge && !xe_tt->purgeable))
1142		return -EBUSY;
1143
1144	if (!xe_bo_eviction_valuable(bo, &place))
1145		return -EBUSY;
1146
1147	if (!xe_bo_is_xe_bo(bo) || !xe_bo_get_unless_zero(xe_bo))
1148		return xe_bo_shrink_purge(ctx, bo, scanned);
1149
1150	if (xe_tt->purgeable) {
1151		if (bo->resource->mem_type != XE_PL_SYSTEM)
1152			lret = xe_bo_move_notify(xe_bo, ctx);
1153		if (!lret)
1154			lret = xe_bo_shrink_purge(ctx, bo, scanned);
1155		goto out_unref;
1156	}
1157
1158	/* System CCS needs gpu copy when moving PL_TT -> PL_SYSTEM */
1159	needs_rpm = (!IS_DGFX(xe) && bo->resource->mem_type != XE_PL_SYSTEM &&
1160		     xe_bo_needs_ccs_pages(xe_bo));
1161	if (needs_rpm && !xe_pm_runtime_get_if_active(xe))
1162		goto out_unref;
1163
1164	*scanned += tt->num_pages;
1165	lret = ttm_bo_shrink(ctx, bo, (struct ttm_bo_shrink_flags)
1166			     {.purge = false,
1167			      .writeback = flags.writeback,
1168			      .allow_move = true});
1169	if (needs_rpm)
1170		xe_pm_runtime_put(xe);
1171
1172	if (lret > 0) {
1173		xe_ttm_tt_account_subtract(xe, tt);
1174		update_global_total_pages(bo->bdev, -(long)tt->num_pages);
1175	}
1176
1177out_unref:
1178	xe_bo_put(xe_bo);
1179
1180	return lret;
1181}
1182
1183/**
1184 * xe_bo_notifier_prepare_pinned() - Prepare a pinned VRAM object to be backed
1185 * up in system memory.
1186 * @bo: The buffer object to prepare.
1187 *
1188 * On successful completion, the object backup pages are allocated. Expectation
1189 * is that this is called from the PM notifier, prior to suspend/hibernation.
1190 *
1191 * Return: 0 on success. Negative error code on failure.
1192 */
1193int xe_bo_notifier_prepare_pinned(struct xe_bo *bo)
1194{
1195	struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
1196	struct xe_validation_ctx ctx;
1197	struct drm_exec exec;
1198	struct xe_bo *backup;
1199	int ret = 0;
1200
1201	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.exclusive = true}, ret) {
1202		ret = drm_exec_lock_obj(&exec, &bo->ttm.base);
1203		drm_exec_retry_on_contention(&exec);
1204		xe_assert(xe, !ret);
1205		xe_assert(xe, !bo->backup_obj);
1206
1207		/*
1208		 * Since this is called from the PM notifier we might have raced with
1209		 * someone unpinning this after we dropped the pinned list lock and
1210		 * grabbing the above bo lock.
1211		 */
1212		if (!xe_bo_is_pinned(bo))
1213			break;
1214
1215		if (!xe_bo_is_vram(bo))
1216			break;
1217
1218		if (bo->flags & XE_BO_FLAG_PINNED_NORESTORE)
1219			break;
1220
1221		backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, xe_bo_size(bo),
1222					   DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel,
1223					   XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1224					   XE_BO_FLAG_PINNED, &exec);
1225		if (IS_ERR(backup)) {
1226			drm_exec_retry_on_contention(&exec);
1227			ret = PTR_ERR(backup);
1228			xe_validation_retry_on_oom(&ctx, &ret);
1229			break;
1230		}
1231
1232		backup->parent_obj = xe_bo_get(bo); /* Released by bo_destroy */
1233		ttm_bo_pin(&backup->ttm);
1234		bo->backup_obj = backup;
1235	}
1236
1237	return ret;
1238}
1239
1240/**
1241 * xe_bo_notifier_unprepare_pinned() - Undo the previous prepare operation.
1242 * @bo: The buffer object to undo the prepare for.
1243 *
1244 * Always returns 0. The backup object is removed, if still present. Expectation
1245 * it that this called from the PM notifier when undoing the prepare step.
1246 *
1247 * Return: Always returns 0.
1248 */
1249int xe_bo_notifier_unprepare_pinned(struct xe_bo *bo)
1250{
1251	xe_bo_lock(bo, false);
1252	if (bo->backup_obj) {
1253		ttm_bo_unpin(&bo->backup_obj->ttm);
1254		xe_bo_put(bo->backup_obj);
1255		bo->backup_obj = NULL;
1256	}
1257	xe_bo_unlock(bo);
1258
1259	return 0;
1260}
1261
1262static int xe_bo_evict_pinned_copy(struct xe_bo *bo, struct xe_bo *backup)
1263{
1264	struct xe_device *xe = xe_bo_device(bo);
1265	bool unmap = false;
1266	int ret = 0;
1267
1268	if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) {
1269		struct xe_migrate *migrate;
1270		struct dma_fence *fence;
1271
1272		if (bo->tile)
1273			migrate = bo->tile->migrate;
1274		else
1275			migrate = mem_type_to_migrate(xe, bo->ttm.resource->mem_type);
1276
1277		xe_assert(xe, bo->ttm.base.resv == backup->ttm.base.resv);
1278		ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
1279		if (ret)
1280			goto out_backup;
1281
1282		fence = xe_migrate_copy(migrate, bo, backup, bo->ttm.resource,
1283					backup->ttm.resource, false);
1284		if (IS_ERR(fence)) {
1285			ret = PTR_ERR(fence);
1286			goto out_backup;
1287		}
1288
1289		dma_resv_add_fence(bo->ttm.base.resv, fence,
1290				   DMA_RESV_USAGE_KERNEL);
1291		dma_fence_put(fence);
1292	} else {
1293		ret = xe_bo_vmap(backup);
1294		if (ret)
1295			goto out_backup;
1296
1297		if (iosys_map_is_null(&bo->vmap)) {
1298			ret = xe_bo_vmap(bo);
1299			if (ret)
1300				goto out_vunmap;
1301			unmap = true;
1302		}
1303
1304		xe_map_memcpy_from(xe, backup->vmap.vaddr, &bo->vmap, 0,
1305				   xe_bo_size(bo));
1306	}
1307
1308	if (!bo->backup_obj)
1309		bo->backup_obj = backup;
1310out_vunmap:
1311	xe_bo_vunmap(backup);
1312out_backup:
1313	if (unmap)
1314		xe_bo_vunmap(bo);
1315
1316	return ret;
1317}
1318
1319/**
1320 * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory
1321 * @bo: The buffer object to move.
1322 *
1323 * On successful completion, the object memory will be moved to system memory.
1324 *
1325 * This is needed to for special handling of pinned VRAM object during
1326 * suspend-resume.
1327 *
1328 * Return: 0 on success. Negative error code on failure.
1329 */
1330int xe_bo_evict_pinned(struct xe_bo *bo)
1331{
1332	struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
1333	struct xe_validation_ctx ctx;
1334	struct drm_exec exec;
1335	struct xe_bo *backup = bo->backup_obj;
1336	bool backup_created = false;
1337	int ret = 0;
1338
1339	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.exclusive = true}, ret) {
1340		ret = drm_exec_lock_obj(&exec, &bo->ttm.base);
1341		drm_exec_retry_on_contention(&exec);
1342		xe_assert(xe, !ret);
1343
1344		if (WARN_ON(!bo->ttm.resource)) {
1345			ret = -EINVAL;
1346			break;
1347		}
1348
1349		if (WARN_ON(!xe_bo_is_pinned(bo))) {
1350			ret = -EINVAL;
1351			break;
1352		}
1353
1354		if (!xe_bo_is_vram(bo))
1355			break;
1356
1357		if (bo->flags & XE_BO_FLAG_PINNED_NORESTORE)
1358			break;
1359
1360		if (!backup) {
1361			backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL,
1362						   xe_bo_size(bo),
1363						   DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel,
1364						   XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1365						   XE_BO_FLAG_PINNED, &exec);
1366			if (IS_ERR(backup)) {
1367				drm_exec_retry_on_contention(&exec);
1368				ret = PTR_ERR(backup);
1369				xe_validation_retry_on_oom(&ctx, &ret);
1370				break;
1371			}
1372			backup->parent_obj = xe_bo_get(bo); /* Released by bo_destroy */
1373			backup_created = true;
1374		}
1375
1376		ret = xe_bo_evict_pinned_copy(bo, backup);
1377	}
1378
1379	if (ret && backup_created)
1380		xe_bo_put(backup);
1381
1382	return ret;
1383}
1384
1385/**
1386 * xe_bo_restore_pinned() - Restore a pinned VRAM object
1387 * @bo: The buffer object to move.
1388 *
1389 * On successful completion, the object memory will be moved back to VRAM.
1390 *
1391 * This is needed to for special handling of pinned VRAM object during
1392 * suspend-resume.
1393 *
1394 * Return: 0 on success. Negative error code on failure.
1395 */
1396int xe_bo_restore_pinned(struct xe_bo *bo)
1397{
1398	struct ttm_operation_ctx ctx = {
1399		.interruptible = false,
1400		.gfp_retry_mayfail = false,
1401	};
1402	struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
1403	struct xe_bo *backup = bo->backup_obj;
1404	bool unmap = false;
1405	int ret;
1406
1407	if (!backup)
1408		return 0;
1409
1410	xe_bo_lock(bo, false);
1411
1412	if (!xe_bo_is_pinned(backup)) {
1413		ret = ttm_bo_validate(&backup->ttm, &backup->placement, &ctx);
1414		if (ret)
1415			goto out_unlock_bo;
1416	}
1417
1418	if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) {
1419		struct xe_migrate *migrate;
1420		struct dma_fence *fence;
1421
1422		if (bo->tile)
1423			migrate = bo->tile->migrate;
1424		else
1425			migrate = mem_type_to_migrate(xe, bo->ttm.resource->mem_type);
1426
1427		ret = dma_resv_reserve_fences(bo->ttm.base.resv, 1);
1428		if (ret)
1429			goto out_unlock_bo;
1430
1431		fence = xe_migrate_copy(migrate, backup, bo,
1432					backup->ttm.resource, bo->ttm.resource,
1433					false);
1434		if (IS_ERR(fence)) {
1435			ret = PTR_ERR(fence);
1436			goto out_unlock_bo;
1437		}
1438
1439		dma_resv_add_fence(bo->ttm.base.resv, fence,
1440				   DMA_RESV_USAGE_KERNEL);
1441		dma_fence_put(fence);
1442	} else {
1443		ret = xe_bo_vmap(backup);
1444		if (ret)
1445			goto out_unlock_bo;
1446
1447		if (iosys_map_is_null(&bo->vmap)) {
1448			ret = xe_bo_vmap(bo);
1449			if (ret)
1450				goto out_backup;
1451			unmap = true;
1452		}
1453
1454		xe_map_memcpy_to(xe, &bo->vmap, 0, backup->vmap.vaddr,
1455				 xe_bo_size(bo));
1456	}
1457
1458	bo->backup_obj = NULL;
1459
1460out_backup:
1461	xe_bo_vunmap(backup);
1462	if (!bo->backup_obj) {
1463		if (xe_bo_is_pinned(backup))
1464			ttm_bo_unpin(&backup->ttm);
1465		xe_bo_put(backup);
1466	}
1467out_unlock_bo:
1468	if (unmap)
1469		xe_bo_vunmap(bo);
1470	xe_bo_unlock(bo);
1471	return ret;
1472}
1473
1474int xe_bo_dma_unmap_pinned(struct xe_bo *bo)
1475{
1476	struct ttm_buffer_object *ttm_bo = &bo->ttm;
1477	struct ttm_tt *tt = ttm_bo->ttm;
1478
1479	if (tt) {
1480		struct xe_ttm_tt *xe_tt = container_of(tt, typeof(*xe_tt), ttm);
1481
1482		if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) {
1483			dma_buf_unmap_attachment(ttm_bo->base.import_attach,
1484						 ttm_bo->sg,
1485						 DMA_BIDIRECTIONAL);
1486			ttm_bo->sg = NULL;
1487			xe_tt->sg = NULL;
1488		} else if (xe_tt->sg) {
1489			dma_unmap_sgtable(ttm_to_xe_device(ttm_bo->bdev)->drm.dev,
1490					  xe_tt->sg,
1491					  DMA_BIDIRECTIONAL, 0);
1492			sg_free_table(xe_tt->sg);
1493			xe_tt->sg = NULL;
1494		}
1495	}
1496
1497	return 0;
1498}
1499
1500static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,
1501				       unsigned long page_offset)
1502{
1503	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
1504	struct xe_res_cursor cursor;
1505	struct xe_vram_region *vram;
1506
1507	if (ttm_bo->resource->mem_type == XE_PL_STOLEN)
1508		return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;
1509
1510	vram = res_to_mem_region(ttm_bo->resource);
1511	xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);
1512	return (vram->io_start + cursor.start) >> PAGE_SHIFT;
1513}
1514
1515static void __xe_bo_vunmap(struct xe_bo *bo);
1516
1517/*
1518 * TODO: Move this function to TTM so we don't rely on how TTM does its
1519 * locking, thereby abusing TTM internals.
1520 */
1521static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo)
1522{
1523	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1524	bool locked;
1525
1526	xe_assert(xe, !kref_read(&ttm_bo->kref));
1527
1528	/*
1529	 * We can typically only race with TTM trylocking under the
1530	 * lru_lock, which will immediately be unlocked again since
1531	 * the ttm_bo refcount is zero at this point. So trylocking *should*
1532	 * always succeed here, as long as we hold the lru lock.
1533	 */
1534	spin_lock(&ttm_bo->bdev->lru_lock);
1535	locked = dma_resv_trylock(&ttm_bo->base._resv);
1536	spin_unlock(&ttm_bo->bdev->lru_lock);
1537	xe_assert(xe, locked);
1538
1539	return locked;
1540}
1541
1542static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo)
1543{
1544	struct dma_resv_iter cursor;
1545	struct dma_fence *fence;
1546	struct dma_fence *replacement = NULL;
1547	struct xe_bo *bo;
1548
1549	if (!xe_bo_is_xe_bo(ttm_bo))
1550		return;
1551
1552	bo = ttm_to_xe_bo(ttm_bo);
1553	xe_assert(xe_bo_device(bo), !(bo->created && kref_read(&ttm_bo->base.refcount)));
1554
1555	if (!xe_ttm_bo_lock_in_destructor(ttm_bo))
1556		return;
1557
1558	/*
1559	 * Scrub the preempt fences if any. The unbind fence is already
1560	 * attached to the resv.
1561	 * TODO: Don't do this for external bos once we scrub them after
1562	 * unbind.
1563	 */
1564	dma_resv_for_each_fence(&cursor, &ttm_bo->base._resv,
1565				DMA_RESV_USAGE_BOOKKEEP, fence) {
1566		if (xe_fence_is_xe_preempt(fence) &&
1567		    !dma_fence_is_signaled(fence)) {
1568			if (!replacement)
1569				replacement = dma_fence_get_stub();
1570
1571			dma_resv_replace_fences(&ttm_bo->base._resv,
1572						fence->context,
1573						replacement,
1574						DMA_RESV_USAGE_BOOKKEEP);
1575		}
1576	}
1577	dma_fence_put(replacement);
1578
1579	dma_resv_unlock(&ttm_bo->base._resv);
1580}
1581
1582static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo)
1583{
1584	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
1585
1586	if (!xe_bo_is_xe_bo(ttm_bo))
1587		return;
1588
1589	if (IS_VF_CCS_READY(ttm_to_xe_device(ttm_bo->bdev)))
1590		xe_sriov_vf_ccs_detach_bo(bo);
1591
1592	/*
1593	 * Object is idle and about to be destroyed. Release the
1594	 * dma-buf attachment.
1595	 */
1596	if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) {
1597		struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm,
1598						       struct xe_ttm_tt, ttm);
1599
1600		dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg,
1601					 DMA_BIDIRECTIONAL);
1602		ttm_bo->sg = NULL;
1603		xe_tt->sg = NULL;
1604	}
1605}
1606
1607static void xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx)
1608{
1609	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1610
1611	if (ttm_bo->ttm) {
1612		struct ttm_placement place = {};
1613		int ret = ttm_bo_validate(ttm_bo, &place, ctx);
1614
1615		drm_WARN_ON(&xe->drm, ret);
1616	}
1617}
1618
1619static void xe_ttm_bo_swap_notify(struct ttm_buffer_object *ttm_bo)
1620{
1621	struct ttm_operation_ctx ctx = {
1622		.interruptible = false,
1623		.gfp_retry_mayfail = false,
1624	};
1625
1626	if (ttm_bo->ttm) {
1627		struct xe_ttm_tt *xe_tt =
1628			container_of(ttm_bo->ttm, struct xe_ttm_tt, ttm);
1629
1630		if (xe_tt->purgeable)
1631			xe_ttm_bo_purge(ttm_bo, &ctx);
1632	}
1633}
1634
1635static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo,
1636				unsigned long offset, void *buf, int len,
1637				int write)
1638{
1639	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
1640	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1641	struct iosys_map vmap;
1642	struct xe_res_cursor cursor;
1643	struct xe_vram_region *vram;
1644	int bytes_left = len;
1645	int err = 0;
1646
1647	xe_bo_assert_held(bo);
1648	xe_device_assert_mem_access(xe);
1649
1650	if (!mem_type_is_vram(ttm_bo->resource->mem_type))
1651		return -EIO;
1652
1653	if (!xe_bo_is_visible_vram(bo) || len >= SZ_16K) {
1654		struct xe_migrate *migrate =
1655			mem_type_to_migrate(xe, ttm_bo->resource->mem_type);
1656
1657		err = xe_migrate_access_memory(migrate, bo, offset, buf, len,
1658					       write);
1659		goto out;
1660	}
1661
1662	vram = res_to_mem_region(ttm_bo->resource);
1663	xe_res_first(ttm_bo->resource, offset & PAGE_MASK,
1664		     xe_bo_size(bo) - (offset & PAGE_MASK), &cursor);
1665
1666	do {
1667		unsigned long page_offset = (offset & ~PAGE_MASK);
1668		int byte_count = min((int)(PAGE_SIZE - page_offset), bytes_left);
1669
1670		iosys_map_set_vaddr_iomem(&vmap, (u8 __iomem *)vram->mapping +
1671					  cursor.start);
1672		if (write)
1673			xe_map_memcpy_to(xe, &vmap, page_offset, buf, byte_count);
1674		else
1675			xe_map_memcpy_from(xe, buf, &vmap, page_offset, byte_count);
1676
1677		buf += byte_count;
1678		offset += byte_count;
1679		bytes_left -= byte_count;
1680		if (bytes_left)
1681			xe_res_next(&cursor, PAGE_SIZE);
1682	} while (bytes_left);
1683
1684out:
1685	return err ?: len;
1686}
1687
1688const struct ttm_device_funcs xe_ttm_funcs = {
1689	.ttm_tt_create = xe_ttm_tt_create,
1690	.ttm_tt_populate = xe_ttm_tt_populate,
1691	.ttm_tt_unpopulate = xe_ttm_tt_unpopulate,
1692	.ttm_tt_destroy = xe_ttm_tt_destroy,
1693	.evict_flags = xe_evict_flags,
1694	.move = xe_bo_move,
1695	.io_mem_reserve = xe_ttm_io_mem_reserve,
1696	.io_mem_pfn = xe_ttm_io_mem_pfn,
1697	.access_memory = xe_ttm_access_memory,
1698	.release_notify = xe_ttm_bo_release_notify,
1699	.eviction_valuable = xe_bo_eviction_valuable,
1700	.delete_mem_notify = xe_ttm_bo_delete_mem_notify,
1701	.swap_notify = xe_ttm_bo_swap_notify,
1702};
1703
1704static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
1705{
1706	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
1707	struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
1708	struct xe_tile *tile;
1709	u8 id;
1710
1711	if (bo->ttm.base.import_attach)
1712		drm_prime_gem_destroy(&bo->ttm.base, NULL);
1713	drm_gem_object_release(&bo->ttm.base);
1714
1715	xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list));
1716
1717	for_each_tile(tile, xe, id)
1718		if (bo->ggtt_node[id] && bo->ggtt_node[id]->base.size)
1719			xe_ggtt_remove_bo(tile->mem.ggtt, bo);
1720
1721#ifdef CONFIG_PROC_FS
1722	if (bo->client)
1723		xe_drm_client_remove_bo(bo);
1724#endif
1725
1726	if (bo->vm && xe_bo_is_user(bo))
1727		xe_vm_put(bo->vm);
1728
1729	if (bo->parent_obj)
1730		xe_bo_put(bo->parent_obj);
1731
1732	mutex_lock(&xe->mem_access.vram_userfault.lock);
1733	if (!list_empty(&bo->vram_userfault_link))
1734		list_del(&bo->vram_userfault_link);
1735	mutex_unlock(&xe->mem_access.vram_userfault.lock);
1736
1737	kfree(bo);
1738}
1739
1740static void xe_gem_object_free(struct drm_gem_object *obj)
1741{
1742	/* Our BO reference counting scheme works as follows:
1743	 *
1744	 * The gem object kref is typically used throughout the driver,
1745	 * and the gem object holds a ttm_buffer_object refcount, so
1746	 * that when the last gem object reference is put, which is when
1747	 * we end up in this function, we put also that ttm_buffer_object
1748	 * refcount. Anything using gem interfaces is then no longer
1749	 * allowed to access the object in a way that requires a gem
1750	 * refcount, including locking the object.
1751	 *
1752	 * driver ttm callbacks is allowed to use the ttm_buffer_object
1753	 * refcount directly if needed.
1754	 */
1755	__xe_bo_vunmap(gem_to_xe_bo(obj));
1756	ttm_bo_fini(container_of(obj, struct ttm_buffer_object, base));
1757}
1758
1759static void xe_gem_object_close(struct drm_gem_object *obj,
1760				struct drm_file *file_priv)
1761{
1762	struct xe_bo *bo = gem_to_xe_bo(obj);
1763
1764	if (bo->vm && !xe_vm_in_fault_mode(bo->vm)) {
1765		xe_assert(xe_bo_device(bo), xe_bo_is_user(bo));
1766
1767		xe_bo_lock(bo, false);
1768		ttm_bo_set_bulk_move(&bo->ttm, NULL);
1769		xe_bo_unlock(bo);
1770	}
1771}
1772
1773static bool should_migrate_to_smem(struct xe_bo *bo)
1774{
1775	/*
1776	 * NOTE: The following atomic checks are platform-specific. For example,
1777	 * if a device supports CXL atomics, these may not be necessary or
1778	 * may behave differently.
1779	 */
1780
1781	return bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL ||
1782	       bo->attr.atomic_access == DRM_XE_ATOMIC_CPU;
1783}
1784
1785static int xe_bo_wait_usage_kernel(struct xe_bo *bo, struct ttm_operation_ctx *ctx)
1786{
1787	long lerr;
1788
1789	if (ctx->no_wait_gpu)
1790		return dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL) ?
1791			0 : -EBUSY;
1792
1793	lerr = dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
1794				     ctx->interruptible, MAX_SCHEDULE_TIMEOUT);
1795	if (lerr < 0)
1796		return lerr;
1797	if (lerr == 0)
1798		return -EBUSY;
1799
1800	return 0;
1801}
1802
1803/* Populate the bo if swapped out, or migrate if the access mode requires that. */
1804static int xe_bo_fault_migrate(struct xe_bo *bo, struct ttm_operation_ctx *ctx,
1805			       struct drm_exec *exec)
1806{
1807	struct ttm_buffer_object *tbo = &bo->ttm;
1808	int err = 0;
1809
1810	if (ttm_manager_type(tbo->bdev, tbo->resource->mem_type)->use_tt) {
1811		err = xe_bo_wait_usage_kernel(bo, ctx);
1812		if (!err)
1813			err = ttm_bo_populate(&bo->ttm, ctx);
1814	} else if (should_migrate_to_smem(bo)) {
1815		xe_assert(xe_bo_device(bo), bo->flags & XE_BO_FLAG_SYSTEM);
1816		err = xe_bo_migrate(bo, XE_PL_TT, ctx, exec);
1817	}
1818
1819	return err;
1820}
1821
1822/* Call into TTM to populate PTEs, and register bo for PTE removal on runtime suspend. */
1823static vm_fault_t __xe_bo_cpu_fault(struct vm_fault *vmf, struct xe_device *xe, struct xe_bo *bo)
1824{
1825	vm_fault_t ret;
1826
1827	trace_xe_bo_cpu_fault(bo);
1828
1829	ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
1830				       TTM_BO_VM_NUM_PREFAULT);
1831	/*
1832	 * When TTM is actually called to insert PTEs, ensure no blocking conditions
1833	 * remain, in which case TTM may drop locks and return VM_FAULT_RETRY.
1834	 */
1835	xe_assert(xe, ret != VM_FAULT_RETRY);
1836
1837	if (ret == VM_FAULT_NOPAGE &&
1838	    mem_type_is_vram(bo->ttm.resource->mem_type)) {
1839		mutex_lock(&xe->mem_access.vram_userfault.lock);
1840		if (list_empty(&bo->vram_userfault_link))
1841			list_add(&bo->vram_userfault_link,
1842				 &xe->mem_access.vram_userfault.list);
1843		mutex_unlock(&xe->mem_access.vram_userfault.lock);
1844	}
1845
1846	return ret;
1847}
1848
1849static vm_fault_t xe_err_to_fault_t(int err)
1850{
1851	switch (err) {
1852	case 0:
1853	case -EINTR:
1854	case -ERESTARTSYS:
1855	case -EAGAIN:
1856		return VM_FAULT_NOPAGE;
1857	case -ENOMEM:
1858	case -ENOSPC:
1859		return VM_FAULT_OOM;
1860	default:
1861		break;
1862	}
1863	return VM_FAULT_SIGBUS;
1864}
1865
1866static bool xe_ttm_bo_is_imported(struct ttm_buffer_object *tbo)
1867{
1868	dma_resv_assert_held(tbo->base.resv);
1869
1870	return tbo->ttm &&
1871		(tbo->ttm->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE)) ==
1872		TTM_TT_FLAG_EXTERNAL;
1873}
1874
1875static vm_fault_t xe_bo_cpu_fault_fastpath(struct vm_fault *vmf, struct xe_device *xe,
1876					   struct xe_bo *bo, bool needs_rpm)
1877{
1878	struct ttm_buffer_object *tbo = &bo->ttm;
1879	vm_fault_t ret = VM_FAULT_RETRY;
1880	struct xe_validation_ctx ctx;
1881	struct ttm_operation_ctx tctx = {
1882		.interruptible = true,
1883		.no_wait_gpu = true,
1884		.gfp_retry_mayfail = true,
1885
1886	};
1887	int err;
1888
1889	if (needs_rpm && !xe_pm_runtime_get_if_active(xe))
1890		return VM_FAULT_RETRY;
1891
1892	err = xe_validation_ctx_init(&ctx, &xe->val, NULL,
1893				     (struct xe_val_flags) {
1894					     .interruptible = true,
1895					     .no_block = true
1896				     });
1897	if (err)
1898		goto out_pm;
1899
1900	if (!dma_resv_trylock(tbo->base.resv))
1901		goto out_validation;
1902
1903	if (xe_ttm_bo_is_imported(tbo)) {
1904		ret = VM_FAULT_SIGBUS;
1905		drm_dbg(&xe->drm, "CPU trying to access an imported buffer object.\n");
1906		goto out_unlock;
1907	}
1908
1909	err = xe_bo_fault_migrate(bo, &tctx, NULL);
1910	if (err) {
1911		/* Return VM_FAULT_RETRY on these errors. */
1912		if (err != -ENOMEM && err != -ENOSPC && err != -EBUSY)
1913			ret = xe_err_to_fault_t(err);
1914		goto out_unlock;
1915	}
1916
1917	if (dma_resv_test_signaled(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL))
1918		ret = __xe_bo_cpu_fault(vmf, xe, bo);
1919
1920out_unlock:
1921	dma_resv_unlock(tbo->base.resv);
1922out_validation:
1923	xe_validation_ctx_fini(&ctx);
1924out_pm:
1925	if (needs_rpm)
1926		xe_pm_runtime_put(xe);
1927
1928	return ret;
1929}
1930
1931static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf)
1932{
1933	struct ttm_buffer_object *tbo = vmf->vma->vm_private_data;
1934	struct drm_device *ddev = tbo->base.dev;
1935	struct xe_device *xe = to_xe_device(ddev);
1936	struct xe_bo *bo = ttm_to_xe_bo(tbo);
1937	bool needs_rpm = bo->flags & XE_BO_FLAG_VRAM_MASK;
1938	bool retry_after_wait = false;
1939	struct xe_validation_ctx ctx;
1940	struct drm_exec exec;
1941	vm_fault_t ret;
1942	int err = 0;
1943	int idx;
1944
1945	if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx))
1946		return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
1947
1948	ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm);
1949	if (ret != VM_FAULT_RETRY)
1950		goto out;
1951
1952	if (fault_flag_allow_retry_first(vmf->flags)) {
1953		if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
1954			goto out;
1955		retry_after_wait = true;
1956		xe_bo_get(bo);
1957		mmap_read_unlock(vmf->vma->vm_mm);
1958	} else {
1959		ret = VM_FAULT_NOPAGE;
1960	}
1961
1962	/*
1963	 * The fastpath failed and we were not required to return and retry immediately.
1964	 * We're now running in one of two modes:
1965	 *
1966	 * 1) retry_after_wait == true: The mmap_read_lock() is dropped, and we're trying
1967	 * to resolve blocking waits. But we can't resolve the fault since the
1968	 * mmap_read_lock() is dropped. After retrying the fault, the aim is that the fastpath
1969	 * should succeed. But it may fail since we drop the bo lock.
1970	 *
1971	 * 2) retry_after_wait == false: The fastpath failed, typically even after
1972	 * a retry. Do whatever's necessary to resolve the fault.
1973	 *
1974	 * This construct is recommended to avoid excessive waits under the mmap_lock.
1975	 */
1976
1977	if (needs_rpm)
1978		xe_pm_runtime_get(xe);
1979
1980	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true},
1981			    err) {
1982		struct ttm_operation_ctx tctx = {
1983			.interruptible = true,
1984			.no_wait_gpu = false,
1985			.gfp_retry_mayfail = retry_after_wait,
1986		};
1987
1988		err = drm_exec_lock_obj(&exec, &tbo->base);
1989		drm_exec_retry_on_contention(&exec);
1990		if (err)
1991			break;
1992
1993		if (xe_ttm_bo_is_imported(tbo)) {
1994			err = -EFAULT;
1995			drm_dbg(&xe->drm, "CPU trying to access an imported buffer object.\n");
1996			break;
1997		}
1998
1999		err = xe_bo_fault_migrate(bo, &tctx, &exec);
2000		if (err) {
2001			drm_exec_retry_on_contention(&exec);
2002			xe_validation_retry_on_oom(&ctx, &err);
2003			break;
2004		}
2005
2006		err = xe_bo_wait_usage_kernel(bo, &tctx);
2007		if (err)
2008			break;
2009
2010		if (!retry_after_wait)
2011			ret = __xe_bo_cpu_fault(vmf, xe, bo);
2012	}
2013	/* if retry_after_wait == true, we *must* return VM_FAULT_RETRY. */
2014	if (err && !retry_after_wait)
2015		ret = xe_err_to_fault_t(err);
2016
2017	if (needs_rpm)
2018		xe_pm_runtime_put(xe);
2019
2020	if (retry_after_wait)
2021		xe_bo_put(bo);
2022out:
2023	drm_dev_exit(idx);
2024
2025	return ret;
2026}
2027
2028static int xe_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
2029			   void *buf, int len, int write)
2030{
2031	struct ttm_buffer_object *ttm_bo = vma->vm_private_data;
2032	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
2033	struct xe_device *xe = xe_bo_device(bo);
2034	int ret;
2035
2036	xe_pm_runtime_get(xe);
2037	ret = ttm_bo_vm_access(vma, addr, buf, len, write);
2038	xe_pm_runtime_put(xe);
2039
2040	return ret;
2041}
2042
2043/**
2044 * xe_bo_read() - Read from an xe_bo
2045 * @bo: The buffer object to read from.
2046 * @offset: The byte offset to start reading from.
2047 * @dst: Location to store the read.
2048 * @size: Size in bytes for the read.
2049 *
2050 * Read @size bytes from the @bo, starting from @offset, storing into @dst.
2051 *
2052 * Return: Zero on success, or negative error.
2053 */
2054int xe_bo_read(struct xe_bo *bo, u64 offset, void *dst, int size)
2055{
2056	int ret;
2057
2058	ret = ttm_bo_access(&bo->ttm, offset, dst, size, 0);
2059	if (ret >= 0 && ret != size)
2060		ret = -EIO;
2061	else if (ret == size)
2062		ret = 0;
2063
2064	return ret;
2065}
2066
2067static const struct vm_operations_struct xe_gem_vm_ops = {
2068	.fault = xe_bo_cpu_fault,
2069	.open = ttm_bo_vm_open,
2070	.close = ttm_bo_vm_close,
2071	.access = xe_bo_vm_access,
2072};
2073
2074static const struct drm_gem_object_funcs xe_gem_object_funcs = {
2075	.free = xe_gem_object_free,
2076	.close = xe_gem_object_close,
2077	.mmap = drm_gem_ttm_mmap,
2078	.export = xe_gem_prime_export,
2079	.vm_ops = &xe_gem_vm_ops,
2080};
2081
2082/**
2083 * xe_bo_alloc - Allocate storage for a struct xe_bo
2084 *
2085 * This function is intended to allocate storage to be used for input
2086 * to __xe_bo_create_locked(), in the case a pointer to the bo to be
2087 * created is needed before the call to __xe_bo_create_locked().
2088 * If __xe_bo_create_locked ends up never to be called, then the
2089 * storage allocated with this function needs to be freed using
2090 * xe_bo_free().
2091 *
2092 * Return: A pointer to an uninitialized struct xe_bo on success,
2093 * ERR_PTR(-ENOMEM) on error.
2094 */
2095struct xe_bo *xe_bo_alloc(void)
2096{
2097	struct xe_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL);
2098
2099	if (!bo)
2100		return ERR_PTR(-ENOMEM);
2101
2102	return bo;
2103}
2104
2105/**
2106 * xe_bo_free - Free storage allocated using xe_bo_alloc()
2107 * @bo: The buffer object storage.
2108 *
2109 * Refer to xe_bo_alloc() documentation for valid use-cases.
2110 */
2111void xe_bo_free(struct xe_bo *bo)
2112{
2113	kfree(bo);
2114}
2115
2116/**
2117 * xe_bo_init_locked() - Initialize or create an xe_bo.
2118 * @xe: The xe device.
2119 * @bo: An already allocated buffer object or NULL
2120 * if the function should allocate a new one.
2121 * @tile: The tile to select for migration of this bo, and the tile used for
2122 * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos.
2123 * @resv: Pointer to a locked shared reservation object to use for this bo,
2124 * or NULL for the xe_bo to use its own.
2125 * @bulk: The bulk move to use for LRU bumping, or NULL for external bos.
2126 * @size: The storage size to use for the bo.
2127 * @cpu_caching: The cpu caching used for system memory backing store.
2128 * @type: The TTM buffer object type.
2129 * @flags: XE_BO_FLAG_ flags.
2130 * @exec: The drm_exec transaction to use for exhaustive eviction.
2131 *
2132 * Initialize or create an xe buffer object. On failure, any allocated buffer
2133 * object passed in @bo will have been unreferenced.
2134 *
2135 * Return: The buffer object on success. Negative error pointer on failure.
2136 */
2137struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
2138				struct xe_tile *tile, struct dma_resv *resv,
2139				struct ttm_lru_bulk_move *bulk, size_t size,
2140				u16 cpu_caching, enum ttm_bo_type type,
2141				u32 flags, struct drm_exec *exec)
2142{
2143	struct ttm_operation_ctx ctx = {
2144		.interruptible = true,
2145		.no_wait_gpu = false,
2146		.gfp_retry_mayfail = true,
2147	};
2148	struct ttm_placement *placement;
2149	uint32_t alignment;
2150	size_t aligned_size;
2151	int err;
2152
2153	/* Only kernel objects should set GT */
2154	xe_assert(xe, !tile || type == ttm_bo_type_kernel);
2155
2156	if (XE_WARN_ON(!size)) {
2157		xe_bo_free(bo);
2158		return ERR_PTR(-EINVAL);
2159	}
2160
2161	/* XE_BO_FLAG_GGTTx requires XE_BO_FLAG_GGTT also be set */
2162	if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT))
2163		return ERR_PTR(-EINVAL);
2164
2165	if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) &&
2166	    !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) &&
2167	    ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) ||
2168	     (flags & (XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M)))) {
2169		size_t align = flags & XE_BO_FLAG_NEEDS_2M ? SZ_2M : SZ_64K;
2170
2171		aligned_size = ALIGN(size, align);
2172		if (type != ttm_bo_type_device)
2173			size = ALIGN(size, align);
2174		flags |= XE_BO_FLAG_INTERNAL_64K;
2175		alignment = align >> PAGE_SHIFT;
2176	} else {
2177		aligned_size = ALIGN(size, SZ_4K);
2178		flags &= ~XE_BO_FLAG_INTERNAL_64K;
2179		alignment = SZ_4K >> PAGE_SHIFT;
2180	}
2181
2182	if (type == ttm_bo_type_device && aligned_size != size)
2183		return ERR_PTR(-EINVAL);
2184
2185	if (!bo) {
2186		bo = xe_bo_alloc();
2187		if (IS_ERR(bo))
2188			return bo;
2189	}
2190
2191	bo->ccs_cleared = false;
2192	bo->tile = tile;
2193	bo->flags = flags;
2194	bo->cpu_caching = cpu_caching;
2195	bo->ttm.base.funcs = &xe_gem_object_funcs;
2196	bo->ttm.priority = XE_BO_PRIORITY_NORMAL;
2197	INIT_LIST_HEAD(&bo->pinned_link);
2198#ifdef CONFIG_PROC_FS
2199	INIT_LIST_HEAD(&bo->client_link);
2200#endif
2201	INIT_LIST_HEAD(&bo->vram_userfault_link);
2202
2203	drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size);
2204
2205	if (resv) {
2206		ctx.allow_res_evict = !(flags & XE_BO_FLAG_NO_RESV_EVICT);
2207		ctx.resv = resv;
2208	}
2209
2210	xe_validation_assert_exec(xe, exec, &bo->ttm.base);
2211	if (!(flags & XE_BO_FLAG_FIXED_PLACEMENT)) {
2212		err = __xe_bo_placement_for_flags(xe, bo, bo->flags, type);
2213		if (WARN_ON(err)) {
2214			xe_ttm_bo_destroy(&bo->ttm);
2215			return ERR_PTR(err);
2216		}
2217	}
2218
2219	/* Defer populating type_sg bos */
2220	placement = (type == ttm_bo_type_sg ||
2221		     bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement :
2222		&bo->placement;
2223	err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type,
2224				   placement, alignment,
2225				   &ctx, NULL, resv, xe_ttm_bo_destroy);
2226	if (err)
2227		return ERR_PTR(err);
2228
2229	/*
2230	 * The VRAM pages underneath are potentially still being accessed by the
2231	 * GPU, as per async GPU clearing and async evictions. However TTM makes
2232	 * sure to add any corresponding move/clear fences into the objects
2233	 * dma-resv using the DMA_RESV_USAGE_KERNEL slot.
2234	 *
2235	 * For KMD internal buffers we don't care about GPU clearing, however we
2236	 * still need to handle async evictions, where the VRAM is still being
2237	 * accessed by the GPU. Most internal callers are not expecting this,
2238	 * since they are missing the required synchronisation before accessing
2239	 * the memory. To keep things simple just sync wait any kernel fences
2240	 * here, if the buffer is designated KMD internal.
2241	 *
2242	 * For normal userspace objects we should already have the required
2243	 * pipelining or sync waiting elsewhere, since we already have to deal
2244	 * with things like async GPU clearing.
2245	 */
2246	if (type == ttm_bo_type_kernel) {
2247		long timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
2248						     DMA_RESV_USAGE_KERNEL,
2249						     ctx.interruptible,
2250						     MAX_SCHEDULE_TIMEOUT);
2251
2252		if (timeout < 0) {
2253			if (!resv)
2254				dma_resv_unlock(bo->ttm.base.resv);
2255			xe_bo_put(bo);
2256			return ERR_PTR(timeout);
2257		}
2258	}
2259
2260	bo->created = true;
2261	if (bulk)
2262		ttm_bo_set_bulk_move(&bo->ttm, bulk);
2263	else
2264		ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
2265
2266	return bo;
2267}
2268
2269static int __xe_bo_fixed_placement(struct xe_device *xe,
2270				   struct xe_bo *bo, enum ttm_bo_type type,
2271				   u32 flags,
2272				   u64 start, u64 end, u64 size)
2273{
2274	struct ttm_place *place = bo->placements;
2275	u32 vram_flag, vram_stolen_flags;
2276
2277	/*
2278	 * to allow fixed placement in GGTT of a VF, post-migration fixups would have to
2279	 * include selecting a new fixed offset and shifting the page ranges for it
2280	 */
2281	xe_assert(xe, !IS_SRIOV_VF(xe) || !(bo->flags & XE_BO_FLAG_GGTT));
2282
2283	if (flags & (XE_BO_FLAG_USER | XE_BO_FLAG_SYSTEM))
2284		return -EINVAL;
2285
2286	vram_flag = flags & XE_BO_FLAG_VRAM_MASK;
2287	vram_stolen_flags = (flags & (XE_BO_FLAG_STOLEN)) | vram_flag;
2288
2289	/* check if more than one VRAM/STOLEN flag is set */
2290	if (hweight32(vram_stolen_flags) > 1)
2291		return -EINVAL;
2292
2293	place->flags = TTM_PL_FLAG_CONTIGUOUS;
2294	place->fpfn = start >> PAGE_SHIFT;
2295	place->lpfn = end >> PAGE_SHIFT;
2296
2297	if (flags & XE_BO_FLAG_STOLEN)
2298		place->mem_type = XE_PL_STOLEN;
2299	else
2300		place->mem_type = bo_vram_flags_to_vram_placement(xe, flags, vram_flag, type);
2301
2302	bo->placement = (struct ttm_placement) {
2303		.num_placement = 1,
2304		.placement = place,
2305	};
2306
2307	return 0;
2308}
2309
2310static struct xe_bo *
2311__xe_bo_create_locked(struct xe_device *xe,
2312		      struct xe_tile *tile, struct xe_vm *vm,
2313		      size_t size, u64 start, u64 end,
2314		      u16 cpu_caching, enum ttm_bo_type type, u32 flags,
2315		      u64 alignment, struct drm_exec *exec)
2316{
2317	struct xe_bo *bo = NULL;
2318	int err;
2319
2320	if (vm)
2321		xe_vm_assert_held(vm);
2322
2323	if (start || end != ~0ULL) {
2324		bo = xe_bo_alloc();
2325		if (IS_ERR(bo))
2326			return bo;
2327
2328		flags |= XE_BO_FLAG_FIXED_PLACEMENT;
2329		err = __xe_bo_fixed_placement(xe, bo, type, flags, start, end, size);
2330		if (err) {
2331			xe_bo_free(bo);
2332			return ERR_PTR(err);
2333		}
2334	}
2335
2336	bo = xe_bo_init_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,
2337			       vm && !xe_vm_in_fault_mode(vm) &&
2338			       flags & XE_BO_FLAG_USER ?
2339			       &vm->lru_bulk_move : NULL, size,
2340			       cpu_caching, type, flags, exec);
2341	if (IS_ERR(bo))
2342		return bo;
2343
2344	bo->min_align = alignment;
2345
2346	/*
2347	 * Note that instead of taking a reference no the drm_gpuvm_resv_bo(),
2348	 * to ensure the shared resv doesn't disappear under the bo, the bo
2349	 * will keep a reference to the vm, and avoid circular references
2350	 * by having all the vm's bo refereferences released at vm close
2351	 * time.
2352	 */
2353	if (vm && xe_bo_is_user(bo))
2354		xe_vm_get(vm);
2355	bo->vm = vm;
2356
2357	if (bo->flags & XE_BO_FLAG_GGTT) {
2358		struct xe_tile *t;
2359		u8 id;
2360
2361		if (!(bo->flags & XE_BO_FLAG_GGTT_ALL)) {
2362			if (!tile && flags & XE_BO_FLAG_STOLEN)
2363				tile = xe_device_get_root_tile(xe);
2364
2365			xe_assert(xe, tile);
2366		}
2367
2368		for_each_tile(t, xe, id) {
2369			if (t != tile && !(bo->flags & XE_BO_FLAG_GGTTx(t)))
2370				continue;
2371
2372			if (flags & XE_BO_FLAG_FIXED_PLACEMENT) {
2373				err = xe_ggtt_insert_bo_at(t->mem.ggtt, bo,
2374							   start + xe_bo_size(bo), U64_MAX,
2375							   exec);
2376			} else {
2377				err = xe_ggtt_insert_bo(t->mem.ggtt, bo, exec);
2378			}
2379			if (err)
2380				goto err_unlock_put_bo;
2381		}
2382	}
2383
2384	trace_xe_bo_create(bo);
2385	return bo;
2386
2387err_unlock_put_bo:
2388	__xe_bo_unset_bulk_move(bo);
2389	xe_bo_unlock_vm_held(bo);
2390	xe_bo_put(bo);
2391	return ERR_PTR(err);
2392}
2393
2394/**
2395 * xe_bo_create_locked() - Create a BO
2396 * @xe: The xe device.
2397 * @tile: The tile to select for migration of this bo, and the tile used for
2398 * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos.
2399 * @vm: The local vm or NULL for external objects.
2400 * @size: The storage size to use for the bo.
2401 * @type: The TTM buffer object type.
2402 * @flags: XE_BO_FLAG_ flags.
2403 * @exec: The drm_exec transaction to use for exhaustive eviction.
2404 *
2405 * Create a locked xe BO with no range- nor alignment restrictions.
2406 *
2407 * Return: The buffer object on success. Negative error pointer on failure.
2408 */
2409struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,
2410				  struct xe_vm *vm, size_t size,
2411				  enum ttm_bo_type type, u32 flags,
2412				  struct drm_exec *exec)
2413{
2414	return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type,
2415				     flags, 0, exec);
2416}
2417
2418static struct xe_bo *xe_bo_create_novm(struct xe_device *xe, struct xe_tile *tile,
2419				       size_t size, u16 cpu_caching,
2420				       enum ttm_bo_type type, u32 flags,
2421				       u64 alignment, bool intr)
2422{
2423	struct xe_validation_ctx ctx;
2424	struct drm_exec exec;
2425	struct xe_bo *bo;
2426	int ret = 0;
2427
2428	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = intr},
2429			    ret) {
2430		bo = __xe_bo_create_locked(xe, tile, NULL, size, 0, ~0ULL,
2431					   cpu_caching, type, flags, alignment, &exec);
2432		drm_exec_retry_on_contention(&exec);
2433		if (IS_ERR(bo)) {
2434			ret = PTR_ERR(bo);
2435			xe_validation_retry_on_oom(&ctx, &ret);
2436		} else {
2437			xe_bo_unlock(bo);
2438		}
2439	}
2440
2441	return ret ? ERR_PTR(ret) : bo;
2442}
2443
2444/**
2445 * xe_bo_create_user() - Create a user BO
2446 * @xe: The xe device.
2447 * @vm: The local vm or NULL for external objects.
2448 * @size: The storage size to use for the bo.
2449 * @cpu_caching: The caching mode to be used for system backing store.
2450 * @flags: XE_BO_FLAG_ flags.
2451 * @exec: The drm_exec transaction to use for exhaustive eviction, or NULL
2452 * if such a transaction should be initiated by the call.
2453 *
2454 * Create a bo on behalf of user-space.
2455 *
2456 * Return: The buffer object on success. Negative error pointer on failure.
2457 */
2458struct xe_bo *xe_bo_create_user(struct xe_device *xe,
2459				struct xe_vm *vm, size_t size,
2460				u16 cpu_caching,
2461				u32 flags, struct drm_exec *exec)
2462{
2463	struct xe_bo *bo;
2464
2465	flags |= XE_BO_FLAG_USER;
2466
2467	if (vm || exec) {
2468		xe_assert(xe, exec);
2469		bo = __xe_bo_create_locked(xe, NULL, vm, size, 0, ~0ULL,
2470					   cpu_caching, ttm_bo_type_device,
2471					   flags, 0, exec);
2472		if (!IS_ERR(bo))
2473			xe_bo_unlock_vm_held(bo);
2474	} else {
2475		bo = xe_bo_create_novm(xe, NULL, size, cpu_caching,
2476				       ttm_bo_type_device, flags, 0, true);
2477	}
2478
2479	return bo;
2480}
2481
2482/**
2483 * xe_bo_create_pin_range_novm() - Create and pin a BO with range options.
2484 * @xe: The xe device.
2485 * @tile: The tile to select for migration of this bo, and the tile used for
2486 * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos.
2487 * @size: The storage size to use for the bo.
2488 * @start: Start of fixed VRAM range or 0.
2489 * @end: End of fixed VRAM range or ~0ULL.
2490 * @type: The TTM buffer object type.
2491 * @flags: XE_BO_FLAG_ flags.
2492 *
2493 * Create an Xe BO with range- and options. If @start and @end indicate
2494 * a fixed VRAM range, this must be a ttm_bo_type_kernel bo with VRAM placement
2495 * only.
2496 *
2497 * Return: The buffer object on success. Negative error pointer on failure.
2498 */
2499struct xe_bo *xe_bo_create_pin_range_novm(struct xe_device *xe, struct xe_tile *tile,
2500					  size_t size, u64 start, u64 end,
2501					  enum ttm_bo_type type, u32 flags)
2502{
2503	struct xe_validation_ctx ctx;
2504	struct drm_exec exec;
2505	struct xe_bo *bo;
2506	int err = 0;
2507
2508	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) {
2509		bo = __xe_bo_create_locked(xe, tile, NULL, size, start, end,
2510					   0, type, flags, 0, &exec);
2511		if (IS_ERR(bo)) {
2512			drm_exec_retry_on_contention(&exec);
2513			err = PTR_ERR(bo);
2514			xe_validation_retry_on_oom(&ctx, &err);
2515			break;
2516		}
2517
2518		err = xe_bo_pin(bo, &exec);
2519		xe_bo_unlock(bo);
2520		if (err) {
2521			xe_bo_put(bo);
2522			drm_exec_retry_on_contention(&exec);
2523			xe_validation_retry_on_oom(&ctx, &err);
2524			break;
2525		}
2526	}
2527
2528	return err ? ERR_PTR(err) : bo;
2529}
2530
2531static struct xe_bo *xe_bo_create_pin_map_at_aligned(struct xe_device *xe,
2532						     struct xe_tile *tile,
2533						     struct xe_vm *vm,
2534						     size_t size, u64 offset,
2535						     enum ttm_bo_type type, u32 flags,
2536						     u64 alignment, struct drm_exec *exec)
2537{
2538	struct xe_bo *bo;
2539	int err;
2540	u64 start = offset == ~0ull ? 0 : offset;
2541	u64 end = offset == ~0ull ? ~0ull : start + size;
2542
2543	if (flags & XE_BO_FLAG_STOLEN &&
2544	    xe_ttm_stolen_cpu_access_needs_ggtt(xe))
2545		flags |= XE_BO_FLAG_GGTT;
2546
2547	bo = __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type,
2548				   flags | XE_BO_FLAG_NEEDS_CPU_ACCESS | XE_BO_FLAG_PINNED,
2549				   alignment, exec);
2550	if (IS_ERR(bo))
2551		return bo;
2552
2553	err = xe_bo_pin(bo, exec);
2554	if (err)
2555		goto err_put;
2556
2557	err = xe_bo_vmap(bo);
2558	if (err)
2559		goto err_unpin;
2560
2561	xe_bo_unlock_vm_held(bo);
2562
2563	return bo;
2564
2565err_unpin:
2566	xe_bo_unpin(bo);
2567err_put:
2568	xe_bo_unlock_vm_held(bo);
2569	xe_bo_put(bo);
2570	return ERR_PTR(err);
2571}
2572
2573/**
2574 * xe_bo_create_pin_map_at_novm() - Create pinned and mapped bo at optional VRAM offset
2575 * @xe: The xe device.
2576 * @tile: The tile to select for migration of this bo, and the tile used for
2577 * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos.
2578 * @size: The storage size to use for the bo.
2579 * @offset: Optional VRAM offset or %~0ull for don't care.
2580 * @type: The TTM buffer object type.
2581 * @flags: XE_BO_FLAG_ flags.
2582 * @alignment: GGTT alignment.
2583 * @intr: Whether to execute any waits for backing store interruptible.
2584 *
2585 * Create a pinned and optionally mapped bo with VRAM offset and GGTT alignment
2586 * options. The bo will be external and not associated with a VM.
2587 *
2588 * Return: The buffer object on success. Negative error pointer on failure.
2589 * In particular, the function may return ERR_PTR(%-EINTR) if @intr was set
2590 * to true on entry.
2591 */
2592struct xe_bo *
2593xe_bo_create_pin_map_at_novm(struct xe_device *xe, struct xe_tile *tile,
2594			     size_t size, u64 offset, enum ttm_bo_type type, u32 flags,
2595			     u64 alignment, bool intr)
2596{
2597	struct xe_validation_ctx ctx;
2598	struct drm_exec exec;
2599	struct xe_bo *bo;
2600	int ret = 0;
2601
2602	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = intr},
2603			    ret) {
2604		bo = xe_bo_create_pin_map_at_aligned(xe, tile, NULL, size, offset,
2605						     type, flags, alignment, &exec);
2606		if (IS_ERR(bo)) {
2607			drm_exec_retry_on_contention(&exec);
2608			ret = PTR_ERR(bo);
2609			xe_validation_retry_on_oom(&ctx, &ret);
2610		}
2611	}
2612
2613	return ret ? ERR_PTR(ret) : bo;
2614}
2615
2616/**
2617 * xe_bo_create_pin_map() - Create pinned and mapped bo
2618 * @xe: The xe device.
2619 * @tile: The tile to select for migration of this bo, and the tile used for
2620 * @vm: The vm to associate the buffer object with. The vm's resv must be locked
2621 * with the transaction represented by @exec.
2622 * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos.
2623 * @size: The storage size to use for the bo.
2624 * @type: The TTM buffer object type.
2625 * @flags: XE_BO_FLAG_ flags.
2626 * @exec: The drm_exec transaction to use for exhaustive eviction, and
2627 * previously used for locking @vm's resv.
2628 *
2629 * Create a pinned and mapped bo. The bo will be external and not associated
2630 * with a VM.
2631 *
2632 * Return: The buffer object on success. Negative error pointer on failure.
2633 * In particular, the function may return ERR_PTR(%-EINTR) if @exec was
2634 * configured for interruptible locking.
2635 */
2636struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
2637				   struct xe_vm *vm, size_t size,
2638				   enum ttm_bo_type type, u32 flags,
2639				   struct drm_exec *exec)
2640{
2641	return xe_bo_create_pin_map_at_aligned(xe, tile, vm, size, ~0ull, type, flags,
2642					       0, exec);
2643}
2644
2645/**
2646 * xe_bo_create_pin_map_novm() - Create pinned and mapped bo
2647 * @xe: The xe device.
2648 * @tile: The tile to select for migration of this bo, and the tile used for
2649 * GGTT binding if any. Only to be non-NULL for ttm_bo_type_kernel bos.
2650 * @size: The storage size to use for the bo.
2651 * @type: The TTM buffer object type.
2652 * @flags: XE_BO_FLAG_ flags.
2653 * @intr: Whether to execute any waits for backing store interruptible.
2654 *
2655 * Create a pinned and mapped bo. The bo will be external and not associated
2656 * with a VM.
2657 *
2658 * Return: The buffer object on success. Negative error pointer on failure.
2659 * In particular, the function may return ERR_PTR(%-EINTR) if @intr was set
2660 * to true on entry.
2661 */
2662struct xe_bo *xe_bo_create_pin_map_novm(struct xe_device *xe, struct xe_tile *tile,
2663					size_t size, enum ttm_bo_type type, u32 flags,
2664					bool intr)
2665{
2666	return xe_bo_create_pin_map_at_novm(xe, tile, size, ~0ull, type, flags, 0, intr);
2667}
2668
2669static void __xe_bo_unpin_map_no_vm(void *arg)
2670{
2671	xe_bo_unpin_map_no_vm(arg);
2672}
2673
2674struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile,
2675					   size_t size, u32 flags)
2676{
2677	struct xe_bo *bo;
2678	int ret;
2679
2680	KUNIT_STATIC_STUB_REDIRECT(xe_managed_bo_create_pin_map, xe, tile, size, flags);
2681	bo = xe_bo_create_pin_map_novm(xe, tile, size, ttm_bo_type_kernel, flags, true);
2682	if (IS_ERR(bo))
2683		return bo;
2684
2685	ret = devm_add_action_or_reset(xe->drm.dev, __xe_bo_unpin_map_no_vm, bo);
2686	if (ret)
2687		return ERR_PTR(ret);
2688
2689	return bo;
2690}
2691
2692void xe_managed_bo_unpin_map_no_vm(struct xe_bo *bo)
2693{
2694	devm_release_action(xe_bo_device(bo)->drm.dev, __xe_bo_unpin_map_no_vm, bo);
2695}
2696
2697struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile,
2698					     const void *data, size_t size, u32 flags)
2699{
2700	struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags);
2701
2702	if (IS_ERR(bo))
2703		return bo;
2704
2705	xe_map_memcpy_to(xe, &bo->vmap, 0, data, size);
2706
2707	return bo;
2708}
2709
2710/**
2711 * xe_managed_bo_reinit_in_vram
2712 * @xe: xe device
2713 * @tile: Tile where the new buffer will be created
2714 * @src: Managed buffer object allocated in system memory
2715 *
2716 * Replace a managed src buffer object allocated in system memory with a new
2717 * one allocated in vram, copying the data between them.
2718 * Buffer object in VRAM is not going to have the same GGTT address, the caller
2719 * is responsible for making sure that any old references to it are updated.
2720 *
2721 * Returns 0 for success, negative error code otherwise.
2722 */
2723int xe_managed_bo_reinit_in_vram(struct xe_device *xe, struct xe_tile *tile, struct xe_bo **src)
2724{
2725	struct xe_bo *bo;
2726	u32 dst_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT;
2727
2728	dst_flags |= (*src)->flags & (XE_BO_FLAG_GGTT_INVALIDATE |
2729				      XE_BO_FLAG_PINNED_NORESTORE);
2730
2731	xe_assert(xe, IS_DGFX(xe));
2732	xe_assert(xe, !(*src)->vmap.is_iomem);
2733
2734	bo = xe_managed_bo_create_from_data(xe, tile, (*src)->vmap.vaddr,
2735					    xe_bo_size(*src), dst_flags);
2736	if (IS_ERR(bo))
2737		return PTR_ERR(bo);
2738
2739	devm_release_action(xe->drm.dev, __xe_bo_unpin_map_no_vm, *src);
2740	*src = bo;
2741
2742	return 0;
2743}
2744
2745/*
2746 * XXX: This is in the VM bind data path, likely should calculate this once and
2747 * store, with a recalculation if the BO is moved.
2748 */
2749uint64_t vram_region_gpu_offset(struct ttm_resource *res)
2750{
2751	struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
2752
2753	switch (res->mem_type) {
2754	case XE_PL_STOLEN:
2755		return xe_ttm_stolen_gpu_offset(xe);
2756	case XE_PL_TT:
2757	case XE_PL_SYSTEM:
2758		return 0;
2759	default:
2760		return res_to_mem_region(res)->dpa_base;
2761	}
2762	return 0;
2763}
2764
2765/**
2766 * xe_bo_pin_external - pin an external BO
2767 * @bo: buffer object to be pinned
2768 * @in_place: Pin in current placement, don't attempt to migrate.
2769 * @exec: The drm_exec transaction to use for exhaustive eviction.
2770 *
2771 * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD)
2772 * BO. Unique call compared to xe_bo_pin as this function has it own set of
2773 * asserts and code to ensure evict / restore on suspend / resume.
2774 *
2775 * Returns 0 for success, negative error code otherwise.
2776 */
2777int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec)
2778{
2779	struct xe_device *xe = xe_bo_device(bo);
2780	int err;
2781
2782	xe_assert(xe, !bo->vm);
2783	xe_assert(xe, xe_bo_is_user(bo));
2784
2785	if (!xe_bo_is_pinned(bo)) {
2786		if (!in_place) {
2787			err = xe_bo_validate(bo, NULL, false, exec);
2788			if (err)
2789				return err;
2790		}
2791
2792		spin_lock(&xe->pinned.lock);
2793		list_add_tail(&bo->pinned_link, &xe->pinned.late.external);
2794		spin_unlock(&xe->pinned.lock);
2795	}
2796
2797	ttm_bo_pin(&bo->ttm);
2798	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
2799		xe_ttm_tt_account_subtract(xe, bo->ttm.ttm);
2800
2801	/*
2802	 * FIXME: If we always use the reserve / unreserve functions for locking
2803	 * we do not need this.
2804	 */
2805	ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
2806
2807	return 0;
2808}
2809
2810/**
2811 * xe_bo_pin() - Pin a kernel bo after potentially migrating it
2812 * @bo: The kernel bo to pin.
2813 * @exec: The drm_exec transaction to use for exhaustive eviction.
2814 *
2815 * Attempts to migrate a bo to @bo->placement. If that succeeds,
2816 * pins the bo.
2817 *
2818 * Return: %0 on success, negative error code on migration failure.
2819 */
2820int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec)
2821{
2822	struct ttm_place *place = &bo->placements[0];
2823	struct xe_device *xe = xe_bo_device(bo);
2824	int err;
2825
2826	/* We currently don't expect user BO to be pinned */
2827	xe_assert(xe, !xe_bo_is_user(bo));
2828
2829	/* Pinned object must be in GGTT or have pinned flag */
2830	xe_assert(xe, bo->flags & (XE_BO_FLAG_PINNED |
2831				   XE_BO_FLAG_GGTT));
2832
2833	/*
2834	 * No reason we can't support pinning imported dma-bufs we just don't
2835	 * expect to pin an imported dma-buf.
2836	 */
2837	xe_assert(xe, !bo->ttm.base.import_attach);
2838
2839	/* We only expect at most 1 pin */
2840	xe_assert(xe, !xe_bo_is_pinned(bo));
2841
2842	err = xe_bo_validate(bo, NULL, false, exec);
2843	if (err)
2844		return err;
2845
2846	if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {
2847		spin_lock(&xe->pinned.lock);
2848		if (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)
2849			list_add_tail(&bo->pinned_link, &xe->pinned.late.kernel_bo_present);
2850		else
2851			list_add_tail(&bo->pinned_link, &xe->pinned.early.kernel_bo_present);
2852		spin_unlock(&xe->pinned.lock);
2853	}
2854
2855	ttm_bo_pin(&bo->ttm);
2856	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
2857		xe_ttm_tt_account_subtract(xe, bo->ttm.ttm);
2858
2859	/*
2860	 * FIXME: If we always use the reserve / unreserve functions for locking
2861	 * we do not need this.
2862	 */
2863	ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
2864
2865	return 0;
2866}
2867
2868/**
2869 * xe_bo_unpin_external - unpin an external BO
2870 * @bo: buffer object to be unpinned
2871 *
2872 * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD)
2873 * BO. Unique call compared to xe_bo_unpin as this function has it own set of
2874 * asserts and code to ensure evict / restore on suspend / resume.
2875 *
2876 * Returns 0 for success, negative error code otherwise.
2877 */
2878void xe_bo_unpin_external(struct xe_bo *bo)
2879{
2880	struct xe_device *xe = xe_bo_device(bo);
2881
2882	xe_assert(xe, !bo->vm);
2883	xe_assert(xe, xe_bo_is_pinned(bo));
2884	xe_assert(xe, xe_bo_is_user(bo));
2885
2886	spin_lock(&xe->pinned.lock);
2887	if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link))
2888		list_del_init(&bo->pinned_link);
2889	spin_unlock(&xe->pinned.lock);
2890
2891	ttm_bo_unpin(&bo->ttm);
2892	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
2893		xe_ttm_tt_account_add(xe, bo->ttm.ttm);
2894
2895	/*
2896	 * FIXME: If we always use the reserve / unreserve functions for locking
2897	 * we do not need this.
2898	 */
2899	ttm_bo_move_to_lru_tail_unlocked(&bo->ttm);
2900}
2901
2902void xe_bo_unpin(struct xe_bo *bo)
2903{
2904	struct ttm_place *place = &bo->placements[0];
2905	struct xe_device *xe = xe_bo_device(bo);
2906
2907	xe_assert(xe, !bo->ttm.base.import_attach);
2908	xe_assert(xe, xe_bo_is_pinned(bo));
2909
2910	if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {
2911		spin_lock(&xe->pinned.lock);
2912		xe_assert(xe, !list_empty(&bo->pinned_link));
2913		list_del_init(&bo->pinned_link);
2914		spin_unlock(&xe->pinned.lock);
2915
2916		if (bo->backup_obj) {
2917			if (xe_bo_is_pinned(bo->backup_obj))
2918				ttm_bo_unpin(&bo->backup_obj->ttm);
2919			xe_bo_put(bo->backup_obj);
2920			bo->backup_obj = NULL;
2921		}
2922	}
2923	ttm_bo_unpin(&bo->ttm);
2924	if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm))
2925		xe_ttm_tt_account_add(xe, bo->ttm.ttm);
2926}
2927
2928/**
2929 * xe_bo_validate() - Make sure the bo is in an allowed placement
2930 * @bo: The bo,
2931 * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or
2932 *      NULL. Used together with @allow_res_evict.
2933 * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's
2934 *                   reservation object.
2935 * @exec: The drm_exec transaction to use for exhaustive eviction.
2936 *
2937 * Make sure the bo is in allowed placement, migrating it if necessary. If
2938 * needed, other bos will be evicted. If bos selected for eviction shares
2939 * the @vm's reservation object, they can be evicted iff @allow_res_evict is
2940 * set to true, otherwise they will be bypassed.
2941 *
2942 * Return: 0 on success, negative error code on failure. May return
2943 * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal.
2944 */
2945int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict,
2946		   struct drm_exec *exec)
2947{
2948	struct ttm_operation_ctx ctx = {
2949		.interruptible = true,
2950		.no_wait_gpu = false,
2951		.gfp_retry_mayfail = true,
2952	};
2953	int ret;
2954
2955	if (xe_bo_is_pinned(bo))
2956		return 0;
2957
2958	if (vm) {
2959		lockdep_assert_held(&vm->lock);
2960		xe_vm_assert_held(vm);
2961
2962		ctx.allow_res_evict = allow_res_evict;
2963		ctx.resv = xe_vm_resv(vm);
2964	}
2965
2966	xe_vm_set_validating(vm, allow_res_evict);
2967	trace_xe_bo_validate(bo);
2968	xe_validation_assert_exec(xe_bo_device(bo), exec, &bo->ttm.base);
2969	ret = ttm_bo_validate(&bo->ttm, &bo->placement, &ctx);
2970	xe_vm_clear_validating(vm, allow_res_evict);
2971
2972	return ret;
2973}
2974
2975bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo)
2976{
2977	if (bo->destroy == &xe_ttm_bo_destroy)
2978		return true;
2979
2980	return false;
2981}
2982
2983/*
2984 * Resolve a BO address. There is no assert to check if the proper lock is held
2985 * so it should only be used in cases where it is not fatal to get the wrong
2986 * address, such as printing debug information, but not in cases where memory is
2987 * written based on this result.
2988 */
2989dma_addr_t __xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)
2990{
2991	struct xe_device *xe = xe_bo_device(bo);
2992	struct xe_res_cursor cur;
2993	u64 page;
2994
2995	xe_assert(xe, page_size <= PAGE_SIZE);
2996	page = offset >> PAGE_SHIFT;
2997	offset &= (PAGE_SIZE - 1);
2998
2999	if (!xe_bo_is_vram(bo) && !xe_bo_is_stolen(bo)) {
3000		xe_assert(xe, bo->ttm.ttm);
3001
3002		xe_res_first_sg(xe_bo_sg(bo), page << PAGE_SHIFT,
3003				page_size, &cur);
3004		return xe_res_dma(&cur) + offset;
3005	} else {
3006		struct xe_res_cursor cur;
3007
3008		xe_res_first(bo->ttm.resource, page << PAGE_SHIFT,
3009			     page_size, &cur);
3010		return cur.start + offset + vram_region_gpu_offset(bo->ttm.resource);
3011	}
3012}
3013
3014dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, size_t page_size)
3015{
3016	if (!READ_ONCE(bo->ttm.pin_count))
3017		xe_bo_assert_held(bo);
3018	return __xe_bo_addr(bo, offset, page_size);
3019}
3020
3021int xe_bo_vmap(struct xe_bo *bo)
3022{
3023	struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
3024	void *virtual;
3025	bool is_iomem;
3026	int ret;
3027
3028	xe_bo_assert_held(bo);
3029
3030	if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) ||
3031			!force_contiguous(bo->flags)))
3032		return -EINVAL;
3033
3034	if (!iosys_map_is_null(&bo->vmap))
3035		return 0;
3036
3037	/*
3038	 * We use this more or less deprecated interface for now since
3039	 * ttm_bo_vmap() doesn't offer the optimization of kmapping
3040	 * single page bos, which is done here.
3041	 * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap
3042	 * to use struct iosys_map.
3043	 */
3044	ret = ttm_bo_kmap(&bo->ttm, 0, xe_bo_size(bo) >> PAGE_SHIFT, &bo->kmap);
3045	if (ret)
3046		return ret;
3047
3048	virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
3049	if (is_iomem)
3050		iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual);
3051	else
3052		iosys_map_set_vaddr(&bo->vmap, virtual);
3053
3054	return 0;
3055}
3056
3057static void __xe_bo_vunmap(struct xe_bo *bo)
3058{
3059	if (!iosys_map_is_null(&bo->vmap)) {
3060		iosys_map_clear(&bo->vmap);
3061		ttm_bo_kunmap(&bo->kmap);
3062	}
3063}
3064
3065void xe_bo_vunmap(struct xe_bo *bo)
3066{
3067	xe_bo_assert_held(bo);
3068	__xe_bo_vunmap(bo);
3069}
3070
3071static int gem_create_set_pxp_type(struct xe_device *xe, struct xe_bo *bo, u64 value)
3072{
3073	if (value == DRM_XE_PXP_TYPE_NONE)
3074		return 0;
3075
3076	/* we only support DRM_XE_PXP_TYPE_HWDRM for now */
3077	if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM))
3078		return -EINVAL;
3079
3080	return xe_pxp_key_assign(xe->pxp, bo);
3081}
3082
3083typedef int (*xe_gem_create_set_property_fn)(struct xe_device *xe,
3084					     struct xe_bo *bo,
3085					     u64 value);
3086
3087static const xe_gem_create_set_property_fn gem_create_set_property_funcs[] = {
3088	[DRM_XE_GEM_CREATE_SET_PROPERTY_PXP_TYPE] = gem_create_set_pxp_type,
3089};
3090
3091static int gem_create_user_ext_set_property(struct xe_device *xe,
3092					    struct xe_bo *bo,
3093					    u64 extension)
3094{
3095	u64 __user *address = u64_to_user_ptr(extension);
3096	struct drm_xe_ext_set_property ext;
3097	int err;
3098	u32 idx;
3099
3100	err = copy_from_user(&ext, address, sizeof(ext));
3101	if (XE_IOCTL_DBG(xe, err))
3102		return -EFAULT;
3103
3104	if (XE_IOCTL_DBG(xe, ext.property >=
3105			 ARRAY_SIZE(gem_create_set_property_funcs)) ||
3106	    XE_IOCTL_DBG(xe, ext.pad) ||
3107	    XE_IOCTL_DBG(xe, ext.property != DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY))
3108		return -EINVAL;
3109
3110	idx = array_index_nospec(ext.property, ARRAY_SIZE(gem_create_set_property_funcs));
3111	if (!gem_create_set_property_funcs[idx])
3112		return -EINVAL;
3113
3114	return gem_create_set_property_funcs[idx](xe, bo, ext.value);
3115}
3116
3117typedef int (*xe_gem_create_user_extension_fn)(struct xe_device *xe,
3118					       struct xe_bo *bo,
3119					       u64 extension);
3120
3121static const xe_gem_create_user_extension_fn gem_create_user_extension_funcs[] = {
3122	[DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY] = gem_create_user_ext_set_property,
3123};
3124
3125#define MAX_USER_EXTENSIONS	16
3126static int gem_create_user_extensions(struct xe_device *xe, struct xe_bo *bo,
3127				      u64 extensions, int ext_number)
3128{
3129	u64 __user *address = u64_to_user_ptr(extensions);
3130	struct drm_xe_user_extension ext;
3131	int err;
3132	u32 idx;
3133
3134	if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
3135		return -E2BIG;
3136
3137	err = copy_from_user(&ext, address, sizeof(ext));
3138	if (XE_IOCTL_DBG(xe, err))
3139		return -EFAULT;
3140
3141	if (XE_IOCTL_DBG(xe, ext.pad) ||
3142	    XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(gem_create_user_extension_funcs)))
3143		return -EINVAL;
3144
3145	idx = array_index_nospec(ext.name,
3146				 ARRAY_SIZE(gem_create_user_extension_funcs));
3147	err = gem_create_user_extension_funcs[idx](xe, bo, extensions);
3148	if (XE_IOCTL_DBG(xe, err))
3149		return err;
3150
3151	if (ext.next_extension)
3152		return gem_create_user_extensions(xe, bo, ext.next_extension,
3153						  ++ext_number);
3154
3155	return 0;
3156}
3157
3158int xe_gem_create_ioctl(struct drm_device *dev, void *data,
3159			struct drm_file *file)
3160{
3161	struct xe_device *xe = to_xe_device(dev);
3162	struct xe_file *xef = to_xe_file(file);
3163	struct drm_xe_gem_create *args = data;
3164	struct xe_validation_ctx ctx;
3165	struct drm_exec exec;
3166	struct xe_vm *vm = NULL;
3167	struct xe_bo *bo;
3168	unsigned int bo_flags;
3169	u32 handle;
3170	int err;
3171
3172	if (XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
3173	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
3174		return -EINVAL;
3175
3176	/* at least one valid memory placement must be specified */
3177	if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) ||
3178			 !args->placement))
3179		return -EINVAL;
3180
3181	if (XE_IOCTL_DBG(xe, args->flags &
3182			 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING |
3183			   DRM_XE_GEM_CREATE_FLAG_SCANOUT |
3184			   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))
3185		return -EINVAL;
3186
3187	if (XE_IOCTL_DBG(xe, args->handle))
3188		return -EINVAL;
3189
3190	if (XE_IOCTL_DBG(xe, !args->size))
3191		return -EINVAL;
3192
3193	if (XE_IOCTL_DBG(xe, args->size > SIZE_MAX))
3194		return -EINVAL;
3195
3196	if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK))
3197		return -EINVAL;
3198
3199	bo_flags = 0;
3200	if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING)
3201		bo_flags |= XE_BO_FLAG_DEFER_BACKING;
3202
3203	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
3204		bo_flags |= XE_BO_FLAG_SCANOUT;
3205
3206	bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);
3207
3208	/* CCS formats need physical placement at a 64K alignment in VRAM. */
3209	if ((bo_flags & XE_BO_FLAG_VRAM_MASK) &&
3210	    (bo_flags & XE_BO_FLAG_SCANOUT) &&
3211	    !(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) &&
3212	    IS_ALIGNED(args->size, SZ_64K))
3213		bo_flags |= XE_BO_FLAG_NEEDS_64K;
3214
3215	if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) {
3216		if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_FLAG_VRAM_MASK)))
3217			return -EINVAL;
3218
3219		bo_flags |= XE_BO_FLAG_NEEDS_CPU_ACCESS;
3220	}
3221
3222	if (XE_IOCTL_DBG(xe, !args->cpu_caching ||
3223			 args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC))
3224		return -EINVAL;
3225
3226	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_VRAM_MASK &&
3227			 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC))
3228		return -EINVAL;
3229
3230	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_SCANOUT &&
3231			 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
3232		return -EINVAL;
3233
3234	if (args->vm_id) {
3235		vm = xe_vm_lookup(xef, args->vm_id);
3236		if (XE_IOCTL_DBG(xe, !vm))
3237			return -ENOENT;
3238	}
3239
3240	err = 0;
3241	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true},
3242			    err) {
3243		if (vm) {
3244			err = xe_vm_drm_exec_lock(vm, &exec);
3245			drm_exec_retry_on_contention(&exec);
3246			if (err)
3247				break;
3248		}
3249		bo = xe_bo_create_user(xe, vm, args->size, args->cpu_caching,
3250				       bo_flags, &exec);
3251		drm_exec_retry_on_contention(&exec);
3252		if (IS_ERR(bo)) {
3253			err = PTR_ERR(bo);
3254			xe_validation_retry_on_oom(&ctx, &err);
3255			break;
3256		}
3257	}
3258	if (err)
3259		goto out_vm;
3260
3261	if (args->extensions) {
3262		err = gem_create_user_extensions(xe, bo, args->extensions, 0);
3263		if (err)
3264			goto out_bulk;
3265	}
3266
3267	err = drm_gem_handle_create(file, &bo->ttm.base, &handle);
3268	if (err)
3269		goto out_bulk;
3270
3271	args->handle = handle;
3272	goto out_put;
3273
3274out_bulk:
3275	if (vm && !xe_vm_in_fault_mode(vm)) {
3276		xe_vm_lock(vm, false);
3277		__xe_bo_unset_bulk_move(bo);
3278		xe_vm_unlock(vm);
3279	}
3280out_put:
3281	xe_bo_put(bo);
3282out_vm:
3283	if (vm)
3284		xe_vm_put(vm);
3285
3286	return err;
3287}
3288
3289int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
3290			     struct drm_file *file)
3291{
3292	struct xe_device *xe = to_xe_device(dev);
3293	struct drm_xe_gem_mmap_offset *args = data;
3294	struct drm_gem_object *gem_obj;
3295
3296	if (XE_IOCTL_DBG(xe, args->extensions) ||
3297	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
3298		return -EINVAL;
3299
3300	if (XE_IOCTL_DBG(xe, args->flags &
3301			 ~DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER))
3302		return -EINVAL;
3303
3304	if (args->flags & DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER) {
3305		if (XE_IOCTL_DBG(xe, !IS_DGFX(xe)))
3306			return -EINVAL;
3307
3308		if (XE_IOCTL_DBG(xe, args->handle))
3309			return -EINVAL;
3310
3311		if (XE_IOCTL_DBG(xe, PAGE_SIZE > SZ_4K))
3312			return -EINVAL;
3313
3314		BUILD_BUG_ON(((XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT) +
3315			      SZ_4K) >= DRM_FILE_PAGE_OFFSET_START);
3316		args->offset = XE_PCI_BARRIER_MMAP_OFFSET;
3317		return 0;
3318	}
3319
3320	gem_obj = drm_gem_object_lookup(file, args->handle);
3321	if (XE_IOCTL_DBG(xe, !gem_obj))
3322		return -ENOENT;
3323
3324	/* The mmap offset was set up at BO allocation time. */
3325	args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
3326
3327	xe_bo_put(gem_to_xe_bo(gem_obj));
3328	return 0;
3329}
3330
3331/**
3332 * xe_bo_lock() - Lock the buffer object's dma_resv object
3333 * @bo: The struct xe_bo whose lock is to be taken
3334 * @intr: Whether to perform any wait interruptible
3335 *
3336 * Locks the buffer object's dma_resv object. If the buffer object is
3337 * pointing to a shared dma_resv object, that shared lock is locked.
3338 *
3339 * Return: 0 on success, -EINTR if @intr is true and the wait for a
3340 * contended lock was interrupted. If @intr is set to false, the
3341 * function always returns 0.
3342 */
3343int xe_bo_lock(struct xe_bo *bo, bool intr)
3344{
3345	if (intr)
3346		return dma_resv_lock_interruptible(bo->ttm.base.resv, NULL);
3347
3348	dma_resv_lock(bo->ttm.base.resv, NULL);
3349
3350	return 0;
3351}
3352
3353/**
3354 * xe_bo_unlock() - Unlock the buffer object's dma_resv object
3355 * @bo: The struct xe_bo whose lock is to be released.
3356 *
3357 * Unlock a buffer object lock that was locked by xe_bo_lock().
3358 */
3359void xe_bo_unlock(struct xe_bo *bo)
3360{
3361	dma_resv_unlock(bo->ttm.base.resv);
3362}
3363
3364/**
3365 * xe_bo_can_migrate - Whether a buffer object likely can be migrated
3366 * @bo: The buffer object to migrate
3367 * @mem_type: The TTM memory type intended to migrate to
3368 *
3369 * Check whether the buffer object supports migration to the
3370 * given memory type. Note that pinning may affect the ability to migrate as
3371 * returned by this function.
3372 *
3373 * This function is primarily intended as a helper for checking the
3374 * possibility to migrate buffer objects and can be called without
3375 * the object lock held.
3376 *
3377 * Return: true if migration is possible, false otherwise.
3378 */
3379bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type)
3380{
3381	unsigned int cur_place;
3382
3383	if (bo->ttm.type == ttm_bo_type_kernel)
3384		return true;
3385
3386	if (bo->ttm.type == ttm_bo_type_sg)
3387		return false;
3388
3389	for (cur_place = 0; cur_place < bo->placement.num_placement;
3390	     cur_place++) {
3391		if (bo->placements[cur_place].mem_type == mem_type)
3392			return true;
3393	}
3394
3395	return false;
3396}
3397
3398static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place)
3399{
3400	memset(place, 0, sizeof(*place));
3401	place->mem_type = mem_type;
3402}
3403
3404/**
3405 * xe_bo_migrate - Migrate an object to the desired region id
3406 * @bo: The buffer object to migrate.
3407 * @mem_type: The TTM region type to migrate to.
3408 * @tctx: A pointer to a struct ttm_operation_ctx or NULL if
3409 * a default interruptibe ctx is to be used.
3410 * @exec: The drm_exec transaction to use for exhaustive eviction.
3411 *
3412 * Attempt to migrate the buffer object to the desired memory region. The
3413 * buffer object may not be pinned, and must be locked.
3414 * On successful completion, the object memory type will be updated,
3415 * but an async migration task may not have completed yet, and to
3416 * accomplish that, the object's kernel fences must be signaled with
3417 * the object lock held.
3418 *
3419 * Return: 0 on success. Negative error code on failure. In particular may
3420 * return -EINTR or -ERESTARTSYS if signal pending.
3421 */
3422int xe_bo_migrate(struct xe_bo *bo, u32 mem_type, struct ttm_operation_ctx *tctx,
3423		  struct drm_exec *exec)
3424{
3425	struct xe_device *xe = ttm_to_xe_device(bo->ttm.bdev);
3426	struct ttm_operation_ctx ctx = {
3427		.interruptible = true,
3428		.no_wait_gpu = false,
3429		.gfp_retry_mayfail = true,
3430	};
3431	struct ttm_placement placement;
3432	struct ttm_place requested;
3433
3434	xe_bo_assert_held(bo);
3435	tctx = tctx ? tctx : &ctx;
3436
3437	if (bo->ttm.resource->mem_type == mem_type)
3438		return 0;
3439
3440	if (xe_bo_is_pinned(bo))
3441		return -EBUSY;
3442
3443	if (!xe_bo_can_migrate(bo, mem_type))
3444		return -EINVAL;
3445
3446	xe_place_from_ttm_type(mem_type, &requested);
3447	placement.num_placement = 1;
3448	placement.placement = &requested;
3449
3450	/*
3451	 * Stolen needs to be handled like below VRAM handling if we ever need
3452	 * to support it.
3453	 */
3454	drm_WARN_ON(&xe->drm, mem_type == XE_PL_STOLEN);
3455
3456	if (mem_type_is_vram(mem_type)) {
3457		u32 c = 0;
3458
3459		add_vram(xe, bo, &requested, bo->flags, mem_type, &c);
3460	}
3461
3462	if (!tctx->no_wait_gpu)
3463		xe_validation_assert_exec(xe_bo_device(bo), exec, &bo->ttm.base);
3464	return ttm_bo_validate(&bo->ttm, &placement, tctx);
3465}
3466
3467/**
3468 * xe_bo_evict - Evict an object to evict placement
3469 * @bo: The buffer object to migrate.
3470 * @exec: The drm_exec transaction to use for exhaustive eviction.
3471 *
3472 * On successful completion, the object memory will be moved to evict
3473 * placement. This function blocks until the object has been fully moved.
3474 *
3475 * Return: 0 on success. Negative error code on failure.
3476 */
3477int xe_bo_evict(struct xe_bo *bo, struct drm_exec *exec)
3478{
3479	struct ttm_operation_ctx ctx = {
3480		.interruptible = false,
3481		.no_wait_gpu = false,
3482		.gfp_retry_mayfail = true,
3483	};
3484	struct ttm_placement placement;
3485	int ret;
3486
3487	xe_evict_flags(&bo->ttm, &placement);
3488	ret = ttm_bo_validate(&bo->ttm, &placement, &ctx);
3489	if (ret)
3490		return ret;
3491
3492	dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
3493			      false, MAX_SCHEDULE_TIMEOUT);
3494
3495	return 0;
3496}
3497
3498/**
3499 * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when
3500 * placed in system memory.
3501 * @bo: The xe_bo
3502 *
3503 * Return: true if extra pages need to be allocated, false otherwise.
3504 */
3505bool xe_bo_needs_ccs_pages(struct xe_bo *bo)
3506{
3507	struct xe_device *xe = xe_bo_device(bo);
3508
3509	if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))
3510		return false;
3511
3512	if (!xe_device_has_flat_ccs(xe) || bo->ttm.type != ttm_bo_type_device)
3513		return false;
3514
3515	/* On discrete GPUs, if the GPU can access this buffer from
3516	 * system memory (i.e., it allows XE_PL_TT placement), FlatCCS
3517	 * can't be used since there's no CCS storage associated with
3518	 * non-VRAM addresses.
3519	 */
3520	if (IS_DGFX(xe) && (bo->flags & XE_BO_FLAG_SYSTEM))
3521		return false;
3522
3523	/*
3524	 * Compression implies coh_none, therefore we know for sure that WB
3525	 * memory can't currently use compression, which is likely one of the
3526	 * common cases.
3527	 */
3528	if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)
3529		return false;
3530
3531	return true;
3532}
3533
3534/**
3535 * __xe_bo_release_dummy() - Dummy kref release function
3536 * @kref: The embedded struct kref.
3537 *
3538 * Dummy release function for xe_bo_put_deferred(). Keep off.
3539 */
3540void __xe_bo_release_dummy(struct kref *kref)
3541{
3542}
3543
3544/**
3545 * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().
3546 * @deferred: The lockless list used for the call to xe_bo_put_deferred().
3547 *
3548 * Puts all bos whose put was deferred by xe_bo_put_deferred().
3549 * The @deferred list can be either an onstack local list or a global
3550 * shared list used by a workqueue.
3551 */
3552void xe_bo_put_commit(struct llist_head *deferred)
3553{
3554	struct llist_node *freed;
3555	struct xe_bo *bo, *next;
3556
3557	if (!deferred)
3558		return;
3559
3560	freed = llist_del_all(deferred);
3561	if (!freed)
3562		return;
3563
3564	llist_for_each_entry_safe(bo, next, freed, freed)
3565		drm_gem_object_free(&bo->ttm.base.refcount);
3566}
3567
3568static void xe_bo_dev_work_func(struct work_struct *work)
3569{
3570	struct xe_bo_dev *bo_dev = container_of(work, typeof(*bo_dev), async_free);
3571
3572	xe_bo_put_commit(&bo_dev->async_list);
3573}
3574
3575/**
3576 * xe_bo_dev_init() - Initialize BO dev to manage async BO freeing
3577 * @bo_dev: The BO dev structure
3578 */
3579void xe_bo_dev_init(struct xe_bo_dev *bo_dev)
3580{
3581	INIT_WORK(&bo_dev->async_free, xe_bo_dev_work_func);
3582}
3583
3584/**
3585 * xe_bo_dev_fini() - Finalize BO dev managing async BO freeing
3586 * @bo_dev: The BO dev structure
3587 */
3588void xe_bo_dev_fini(struct xe_bo_dev *bo_dev)
3589{
3590	flush_work(&bo_dev->async_free);
3591}
3592
3593void xe_bo_put(struct xe_bo *bo)
3594{
3595	struct xe_tile *tile;
3596	u8 id;
3597
3598	might_sleep();
3599	if (bo) {
3600#ifdef CONFIG_PROC_FS
3601		if (bo->client)
3602			might_lock(&bo->client->bos_lock);
3603#endif
3604		for_each_tile(tile, xe_bo_device(bo), id)
3605			if (bo->ggtt_node[id] && bo->ggtt_node[id]->ggtt)
3606				xe_ggtt_might_lock(bo->ggtt_node[id]->ggtt);
3607		drm_gem_object_put(&bo->ttm.base);
3608	}
3609}
3610
3611/**
3612 * xe_bo_dumb_create - Create a dumb bo as backing for a fb
3613 * @file_priv: ...
3614 * @dev: ...
3615 * @args: ...
3616 *
3617 * See dumb_create() hook in include/drm/drm_drv.h
3618 *
3619 * Return: ...
3620 */
3621int xe_bo_dumb_create(struct drm_file *file_priv,
3622		      struct drm_device *dev,
3623		      struct drm_mode_create_dumb *args)
3624{
3625	struct xe_device *xe = to_xe_device(dev);
3626	struct xe_bo *bo;
3627	uint32_t handle;
3628	int err;
3629	u32 page_size = max_t(u32, PAGE_SIZE,
3630		xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K);
3631
3632	err = drm_mode_size_dumb(dev, args, SZ_64, page_size);
3633	if (err)
3634		return err;
3635
3636	bo = xe_bo_create_user(xe, NULL, args->size,
3637			       DRM_XE_GEM_CPU_CACHING_WC,
3638			       XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
3639			       XE_BO_FLAG_SCANOUT |
3640			       XE_BO_FLAG_NEEDS_CPU_ACCESS, NULL);
3641	if (IS_ERR(bo))
3642		return PTR_ERR(bo);
3643
3644	err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle);
3645	/* drop reference from allocate - handle holds it now */
3646	drm_gem_object_put(&bo->ttm.base);
3647	if (!err)
3648		args->handle = handle;
3649	return err;
3650}
3651
3652void xe_bo_runtime_pm_release_mmap_offset(struct xe_bo *bo)
3653{
3654	struct ttm_buffer_object *tbo = &bo->ttm;
3655	struct ttm_device *bdev = tbo->bdev;
3656
3657	drm_vma_node_unmap(&tbo->base.vma_node, bdev->dev_mapping);
3658
3659	list_del_init(&bo->vram_userfault_link);
3660}
3661
3662#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
3663#include "tests/xe_bo.c"
3664#endif