FILE
xe_migrate.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/xe/xe_migrate.c
1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6#include "xe_migrate.h" 7 8#include <linux/bitfield.h> 9#include <linux/sizes.h> 10 11#include <drm/drm_managed.h> 12#include <drm/drm_pagemap.h> 13#include <drm/ttm/ttm_tt.h> 14#include <uapi/drm/xe_drm.h> 15 16#include <generated/xe_wa_oob.h> 17 18#include "instructions/xe_gpu_commands.h" 19#include "instructions/xe_mi_commands.h" 20#include "regs/xe_gtt_defs.h" 21#include "tests/xe_test.h" 22#include "xe_assert.h" 23#include "xe_bb.h" 24#include "xe_bo.h" 25#include "xe_exec_queue.h" 26#include "xe_ggtt.h" 27#include "xe_gt.h" 28#include "xe_hw_engine.h" 29#include "xe_lrc.h" 30#include "xe_map.h" 31#include "xe_mocs.h" 32#include "xe_printk.h" 33#include "xe_pt.h" 34#include "xe_res_cursor.h" 35#include "xe_sa.h" 36#include "xe_sched_job.h" 37#include "xe_sync.h" 38#include "xe_trace_bo.h" 39#include "xe_validation.h" 40#include "xe_vm.h" 41#include "xe_vram.h" 42 43/** 44 * struct xe_migrate - migrate context. 45 */ 46struct xe_migrate { 47 /** @q: Default exec queue used for migration */ 48 struct xe_exec_queue *q; 49 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ 50 struct xe_tile *tile; 51 /** @job_mutex: Timeline mutex for @eng. */ 52 struct mutex job_mutex; 53 /** @pt_bo: Page-table buffer object. */ 54 struct xe_bo *pt_bo; 55 /** @batch_base_ofs: VM offset of the migration batch buffer */ 56 u64 batch_base_ofs; 57 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ 58 u64 usm_batch_base_ofs; 59 /** @cleared_mem_ofs: VM offset of @cleared_bo. */ 60 u64 cleared_mem_ofs; 61 /** @large_page_copy_ofs: VM offset of 2M pages used for large copies */ 62 u64 large_page_copy_ofs; 63 /** 64 * @large_page_copy_pdes: BO offset to writeout 2M pages (PDEs) used for 65 * large copies 66 */ 67 u64 large_page_copy_pdes; 68 /** 69 * @fence: dma-fence representing the last migration job batch. 70 * Protected by @job_mutex. 71 */ 72 struct dma_fence *fence; 73 /** 74 * @vm_update_sa: For integrated, used to suballocate page-tables 75 * out of the pt_bo. 76 */ 77 struct drm_suballoc_manager vm_update_sa; 78 /** @min_chunk_size: For dgfx, Minimum chunk size */ 79 u64 min_chunk_size; 80}; 81 82#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ 83#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ 84#define NUM_KERNEL_PDE 15 85#define NUM_PT_SLOTS 32 86#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M 87#define MAX_NUM_PTE 512 88#define IDENTITY_OFFSET 256ULL 89 90/* 91 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest 92 * legal value accepted. Since that instruction field is always stored in 93 * (val-2) format, this translates to 0x400 dwords for the true maximum length 94 * of the instruction. Subtracting the instruction header (1 dword) and 95 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. 96 */ 97#define MAX_PTE_PER_SDI 0x1FEU 98 99static void xe_migrate_fini(void *arg) 100{ 101 struct xe_migrate *m = arg; 102 103 xe_vm_lock(m->q->vm, false); 104 xe_bo_unpin(m->pt_bo); 105 xe_vm_unlock(m->q->vm); 106 107 dma_fence_put(m->fence); 108 xe_bo_put(m->pt_bo); 109 drm_suballoc_manager_fini(&m->vm_update_sa); 110 mutex_destroy(&m->job_mutex); 111 xe_vm_close_and_put(m->q->vm); 112 xe_exec_queue_put(m->q); 113} 114 115static u64 xe_migrate_vm_addr(u64 slot, u32 level) 116{ 117 XE_WARN_ON(slot >= NUM_PT_SLOTS); 118 119 /* First slot is reserved for mapping of PT bo and bb, start from 1 */ 120 return (slot + 1ULL) << xe_pt_shift(level + 1); 121} 122 123static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte) 124{ 125 /* 126 * Remove the DPA to get a correct offset into identity table for the 127 * migrate offset 128 */ 129 u64 identity_offset = IDENTITY_OFFSET; 130 131 if (GRAPHICS_VER(xe) >= 20 && is_comp_pte) 132 identity_offset += DIV_ROUND_UP_ULL(xe_vram_region_actual_physical_size 133 (xe->mem.vram), SZ_1G); 134 135 addr -= xe_vram_region_dpa_base(xe->mem.vram); 136 return addr + (identity_offset << xe_pt_shift(2)); 137} 138 139static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo, 140 u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs) 141{ 142 struct xe_vram_region *vram = xe->mem.vram; 143 resource_size_t dpa_base = xe_vram_region_dpa_base(vram); 144 u64 pos, ofs, flags; 145 u64 entry; 146 /* XXX: Unclear if this should be usable_size? */ 147 u64 vram_limit = xe_vram_region_actual_physical_size(vram) + dpa_base; 148 u32 level = 2; 149 150 ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8; 151 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, 152 true, 0); 153 154 xe_assert(xe, IS_ALIGNED(xe_vram_region_usable_size(vram), SZ_2M)); 155 156 /* 157 * Use 1GB pages when possible, last chunk always use 2M 158 * pages as mixing reserved memory (stolen, WOCPM) with a single 159 * mapping is not allowed on certain platforms. 160 */ 161 for (pos = dpa_base; pos < vram_limit; 162 pos += SZ_1G, ofs += 8) { 163 if (pos + SZ_1G >= vram_limit) { 164 entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs); 165 xe_map_wr(xe, &bo->vmap, ofs, u64, entry); 166 167 flags = vm->pt_ops->pte_encode_addr(xe, 0, 168 pat_index, 169 level - 1, 170 true, 0); 171 172 for (ofs = pt_2m_ofs; pos < vram_limit; 173 pos += SZ_2M, ofs += 8) 174 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 175 break; /* Ensure pos == vram_limit assert correct */ 176 } 177 178 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 179 } 180 181 xe_assert(xe, pos == vram_limit); 182} 183 184static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, 185 struct xe_vm *vm, struct drm_exec *exec) 186{ 187 struct xe_device *xe = tile_to_xe(tile); 188 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 189 u8 id = tile->id; 190 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; 191#define VRAM_IDENTITY_MAP_COUNT 2 192 u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT; 193#undef VRAM_IDENTITY_MAP_COUNT 194 u32 map_ofs, level, i; 195 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; 196 u64 entry, pt29_ofs; 197 198 /* Can't bump NUM_PT_SLOTS too high */ 199 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); 200 /* Must be a multiple of 64K to support all platforms */ 201 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); 202 /* And one slot reserved for the 4KiB page table updates */ 203 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); 204 205 /* Need to be sure everything fits in the first PT, or create more */ 206 xe_tile_assert(tile, m->batch_base_ofs + xe_bo_size(batch) < SZ_2M); 207 208 bo = xe_bo_create_pin_map(vm->xe, tile, vm, 209 num_entries * XE_PAGE_SIZE, 210 ttm_bo_type_kernel, 211 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 212 XE_BO_FLAG_PAGETABLE, exec); 213 if (IS_ERR(bo)) 214 return PTR_ERR(bo); 215 216 /* PT30 & PT31 reserved for 2M identity map */ 217 pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE; 218 entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs); 219 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); 220 221 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; 222 223 /* Map the entire BO in our level 0 pt */ 224 for (i = 0, level = 0; i < num_entries; level++) { 225 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, 226 pat_index, 0); 227 228 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); 229 230 if (vm->flags & XE_VM_FLAG_64K) 231 i += 16; 232 else 233 i += 1; 234 } 235 236 if (!IS_DGFX(xe)) { 237 /* Write out batch too */ 238 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; 239 for (i = 0; i < xe_bo_size(batch); 240 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 241 XE_PAGE_SIZE) { 242 entry = vm->pt_ops->pte_encode_bo(batch, i, 243 pat_index, 0); 244 245 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 246 entry); 247 level++; 248 } 249 if (xe->info.has_usm) { 250 xe_tile_assert(tile, xe_bo_size(batch) == SZ_1M); 251 252 batch = tile->primary_gt->usm.bb_pool->bo; 253 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; 254 xe_tile_assert(tile, xe_bo_size(batch) == SZ_512K); 255 256 for (i = 0; i < xe_bo_size(batch); 257 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 258 XE_PAGE_SIZE) { 259 entry = vm->pt_ops->pte_encode_bo(batch, i, 260 pat_index, 0); 261 262 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 263 entry); 264 level++; 265 } 266 } 267 } else { 268 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 269 270 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); 271 272 if (xe->info.has_usm) { 273 batch = tile->primary_gt->usm.bb_pool->bo; 274 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 275 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); 276 } 277 } 278 279 for (level = 1; level < num_level; level++) { 280 u32 flags = 0; 281 282 if (vm->flags & XE_VM_FLAG_64K && level == 1) 283 flags = XE_PDE_64K; 284 285 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * 286 XE_PAGE_SIZE); 287 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, 288 entry | flags); 289 } 290 291 /* Write PDE's that point to our BO. */ 292 for (i = 0; i < map_ofs / XE_PAGE_SIZE; i++) { 293 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE); 294 295 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + 296 (i + 1) * 8, u64, entry); 297 } 298 299 /* Reserve 2M PDEs */ 300 level = 1; 301 m->large_page_copy_ofs = NUM_PT_SLOTS << xe_pt_shift(level); 302 m->large_page_copy_pdes = map_ofs + XE_PAGE_SIZE * level + 303 NUM_PT_SLOTS * 8; 304 305 /* Set up a 1GiB NULL mapping at 255GiB offset. */ 306 level = 2; 307 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, 308 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) 309 | XE_PTE_NULL); 310 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); 311 312 /* Identity map the entire vram at 256GiB offset */ 313 if (IS_DGFX(xe)) { 314 u64 pt30_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE; 315 resource_size_t actual_phy_size = xe_vram_region_actual_physical_size(xe->mem.vram); 316 317 xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET, 318 pat_index, pt30_ofs); 319 xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); 320 321 /* 322 * Identity map the entire vram for compressed pat_index for xe2+ 323 * if flat ccs is enabled. 324 */ 325 if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { 326 u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION]; 327 u64 vram_offset = IDENTITY_OFFSET + 328 DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G); 329 u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE; 330 331 xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET - 332 IDENTITY_OFFSET / 2) * SZ_1G); 333 xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset, 334 comp_pat_index, pt31_ofs); 335 } 336 } 337 338 /* 339 * Example layout created above, with root level = 3: 340 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's 341 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's 342 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's 343 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map] 344 * 345 * This makes the lowest part of the VM point to the pagetables. 346 * Hence the lowest 2M in the vm should point to itself, with a few writes 347 * and flushes, other parts of the VM can be used either for copying and 348 * clearing. 349 * 350 * For performance, the kernel reserves PDE's, so about 20 are left 351 * for async VM updates. 352 * 353 * To make it easier to work, each scratch PT is put in slot (1 + PT #) 354 * everywhere, this allows lockless updates to scratch pages by using 355 * the different addresses in VM. 356 */ 357#define NUM_VMUSA_UNIT_PER_PAGE 32 358#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) 359#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) 360 drm_suballoc_manager_init(&m->vm_update_sa, 361 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * 362 NUM_VMUSA_UNIT_PER_PAGE, 0); 363 364 m->pt_bo = bo; 365 return 0; 366} 367 368/* 369 * Including the reserved copy engine is required to avoid deadlocks due to 370 * migrate jobs servicing the faults gets stuck behind the job that faulted. 371 */ 372static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) 373{ 374 u32 logical_mask = 0; 375 struct xe_hw_engine *hwe; 376 enum xe_hw_engine_id id; 377 378 for_each_hw_engine(hwe, gt, id) { 379 if (hwe->class != XE_ENGINE_CLASS_COPY) 380 continue; 381 382 if (xe_gt_is_usm_hwe(gt, hwe)) 383 logical_mask |= BIT(hwe->logical_instance); 384 } 385 386 return logical_mask; 387} 388 389static bool xe_migrate_needs_ccs_emit(struct xe_device *xe) 390{ 391 return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)); 392} 393 394/** 395 * xe_migrate_alloc - Allocate a migrate struct for a given &xe_tile 396 * @tile: &xe_tile 397 * 398 * Allocates a &xe_migrate for a given tile. 399 * 400 * Return: &xe_migrate on success, or NULL when out of memory. 401 */ 402struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile) 403{ 404 struct xe_migrate *m = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*m), GFP_KERNEL); 405 406 if (m) 407 m->tile = tile; 408 return m; 409} 410 411static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, struct xe_vm *vm) 412{ 413 struct xe_device *xe = tile_to_xe(tile); 414 struct xe_validation_ctx ctx; 415 struct drm_exec exec; 416 int err = 0; 417 418 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { 419 err = xe_vm_drm_exec_lock(vm, &exec); 420 drm_exec_retry_on_contention(&exec); 421 err = xe_migrate_prepare_vm(tile, m, vm, &exec); 422 drm_exec_retry_on_contention(&exec); 423 xe_validation_retry_on_oom(&ctx, &err); 424 } 425 426 return err; 427} 428 429/** 430 * xe_migrate_init() - Initialize a migrate context 431 * @m: The migration context 432 * 433 * Return: 0 if successful, negative error code on failure 434 */ 435int xe_migrate_init(struct xe_migrate *m) 436{ 437 struct xe_tile *tile = m->tile; 438 struct xe_gt *primary_gt = tile->primary_gt; 439 struct xe_device *xe = tile_to_xe(tile); 440 struct xe_vm *vm; 441 int err; 442 443 /* Special layout, prepared below.. */ 444 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | 445 XE_VM_FLAG_SET_TILE_ID(tile), NULL); 446 if (IS_ERR(vm)) 447 return PTR_ERR(vm); 448 449 err = xe_migrate_lock_prepare_vm(tile, m, vm); 450 if (err) 451 goto err_out; 452 453 if (xe->info.has_usm) { 454 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, 455 XE_ENGINE_CLASS_COPY, 456 primary_gt->usm.reserved_bcs_instance, 457 false); 458 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); 459 460 if (!hwe || !logical_mask) { 461 err = -EINVAL; 462 goto err_out; 463 } 464 465 /* 466 * XXX: Currently only reserving 1 (likely slow) BCS instance on 467 * PVC, may want to revisit if performance is needed. 468 */ 469 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, 470 EXEC_QUEUE_FLAG_KERNEL | 471 EXEC_QUEUE_FLAG_PERMANENT | 472 EXEC_QUEUE_FLAG_HIGH_PRIORITY | 473 EXEC_QUEUE_FLAG_MIGRATE, 0); 474 } else { 475 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 476 XE_ENGINE_CLASS_COPY, 477 EXEC_QUEUE_FLAG_KERNEL | 478 EXEC_QUEUE_FLAG_PERMANENT | 479 EXEC_QUEUE_FLAG_MIGRATE, 0); 480 } 481 if (IS_ERR(m->q)) { 482 err = PTR_ERR(m->q); 483 goto err_out; 484 } 485 486 mutex_init(&m->job_mutex); 487 fs_reclaim_acquire(GFP_KERNEL); 488 might_lock(&m->job_mutex); 489 fs_reclaim_release(GFP_KERNEL); 490 491 err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m); 492 if (err) 493 return err; 494 495 if (IS_DGFX(xe)) { 496 if (xe_migrate_needs_ccs_emit(xe)) 497 /* min chunk size corresponds to 4K of CCS Metadata */ 498 m->min_chunk_size = SZ_4K * SZ_64K / 499 xe_device_ccs_bytes(xe, SZ_64K); 500 else 501 /* Somewhat arbitrary to avoid a huge amount of blits */ 502 m->min_chunk_size = SZ_64K; 503 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); 504 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", 505 (unsigned long long)m->min_chunk_size); 506 } 507 508 return err; 509 510err_out: 511 xe_vm_close_and_put(vm); 512 return err; 513 514} 515 516static u64 max_mem_transfer_per_pass(struct xe_device *xe) 517{ 518 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 519 return MAX_CCS_LIMITED_TRANSFER; 520 521 return MAX_PREEMPTDISABLE_TRANSFER; 522} 523 524static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) 525{ 526 struct xe_device *xe = tile_to_xe(m->tile); 527 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); 528 529 if (mem_type_is_vram(cur->mem_type)) { 530 /* 531 * VRAM we want to blit in chunks with sizes aligned to 532 * min_chunk_size in order for the offset to CCS metadata to be 533 * page-aligned. If it's the last chunk it may be smaller. 534 * 535 * Another constraint is that we need to limit the blit to 536 * the VRAM block size, unless size is smaller than 537 * min_chunk_size. 538 */ 539 u64 chunk = max_t(u64, cur->size, m->min_chunk_size); 540 541 size = min_t(u64, size, chunk); 542 if (size > m->min_chunk_size) 543 size = round_down(size, m->min_chunk_size); 544 } 545 546 return size; 547} 548 549static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) 550{ 551 /* If the chunk is not fragmented, allow identity map. */ 552 return cur->size >= size; 553} 554 555#define PTE_UPDATE_FLAG_IS_VRAM BIT(0) 556#define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1) 557 558static u32 pte_update_size(struct xe_migrate *m, 559 u32 flags, 560 struct ttm_resource *res, 561 struct xe_res_cursor *cur, 562 u64 *L0, u64 *L0_ofs, u32 *L0_pt, 563 u32 cmd_size, u32 pt_ofs, u32 avail_pts) 564{ 565 u32 cmds = 0; 566 bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags; 567 bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags; 568 569 *L0_pt = pt_ofs; 570 if (is_vram && xe_migrate_allow_identity(*L0, cur)) { 571 /* Offset into identity map. */ 572 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), 573 cur->start + vram_region_gpu_offset(res), 574 is_comp_pte); 575 cmds += cmd_size; 576 } else { 577 /* Clip L0 to available size */ 578 u64 size = min(*L0, (u64)avail_pts * SZ_2M); 579 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; 580 581 *L0 = size; 582 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); 583 584 /* MI_STORE_DATA_IMM */ 585 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); 586 587 /* PDE qwords */ 588 cmds += num_4k_pages * 2; 589 590 /* Each chunk has a single blit command */ 591 cmds += cmd_size; 592 } 593 594 return cmds; 595} 596 597static void emit_pte(struct xe_migrate *m, 598 struct xe_bb *bb, u32 at_pt, 599 bool is_vram, bool is_comp_pte, 600 struct xe_res_cursor *cur, 601 u32 size, struct ttm_resource *res) 602{ 603 struct xe_device *xe = tile_to_xe(m->tile); 604 struct xe_vm *vm = m->q->vm; 605 u16 pat_index; 606 u32 ptes; 607 u64 ofs = (u64)at_pt * XE_PAGE_SIZE; 608 u64 cur_ofs; 609 610 /* Indirect access needs compression enabled uncached PAT index */ 611 if (GRAPHICS_VERx100(xe) >= 2000) 612 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : 613 xe->pat.idx[XE_CACHE_WB]; 614 else 615 pat_index = xe->pat.idx[XE_CACHE_WB]; 616 617 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); 618 619 while (ptes) { 620 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 621 622 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 623 bb->cs[bb->len++] = ofs; 624 bb->cs[bb->len++] = 0; 625 626 cur_ofs = ofs; 627 ofs += chunk * 8; 628 ptes -= chunk; 629 630 while (chunk--) { 631 u64 addr, flags = 0; 632 bool devmem = false; 633 634 addr = xe_res_dma(cur) & PAGE_MASK; 635 if (is_vram) { 636 if (vm->flags & XE_VM_FLAG_64K) { 637 u64 va = cur_ofs * XE_PAGE_SIZE / 8; 638 639 xe_assert(xe, (va & (SZ_64K - 1)) == 640 (addr & (SZ_64K - 1))); 641 642 flags |= XE_PTE_PS64; 643 } 644 645 addr += vram_region_gpu_offset(res); 646 devmem = true; 647 } 648 649 addr = vm->pt_ops->pte_encode_addr(m->tile->xe, 650 addr, pat_index, 651 0, devmem, flags); 652 bb->cs[bb->len++] = lower_32_bits(addr); 653 bb->cs[bb->len++] = upper_32_bits(addr); 654 655 xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); 656 cur_ofs += 8; 657 } 658 } 659} 660 661#define EMIT_COPY_CCS_DW 5 662static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, 663 u64 dst_ofs, bool dst_is_indirect, 664 u64 src_ofs, bool src_is_indirect, 665 u32 size) 666{ 667 struct xe_device *xe = gt_to_xe(gt); 668 u32 *cs = bb->cs + bb->len; 669 u32 num_ccs_blks; 670 u32 num_pages; 671 u32 ccs_copy_size; 672 u32 mocs; 673 674 if (GRAPHICS_VERx100(xe) >= 2000) { 675 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); 676 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); 677 678 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); 679 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); 680 681 } else { 682 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), 683 NUM_CCS_BYTES_PER_BLOCK); 684 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); 685 686 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); 687 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); 688 } 689 690 *cs++ = XY_CTRL_SURF_COPY_BLT | 691 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | 692 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | 693 ccs_copy_size; 694 *cs++ = lower_32_bits(src_ofs); 695 *cs++ = upper_32_bits(src_ofs) | mocs; 696 *cs++ = lower_32_bits(dst_ofs); 697 *cs++ = upper_32_bits(dst_ofs) | mocs; 698 699 bb->len = cs - bb->cs; 700} 701 702#define EMIT_COPY_DW 10 703static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 704 u64 dst_ofs, unsigned int size, 705 unsigned int pitch) 706{ 707 struct xe_device *xe = gt_to_xe(gt); 708 u32 mocs = 0; 709 u32 tile_y = 0; 710 711 xe_gt_assert(gt, !(pitch & 3)); 712 xe_gt_assert(gt, size / pitch <= S16_MAX); 713 xe_gt_assert(gt, pitch / 4 <= S16_MAX); 714 xe_gt_assert(gt, pitch <= U16_MAX); 715 716 if (GRAPHICS_VER(xe) >= 20) 717 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); 718 719 if (GRAPHICS_VERx100(xe) >= 1250) 720 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; 721 722 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); 723 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; 724 bb->cs[bb->len++] = 0; 725 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; 726 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 727 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 728 bb->cs[bb->len++] = 0; 729 bb->cs[bb->len++] = pitch | mocs; 730 bb->cs[bb->len++] = lower_32_bits(src_ofs); 731 bb->cs[bb->len++] = upper_32_bits(src_ofs); 732} 733 734#define PAGE_COPY_MODE_PS SZ_256 /* hw uses 256 bytes as the page-size */ 735static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 736 u64 dst_ofs, unsigned int size, unsigned int pitch) 737{ 738 u32 mode, copy_type, width; 739 740 xe_gt_assert(gt, IS_ALIGNED(size, pitch)); 741 xe_gt_assert(gt, pitch <= U16_MAX); 742 xe_gt_assert(gt, pitch); 743 xe_gt_assert(gt, size); 744 745 if (IS_ALIGNED(size, PAGE_COPY_MODE_PS) && 746 IS_ALIGNED(lower_32_bits(src_ofs), PAGE_COPY_MODE_PS) && 747 IS_ALIGNED(lower_32_bits(dst_ofs), PAGE_COPY_MODE_PS)) { 748 mode = MEM_COPY_PAGE_COPY_MODE; 749 copy_type = 0; /* linear copy */ 750 width = size / PAGE_COPY_MODE_PS; 751 } else if (pitch > 1) { 752 xe_gt_assert(gt, size / pitch <= U16_MAX); 753 mode = 0; /* BYTE_COPY */ 754 copy_type = MEM_COPY_MATRIX_COPY; 755 width = pitch; 756 } else { 757 mode = 0; /* BYTE_COPY */ 758 copy_type = 0; /* linear copy */ 759 width = size; 760 } 761 762 xe_gt_assert(gt, width <= U16_MAX); 763 764 bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type; 765 bb->cs[bb->len++] = width - 1; 766 bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */ 767 bb->cs[bb->len++] = pitch - 1; 768 bb->cs[bb->len++] = pitch - 1; 769 bb->cs[bb->len++] = lower_32_bits(src_ofs); 770 bb->cs[bb->len++] = upper_32_bits(src_ofs); 771 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 772 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 773 bb->cs[bb->len++] = FIELD_PREP(MEM_COPY_SRC_MOCS_INDEX_MASK, gt->mocs.uc_index) | 774 FIELD_PREP(MEM_COPY_DST_MOCS_INDEX_MASK, gt->mocs.uc_index); 775} 776 777static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, 778 u64 src_ofs, u64 dst_ofs, unsigned int size, 779 unsigned int pitch) 780{ 781 struct xe_device *xe = gt_to_xe(gt); 782 783 if (xe->info.has_mem_copy_instr) 784 emit_mem_copy(gt, bb, src_ofs, dst_ofs, size, pitch); 785 else 786 emit_xy_fast_copy(gt, bb, src_ofs, dst_ofs, size, pitch); 787} 788 789static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) 790{ 791 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; 792} 793 794static u32 xe_migrate_ccs_copy(struct xe_migrate *m, 795 struct xe_bb *bb, 796 u64 src_ofs, bool src_is_indirect, 797 u64 dst_ofs, bool dst_is_indirect, u32 dst_size, 798 u64 ccs_ofs, bool copy_ccs) 799{ 800 struct xe_gt *gt = m->tile->primary_gt; 801 u32 flush_flags = 0; 802 803 if (!copy_ccs && dst_is_indirect) { 804 /* 805 * If the src is already in vram, then it should already 806 * have been cleared by us, or has been populated by the 807 * user. Make sure we copy the CCS aux state as-is. 808 * 809 * Otherwise if the bo doesn't have any CCS metadata attached, 810 * we still need to clear it for security reasons. 811 */ 812 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; 813 814 emit_copy_ccs(gt, bb, 815 dst_ofs, true, 816 ccs_src_ofs, src_is_indirect, dst_size); 817 818 flush_flags = MI_FLUSH_DW_CCS; 819 } else if (copy_ccs) { 820 if (!src_is_indirect) 821 src_ofs = ccs_ofs; 822 else if (!dst_is_indirect) 823 dst_ofs = ccs_ofs; 824 825 xe_gt_assert(gt, src_is_indirect || dst_is_indirect); 826 827 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, 828 src_is_indirect, dst_size); 829 if (dst_is_indirect) 830 flush_flags = MI_FLUSH_DW_CCS; 831 } 832 833 return flush_flags; 834} 835 836/** 837 * xe_migrate_copy() - Copy content of TTM resources. 838 * @m: The migration context. 839 * @src_bo: The buffer object @src is currently bound to. 840 * @dst_bo: If copying between resources created for the same bo, set this to 841 * the same value as @src_bo. If copying between buffer objects, set it to 842 * the buffer object @dst is currently bound to. 843 * @src: The source TTM resource. 844 * @dst: The dst TTM resource. 845 * @copy_only_ccs: If true copy only CCS metadata 846 * 847 * Copies the contents of @src to @dst: On flat CCS devices, 848 * the CCS metadata is copied as well if needed, or if not present, 849 * the CCS metadata of @dst is cleared for security reasons. 850 * 851 * Return: Pointer to a dma_fence representing the last copy batch, or 852 * an error pointer on failure. If there is a failure, any copy operation 853 * started by the function call has been synced. 854 */ 855struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 856 struct xe_bo *src_bo, 857 struct xe_bo *dst_bo, 858 struct ttm_resource *src, 859 struct ttm_resource *dst, 860 bool copy_only_ccs) 861{ 862 struct xe_gt *gt = m->tile->primary_gt; 863 struct xe_device *xe = gt_to_xe(gt); 864 struct dma_fence *fence = NULL; 865 u64 size = xe_bo_size(src_bo); 866 struct xe_res_cursor src_it, dst_it, ccs_it; 867 u64 src_L0_ofs, dst_L0_ofs; 868 u32 src_L0_pt, dst_L0_pt; 869 u64 src_L0, dst_L0; 870 int pass = 0; 871 int err; 872 bool src_is_pltt = src->mem_type == XE_PL_TT; 873 bool dst_is_pltt = dst->mem_type == XE_PL_TT; 874 bool src_is_vram = mem_type_is_vram(src->mem_type); 875 bool dst_is_vram = mem_type_is_vram(dst->mem_type); 876 bool type_device = src_bo->ttm.type == ttm_bo_type_device; 877 bool needs_ccs_emit = type_device && xe_migrate_needs_ccs_emit(xe); 878 bool copy_ccs = xe_device_has_flat_ccs(xe) && 879 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); 880 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); 881 bool use_comp_pat = type_device && xe_device_has_flat_ccs(xe) && 882 GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram; 883 884 /* Copying CCS between two different BOs is not supported yet. */ 885 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) 886 return ERR_PTR(-EINVAL); 887 888 if (src_bo != dst_bo && XE_WARN_ON(xe_bo_size(src_bo) != xe_bo_size(dst_bo))) 889 return ERR_PTR(-EINVAL); 890 891 if (!src_is_vram) 892 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 893 else 894 xe_res_first(src, 0, size, &src_it); 895 if (!dst_is_vram) 896 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); 897 else 898 xe_res_first(dst, 0, size, &dst_it); 899 900 if (copy_system_ccs) 901 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 902 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 903 &ccs_it); 904 905 while (size) { 906 u32 batch_size = 1; /* MI_BATCH_BUFFER_END */ 907 struct xe_sched_job *job; 908 struct xe_bb *bb; 909 u32 flush_flags = 0; 910 u32 update_idx; 911 u64 ccs_ofs, ccs_size; 912 u32 ccs_pt; 913 u32 pte_flags; 914 915 bool usm = xe->info.has_usm; 916 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 917 918 src_L0 = xe_migrate_res_sizes(m, &src_it); 919 dst_L0 = xe_migrate_res_sizes(m, &dst_it); 920 921 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", 922 pass++, src_L0, dst_L0); 923 924 src_L0 = min(src_L0, dst_L0); 925 926 pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 927 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0; 928 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0, 929 &src_L0_ofs, &src_L0_pt, 0, 0, 930 avail_pts); 931 if (copy_only_ccs) { 932 dst_L0_ofs = src_L0_ofs; 933 } else { 934 pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 935 batch_size += pte_update_size(m, pte_flags, dst, 936 &dst_it, &src_L0, 937 &dst_L0_ofs, &dst_L0_pt, 938 0, avail_pts, avail_pts); 939 } 940 941 if (copy_system_ccs) { 942 xe_assert(xe, type_device); 943 ccs_size = xe_device_ccs_bytes(xe, src_L0); 944 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, 945 &ccs_ofs, &ccs_pt, 0, 946 2 * avail_pts, 947 avail_pts); 948 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 949 } 950 951 /* Add copy commands size here */ 952 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + 953 ((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0)); 954 955 bb = xe_bb_new(gt, batch_size, usm); 956 if (IS_ERR(bb)) { 957 err = PTR_ERR(bb); 958 goto err_sync; 959 } 960 961 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) 962 xe_res_next(&src_it, src_L0); 963 else 964 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs || use_comp_pat, 965 &src_it, src_L0, src); 966 967 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) 968 xe_res_next(&dst_it, src_L0); 969 else if (!copy_only_ccs) 970 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, 971 &dst_it, src_L0, dst); 972 973 if (copy_system_ccs) 974 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 975 976 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 977 update_idx = bb->len; 978 979 if (!copy_only_ccs) 980 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); 981 982 if (needs_ccs_emit) 983 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, 984 IS_DGFX(xe) ? src_is_vram : src_is_pltt, 985 dst_L0_ofs, 986 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, 987 src_L0, ccs_ofs, copy_ccs); 988 989 job = xe_bb_create_migration_job(m->q, bb, 990 xe_migrate_batch_base(m, usm), 991 update_idx); 992 if (IS_ERR(job)) { 993 err = PTR_ERR(job); 994 goto err; 995 } 996 997 xe_sched_job_add_migrate_flush(job, flush_flags | MI_INVALIDATE_TLB); 998 if (!fence) { 999 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv, 1000 DMA_RESV_USAGE_BOOKKEEP); 1001 if (!err && src_bo->ttm.base.resv != dst_bo->ttm.base.resv) 1002 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv, 1003 DMA_RESV_USAGE_BOOKKEEP); 1004 if (err) 1005 goto err_job; 1006 } 1007 1008 mutex_lock(&m->job_mutex); 1009 xe_sched_job_arm(job); 1010 dma_fence_put(fence); 1011 fence = dma_fence_get(&job->drm.s_fence->finished); 1012 xe_sched_job_push(job); 1013 1014 dma_fence_put(m->fence); 1015 m->fence = dma_fence_get(fence); 1016 1017 mutex_unlock(&m->job_mutex); 1018 1019 xe_bb_free(bb, fence); 1020 size -= src_L0; 1021 continue; 1022 1023err_job: 1024 xe_sched_job_put(job); 1025err: 1026 xe_bb_free(bb, NULL); 1027 1028err_sync: 1029 /* Sync partial copy if any. FIXME: under job_mutex? */ 1030 if (fence) { 1031 dma_fence_wait(fence, false); 1032 dma_fence_put(fence); 1033 } 1034 1035 return ERR_PTR(err); 1036 } 1037 1038 return fence; 1039} 1040 1041/** 1042 * xe_migrate_lrc() - Get the LRC from migrate context. 1043 * @migrate: Migrate context. 1044 * 1045 * Return: Pointer to LRC on success, error on failure 1046 */ 1047struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate) 1048{ 1049 return migrate->q->lrc[0]; 1050} 1051 1052static u64 migrate_vm_ppgtt_addr_tlb_inval(void) 1053{ 1054 /* 1055 * The migrate VM is self-referential so it can modify its own PTEs (see 1056 * pte_update_size() or emit_pte() functions). We reserve NUM_KERNEL_PDE 1057 * entries for kernel operations (copies, clears, CCS migrate), and 1058 * suballocate the rest to user operations (binds/unbinds). With 1059 * NUM_KERNEL_PDE = 15, NUM_KERNEL_PDE - 1 is already used for PTE updates, 1060 * so assign NUM_KERNEL_PDE - 2 for TLB invalidation. 1061 */ 1062 return (NUM_KERNEL_PDE - 2) * XE_PAGE_SIZE; 1063} 1064 1065static int emit_flush_invalidate(u32 *dw, int i, u32 flags) 1066{ 1067 u64 addr = migrate_vm_ppgtt_addr_tlb_inval(); 1068 1069 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | 1070 MI_FLUSH_IMM_DW | flags; 1071 dw[i++] = lower_32_bits(addr); 1072 dw[i++] = upper_32_bits(addr); 1073 dw[i++] = MI_NOOP; 1074 dw[i++] = MI_NOOP; 1075 1076 return i; 1077} 1078 1079/** 1080 * xe_migrate_ccs_rw_copy() - Copy content of TTM resources. 1081 * @tile: Tile whose migration context to be used. 1082 * @q : Execution to be used along with migration context. 1083 * @src_bo: The buffer object @src is currently bound to. 1084 * @read_write : Creates BB commands for CCS read/write. 1085 * 1086 * Creates batch buffer instructions to copy CCS metadata from CCS pool to 1087 * memory and vice versa. 1088 * 1089 * This function should only be called for IGPU. 1090 * 1091 * Return: 0 if successful, negative error code on failure. 1092 */ 1093int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, 1094 struct xe_bo *src_bo, 1095 enum xe_sriov_vf_ccs_rw_ctxs read_write) 1096 1097{ 1098 bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX; 1099 bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX; 1100 struct ttm_resource *src = src_bo->ttm.resource; 1101 struct xe_migrate *m = tile->migrate; 1102 struct xe_gt *gt = tile->primary_gt; 1103 u32 batch_size, batch_size_allocated; 1104 struct xe_device *xe = gt_to_xe(gt); 1105 struct xe_res_cursor src_it, ccs_it; 1106 u64 size = xe_bo_size(src_bo); 1107 struct xe_bb *bb = NULL; 1108 u64 src_L0, src_L0_ofs; 1109 u32 src_L0_pt; 1110 int err; 1111 1112 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 1113 1114 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 1115 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 1116 &ccs_it); 1117 1118 /* Calculate Batch buffer size */ 1119 batch_size = 0; 1120 while (size) { 1121 batch_size += 10; /* Flush + ggtt addr + 2 NOP */ 1122 u64 ccs_ofs, ccs_size; 1123 u32 ccs_pt; 1124 1125 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1126 1127 src_L0 = min_t(u64, max_mem_transfer_per_pass(xe), size); 1128 1129 batch_size += pte_update_size(m, false, src, &src_it, &src_L0, 1130 &src_L0_ofs, &src_L0_pt, 0, 0, 1131 avail_pts); 1132 1133 ccs_size = xe_device_ccs_bytes(xe, src_L0); 1134 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs, 1135 &ccs_pt, 0, avail_pts, avail_pts); 1136 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 1137 1138 /* Add copy commands size here */ 1139 batch_size += EMIT_COPY_CCS_DW; 1140 1141 size -= src_L0; 1142 } 1143 1144 bb = xe_bb_ccs_new(gt, batch_size, read_write); 1145 if (IS_ERR(bb)) { 1146 drm_err(&xe->drm, "BB allocation failed.\n"); 1147 err = PTR_ERR(bb); 1148 goto err_ret; 1149 } 1150 1151 batch_size_allocated = batch_size; 1152 size = xe_bo_size(src_bo); 1153 batch_size = 0; 1154 1155 /* 1156 * Emit PTE and copy commands here. 1157 * The CCS copy command can only support limited size. If the size to be 1158 * copied is more than the limit, divide copy into chunks. So, calculate 1159 * sizes here again before copy command is emitted. 1160 */ 1161 while (size) { 1162 batch_size += 10; /* Flush + ggtt addr + 2 NOP */ 1163 u32 flush_flags = 0; 1164 u64 ccs_ofs, ccs_size; 1165 u32 ccs_pt; 1166 1167 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1168 1169 src_L0 = xe_migrate_res_sizes(m, &src_it); 1170 1171 batch_size += pte_update_size(m, false, src, &src_it, &src_L0, 1172 &src_L0_ofs, &src_L0_pt, 0, 0, 1173 avail_pts); 1174 1175 ccs_size = xe_device_ccs_bytes(xe, src_L0); 1176 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs, 1177 &ccs_pt, 0, avail_pts, avail_pts); 1178 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 1179 batch_size += EMIT_COPY_CCS_DW; 1180 1181 emit_pte(m, bb, src_L0_pt, false, true, &src_it, src_L0, src); 1182 1183 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 1184 1185 bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags); 1186 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_pltt, 1187 src_L0_ofs, dst_is_pltt, 1188 src_L0, ccs_ofs, true); 1189 bb->len = emit_flush_invalidate(bb->cs, bb->len, flush_flags); 1190 1191 size -= src_L0; 1192 } 1193 1194 xe_assert(xe, (batch_size_allocated == bb->len)); 1195 src_bo->bb_ccs[read_write] = bb; 1196 1197 return 0; 1198 1199err_ret: 1200 return err; 1201} 1202 1203/** 1204 * xe_migrate_exec_queue() - Get the execution queue from migrate context. 1205 * @migrate: Migrate context. 1206 * 1207 * Return: Pointer to execution queue on success, error on failure 1208 */ 1209struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate) 1210{ 1211 return migrate->q; 1212} 1213 1214/** 1215 * xe_migrate_vram_copy_chunk() - Copy a chunk of a VRAM buffer object. 1216 * @vram_bo: The VRAM buffer object. 1217 * @vram_offset: The VRAM offset. 1218 * @sysmem_bo: The sysmem buffer object. 1219 * @sysmem_offset: The sysmem offset. 1220 * @size: The size of VRAM chunk to copy. 1221 * @dir: The direction of the copy operation. 1222 * 1223 * Copies a portion of a buffer object between VRAM and system memory. 1224 * On Xe2 platforms that support flat CCS, VRAM data is decompressed when 1225 * copying to system memory. 1226 * 1227 * Return: Pointer to a dma_fence representing the last copy batch, or 1228 * an error pointer on failure. If there is a failure, any copy operation 1229 * started by the function call has been synced. 1230 */ 1231struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_offset, 1232 struct xe_bo *sysmem_bo, u64 sysmem_offset, 1233 u64 size, enum xe_migrate_copy_dir dir) 1234{ 1235 struct xe_device *xe = xe_bo_device(vram_bo); 1236 struct xe_tile *tile = vram_bo->tile; 1237 struct xe_gt *gt = tile->primary_gt; 1238 struct xe_migrate *m = tile->migrate; 1239 struct dma_fence *fence = NULL; 1240 struct ttm_resource *vram = vram_bo->ttm.resource; 1241 struct ttm_resource *sysmem = sysmem_bo->ttm.resource; 1242 struct xe_res_cursor vram_it, sysmem_it; 1243 u64 vram_L0_ofs, sysmem_L0_ofs; 1244 u32 vram_L0_pt, sysmem_L0_pt; 1245 u64 vram_L0, sysmem_L0; 1246 bool to_sysmem = (dir == XE_MIGRATE_COPY_TO_SRAM); 1247 bool use_comp_pat = to_sysmem && 1248 GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe); 1249 int pass = 0; 1250 int err; 1251 1252 xe_assert(xe, IS_ALIGNED(vram_offset | sysmem_offset | size, PAGE_SIZE)); 1253 xe_assert(xe, xe_bo_is_vram(vram_bo)); 1254 xe_assert(xe, !xe_bo_is_vram(sysmem_bo)); 1255 xe_assert(xe, !range_overflows(vram_offset, size, (u64)vram_bo->ttm.base.size)); 1256 xe_assert(xe, !range_overflows(sysmem_offset, size, (u64)sysmem_bo->ttm.base.size)); 1257 1258 xe_res_first(vram, vram_offset, size, &vram_it); 1259 xe_res_first_sg(xe_bo_sg(sysmem_bo), sysmem_offset, size, &sysmem_it); 1260 1261 while (size) { 1262 u32 pte_flags = PTE_UPDATE_FLAG_IS_VRAM; 1263 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ 1264 struct xe_sched_job *job; 1265 struct xe_bb *bb; 1266 u32 update_idx; 1267 bool usm = xe->info.has_usm; 1268 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1269 1270 sysmem_L0 = xe_migrate_res_sizes(m, &sysmem_it); 1271 vram_L0 = min(xe_migrate_res_sizes(m, &vram_it), sysmem_L0); 1272 1273 xe_dbg(xe, "Pass %u, size: %llu\n", pass++, vram_L0); 1274 1275 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0; 1276 batch_size += pte_update_size(m, pte_flags, vram, &vram_it, &vram_L0, 1277 &vram_L0_ofs, &vram_L0_pt, 0, 0, avail_pts); 1278 1279 batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs, 1280 &sysmem_L0_pt, 0, avail_pts, avail_pts); 1281 batch_size += EMIT_COPY_DW; 1282 1283 bb = xe_bb_new(gt, batch_size, usm); 1284 if (IS_ERR(bb)) { 1285 err = PTR_ERR(bb); 1286 return ERR_PTR(err); 1287 } 1288 1289 if (xe_migrate_allow_identity(vram_L0, &vram_it)) 1290 xe_res_next(&vram_it, vram_L0); 1291 else 1292 emit_pte(m, bb, vram_L0_pt, true, use_comp_pat, &vram_it, vram_L0, vram); 1293 1294 emit_pte(m, bb, sysmem_L0_pt, false, false, &sysmem_it, vram_L0, sysmem); 1295 1296 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1297 update_idx = bb->len; 1298 1299 if (to_sysmem) 1300 emit_copy(gt, bb, vram_L0_ofs, sysmem_L0_ofs, vram_L0, XE_PAGE_SIZE); 1301 else 1302 emit_copy(gt, bb, sysmem_L0_ofs, vram_L0_ofs, vram_L0, XE_PAGE_SIZE); 1303 1304 job = xe_bb_create_migration_job(m->q, bb, xe_migrate_batch_base(m, usm), 1305 update_idx); 1306 if (IS_ERR(job)) { 1307 xe_bb_free(bb, NULL); 1308 err = PTR_ERR(job); 1309 return ERR_PTR(err); 1310 } 1311 1312 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); 1313 1314 xe_assert(xe, dma_resv_test_signaled(vram_bo->ttm.base.resv, 1315 DMA_RESV_USAGE_BOOKKEEP)); 1316 xe_assert(xe, dma_resv_test_signaled(sysmem_bo->ttm.base.resv, 1317 DMA_RESV_USAGE_BOOKKEEP)); 1318 1319 scoped_guard(mutex, &m->job_mutex) { 1320 xe_sched_job_arm(job); 1321 dma_fence_put(fence); 1322 fence = dma_fence_get(&job->drm.s_fence->finished); 1323 xe_sched_job_push(job); 1324 1325 dma_fence_put(m->fence); 1326 m->fence = dma_fence_get(fence); 1327 } 1328 1329 xe_bb_free(bb, fence); 1330 size -= vram_L0; 1331 } 1332 1333 return fence; 1334} 1335 1336static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 1337 u32 size, u32 pitch) 1338{ 1339 struct xe_device *xe = gt_to_xe(gt); 1340 u32 *cs = bb->cs + bb->len; 1341 u32 len = PVC_MEM_SET_CMD_LEN_DW; 1342 1343 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); 1344 *cs++ = pitch - 1; 1345 *cs++ = (size / pitch) - 1; 1346 *cs++ = pitch - 1; 1347 *cs++ = lower_32_bits(src_ofs); 1348 *cs++ = upper_32_bits(src_ofs); 1349 if (GRAPHICS_VERx100(xe) >= 2000) 1350 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 1351 else 1352 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 1353 1354 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 1355 1356 bb->len += len; 1357} 1358 1359static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, 1360 u64 src_ofs, u32 size, u32 pitch, bool is_vram) 1361{ 1362 struct xe_device *xe = gt_to_xe(gt); 1363 u32 *cs = bb->cs + bb->len; 1364 u32 len = XY_FAST_COLOR_BLT_DW; 1365 1366 if (GRAPHICS_VERx100(xe) < 1250) 1367 len = 11; 1368 1369 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 1370 (len - 2); 1371 if (GRAPHICS_VERx100(xe) >= 2000) 1372 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | 1373 (pitch - 1); 1374 else 1375 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | 1376 (pitch - 1); 1377 *cs++ = 0; 1378 *cs++ = (size / pitch) << 16 | pitch / 4; 1379 *cs++ = lower_32_bits(src_ofs); 1380 *cs++ = upper_32_bits(src_ofs); 1381 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 1382 *cs++ = 0; 1383 *cs++ = 0; 1384 *cs++ = 0; 1385 *cs++ = 0; 1386 1387 if (len > 11) { 1388 *cs++ = 0; 1389 *cs++ = 0; 1390 *cs++ = 0; 1391 *cs++ = 0; 1392 *cs++ = 0; 1393 } 1394 1395 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 1396 1397 bb->len += len; 1398} 1399 1400static bool has_service_copy_support(struct xe_gt *gt) 1401{ 1402 /* 1403 * What we care about is whether the architecture was designed with 1404 * service copy functionality (specifically the new MEM_SET / MEM_COPY 1405 * instructions) so check the architectural engine list rather than the 1406 * actual list since these instructions are usable on BCS0 even if 1407 * all of the actual service copy engines (BCS1-BCS8) have been fused 1408 * off. 1409 */ 1410 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 1411 XE_HW_ENGINE_BCS1); 1412} 1413 1414static u32 emit_clear_cmd_len(struct xe_gt *gt) 1415{ 1416 if (has_service_copy_support(gt)) 1417 return PVC_MEM_SET_CMD_LEN_DW; 1418 else 1419 return XY_FAST_COLOR_BLT_DW; 1420} 1421 1422static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 1423 u32 size, u32 pitch, bool is_vram) 1424{ 1425 if (has_service_copy_support(gt)) 1426 emit_clear_link_copy(gt, bb, src_ofs, size, pitch); 1427 else 1428 emit_clear_main_copy(gt, bb, src_ofs, size, pitch, 1429 is_vram); 1430} 1431 1432/** 1433 * xe_migrate_clear() - Copy content of TTM resources. 1434 * @m: The migration context. 1435 * @bo: The buffer object @dst is currently bound to. 1436 * @dst: The dst TTM resource to be cleared. 1437 * @clear_flags: flags to specify which data to clear: CCS, BO, or both. 1438 * 1439 * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set. 1440 * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA. 1441 * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata. 1442 * TODO: Eliminate the @bo argument. 1443 * 1444 * Return: Pointer to a dma_fence representing the last clear batch, or 1445 * an error pointer on failure. If there is a failure, any clear operation 1446 * started by the function call has been synced. 1447 */ 1448struct dma_fence *xe_migrate_clear(struct xe_migrate *m, 1449 struct xe_bo *bo, 1450 struct ttm_resource *dst, 1451 u32 clear_flags) 1452{ 1453 bool clear_vram = mem_type_is_vram(dst->mem_type); 1454 bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags; 1455 bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags; 1456 struct xe_gt *gt = m->tile->primary_gt; 1457 struct xe_device *xe = gt_to_xe(gt); 1458 bool clear_only_system_ccs = false; 1459 struct dma_fence *fence = NULL; 1460 u64 size = xe_bo_size(bo); 1461 struct xe_res_cursor src_it; 1462 struct ttm_resource *src = dst; 1463 int err; 1464 1465 if (WARN_ON(!clear_bo_data && !clear_ccs)) 1466 return NULL; 1467 1468 if (!clear_bo_data && clear_ccs && !IS_DGFX(xe)) 1469 clear_only_system_ccs = true; 1470 1471 if (!clear_vram) 1472 xe_res_first_sg(xe_bo_sg(bo), 0, xe_bo_size(bo), &src_it); 1473 else 1474 xe_res_first(src, 0, xe_bo_size(bo), &src_it); 1475 1476 while (size) { 1477 u64 clear_L0_ofs; 1478 u32 clear_L0_pt; 1479 u32 flush_flags = 0; 1480 u64 clear_L0; 1481 struct xe_sched_job *job; 1482 struct xe_bb *bb; 1483 u32 batch_size, update_idx; 1484 u32 pte_flags; 1485 1486 bool usm = xe->info.has_usm; 1487 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1488 1489 clear_L0 = xe_migrate_res_sizes(m, &src_it); 1490 1491 /* Calculate final sizes and batch size.. */ 1492 pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 1493 batch_size = 1 + 1494 pte_update_size(m, pte_flags, src, &src_it, 1495 &clear_L0, &clear_L0_ofs, &clear_L0_pt, 1496 clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0, 1497 avail_pts); 1498 1499 if (xe_migrate_needs_ccs_emit(xe)) 1500 batch_size += EMIT_COPY_CCS_DW; 1501 1502 /* Clear commands */ 1503 1504 if (WARN_ON_ONCE(!clear_L0)) 1505 break; 1506 1507 bb = xe_bb_new(gt, batch_size, usm); 1508 if (IS_ERR(bb)) { 1509 err = PTR_ERR(bb); 1510 goto err_sync; 1511 } 1512 1513 size -= clear_L0; 1514 /* Preemption is enabled again by the ring ops. */ 1515 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) { 1516 xe_res_next(&src_it, clear_L0); 1517 } else { 1518 emit_pte(m, bb, clear_L0_pt, clear_vram, 1519 clear_only_system_ccs, &src_it, clear_L0, dst); 1520 flush_flags |= MI_INVALIDATE_TLB; 1521 } 1522 1523 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1524 update_idx = bb->len; 1525 1526 if (clear_bo_data) 1527 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); 1528 1529 if (xe_migrate_needs_ccs_emit(xe)) { 1530 emit_copy_ccs(gt, bb, clear_L0_ofs, true, 1531 m->cleared_mem_ofs, false, clear_L0); 1532 flush_flags |= MI_FLUSH_DW_CCS; 1533 } 1534 1535 job = xe_bb_create_migration_job(m->q, bb, 1536 xe_migrate_batch_base(m, usm), 1537 update_idx); 1538 if (IS_ERR(job)) { 1539 err = PTR_ERR(job); 1540 goto err; 1541 } 1542 1543 xe_sched_job_add_migrate_flush(job, flush_flags); 1544 if (!fence) { 1545 /* 1546 * There can't be anything userspace related at this 1547 * point, so we just need to respect any potential move 1548 * fences, which are always tracked as 1549 * DMA_RESV_USAGE_KERNEL. 1550 */ 1551 err = xe_sched_job_add_deps(job, bo->ttm.base.resv, 1552 DMA_RESV_USAGE_KERNEL); 1553 if (err) 1554 goto err_job; 1555 } 1556 1557 mutex_lock(&m->job_mutex); 1558 xe_sched_job_arm(job); 1559 dma_fence_put(fence); 1560 fence = dma_fence_get(&job->drm.s_fence->finished); 1561 xe_sched_job_push(job); 1562 1563 dma_fence_put(m->fence); 1564 m->fence = dma_fence_get(fence); 1565 1566 mutex_unlock(&m->job_mutex); 1567 1568 xe_bb_free(bb, fence); 1569 continue; 1570 1571err_job: 1572 xe_sched_job_put(job); 1573err: 1574 xe_bb_free(bb, NULL); 1575err_sync: 1576 /* Sync partial copies if any. FIXME: job_mutex? */ 1577 if (fence) { 1578 dma_fence_wait(fence, false); 1579 dma_fence_put(fence); 1580 } 1581 1582 return ERR_PTR(err); 1583 } 1584 1585 if (clear_ccs) 1586 bo->ccs_cleared = true; 1587 1588 return fence; 1589} 1590 1591static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, 1592 const struct xe_vm_pgtable_update_op *pt_op, 1593 const struct xe_vm_pgtable_update *update, 1594 struct xe_migrate_pt_update *pt_update) 1595{ 1596 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1597 u32 chunk; 1598 u32 ofs = update->ofs, size = update->qwords; 1599 1600 /* 1601 * If we have 512 entries (max), we would populate it ourselves, 1602 * and update the PDE above it to the new pointer. 1603 * The only time this can only happen if we have to update the top 1604 * PDE. This requires a BO that is almost vm->size big. 1605 * 1606 * This shouldn't be possible in practice.. might change when 16K 1607 * pages are used. Hence the assert. 1608 */ 1609 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); 1610 if (!ppgtt_ofs) 1611 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), 1612 xe_bo_addr(update->pt_bo, 0, 1613 XE_PAGE_SIZE), false); 1614 1615 do { 1616 u64 addr = ppgtt_ofs + ofs * 8; 1617 1618 chunk = min(size, MAX_PTE_PER_SDI); 1619 1620 /* Ensure populatefn can do memset64 by aligning bb->cs */ 1621 if (!(bb->len & 1)) 1622 bb->cs[bb->len++] = MI_NOOP; 1623 1624 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1625 bb->cs[bb->len++] = lower_32_bits(addr); 1626 bb->cs[bb->len++] = upper_32_bits(addr); 1627 if (pt_op->bind) 1628 ops->populate(pt_update, tile, NULL, bb->cs + bb->len, 1629 ofs, chunk, update); 1630 else 1631 ops->clear(pt_update, tile, NULL, bb->cs + bb->len, 1632 ofs, chunk, update); 1633 1634 bb->len += chunk * 2; 1635 ofs += chunk; 1636 size -= chunk; 1637 } while (size); 1638} 1639 1640struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) 1641{ 1642 return xe_vm_get(m->q->vm); 1643} 1644 1645#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1646struct migrate_test_params { 1647 struct xe_test_priv base; 1648 bool force_gpu; 1649}; 1650 1651#define to_migrate_test_params(_priv) \ 1652 container_of(_priv, struct migrate_test_params, base) 1653#endif 1654 1655static struct dma_fence * 1656xe_migrate_update_pgtables_cpu(struct xe_migrate *m, 1657 struct xe_migrate_pt_update *pt_update) 1658{ 1659 XE_TEST_DECLARE(struct migrate_test_params *test = 1660 to_migrate_test_params 1661 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) 1662 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1663 struct xe_vm *vm = pt_update->vops->vm; 1664 struct xe_vm_pgtable_update_ops *pt_update_ops = 1665 &pt_update->vops->pt_update_ops[pt_update->tile_id]; 1666 int err; 1667 u32 i, j; 1668 1669 if (XE_TEST_ONLY(test && test->force_gpu)) 1670 return ERR_PTR(-ETIME); 1671 1672 if (ops->pre_commit) { 1673 pt_update->job = NULL; 1674 err = ops->pre_commit(pt_update); 1675 if (err) 1676 return ERR_PTR(err); 1677 } 1678 1679 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1680 const struct xe_vm_pgtable_update_op *pt_op = 1681 &pt_update_ops->ops[i]; 1682 1683 for (j = 0; j < pt_op->num_entries; j++) { 1684 const struct xe_vm_pgtable_update *update = 1685 &pt_op->entries[j]; 1686 1687 if (pt_op->bind) 1688 ops->populate(pt_update, m->tile, 1689 &update->pt_bo->vmap, NULL, 1690 update->ofs, update->qwords, 1691 update); 1692 else 1693 ops->clear(pt_update, m->tile, 1694 &update->pt_bo->vmap, NULL, 1695 update->ofs, update->qwords, update); 1696 } 1697 } 1698 1699 trace_xe_vm_cpu_bind(vm); 1700 xe_device_wmb(vm->xe); 1701 1702 return dma_fence_get_stub(); 1703} 1704 1705static struct dma_fence * 1706__xe_migrate_update_pgtables(struct xe_migrate *m, 1707 struct xe_migrate_pt_update *pt_update, 1708 struct xe_vm_pgtable_update_ops *pt_update_ops) 1709{ 1710 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1711 struct xe_tile *tile = m->tile; 1712 struct xe_gt *gt = tile->primary_gt; 1713 struct xe_device *xe = tile_to_xe(tile); 1714 struct xe_sched_job *job; 1715 struct dma_fence *fence; 1716 struct drm_suballoc *sa_bo = NULL; 1717 struct xe_bb *bb; 1718 u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0; 1719 u32 num_updates = 0, current_update = 0; 1720 u64 addr; 1721 int err = 0; 1722 bool is_migrate = pt_update_ops->q == m->q; 1723 bool usm = is_migrate && xe->info.has_usm; 1724 1725 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1726 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; 1727 struct xe_vm_pgtable_update *updates = pt_op->entries; 1728 1729 num_updates += pt_op->num_entries; 1730 for (j = 0; j < pt_op->num_entries; ++j) { 1731 u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, 1732 MAX_PTE_PER_SDI); 1733 1734 /* align noop + MI_STORE_DATA_IMM cmd prefix */ 1735 batch_size += 4 * num_cmds + updates[j].qwords * 2; 1736 } 1737 } 1738 1739 /* fixed + PTE entries */ 1740 if (IS_DGFX(xe)) 1741 batch_size += 2; 1742 else 1743 batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) + 1744 num_updates * 2; 1745 1746 bb = xe_bb_new(gt, batch_size, usm); 1747 if (IS_ERR(bb)) 1748 return ERR_CAST(bb); 1749 1750 /* For sysmem PTE's, need to map them in our hole.. */ 1751 if (!IS_DGFX(xe)) { 1752 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 1753 u32 ptes, ofs; 1754 1755 ppgtt_ofs = NUM_KERNEL_PDE - 1; 1756 if (!is_migrate) { 1757 u32 num_units = DIV_ROUND_UP(num_updates, 1758 NUM_VMUSA_WRITES_PER_UNIT); 1759 1760 if (num_units > m->vm_update_sa.size) { 1761 err = -ENOBUFS; 1762 goto err_bb; 1763 } 1764 sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units, 1765 GFP_KERNEL, true, 0); 1766 if (IS_ERR(sa_bo)) { 1767 err = PTR_ERR(sa_bo); 1768 goto err_bb; 1769 } 1770 1771 ppgtt_ofs = NUM_KERNEL_PDE + 1772 (drm_suballoc_soffset(sa_bo) / 1773 NUM_VMUSA_UNIT_PER_PAGE); 1774 page_ofs = (drm_suballoc_soffset(sa_bo) % 1775 NUM_VMUSA_UNIT_PER_PAGE) * 1776 VM_SA_UPDATE_UNIT_SIZE; 1777 } 1778 1779 /* Map our PT's to gtt */ 1780 i = 0; 1781 j = 0; 1782 ptes = num_updates; 1783 ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; 1784 while (ptes) { 1785 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 1786 u32 idx = 0; 1787 1788 bb->cs[bb->len++] = MI_STORE_DATA_IMM | 1789 MI_SDI_NUM_QW(chunk); 1790 bb->cs[bb->len++] = ofs; 1791 bb->cs[bb->len++] = 0; /* upper_32_bits */ 1792 1793 for (; i < pt_update_ops->num_ops; ++i) { 1794 struct xe_vm_pgtable_update_op *pt_op = 1795 &pt_update_ops->ops[i]; 1796 struct xe_vm_pgtable_update *updates = pt_op->entries; 1797 1798 for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) { 1799 struct xe_vm *vm = pt_update->vops->vm; 1800 struct xe_bo *pt_bo = updates[j].pt_bo; 1801 1802 if (idx == chunk) 1803 goto next_cmd; 1804 1805 xe_tile_assert(tile, xe_bo_size(pt_bo) == SZ_4K); 1806 1807 /* Map a PT at most once */ 1808 if (pt_bo->update_index < 0) 1809 pt_bo->update_index = current_update; 1810 1811 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, 1812 pat_index, 0); 1813 bb->cs[bb->len++] = lower_32_bits(addr); 1814 bb->cs[bb->len++] = upper_32_bits(addr); 1815 } 1816 1817 j = 0; 1818 } 1819 1820next_cmd: 1821 ptes -= chunk; 1822 ofs += chunk * sizeof(u64); 1823 } 1824 1825 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1826 update_idx = bb->len; 1827 1828 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + 1829 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; 1830 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1831 struct xe_vm_pgtable_update_op *pt_op = 1832 &pt_update_ops->ops[i]; 1833 struct xe_vm_pgtable_update *updates = pt_op->entries; 1834 1835 for (j = 0; j < pt_op->num_entries; ++j) { 1836 struct xe_bo *pt_bo = updates[j].pt_bo; 1837 1838 write_pgtable(tile, bb, addr + 1839 pt_bo->update_index * XE_PAGE_SIZE, 1840 pt_op, &updates[j], pt_update); 1841 } 1842 } 1843 } else { 1844 /* phys pages, no preamble required */ 1845 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1846 update_idx = bb->len; 1847 1848 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1849 struct xe_vm_pgtable_update_op *pt_op = 1850 &pt_update_ops->ops[i]; 1851 struct xe_vm_pgtable_update *updates = pt_op->entries; 1852 1853 for (j = 0; j < pt_op->num_entries; ++j) 1854 write_pgtable(tile, bb, 0, pt_op, &updates[j], 1855 pt_update); 1856 } 1857 } 1858 1859 job = xe_bb_create_migration_job(pt_update_ops->q, bb, 1860 xe_migrate_batch_base(m, usm), 1861 update_idx); 1862 if (IS_ERR(job)) { 1863 err = PTR_ERR(job); 1864 goto err_sa; 1865 } 1866 1867 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); 1868 1869 if (ops->pre_commit) { 1870 pt_update->job = job; 1871 err = ops->pre_commit(pt_update); 1872 if (err) 1873 goto err_job; 1874 } 1875 if (is_migrate) 1876 mutex_lock(&m->job_mutex); 1877 1878 xe_sched_job_arm(job); 1879 fence = dma_fence_get(&job->drm.s_fence->finished); 1880 xe_sched_job_push(job); 1881 1882 if (is_migrate) 1883 mutex_unlock(&m->job_mutex); 1884 1885 xe_bb_free(bb, fence); 1886 drm_suballoc_free(sa_bo, fence); 1887 1888 return fence; 1889 1890err_job: 1891 xe_sched_job_put(job); 1892err_sa: 1893 drm_suballoc_free(sa_bo, NULL); 1894err_bb: 1895 xe_bb_free(bb, NULL); 1896 return ERR_PTR(err); 1897} 1898 1899/** 1900 * xe_migrate_update_pgtables() - Pipelined page-table update 1901 * @m: The migrate context. 1902 * @pt_update: PT update arguments 1903 * 1904 * Perform a pipelined page-table update. The update descriptors are typically 1905 * built under the same lock critical section as a call to this function. If 1906 * using the default engine for the updates, they will be performed in the 1907 * order they grab the job_mutex. If different engines are used, external 1908 * synchronization is needed for overlapping updates to maintain page-table 1909 * consistency. Note that the meaning of "overlapping" is that the updates 1910 * touch the same page-table, which might be a higher-level page-directory. 1911 * If no pipelining is needed, then updates may be performed by the cpu. 1912 * 1913 * Return: A dma_fence that, when signaled, indicates the update completion. 1914 */ 1915struct dma_fence * 1916xe_migrate_update_pgtables(struct xe_migrate *m, 1917 struct xe_migrate_pt_update *pt_update) 1918 1919{ 1920 struct xe_vm_pgtable_update_ops *pt_update_ops = 1921 &pt_update->vops->pt_update_ops[pt_update->tile_id]; 1922 struct dma_fence *fence; 1923 1924 fence = xe_migrate_update_pgtables_cpu(m, pt_update); 1925 1926 /* -ETIME indicates a job is needed, anything else is legit error */ 1927 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) 1928 return fence; 1929 1930 return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops); 1931} 1932 1933/** 1934 * xe_migrate_wait() - Complete all operations using the xe_migrate context 1935 * @m: Migrate context to wait for. 1936 * 1937 * Waits until the GPU no longer uses the migrate context's default engine 1938 * or its page-table objects. FIXME: What about separate page-table update 1939 * engines? 1940 */ 1941void xe_migrate_wait(struct xe_migrate *m) 1942{ 1943 if (m->fence) 1944 dma_fence_wait(m->fence, false); 1945} 1946 1947static u32 pte_update_cmd_size(u64 size) 1948{ 1949 u32 num_dword; 1950 u64 entries = DIV_U64_ROUND_UP(size, XE_PAGE_SIZE); 1951 1952 XE_WARN_ON(size > MAX_PREEMPTDISABLE_TRANSFER); 1953 1954 /* 1955 * MI_STORE_DATA_IMM command is used to update page table. Each 1956 * instruction can update maximumly MAX_PTE_PER_SDI pte entries. To 1957 * update n (n <= MAX_PTE_PER_SDI) pte entries, we need: 1958 * 1959 * - 1 dword for the MI_STORE_DATA_IMM command header (opcode etc) 1960 * - 2 dword for the page table's physical location 1961 * - 2*n dword for value of pte to fill (each pte entry is 2 dwords) 1962 */ 1963 num_dword = (1 + 2) * DIV_U64_ROUND_UP(entries, MAX_PTE_PER_SDI); 1964 num_dword += entries * 2; 1965 1966 return num_dword; 1967} 1968 1969static void build_pt_update_batch_sram(struct xe_migrate *m, 1970 struct xe_bb *bb, u32 pt_offset, 1971 struct drm_pagemap_addr *sram_addr, 1972 u32 size, int level) 1973{ 1974 u16 pat_index = tile_to_xe(m->tile)->pat.idx[XE_CACHE_WB]; 1975 u64 gpu_page_size = 0x1ull << xe_pt_shift(level); 1976 u32 ptes; 1977 int i = 0; 1978 1979 xe_tile_assert(m->tile, PAGE_ALIGNED(size)); 1980 1981 ptes = DIV_ROUND_UP(size, gpu_page_size); 1982 while (ptes) { 1983 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 1984 1985 if (!level) 1986 chunk = ALIGN_DOWN(chunk, PAGE_SIZE / XE_PAGE_SIZE); 1987 1988 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1989 bb->cs[bb->len++] = pt_offset; 1990 bb->cs[bb->len++] = 0; 1991 1992 pt_offset += chunk * 8; 1993 ptes -= chunk; 1994 1995 while (chunk--) { 1996 u64 addr = sram_addr[i].addr; 1997 u64 pte; 1998 1999 xe_tile_assert(m->tile, sram_addr[i].proto == 2000 DRM_INTERCONNECT_SYSTEM); 2001 xe_tile_assert(m->tile, addr); 2002 xe_tile_assert(m->tile, PAGE_ALIGNED(addr)); 2003 2004again: 2005 pte = m->q->vm->pt_ops->pte_encode_addr(m->tile->xe, 2006 addr, pat_index, 2007 level, false, 0); 2008 bb->cs[bb->len++] = lower_32_bits(pte); 2009 bb->cs[bb->len++] = upper_32_bits(pte); 2010 2011 if (gpu_page_size < PAGE_SIZE) { 2012 addr += XE_PAGE_SIZE; 2013 if (!PAGE_ALIGNED(addr)) { 2014 chunk--; 2015 goto again; 2016 } 2017 i++; 2018 } else { 2019 i += gpu_page_size / PAGE_SIZE; 2020 } 2021 } 2022 } 2023} 2024 2025static bool xe_migrate_vram_use_pde(struct drm_pagemap_addr *sram_addr, 2026 unsigned long size) 2027{ 2028 u32 large_size = (0x1 << xe_pt_shift(1)); 2029 unsigned long i, incr = large_size / PAGE_SIZE; 2030 2031 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE); i += incr) 2032 if (PAGE_SIZE << sram_addr[i].order != large_size) 2033 return false; 2034 2035 return true; 2036} 2037 2038#define XE_CACHELINE_BYTES 64ull 2039#define XE_CACHELINE_MASK (XE_CACHELINE_BYTES - 1) 2040 2041static u32 xe_migrate_copy_pitch(struct xe_device *xe, u32 len) 2042{ 2043 u32 pitch; 2044 2045 if (IS_ALIGNED(len, PAGE_SIZE)) 2046 pitch = PAGE_SIZE; 2047 else if (IS_ALIGNED(len, SZ_4K)) 2048 pitch = SZ_4K; 2049 else if (IS_ALIGNED(len, SZ_256)) 2050 pitch = SZ_256; 2051 else if (IS_ALIGNED(len, 4)) 2052 pitch = 4; 2053 else 2054 pitch = 1; 2055 2056 xe_assert(xe, pitch > 1 || xe->info.has_mem_copy_instr); 2057 return pitch; 2058} 2059 2060static struct dma_fence *xe_migrate_vram(struct xe_migrate *m, 2061 unsigned long len, 2062 unsigned long sram_offset, 2063 struct drm_pagemap_addr *sram_addr, 2064 u64 vram_addr, 2065 struct dma_fence *deps, 2066 const enum xe_migrate_copy_dir dir) 2067{ 2068 struct xe_gt *gt = m->tile->primary_gt; 2069 struct xe_device *xe = gt_to_xe(gt); 2070 bool use_usm_batch = xe->info.has_usm; 2071 struct dma_fence *fence = NULL; 2072 u32 batch_size = 1; 2073 u64 src_L0_ofs, dst_L0_ofs; 2074 struct xe_sched_job *job; 2075 struct xe_bb *bb; 2076 u32 update_idx, pt_slot = 0; 2077 unsigned long npages = DIV_ROUND_UP(len + sram_offset, PAGE_SIZE); 2078 unsigned int pitch = xe_migrate_copy_pitch(xe, len); 2079 int err; 2080 unsigned long i, j; 2081 bool use_pde = xe_migrate_vram_use_pde(sram_addr, len + sram_offset); 2082 2083 if (!xe->info.has_mem_copy_instr && 2084 drm_WARN_ON(&xe->drm, 2085 (!IS_ALIGNED(len, pitch)) || (sram_offset | vram_addr) & XE_CACHELINE_MASK)) 2086 return ERR_PTR(-EOPNOTSUPP); 2087 2088 xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER); 2089 2090 batch_size += pte_update_cmd_size(npages << PAGE_SHIFT); 2091 batch_size += EMIT_COPY_DW; 2092 2093 bb = xe_bb_new(gt, batch_size, use_usm_batch); 2094 if (IS_ERR(bb)) { 2095 err = PTR_ERR(bb); 2096 return ERR_PTR(err); 2097 } 2098 2099 /* 2100 * If the order of a struct drm_pagemap_addr entry is greater than 0, 2101 * the entry is populated by GPU pagemap but subsequent entries within 2102 * the range of that order are not populated. 2103 * build_pt_update_batch_sram() expects a fully populated array of 2104 * struct drm_pagemap_addr. Ensure this is the case even with higher 2105 * orders. 2106 */ 2107 for (i = 0; !use_pde && i < npages;) { 2108 unsigned int order = sram_addr[i].order; 2109 2110 for (j = 1; j < NR_PAGES(order) && i + j < npages; j++) 2111 if (!sram_addr[i + j].addr) 2112 sram_addr[i + j].addr = sram_addr[i].addr + j * PAGE_SIZE; 2113 2114 i += NR_PAGES(order); 2115 } 2116 2117 if (use_pde) 2118 build_pt_update_batch_sram(m, bb, m->large_page_copy_pdes, 2119 sram_addr, npages << PAGE_SHIFT, 1); 2120 else 2121 build_pt_update_batch_sram(m, bb, pt_slot * XE_PAGE_SIZE, 2122 sram_addr, npages << PAGE_SHIFT, 0); 2123 2124 if (dir == XE_MIGRATE_COPY_TO_VRAM) { 2125 if (use_pde) 2126 src_L0_ofs = m->large_page_copy_ofs + sram_offset; 2127 else 2128 src_L0_ofs = xe_migrate_vm_addr(pt_slot, 0) + sram_offset; 2129 dst_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr, false); 2130 2131 } else { 2132 src_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr, false); 2133 if (use_pde) 2134 dst_L0_ofs = m->large_page_copy_ofs + sram_offset; 2135 else 2136 dst_L0_ofs = xe_migrate_vm_addr(pt_slot, 0) + sram_offset; 2137 } 2138 2139 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 2140 update_idx = bb->len; 2141 2142 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, len, pitch); 2143 2144 job = xe_bb_create_migration_job(m->q, bb, 2145 xe_migrate_batch_base(m, use_usm_batch), 2146 update_idx); 2147 if (IS_ERR(job)) { 2148 err = PTR_ERR(job); 2149 goto err; 2150 } 2151 2152 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB); 2153 2154 if (deps && !dma_fence_is_signaled(deps)) { 2155 dma_fence_get(deps); 2156 err = drm_sched_job_add_dependency(&job->drm, deps); 2157 if (err) 2158 dma_fence_wait(deps, false); 2159 err = 0; 2160 } 2161 2162 mutex_lock(&m->job_mutex); 2163 xe_sched_job_arm(job); 2164 fence = dma_fence_get(&job->drm.s_fence->finished); 2165 xe_sched_job_push(job); 2166 2167 dma_fence_put(m->fence); 2168 m->fence = dma_fence_get(fence); 2169 mutex_unlock(&m->job_mutex); 2170 2171 xe_bb_free(bb, fence); 2172 2173 return fence; 2174 2175err: 2176 xe_bb_free(bb, NULL); 2177 2178 return ERR_PTR(err); 2179} 2180 2181/** 2182 * xe_migrate_to_vram() - Migrate to VRAM 2183 * @m: The migration context. 2184 * @npages: Number of pages to migrate. 2185 * @src_addr: Array of DMA information (source of migrate) 2186 * @dst_addr: Device physical address of VRAM (destination of migrate) 2187 * @deps: struct dma_fence representing the dependencies that need 2188 * to be signaled before migration. 2189 * 2190 * Copy from an array dma addresses to a VRAM device physical address 2191 * 2192 * Return: dma fence for migrate to signal completion on success, ERR_PTR on 2193 * failure 2194 */ 2195struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m, 2196 unsigned long npages, 2197 struct drm_pagemap_addr *src_addr, 2198 u64 dst_addr, 2199 struct dma_fence *deps) 2200{ 2201 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr, 2202 deps, XE_MIGRATE_COPY_TO_VRAM); 2203} 2204 2205/** 2206 * xe_migrate_from_vram() - Migrate from VRAM 2207 * @m: The migration context. 2208 * @npages: Number of pages to migrate. 2209 * @src_addr: Device physical address of VRAM (source of migrate) 2210 * @dst_addr: Array of DMA information (destination of migrate) 2211 * @deps: struct dma_fence representing the dependencies that need 2212 * to be signaled before migration. 2213 * 2214 * Copy from a VRAM device physical address to an array dma addresses 2215 * 2216 * Return: dma fence for migrate to signal completion on success, ERR_PTR on 2217 * failure 2218 */ 2219struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m, 2220 unsigned long npages, 2221 u64 src_addr, 2222 struct drm_pagemap_addr *dst_addr, 2223 struct dma_fence *deps) 2224{ 2225 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr, 2226 deps, XE_MIGRATE_COPY_TO_SRAM); 2227} 2228 2229static void xe_migrate_dma_unmap(struct xe_device *xe, 2230 struct drm_pagemap_addr *pagemap_addr, 2231 int len, int write) 2232{ 2233 unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE); 2234 2235 for (i = 0; i < npages; ++i) { 2236 if (!pagemap_addr[i].addr) 2237 break; 2238 2239 dma_unmap_page(xe->drm.dev, pagemap_addr[i].addr, PAGE_SIZE, 2240 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE); 2241 } 2242 kfree(pagemap_addr); 2243} 2244 2245static struct drm_pagemap_addr *xe_migrate_dma_map(struct xe_device *xe, 2246 void *buf, int len, 2247 int write) 2248{ 2249 struct drm_pagemap_addr *pagemap_addr; 2250 unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE); 2251 2252 pagemap_addr = kcalloc(npages, sizeof(*pagemap_addr), GFP_KERNEL); 2253 if (!pagemap_addr) 2254 return ERR_PTR(-ENOMEM); 2255 2256 for (i = 0; i < npages; ++i) { 2257 dma_addr_t addr; 2258 struct page *page; 2259 enum dma_data_direction dir = write ? DMA_TO_DEVICE : 2260 DMA_FROM_DEVICE; 2261 2262 if (is_vmalloc_addr(buf)) 2263 page = vmalloc_to_page(buf); 2264 else 2265 page = virt_to_page(buf); 2266 2267 addr = dma_map_page(xe->drm.dev, page, 0, PAGE_SIZE, dir); 2268 if (dma_mapping_error(xe->drm.dev, addr)) 2269 goto err_fault; 2270 2271 pagemap_addr[i] = 2272 drm_pagemap_addr_encode(addr, 2273 DRM_INTERCONNECT_SYSTEM, 2274 0, dir); 2275 buf += PAGE_SIZE; 2276 } 2277 2278 return pagemap_addr; 2279 2280err_fault: 2281 xe_migrate_dma_unmap(xe, pagemap_addr, len, write); 2282 return ERR_PTR(-EFAULT); 2283} 2284 2285/** 2286 * xe_migrate_access_memory - Access memory of a BO via GPU 2287 * 2288 * @m: The migration context. 2289 * @bo: buffer object 2290 * @offset: access offset into buffer object 2291 * @buf: pointer to caller memory to read into or write from 2292 * @len: length of access 2293 * @write: write access 2294 * 2295 * Access memory of a BO via GPU either reading in or writing from a passed in 2296 * pointer. Pointer is dma mapped for GPU access and GPU commands are issued to 2297 * read to or write from pointer. 2298 * 2299 * Returns: 2300 * 0 if successful, negative error code on failure. 2301 */ 2302int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, 2303 unsigned long offset, void *buf, int len, 2304 int write) 2305{ 2306 struct xe_tile *tile = m->tile; 2307 struct xe_device *xe = tile_to_xe(tile); 2308 struct xe_res_cursor cursor; 2309 struct dma_fence *fence = NULL; 2310 struct drm_pagemap_addr *pagemap_addr; 2311 unsigned long page_offset = (unsigned long)buf & ~PAGE_MASK; 2312 int bytes_left = len, current_page = 0; 2313 void *orig_buf = buf; 2314 2315 xe_bo_assert_held(bo); 2316 2317 /* Use bounce buffer for small access and unaligned access */ 2318 if (!xe->info.has_mem_copy_instr && 2319 (!IS_ALIGNED(len, 4) || 2320 !IS_ALIGNED(page_offset, XE_CACHELINE_BYTES) || 2321 !IS_ALIGNED(offset, XE_CACHELINE_BYTES))) { 2322 int buf_offset = 0; 2323 void *bounce; 2324 int err; 2325 2326 BUILD_BUG_ON(!is_power_of_2(XE_CACHELINE_BYTES)); 2327 bounce = kmalloc(XE_CACHELINE_BYTES, GFP_KERNEL); 2328 if (!bounce) 2329 return -ENOMEM; 2330 2331 /* 2332 * Less than ideal for large unaligned access but this should be 2333 * fairly rare, can fixup if this becomes common. 2334 */ 2335 do { 2336 int copy_bytes = min_t(int, bytes_left, 2337 XE_CACHELINE_BYTES - 2338 (offset & XE_CACHELINE_MASK)); 2339 int ptr_offset = offset & XE_CACHELINE_MASK; 2340 2341 err = xe_migrate_access_memory(m, bo, 2342 offset & 2343 ~XE_CACHELINE_MASK, 2344 bounce, 2345 XE_CACHELINE_BYTES, 0); 2346 if (err) 2347 break; 2348 2349 if (write) { 2350 memcpy(bounce + ptr_offset, buf + buf_offset, copy_bytes); 2351 2352 err = xe_migrate_access_memory(m, bo, 2353 offset & ~XE_CACHELINE_MASK, 2354 bounce, 2355 XE_CACHELINE_BYTES, write); 2356 if (err) 2357 break; 2358 } else { 2359 memcpy(buf + buf_offset, bounce + ptr_offset, 2360 copy_bytes); 2361 } 2362 2363 bytes_left -= copy_bytes; 2364 buf_offset += copy_bytes; 2365 offset += copy_bytes; 2366 } while (bytes_left); 2367 2368 kfree(bounce); 2369 return err; 2370 } 2371 2372 pagemap_addr = xe_migrate_dma_map(xe, buf, len + page_offset, write); 2373 if (IS_ERR(pagemap_addr)) 2374 return PTR_ERR(pagemap_addr); 2375 2376 xe_res_first(bo->ttm.resource, offset, xe_bo_size(bo) - offset, &cursor); 2377 2378 do { 2379 struct dma_fence *__fence; 2380 u64 vram_addr = vram_region_gpu_offset(bo->ttm.resource) + 2381 cursor.start; 2382 int current_bytes; 2383 u32 pitch; 2384 2385 if (cursor.size > MAX_PREEMPTDISABLE_TRANSFER) 2386 current_bytes = min_t(int, bytes_left, 2387 MAX_PREEMPTDISABLE_TRANSFER); 2388 else 2389 current_bytes = min_t(int, bytes_left, cursor.size); 2390 2391 pitch = xe_migrate_copy_pitch(xe, current_bytes); 2392 if (xe->info.has_mem_copy_instr) 2393 current_bytes = min_t(int, current_bytes, U16_MAX * pitch); 2394 else 2395 current_bytes = min_t(int, current_bytes, 2396 round_down(S16_MAX * pitch, 2397 XE_CACHELINE_BYTES)); 2398 2399 __fence = xe_migrate_vram(m, current_bytes, 2400 (unsigned long)buf & ~PAGE_MASK, 2401 &pagemap_addr[current_page], 2402 vram_addr, NULL, write ? 2403 XE_MIGRATE_COPY_TO_VRAM : 2404 XE_MIGRATE_COPY_TO_SRAM); 2405 if (IS_ERR(__fence)) { 2406 if (fence) { 2407 dma_fence_wait(fence, false); 2408 dma_fence_put(fence); 2409 } 2410 fence = __fence; 2411 goto out_err; 2412 } 2413 2414 dma_fence_put(fence); 2415 fence = __fence; 2416 2417 buf += current_bytes; 2418 offset += current_bytes; 2419 current_page = (int)(buf - orig_buf) / PAGE_SIZE; 2420 bytes_left -= current_bytes; 2421 if (bytes_left) 2422 xe_res_next(&cursor, current_bytes); 2423 } while (bytes_left); 2424 2425 dma_fence_wait(fence, false); 2426 dma_fence_put(fence); 2427 2428out_err: 2429 xe_migrate_dma_unmap(xe, pagemap_addr, len + page_offset, write); 2430 return IS_ERR(fence) ? PTR_ERR(fence) : 0; 2431} 2432 2433/** 2434 * xe_migrate_job_lock() - Lock migrate job lock 2435 * @m: The migration context. 2436 * @q: Queue associated with the operation which requires a lock 2437 * 2438 * Lock the migrate job lock if the queue is a migration queue, otherwise 2439 * assert the VM's dma-resv is held (user queue's have own locking). 2440 */ 2441void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q) 2442{ 2443 bool is_migrate = q == m->q; 2444 2445 if (is_migrate) 2446 mutex_lock(&m->job_mutex); 2447 else 2448 xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ 2449} 2450 2451/** 2452 * xe_migrate_job_unlock() - Unlock migrate job lock 2453 * @m: The migration context. 2454 * @q: Queue associated with the operation which requires a lock 2455 * 2456 * Unlock the migrate job lock if the queue is a migration queue, otherwise 2457 * assert the VM's dma-resv is held (user queue's have own locking). 2458 */ 2459void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q) 2460{ 2461 bool is_migrate = q == m->q; 2462 2463 if (is_migrate) 2464 mutex_unlock(&m->job_mutex); 2465 else 2466 xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */ 2467} 2468 2469#if IS_ENABLED(CONFIG_PROVE_LOCKING) 2470/** 2471 * xe_migrate_job_lock_assert() - Assert migrate job lock held of queue 2472 * @q: Migrate queue 2473 */ 2474void xe_migrate_job_lock_assert(struct xe_exec_queue *q) 2475{ 2476 struct xe_migrate *m = gt_to_tile(q->gt)->migrate; 2477 2478 xe_gt_assert(q->gt, q == m->q); 2479 lockdep_assert_held(&m->job_mutex); 2480} 2481#endif 2482 2483#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 2484#include "tests/xe_migrate.c" 2485#endif
~/
Read-only file preview.