FILE
xe_gt_sriov_pf_migration.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6#include <drm/drm_managed.h> 7 8#include "regs/xe_guc_regs.h" 9 10#include "abi/guc_actions_sriov_abi.h" 11#include "xe_bo.h" 12#include "xe_ggtt.h" 13#include "xe_gt.h" 14#include "xe_gt_sriov_pf.h" 15#include "xe_gt_sriov_pf_config.h" 16#include "xe_gt_sriov_pf_control.h" 17#include "xe_gt_sriov_pf_helpers.h" 18#include "xe_gt_sriov_pf_migration.h" 19#include "xe_gt_sriov_printk.h" 20#include "xe_guc.h" 21#include "xe_guc_buf.h" 22#include "xe_guc_ct.h" 23#include "xe_migrate.h" 24#include "xe_mmio.h" 25#include "xe_sriov.h" 26#include "xe_sriov_packet.h" 27#include "xe_sriov_packet_types.h" 28#include "xe_sriov_pf_migration.h" 29 30#define XE_GT_SRIOV_PF_MIGRATION_RING_SIZE 5 31 32static struct xe_gt_sriov_migration_data *pf_pick_gt_migration(struct xe_gt *gt, unsigned int vfid) 33{ 34 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 35 xe_gt_assert(gt, vfid != PFID); 36 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 37 38 return >->sriov.pf.vfs[vfid].migration; 39} 40 41static void pf_dump_mig_data(struct xe_gt *gt, unsigned int vfid, 42 struct xe_sriov_packet *data, 43 const char *what) 44{ 45 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) { 46 struct drm_printer p = xe_gt_dbg_printer(gt); 47 48 drm_printf(&p, "VF%u %s (%llu bytes)\n", vfid, what, data->hdr.size); 49 drm_print_hex_dump(&p, "mig_hdr: ", (void *)&data->hdr, sizeof(data->hdr)); 50 drm_print_hex_dump(&p, "mig_data: ", data->vaddr, min(SZ_64, data->hdr.size)); 51 } 52} 53 54static ssize_t pf_migration_ggtt_size(struct xe_gt *gt, unsigned int vfid) 55{ 56 if (!xe_gt_is_main_type(gt)) 57 return 0; 58 59 return xe_gt_sriov_pf_config_ggtt_save(gt, vfid, NULL, 0); 60} 61 62static int pf_save_vf_ggtt_mig_data(struct xe_gt *gt, unsigned int vfid) 63{ 64 struct xe_sriov_packet *data; 65 size_t size; 66 int ret; 67 68 size = pf_migration_ggtt_size(gt, vfid); 69 xe_gt_assert(gt, size); 70 71 data = xe_sriov_packet_alloc(gt_to_xe(gt)); 72 if (!data) 73 return -ENOMEM; 74 75 ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, 76 XE_SRIOV_PACKET_TYPE_GGTT, 0, size); 77 if (ret) 78 goto fail; 79 80 ret = xe_gt_sriov_pf_config_ggtt_save(gt, vfid, data->vaddr, size); 81 if (ret) 82 goto fail; 83 84 pf_dump_mig_data(gt, vfid, data, "GGTT data save"); 85 86 ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); 87 if (ret) 88 goto fail; 89 90 return 0; 91 92fail: 93 xe_sriov_packet_free(data); 94 xe_gt_sriov_err(gt, "Failed to save VF%u GGTT data (%pe)\n", vfid, ERR_PTR(ret)); 95 return ret; 96} 97 98static int pf_restore_vf_ggtt_mig_data(struct xe_gt *gt, unsigned int vfid, 99 struct xe_sriov_packet *data) 100{ 101 int ret; 102 103 pf_dump_mig_data(gt, vfid, data, "GGTT data restore"); 104 105 ret = xe_gt_sriov_pf_config_ggtt_restore(gt, vfid, data->vaddr, data->hdr.size); 106 if (ret) { 107 xe_gt_sriov_err(gt, "Failed to restore VF%u GGTT data (%pe)\n", 108 vfid, ERR_PTR(ret)); 109 return ret; 110 } 111 112 return 0; 113} 114 115/** 116 * xe_gt_sriov_pf_migration_ggtt_save() - Save VF GGTT migration data. 117 * @gt: the &xe_gt 118 * @vfid: the VF identifier (can't be 0) 119 * 120 * This function is for PF only. 121 * 122 * Return: 0 on success or a negative error code on failure. 123 */ 124int xe_gt_sriov_pf_migration_ggtt_save(struct xe_gt *gt, unsigned int vfid) 125{ 126 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 127 xe_gt_assert(gt, vfid != PFID); 128 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 129 130 return pf_save_vf_ggtt_mig_data(gt, vfid); 131} 132 133/** 134 * xe_gt_sriov_pf_migration_ggtt_restore() - Restore VF GGTT migration data. 135 * @gt: the &xe_gt 136 * @vfid: the VF identifier (can't be 0) 137 * @data: the &xe_sriov_packet containing migration data 138 * 139 * This function is for PF only. 140 * 141 * Return: 0 on success or a negative error code on failure. 142 */ 143int xe_gt_sriov_pf_migration_ggtt_restore(struct xe_gt *gt, unsigned int vfid, 144 struct xe_sriov_packet *data) 145{ 146 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 147 xe_gt_assert(gt, vfid != PFID); 148 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 149 150 return pf_restore_vf_ggtt_mig_data(gt, vfid, data); 151} 152 153/* Return: number of dwords saved/restored/required or a negative error code on failure */ 154static int guc_action_vf_save_restore(struct xe_guc *guc, u32 vfid, u32 opcode, 155 u64 addr, u32 ndwords) 156{ 157 u32 request[PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_LEN] = { 158 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 159 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 160 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_PF2GUC_SAVE_RESTORE_VF) | 161 FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_0_OPCODE, opcode), 162 FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_1_VFID, vfid), 163 FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_2_ADDR_LO, lower_32_bits(addr)), 164 FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_3_ADDR_HI, upper_32_bits(addr)), 165 FIELD_PREP(PF2GUC_SAVE_RESTORE_VF_REQUEST_MSG_4_SIZE, ndwords), 166 }; 167 168 return xe_guc_ct_send_block(&guc->ct, request, ARRAY_SIZE(request)); 169} 170 171/* Return: size of the state in dwords or a negative error code on failure */ 172static int pf_send_guc_query_vf_mig_data_size(struct xe_gt *gt, unsigned int vfid) 173{ 174 int ret; 175 176 ret = guc_action_vf_save_restore(>->uc.guc, vfid, GUC_PF_OPCODE_VF_SAVE, 0, 0); 177 return ret ?: -ENODATA; 178} 179 180/* Return: number of state dwords saved or a negative error code on failure */ 181static int pf_send_guc_save_vf_mig_data(struct xe_gt *gt, unsigned int vfid, 182 void *dst, size_t size) 183{ 184 const int ndwords = size / sizeof(u32); 185 struct xe_guc *guc = >->uc.guc; 186 CLASS(xe_guc_buf, buf)(&guc->buf, ndwords); 187 int ret; 188 189 xe_gt_assert(gt, size % sizeof(u32) == 0); 190 xe_gt_assert(gt, size == ndwords * sizeof(u32)); 191 192 if (!xe_guc_buf_is_valid(buf)) 193 return -ENOBUFS; 194 195 /* FW expects this buffer to be zero-initialized */ 196 memset(xe_guc_buf_cpu_ptr(buf), 0, size); 197 198 ret = guc_action_vf_save_restore(guc, vfid, GUC_PF_OPCODE_VF_SAVE, 199 xe_guc_buf_flush(buf), ndwords); 200 if (!ret) 201 ret = -ENODATA; 202 else if (ret > ndwords) 203 ret = -EPROTO; 204 else if (ret > 0) 205 memcpy(dst, xe_guc_buf_sync_read(buf), ret * sizeof(u32)); 206 207 return ret; 208} 209 210/* Return: number of state dwords restored or a negative error code on failure */ 211static int pf_send_guc_restore_vf_mig_data(struct xe_gt *gt, unsigned int vfid, 212 const void *src, size_t size) 213{ 214 const int ndwords = size / sizeof(u32); 215 struct xe_guc *guc = >->uc.guc; 216 CLASS(xe_guc_buf_from_data, buf)(&guc->buf, src, size); 217 int ret; 218 219 xe_gt_assert(gt, size % sizeof(u32) == 0); 220 xe_gt_assert(gt, size == ndwords * sizeof(u32)); 221 222 if (!xe_guc_buf_is_valid(buf)) 223 return -ENOBUFS; 224 225 ret = guc_action_vf_save_restore(guc, vfid, GUC_PF_OPCODE_VF_RESTORE, 226 xe_guc_buf_flush(buf), ndwords); 227 if (!ret) 228 ret = -ENODATA; 229 else if (ret > ndwords) 230 ret = -EPROTO; 231 232 return ret; 233} 234 235static bool pf_migration_supported(struct xe_gt *gt) 236{ 237 return xe_sriov_pf_migration_supported(gt_to_xe(gt)); 238} 239 240static int pf_save_vf_guc_mig_data(struct xe_gt *gt, unsigned int vfid) 241{ 242 struct xe_sriov_packet *data; 243 size_t size; 244 int ret; 245 246 ret = pf_send_guc_query_vf_mig_data_size(gt, vfid); 247 if (ret < 0) 248 goto fail; 249 250 size = ret * sizeof(u32); 251 252 data = xe_sriov_packet_alloc(gt_to_xe(gt)); 253 if (!data) { 254 ret = -ENOMEM; 255 goto fail; 256 } 257 258 ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, 259 XE_SRIOV_PACKET_TYPE_GUC, 0, size); 260 if (ret) 261 goto fail_free; 262 263 ret = pf_send_guc_save_vf_mig_data(gt, vfid, data->vaddr, size); 264 if (ret < 0) 265 goto fail_free; 266 size = ret * sizeof(u32); 267 xe_gt_assert(gt, size); 268 xe_gt_assert(gt, size <= data->hdr.size); 269 data->hdr.size = size; 270 data->remaining = size; 271 272 pf_dump_mig_data(gt, vfid, data, "GuC data save"); 273 274 ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); 275 if (ret) 276 goto fail_free; 277 278 return 0; 279 280fail_free: 281 xe_sriov_packet_free(data); 282fail: 283 xe_gt_sriov_err(gt, "Failed to save VF%u GuC data (%pe)\n", 284 vfid, ERR_PTR(ret)); 285 return ret; 286} 287 288static ssize_t pf_migration_guc_size(struct xe_gt *gt, unsigned int vfid) 289{ 290 ssize_t size; 291 292 if (!pf_migration_supported(gt)) 293 return -ENOPKG; 294 295 size = pf_send_guc_query_vf_mig_data_size(gt, vfid); 296 if (size >= 0) 297 size *= sizeof(u32); 298 299 return size; 300} 301 302/** 303 * xe_gt_sriov_pf_migration_guc_save() - Save VF GuC migration data. 304 * @gt: the &xe_gt 305 * @vfid: the VF identifier 306 * 307 * This function is for PF only. 308 * 309 * Return: 0 on success or a negative error code on failure. 310 */ 311int xe_gt_sriov_pf_migration_guc_save(struct xe_gt *gt, unsigned int vfid) 312{ 313 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 314 xe_gt_assert(gt, vfid != PFID); 315 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 316 317 if (!pf_migration_supported(gt)) 318 return -ENOPKG; 319 320 return pf_save_vf_guc_mig_data(gt, vfid); 321} 322 323static int pf_restore_vf_guc_state(struct xe_gt *gt, unsigned int vfid, 324 struct xe_sriov_packet *data) 325{ 326 int ret; 327 328 xe_gt_assert(gt, data->hdr.size); 329 330 pf_dump_mig_data(gt, vfid, data, "GuC data restore"); 331 332 ret = pf_send_guc_restore_vf_mig_data(gt, vfid, data->vaddr, data->hdr.size); 333 if (ret < 0) 334 goto fail; 335 336 return 0; 337 338fail: 339 xe_gt_sriov_err(gt, "Failed to restore VF%u GuC data (%pe)\n", 340 vfid, ERR_PTR(ret)); 341 return ret; 342} 343 344/** 345 * xe_gt_sriov_pf_migration_guc_restore() - Restore VF GuC migration data. 346 * @gt: the &xe_gt 347 * @vfid: the VF identifier 348 * @data: the &xe_sriov_packet containing migration data 349 * 350 * This function is for PF only. 351 * 352 * Return: 0 on success or a negative error code on failure. 353 */ 354int xe_gt_sriov_pf_migration_guc_restore(struct xe_gt *gt, unsigned int vfid, 355 struct xe_sriov_packet *data) 356{ 357 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 358 xe_gt_assert(gt, vfid != PFID); 359 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 360 361 if (!pf_migration_supported(gt)) 362 return -ENOPKG; 363 364 return pf_restore_vf_guc_state(gt, vfid, data); 365} 366 367static ssize_t pf_migration_mmio_size(struct xe_gt *gt, unsigned int vfid) 368{ 369 if (xe_gt_is_media_type(gt)) 370 return MED_VF_SW_FLAG_COUNT * sizeof(u32); 371 else 372 return VF_SW_FLAG_COUNT * sizeof(u32); 373} 374 375static int pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid, void *buf, size_t size) 376{ 377 struct xe_mmio mmio; 378 u32 *regs = buf; 379 int n; 380 381 if (size != pf_migration_mmio_size(gt, vfid)) 382 return -EINVAL; 383 384 xe_mmio_init_vf_view(&mmio, >->mmio, vfid); 385 386 if (xe_gt_is_media_type(gt)) 387 for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) 388 regs[n] = xe_mmio_read32(>->mmio, MED_VF_SW_FLAG(n)); 389 else 390 for (n = 0; n < VF_SW_FLAG_COUNT; n++) 391 regs[n] = xe_mmio_read32(>->mmio, VF_SW_FLAG(n)); 392 393 return 0; 394} 395 396static int pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, 397 const void *buf, size_t size) 398{ 399 const u32 *regs = buf; 400 struct xe_mmio mmio; 401 int n; 402 403 if (size != pf_migration_mmio_size(gt, vfid)) 404 return -EINVAL; 405 406 xe_mmio_init_vf_view(&mmio, >->mmio, vfid); 407 408 if (xe_gt_is_media_type(gt)) 409 for (n = 0; n < MED_VF_SW_FLAG_COUNT; n++) 410 xe_mmio_write32(>->mmio, MED_VF_SW_FLAG(n), regs[n]); 411 else 412 for (n = 0; n < VF_SW_FLAG_COUNT; n++) 413 xe_mmio_write32(>->mmio, VF_SW_FLAG(n), regs[n]); 414 415 return 0; 416} 417 418static int pf_save_vf_mmio_mig_data(struct xe_gt *gt, unsigned int vfid) 419{ 420 struct xe_sriov_packet *data; 421 size_t size; 422 int ret; 423 424 size = pf_migration_mmio_size(gt, vfid); 425 xe_gt_assert(gt, size); 426 427 data = xe_sriov_packet_alloc(gt_to_xe(gt)); 428 if (!data) 429 return -ENOMEM; 430 431 ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, 432 XE_SRIOV_PACKET_TYPE_MMIO, 0, size); 433 if (ret) 434 goto fail; 435 436 ret = pf_migration_mmio_save(gt, vfid, data->vaddr, size); 437 if (ret) 438 goto fail; 439 440 pf_dump_mig_data(gt, vfid, data, "MMIO data save"); 441 442 ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); 443 if (ret) 444 goto fail; 445 446 return 0; 447 448fail: 449 xe_sriov_packet_free(data); 450 xe_gt_sriov_err(gt, "Failed to save VF%u MMIO data (%pe)\n", vfid, ERR_PTR(ret)); 451 return ret; 452} 453 454static int pf_restore_vf_mmio_mig_data(struct xe_gt *gt, unsigned int vfid, 455 struct xe_sriov_packet *data) 456{ 457 int ret; 458 459 pf_dump_mig_data(gt, vfid, data, "MMIO data restore"); 460 461 ret = pf_migration_mmio_restore(gt, vfid, data->vaddr, data->hdr.size); 462 if (ret) { 463 xe_gt_sriov_err(gt, "Failed to restore VF%u MMIO data (%pe)\n", 464 vfid, ERR_PTR(ret)); 465 466 return ret; 467 } 468 469 return 0; 470} 471 472/** 473 * xe_gt_sriov_pf_migration_mmio_save() - Save VF MMIO migration data. 474 * @gt: the &xe_gt 475 * @vfid: the VF identifier (can't be 0) 476 * 477 * This function is for PF only. 478 * 479 * Return: 0 on success or a negative error code on failure. 480 */ 481int xe_gt_sriov_pf_migration_mmio_save(struct xe_gt *gt, unsigned int vfid) 482{ 483 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 484 xe_gt_assert(gt, vfid != PFID); 485 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 486 487 return pf_save_vf_mmio_mig_data(gt, vfid); 488} 489 490/** 491 * xe_gt_sriov_pf_migration_mmio_restore() - Restore VF MMIO migration data. 492 * @gt: the &xe_gt 493 * @vfid: the VF identifier (can't be 0) 494 * @data: the &xe_sriov_packet containing migration data 495 * 496 * This function is for PF only. 497 * 498 * Return: 0 on success or a negative error code on failure. 499 */ 500int xe_gt_sriov_pf_migration_mmio_restore(struct xe_gt *gt, unsigned int vfid, 501 struct xe_sriov_packet *data) 502{ 503 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 504 xe_gt_assert(gt, vfid != PFID); 505 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 506 507 return pf_restore_vf_mmio_mig_data(gt, vfid, data); 508} 509 510static ssize_t pf_migration_vram_size(struct xe_gt *gt, unsigned int vfid) 511{ 512 if (!xe_gt_is_main_type(gt)) 513 return 0; 514 515 return xe_gt_sriov_pf_config_get_lmem(gt, vfid); 516} 517 518static struct dma_fence *__pf_save_restore_vram(struct xe_gt *gt, unsigned int vfid, 519 struct xe_bo *vram, u64 vram_offset, 520 struct xe_bo *sysmem, u64 sysmem_offset, 521 size_t size, bool save) 522{ 523 struct dma_fence *ret = NULL; 524 struct drm_exec exec; 525 int err; 526 527 drm_exec_init(&exec, 0, 0); 528 drm_exec_until_all_locked(&exec) { 529 err = drm_exec_lock_obj(&exec, &vram->ttm.base); 530 drm_exec_retry_on_contention(&exec); 531 if (err) { 532 ret = ERR_PTR(err); 533 goto err; 534 } 535 536 err = drm_exec_lock_obj(&exec, &sysmem->ttm.base); 537 drm_exec_retry_on_contention(&exec); 538 if (err) { 539 ret = ERR_PTR(err); 540 goto err; 541 } 542 } 543 544 ret = xe_migrate_vram_copy_chunk(vram, vram_offset, sysmem, sysmem_offset, size, 545 save ? XE_MIGRATE_COPY_TO_SRAM : XE_MIGRATE_COPY_TO_VRAM); 546 547err: 548 drm_exec_fini(&exec); 549 550 return ret; 551} 552 553#define PF_VRAM_SAVE_RESTORE_TIMEOUT (5 * HZ) 554static int pf_save_vram_chunk(struct xe_gt *gt, unsigned int vfid, 555 struct xe_bo *src_vram, u64 src_vram_offset, 556 size_t size) 557{ 558 struct xe_sriov_packet *data; 559 struct dma_fence *fence; 560 int ret; 561 562 data = xe_sriov_packet_alloc(gt_to_xe(gt)); 563 if (!data) 564 return -ENOMEM; 565 566 ret = xe_sriov_packet_init(data, gt->tile->id, gt->info.id, 567 XE_SRIOV_PACKET_TYPE_VRAM, src_vram_offset, 568 size); 569 if (ret) 570 goto fail; 571 572 fence = __pf_save_restore_vram(gt, vfid, 573 src_vram, src_vram_offset, 574 data->bo, 0, size, true); 575 if (IS_ERR(fence)) { 576 ret = PTR_ERR(fence); 577 goto fail; 578 } 579 580 ret = dma_fence_wait_timeout(fence, false, PF_VRAM_SAVE_RESTORE_TIMEOUT); 581 dma_fence_put(fence); 582 if (!ret) { 583 ret = -ETIME; 584 goto fail; 585 } 586 587 pf_dump_mig_data(gt, vfid, data, "VRAM data save"); 588 589 ret = xe_gt_sriov_pf_migration_save_produce(gt, vfid, data); 590 if (ret) 591 goto fail; 592 593 return 0; 594 595fail: 596 xe_sriov_packet_free(data); 597 return ret; 598} 599 600#define VF_VRAM_STATE_CHUNK_MAX_SIZE SZ_512M 601static int pf_save_vf_vram_mig_data(struct xe_gt *gt, unsigned int vfid) 602{ 603 struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); 604 loff_t *offset = &migration->save.vram_offset; 605 struct xe_bo *vram; 606 size_t vram_size, chunk_size; 607 int ret; 608 609 vram = xe_gt_sriov_pf_config_get_lmem_obj(gt, vfid); 610 if (!vram) 611 return -ENXIO; 612 613 vram_size = xe_bo_size(vram); 614 615 xe_gt_assert(gt, *offset < vram_size); 616 617 chunk_size = min(vram_size - *offset, VF_VRAM_STATE_CHUNK_MAX_SIZE); 618 619 ret = pf_save_vram_chunk(gt, vfid, vram, *offset, chunk_size); 620 if (ret) 621 goto fail; 622 623 *offset += chunk_size; 624 625 xe_bo_put(vram); 626 627 if (*offset < vram_size) 628 return -EAGAIN; 629 630 return 0; 631 632fail: 633 xe_bo_put(vram); 634 xe_gt_sriov_err(gt, "Failed to save VF%u VRAM data (%pe)\n", vfid, ERR_PTR(ret)); 635 return ret; 636} 637 638static int pf_restore_vf_vram_mig_data(struct xe_gt *gt, unsigned int vfid, 639 struct xe_sriov_packet *data) 640{ 641 u64 end = data->hdr.offset + data->hdr.size; 642 struct dma_fence *fence; 643 struct xe_bo *vram; 644 size_t size; 645 int ret = 0; 646 647 vram = xe_gt_sriov_pf_config_get_lmem_obj(gt, vfid); 648 if (!vram) 649 return -ENXIO; 650 651 size = xe_bo_size(vram); 652 653 if (end > size || end < data->hdr.size) { 654 ret = -EINVAL; 655 goto err; 656 } 657 658 pf_dump_mig_data(gt, vfid, data, "VRAM data restore"); 659 660 fence = __pf_save_restore_vram(gt, vfid, vram, data->hdr.offset, 661 data->bo, 0, data->hdr.size, false); 662 if (IS_ERR(fence)) { 663 ret = PTR_ERR(fence); 664 goto err; 665 } 666 667 ret = dma_fence_wait_timeout(fence, false, PF_VRAM_SAVE_RESTORE_TIMEOUT); 668 dma_fence_put(fence); 669 if (!ret) { 670 ret = -ETIME; 671 goto err; 672 } 673 674 xe_bo_put(vram); 675 676 return 0; 677err: 678 xe_bo_put(vram); 679 xe_gt_sriov_err(gt, "Failed to restore VF%u VRAM data (%pe)\n", vfid, ERR_PTR(ret)); 680 return ret; 681} 682 683/** 684 * xe_gt_sriov_pf_migration_vram_save() - Save VF VRAM migration data. 685 * @gt: the &xe_gt 686 * @vfid: the VF identifier (can't be 0) 687 * 688 * This function is for PF only. 689 * 690 * Return: 0 on success or a negative error code on failure. 691 */ 692int xe_gt_sriov_pf_migration_vram_save(struct xe_gt *gt, unsigned int vfid) 693{ 694 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 695 xe_gt_assert(gt, vfid != PFID); 696 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 697 698 return pf_save_vf_vram_mig_data(gt, vfid); 699} 700 701/** 702 * xe_gt_sriov_pf_migration_vram_restore() - Restore VF VRAM migration data. 703 * @gt: the &xe_gt 704 * @vfid: the VF identifier (can't be 0) 705 * @data: the &xe_sriov_packet containing migration data 706 * 707 * This function is for PF only. 708 * 709 * Return: 0 on success or a negative error code on failure. 710 */ 711int xe_gt_sriov_pf_migration_vram_restore(struct xe_gt *gt, unsigned int vfid, 712 struct xe_sriov_packet *data) 713{ 714 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 715 xe_gt_assert(gt, vfid != PFID); 716 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 717 718 return pf_restore_vf_vram_mig_data(gt, vfid, data); 719} 720 721/** 722 * xe_gt_sriov_pf_migration_size() - Total size of migration data from all components within a GT. 723 * @gt: the &xe_gt 724 * @vfid: the VF identifier (can't be 0) 725 * 726 * This function is for PF only. 727 * 728 * Return: total migration data size in bytes or a negative error code on failure. 729 */ 730ssize_t xe_gt_sriov_pf_migration_size(struct xe_gt *gt, unsigned int vfid) 731{ 732 ssize_t total = 0; 733 ssize_t size; 734 735 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 736 xe_gt_assert(gt, vfid != PFID); 737 xe_gt_assert(gt, vfid <= xe_sriov_pf_get_totalvfs(gt_to_xe(gt))); 738 739 size = pf_migration_guc_size(gt, vfid); 740 if (size < 0) 741 return size; 742 if (size > 0) 743 size += sizeof(struct xe_sriov_packet_hdr); 744 total += size; 745 746 size = pf_migration_ggtt_size(gt, vfid); 747 if (size < 0) 748 return size; 749 if (size > 0) 750 size += sizeof(struct xe_sriov_packet_hdr); 751 total += size; 752 753 size = pf_migration_mmio_size(gt, vfid); 754 if (size < 0) 755 return size; 756 if (size > 0) 757 size += sizeof(struct xe_sriov_packet_hdr); 758 total += size; 759 760 size = pf_migration_vram_size(gt, vfid); 761 if (size < 0) 762 return size; 763 if (size > 0) 764 size += sizeof(struct xe_sriov_packet_hdr); 765 total += size; 766 767 return total; 768} 769 770/** 771 * xe_gt_sriov_pf_migration_ring_empty() - Check if a migration ring is empty. 772 * @gt: the &xe_gt 773 * @vfid: the VF identifier 774 * 775 * Return: true if the ring is empty, otherwise false. 776 */ 777bool xe_gt_sriov_pf_migration_ring_empty(struct xe_gt *gt, unsigned int vfid) 778{ 779 return ptr_ring_empty(&pf_pick_gt_migration(gt, vfid)->ring); 780} 781 782/** 783 * xe_gt_sriov_pf_migration_ring_full() - Check if a migration ring is full. 784 * @gt: the &xe_gt 785 * @vfid: the VF identifier 786 * 787 * Return: true if the ring is full, otherwise false. 788 */ 789bool xe_gt_sriov_pf_migration_ring_full(struct xe_gt *gt, unsigned int vfid) 790{ 791 return ptr_ring_full(&pf_pick_gt_migration(gt, vfid)->ring); 792} 793 794/** 795 * xe_gt_sriov_pf_migration_ring_free() - Consume and free all data in migration ring 796 * @gt: the &xe_gt 797 * @vfid: the VF identifier 798 */ 799void xe_gt_sriov_pf_migration_ring_free(struct xe_gt *gt, unsigned int vfid) 800{ 801 struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); 802 struct xe_sriov_packet *data; 803 804 if (ptr_ring_empty(&migration->ring)) 805 return; 806 807 xe_gt_sriov_notice(gt, "VF%u unprocessed migration data left in the ring!\n", vfid); 808 809 while ((data = ptr_ring_consume(&migration->ring))) 810 xe_sriov_packet_free(data); 811} 812 813static void pf_migration_save_data_todo(struct xe_gt *gt, unsigned int vfid, 814 enum xe_sriov_packet_type type) 815{ 816 set_bit(type, &pf_pick_gt_migration(gt, vfid)->save.data_remaining); 817} 818 819/** 820 * xe_gt_sriov_pf_migration_save_init() - Initialize per-GT migration related data. 821 * @gt: the &xe_gt 822 * @vfid: the VF identifier (can't be 0) 823 */ 824void xe_gt_sriov_pf_migration_save_init(struct xe_gt *gt, unsigned int vfid) 825{ 826 struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); 827 828 migration->save.data_remaining = 0; 829 migration->save.vram_offset = 0; 830 831 xe_gt_assert(gt, pf_migration_guc_size(gt, vfid) > 0); 832 pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_GUC); 833 834 if (pf_migration_ggtt_size(gt, vfid) > 0) 835 pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_GGTT); 836 837 xe_gt_assert(gt, pf_migration_mmio_size(gt, vfid) > 0); 838 pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_MMIO); 839 840 if (pf_migration_vram_size(gt, vfid) > 0) 841 pf_migration_save_data_todo(gt, vfid, XE_SRIOV_PACKET_TYPE_VRAM); 842} 843 844/** 845 * xe_gt_sriov_pf_migration_save_data_pending() - Check if migration data type needs to be saved. 846 * @gt: the &xe_gt 847 * @vfid: the VF identifier (can't be 0) 848 * @type: the &xe_sriov_packet_type of data to be checked 849 * 850 * Return: true if the data needs saving, otherwise false. 851 */ 852bool xe_gt_sriov_pf_migration_save_data_pending(struct xe_gt *gt, unsigned int vfid, 853 enum xe_sriov_packet_type type) 854{ 855 return test_bit(type, &pf_pick_gt_migration(gt, vfid)->save.data_remaining); 856} 857 858/** 859 * xe_gt_sriov_pf_migration_save_data_complete() - Complete migration data type save. 860 * @gt: the &xe_gt 861 * @vfid: the VF identifier (can't be 0) 862 * @type: the &xe_sriov_packet_type to be marked as completed. 863 */ 864void xe_gt_sriov_pf_migration_save_data_complete(struct xe_gt *gt, unsigned int vfid, 865 enum xe_sriov_packet_type type) 866{ 867 clear_bit(type, &pf_pick_gt_migration(gt, vfid)->save.data_remaining); 868} 869 870/** 871 * xe_gt_sriov_pf_migration_save_produce() - Add VF save data packet to migration ring. 872 * @gt: the &xe_gt 873 * @vfid: the VF identifier 874 * @data: the &xe_sriov_packet 875 * 876 * Called by the save migration data producer (PF SR-IOV Control worker) when 877 * processing migration data. 878 * Wakes up the save migration data consumer (userspace), that is potentially 879 * waiting for data when the ring was empty. 880 * 881 * Return: 0 on success or a negative error code on failure. 882 */ 883int xe_gt_sriov_pf_migration_save_produce(struct xe_gt *gt, unsigned int vfid, 884 struct xe_sriov_packet *data) 885{ 886 int ret; 887 888 ret = ptr_ring_produce(&pf_pick_gt_migration(gt, vfid)->ring, data); 889 if (ret) 890 return ret; 891 892 wake_up_all(xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid)); 893 894 return 0; 895} 896 897/** 898 * xe_gt_sriov_pf_migration_restore_consume() - Get VF restore data packet from migration ring. 899 * @gt: the &xe_gt 900 * @vfid: the VF identifier 901 * 902 * Called by the restore migration data consumer (PF SR-IOV Control worker) when 903 * processing migration data. 904 * Wakes up the restore migration data producer (userspace), that is 905 * potentially waiting to add more data when the ring is full. 906 * 907 * Return: Pointer to &xe_sriov_packet on success, 908 * NULL if ring is empty. 909 */ 910struct xe_sriov_packet * 911xe_gt_sriov_pf_migration_restore_consume(struct xe_gt *gt, unsigned int vfid) 912{ 913 struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); 914 struct wait_queue_head *wq = xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid); 915 struct xe_sriov_packet *data; 916 917 data = ptr_ring_consume(&migration->ring); 918 if (data) 919 wake_up_all(wq); 920 921 return data; 922} 923 924static bool pf_restore_data_ready(struct xe_gt *gt, unsigned int vfid) 925{ 926 if (xe_gt_sriov_pf_control_check_restore_failed(gt, vfid) || 927 !ptr_ring_full(&pf_pick_gt_migration(gt, vfid)->ring)) 928 return true; 929 930 return false; 931} 932 933/** 934 * xe_gt_sriov_pf_migration_restore_produce() - Add VF restore data packet to migration ring. 935 * @gt: the &xe_gt 936 * @vfid: the VF identifier 937 * @data: the &xe_sriov_packet 938 * 939 * Called by the restore migration data producer (userspace) when processing 940 * migration data. 941 * If the ring is full, waits until there is space. 942 * Queues the restore migration data consumer (PF SR-IOV Control worker), that 943 * is potentially waiting for data when the ring was empty. 944 * 945 * Return: 0 on success or a negative error code on failure. 946 */ 947int xe_gt_sriov_pf_migration_restore_produce(struct xe_gt *gt, unsigned int vfid, 948 struct xe_sriov_packet *data) 949{ 950 int ret; 951 952 xe_gt_assert(gt, data->hdr.tile_id == gt->tile->id); 953 xe_gt_assert(gt, data->hdr.gt_id == gt->info.id); 954 955 for (;;) { 956 if (xe_gt_sriov_pf_control_check_restore_failed(gt, vfid)) 957 return -EIO; 958 959 ret = ptr_ring_produce(&pf_pick_gt_migration(gt, vfid)->ring, data); 960 if (!ret) 961 break; 962 963 ret = wait_event_interruptible(*xe_sriov_pf_migration_waitqueue(gt_to_xe(gt), vfid), 964 pf_restore_data_ready(gt, vfid)); 965 if (ret) 966 return ret; 967 } 968 969 return xe_gt_sriov_pf_control_process_restore_data(gt, vfid); 970} 971 972/** 973 * xe_gt_sriov_pf_migration_save_consume() - Get VF save data packet from migration ring. 974 * @gt: the &xe_gt 975 * @vfid: the VF identifier 976 * 977 * Called by the save migration data consumer (userspace) when 978 * processing migration data. 979 * Queues the save migration data producer (PF SR-IOV Control worker), that is 980 * potentially waiting to add more data when the ring is full. 981 * 982 * Return: Pointer to &xe_sriov_packet on success, 983 * NULL if ring is empty and there's no more data available, 984 * ERR_PTR(-EAGAIN) if the ring is empty, but data is still produced. 985 */ 986struct xe_sriov_packet * 987xe_gt_sriov_pf_migration_save_consume(struct xe_gt *gt, unsigned int vfid) 988{ 989 struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid); 990 struct xe_sriov_packet *data; 991 int ret; 992 993 data = ptr_ring_consume(&migration->ring); 994 if (data) { 995 ret = xe_gt_sriov_pf_control_process_save_data(gt, vfid); 996 if (ret) { 997 xe_sriov_packet_free(data); 998 return ERR_PTR(ret); 999 } 1000 1001 return data; 1002 } 1003 1004 if (xe_gt_sriov_pf_control_check_save_data_done(gt, vfid)) 1005 return NULL; 1006 1007 if (xe_gt_sriov_pf_control_check_save_failed(gt, vfid)) 1008 return ERR_PTR(-EIO); 1009 1010 return ERR_PTR(-EAGAIN); 1011} 1012 1013static void destroy_pf_packet(void *ptr) 1014{ 1015 struct xe_sriov_packet *data = ptr; 1016 1017 xe_sriov_packet_free(data); 1018} 1019 1020static void action_ring_cleanup(void *arg) 1021{ 1022 struct ptr_ring *r = arg; 1023 1024 ptr_ring_cleanup(r, destroy_pf_packet); 1025} 1026 1027static void pf_gt_migration_check_support(struct xe_gt *gt) 1028{ 1029 if (GUC_FIRMWARE_VER(>->uc.guc) < MAKE_GUC_VER(70, 54, 0)) 1030 xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0"); 1031} 1032 1033/** 1034 * xe_gt_sriov_pf_migration_init() - Initialize support for VF migration. 1035 * @gt: the &xe_gt 1036 * 1037 * This function is for PF only. 1038 * 1039 * Return: 0 on success or a negative error code on failure. 1040 */ 1041int xe_gt_sriov_pf_migration_init(struct xe_gt *gt) 1042{ 1043 struct xe_device *xe = gt_to_xe(gt); 1044 unsigned int n, totalvfs; 1045 int err; 1046 1047 xe_gt_assert(gt, IS_SRIOV_PF(xe)); 1048 1049 pf_gt_migration_check_support(gt); 1050 1051 if (!pf_migration_supported(gt)) 1052 return 0; 1053 1054 totalvfs = xe_sriov_pf_get_totalvfs(xe); 1055 for (n = 1; n <= totalvfs; n++) { 1056 struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, n); 1057 1058 err = ptr_ring_init(&migration->ring, 1059 XE_GT_SRIOV_PF_MIGRATION_RING_SIZE, GFP_KERNEL); 1060 if (err) 1061 return err; 1062 1063 err = devm_add_action_or_reset(xe->drm.dev, action_ring_cleanup, &migration->ring); 1064 if (err) 1065 return err; 1066 } 1067 1068 return 0; 1069}
~/
Read-only file preview.