FILE
xe_guc_submit.c
File preview with syntax highlighting.
linux-6.19.14/drivers/gpu/drm/xe/xe_guc_submit.c
1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6#include "xe_guc_submit.h" 7 8#include <linux/bitfield.h> 9#include <linux/bitmap.h> 10#include <linux/circ_buf.h> 11#include <linux/delay.h> 12#include <linux/dma-fence-array.h> 13#include <linux/math64.h> 14 15#include <drm/drm_managed.h> 16 17#include "abi/guc_actions_abi.h" 18#include "abi/guc_actions_slpc_abi.h" 19#include "abi/guc_klvs_abi.h" 20#include "regs/xe_lrc_layout.h" 21#include "xe_assert.h" 22#include "xe_devcoredump.h" 23#include "xe_device.h" 24#include "xe_exec_queue.h" 25#include "xe_force_wake.h" 26#include "xe_gpu_scheduler.h" 27#include "xe_gt.h" 28#include "xe_gt_clock.h" 29#include "xe_gt_printk.h" 30#include "xe_guc.h" 31#include "xe_guc_capture.h" 32#include "xe_guc_ct.h" 33#include "xe_guc_exec_queue_types.h" 34#include "xe_guc_id_mgr.h" 35#include "xe_guc_klv_helpers.h" 36#include "xe_guc_submit_types.h" 37#include "xe_hw_engine.h" 38#include "xe_hw_fence.h" 39#include "xe_lrc.h" 40#include "xe_macros.h" 41#include "xe_map.h" 42#include "xe_mocs.h" 43#include "xe_pm.h" 44#include "xe_ring_ops_types.h" 45#include "xe_sched_job.h" 46#include "xe_trace.h" 47#include "xe_uc_fw.h" 48#include "xe_vm.h" 49 50static struct xe_guc * 51exec_queue_to_guc(struct xe_exec_queue *q) 52{ 53 return &q->gt->uc.guc; 54} 55 56/* 57 * Helpers for engine state, using an atomic as some of the bits can transition 58 * as the same time (e.g. a suspend can be happning at the same time as schedule 59 * engine done being processed). 60 */ 61#define EXEC_QUEUE_STATE_REGISTERED (1 << 0) 62#define EXEC_QUEUE_STATE_ENABLED (1 << 1) 63#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) 64#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) 65#define EXEC_QUEUE_STATE_DESTROYED (1 << 4) 66#define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) 67#define EXEC_QUEUE_STATE_RESET (1 << 6) 68#define EXEC_QUEUE_STATE_KILLED (1 << 7) 69#define EXEC_QUEUE_STATE_WEDGED (1 << 8) 70#define EXEC_QUEUE_STATE_BANNED (1 << 9) 71#define EXEC_QUEUE_STATE_CHECK_TIMEOUT (1 << 10) 72#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11) 73#define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 12) 74#define EXEC_QUEUE_STATE_PENDING_TDR_EXIT (1 << 13) 75 76static bool exec_queue_registered(struct xe_exec_queue *q) 77{ 78 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; 79} 80 81static void set_exec_queue_registered(struct xe_exec_queue *q) 82{ 83 atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); 84} 85 86static void clear_exec_queue_registered(struct xe_exec_queue *q) 87{ 88 atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); 89} 90 91static bool exec_queue_enabled(struct xe_exec_queue *q) 92{ 93 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; 94} 95 96static void set_exec_queue_enabled(struct xe_exec_queue *q) 97{ 98 atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); 99} 100 101static void clear_exec_queue_enabled(struct xe_exec_queue *q) 102{ 103 atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); 104} 105 106static bool exec_queue_pending_enable(struct xe_exec_queue *q) 107{ 108 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; 109} 110 111static void set_exec_queue_pending_enable(struct xe_exec_queue *q) 112{ 113 atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); 114} 115 116static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) 117{ 118 atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); 119} 120 121static bool exec_queue_pending_disable(struct xe_exec_queue *q) 122{ 123 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; 124} 125 126static void set_exec_queue_pending_disable(struct xe_exec_queue *q) 127{ 128 atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); 129} 130 131static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) 132{ 133 atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); 134} 135 136static bool exec_queue_destroyed(struct xe_exec_queue *q) 137{ 138 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; 139} 140 141static void set_exec_queue_destroyed(struct xe_exec_queue *q) 142{ 143 atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); 144} 145 146static void clear_exec_queue_destroyed(struct xe_exec_queue *q) 147{ 148 atomic_and(~EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); 149} 150 151static bool exec_queue_banned(struct xe_exec_queue *q) 152{ 153 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; 154} 155 156static void set_exec_queue_banned(struct xe_exec_queue *q) 157{ 158 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); 159} 160 161static bool exec_queue_suspended(struct xe_exec_queue *q) 162{ 163 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; 164} 165 166static void set_exec_queue_suspended(struct xe_exec_queue *q) 167{ 168 atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); 169} 170 171static void clear_exec_queue_suspended(struct xe_exec_queue *q) 172{ 173 atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); 174} 175 176static bool exec_queue_reset(struct xe_exec_queue *q) 177{ 178 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; 179} 180 181static void set_exec_queue_reset(struct xe_exec_queue *q) 182{ 183 atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); 184} 185 186static bool exec_queue_killed(struct xe_exec_queue *q) 187{ 188 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; 189} 190 191static void set_exec_queue_killed(struct xe_exec_queue *q) 192{ 193 atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); 194} 195 196static bool exec_queue_wedged(struct xe_exec_queue *q) 197{ 198 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; 199} 200 201static void set_exec_queue_wedged(struct xe_exec_queue *q) 202{ 203 atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); 204} 205 206static bool exec_queue_check_timeout(struct xe_exec_queue *q) 207{ 208 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_CHECK_TIMEOUT; 209} 210 211static void set_exec_queue_check_timeout(struct xe_exec_queue *q) 212{ 213 atomic_or(EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); 214} 215 216static void clear_exec_queue_check_timeout(struct xe_exec_queue *q) 217{ 218 atomic_and(~EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); 219} 220 221static bool exec_queue_extra_ref(struct xe_exec_queue *q) 222{ 223 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_EXTRA_REF; 224} 225 226static void set_exec_queue_extra_ref(struct xe_exec_queue *q) 227{ 228 atomic_or(EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state); 229} 230 231static void clear_exec_queue_extra_ref(struct xe_exec_queue *q) 232{ 233 atomic_and(~EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state); 234} 235 236static bool exec_queue_pending_resume(struct xe_exec_queue *q) 237{ 238 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_RESUME; 239} 240 241static void set_exec_queue_pending_resume(struct xe_exec_queue *q) 242{ 243 atomic_or(EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); 244} 245 246static void clear_exec_queue_pending_resume(struct xe_exec_queue *q) 247{ 248 atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); 249} 250 251static bool exec_queue_pending_tdr_exit(struct xe_exec_queue *q) 252{ 253 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_TDR_EXIT; 254} 255 256static void set_exec_queue_pending_tdr_exit(struct xe_exec_queue *q) 257{ 258 atomic_or(EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state); 259} 260 261static void clear_exec_queue_pending_tdr_exit(struct xe_exec_queue *q) 262{ 263 atomic_and(~EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state); 264} 265 266static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) 267{ 268 return (atomic_read(&q->guc->state) & 269 (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | 270 EXEC_QUEUE_STATE_BANNED)); 271} 272 273static void guc_submit_fini(struct drm_device *drm, void *arg) 274{ 275 struct xe_guc *guc = arg; 276 struct xe_device *xe = guc_to_xe(guc); 277 struct xe_gt *gt = guc_to_gt(guc); 278 int ret; 279 280 ret = wait_event_timeout(guc->submission_state.fini_wq, 281 xa_empty(&guc->submission_state.exec_queue_lookup), 282 HZ * 5); 283 284 drain_workqueue(xe->destroy_wq); 285 286 xe_gt_assert(gt, ret); 287 288 xa_destroy(&guc->submission_state.exec_queue_lookup); 289} 290 291static void guc_submit_wedged_fini(void *arg) 292{ 293 struct xe_guc *guc = arg; 294 struct xe_exec_queue *q; 295 unsigned long index; 296 297 mutex_lock(&guc->submission_state.lock); 298 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 299 if (exec_queue_wedged(q)) { 300 mutex_unlock(&guc->submission_state.lock); 301 xe_exec_queue_put(q); 302 mutex_lock(&guc->submission_state.lock); 303 } 304 } 305 mutex_unlock(&guc->submission_state.lock); 306} 307 308static const struct xe_exec_queue_ops guc_exec_queue_ops; 309 310static void primelockdep(struct xe_guc *guc) 311{ 312 if (!IS_ENABLED(CONFIG_LOCKDEP)) 313 return; 314 315 fs_reclaim_acquire(GFP_KERNEL); 316 317 mutex_lock(&guc->submission_state.lock); 318 mutex_unlock(&guc->submission_state.lock); 319 320 fs_reclaim_release(GFP_KERNEL); 321} 322 323/** 324 * xe_guc_submit_init() - Initialize GuC submission. 325 * @guc: the &xe_guc to initialize 326 * @num_ids: number of GuC context IDs to use 327 * 328 * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all 329 * GuC context IDs supported by the GuC firmware should be used for submission. 330 * 331 * Only VF drivers will have to provide explicit number of GuC context IDs 332 * that they can use for submission. 333 * 334 * Return: 0 on success or a negative error code on failure. 335 */ 336int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) 337{ 338 struct xe_device *xe = guc_to_xe(guc); 339 struct xe_gt *gt = guc_to_gt(guc); 340 int err; 341 342 err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); 343 if (err) 344 return err; 345 346 err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); 347 if (err) 348 return err; 349 350 gt->exec_queue_ops = &guc_exec_queue_ops; 351 352 xa_init(&guc->submission_state.exec_queue_lookup); 353 354 init_waitqueue_head(&guc->submission_state.fini_wq); 355 356 primelockdep(guc); 357 358 guc->submission_state.initialized = true; 359 360 return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); 361} 362 363/* 364 * Given that we want to guarantee enough RCS throughput to avoid missing 365 * frames, we set the yield policy to 20% of each 80ms interval. 366 */ 367#define RC_YIELD_DURATION 80 /* in ms */ 368#define RC_YIELD_RATIO 20 /* in percent */ 369static u32 *emit_render_compute_yield_klv(u32 *emit) 370{ 371 *emit++ = PREP_GUC_KLV_TAG(SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD); 372 *emit++ = RC_YIELD_DURATION; 373 *emit++ = RC_YIELD_RATIO; 374 375 return emit; 376} 377 378#define SCHEDULING_POLICY_MAX_DWORDS 16 379static int guc_init_global_schedule_policy(struct xe_guc *guc) 380{ 381 u32 data[SCHEDULING_POLICY_MAX_DWORDS]; 382 u32 *emit = data; 383 u32 count = 0; 384 int ret; 385 386 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0)) 387 return 0; 388 389 *emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV; 390 391 if (CCS_MASK(guc_to_gt(guc))) 392 emit = emit_render_compute_yield_klv(emit); 393 394 count = emit - data; 395 if (count > 1) { 396 xe_assert(guc_to_xe(guc), count <= SCHEDULING_POLICY_MAX_DWORDS); 397 398 ret = xe_guc_ct_send_block(&guc->ct, data, count); 399 if (ret < 0) { 400 xe_gt_err(guc_to_gt(guc), 401 "failed to enable GuC scheduling policies: %pe\n", 402 ERR_PTR(ret)); 403 return ret; 404 } 405 } 406 407 return 0; 408} 409 410int xe_guc_submit_enable(struct xe_guc *guc) 411{ 412 int ret; 413 414 ret = guc_init_global_schedule_policy(guc); 415 if (ret) 416 return ret; 417 418 guc->submission_state.enabled = true; 419 420 return 0; 421} 422 423void xe_guc_submit_disable(struct xe_guc *guc) 424{ 425 guc->submission_state.enabled = false; 426} 427 428static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) 429{ 430 int i; 431 432 lockdep_assert_held(&guc->submission_state.lock); 433 434 for (i = 0; i < xa_count; ++i) 435 xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); 436 437 xe_guc_id_mgr_release_locked(&guc->submission_state.idm, 438 q->guc->id, q->width); 439 440 if (xa_empty(&guc->submission_state.exec_queue_lookup)) 441 wake_up(&guc->submission_state.fini_wq); 442} 443 444static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 445{ 446 int ret; 447 int i; 448 449 /* 450 * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, 451 * worse case user gets -ENOMEM on engine create and has to try again. 452 * 453 * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent 454 * failure. 455 */ 456 lockdep_assert_held(&guc->submission_state.lock); 457 458 ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, 459 q->width); 460 if (ret < 0) 461 return ret; 462 463 q->guc->id = ret; 464 465 for (i = 0; i < q->width; ++i) { 466 ret = xa_err(xa_store(&guc->submission_state.exec_queue_lookup, 467 q->guc->id + i, q, GFP_NOWAIT)); 468 if (ret) 469 goto err_release; 470 } 471 472 return 0; 473 474err_release: 475 __release_guc_id(guc, q, i); 476 477 return ret; 478} 479 480static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 481{ 482 mutex_lock(&guc->submission_state.lock); 483 __release_guc_id(guc, q, q->width); 484 mutex_unlock(&guc->submission_state.lock); 485} 486 487struct exec_queue_policy { 488 u32 count; 489 struct guc_update_exec_queue_policy h2g; 490}; 491 492static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) 493{ 494 size_t bytes = sizeof(policy->h2g.header) + 495 (sizeof(policy->h2g.klv[0]) * policy->count); 496 497 return bytes / sizeof(u32); 498} 499 500static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, 501 u16 guc_id) 502{ 503 policy->h2g.header.action = 504 XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; 505 policy->h2g.header.guc_id = guc_id; 506 policy->count = 0; 507} 508 509#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ 510static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ 511 u32 data) \ 512{ \ 513 XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ 514\ 515 policy->h2g.klv[policy->count].kl = \ 516 FIELD_PREP(GUC_KLV_0_KEY, \ 517 GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ 518 FIELD_PREP(GUC_KLV_0_LEN, 1); \ 519 policy->h2g.klv[policy->count].value = data; \ 520 policy->count++; \ 521} 522 523MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) 524MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) 525MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) 526MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY) 527#undef MAKE_EXEC_QUEUE_POLICY_ADD 528 529static const int xe_exec_queue_prio_to_guc[] = { 530 [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, 531 [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, 532 [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, 533 [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, 534}; 535 536static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) 537{ 538 struct exec_queue_policy policy; 539 enum xe_exec_queue_priority prio = q->sched_props.priority; 540 u32 timeslice_us = q->sched_props.timeslice_us; 541 u32 slpc_exec_queue_freq_req = 0; 542 u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; 543 544 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 545 546 if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) 547 slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; 548 549 __guc_exec_queue_policy_start_klv(&policy, q->guc->id); 550 __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); 551 __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); 552 __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); 553 __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy, 554 slpc_exec_queue_freq_req); 555 556 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, 557 __guc_exec_queue_policy_action_size(&policy), 0, 0); 558} 559 560static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) 561{ 562 struct exec_queue_policy policy; 563 564 __guc_exec_queue_policy_start_klv(&policy, q->guc->id); 565 __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); 566 567 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, 568 __guc_exec_queue_policy_action_size(&policy), 0, 0); 569} 570 571#define parallel_read(xe_, map_, field_) \ 572 xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ 573 field_) 574#define parallel_write(xe_, map_, field_, val_) \ 575 xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ 576 field_, val_) 577 578static void __register_mlrc_exec_queue(struct xe_guc *guc, 579 struct xe_exec_queue *q, 580 struct guc_ctxt_registration_info *info) 581{ 582#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) 583 u32 action[MAX_MLRC_REG_SIZE]; 584 int len = 0; 585 int i; 586 587 xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q)); 588 589 action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; 590 action[len++] = info->flags; 591 action[len++] = info->context_idx; 592 action[len++] = info->engine_class; 593 action[len++] = info->engine_submit_mask; 594 action[len++] = info->wq_desc_lo; 595 action[len++] = info->wq_desc_hi; 596 action[len++] = info->wq_base_lo; 597 action[len++] = info->wq_base_hi; 598 action[len++] = info->wq_size; 599 action[len++] = q->width; 600 action[len++] = info->hwlrca_lo; 601 action[len++] = info->hwlrca_hi; 602 603 for (i = 1; i < q->width; ++i) { 604 struct xe_lrc *lrc = q->lrc[i]; 605 606 action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); 607 action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); 608 } 609 610 /* explicitly checks some fields that we might fixup later */ 611 xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == 612 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER]); 613 xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == 614 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER]); 615 xe_gt_assert(guc_to_gt(guc), q->width == 616 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS]); 617 xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == 618 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR]); 619 xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE); 620#undef MAX_MLRC_REG_SIZE 621 622 xe_guc_ct_send(&guc->ct, action, len, 0, 0); 623} 624 625static void __register_exec_queue(struct xe_guc *guc, 626 struct guc_ctxt_registration_info *info) 627{ 628 u32 action[] = { 629 XE_GUC_ACTION_REGISTER_CONTEXT, 630 info->flags, 631 info->context_idx, 632 info->engine_class, 633 info->engine_submit_mask, 634 info->wq_desc_lo, 635 info->wq_desc_hi, 636 info->wq_base_lo, 637 info->wq_base_hi, 638 info->wq_size, 639 info->hwlrca_lo, 640 info->hwlrca_hi, 641 }; 642 643 /* explicitly checks some fields that we might fixup later */ 644 xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == 645 action[XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER]); 646 xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == 647 action[XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER]); 648 xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == 649 action[XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR]); 650 651 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); 652} 653 654static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) 655{ 656 struct xe_guc *guc = exec_queue_to_guc(q); 657 struct xe_device *xe = guc_to_xe(guc); 658 struct xe_lrc *lrc = q->lrc[0]; 659 struct guc_ctxt_registration_info info; 660 661 xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); 662 xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT); 663 664 memset(&info, 0, sizeof(info)); 665 info.context_idx = q->guc->id; 666 info.engine_class = xe_engine_class_to_guc_class(q->class); 667 info.engine_submit_mask = q->logical_mask; 668 info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); 669 info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); 670 info.flags = CONTEXT_REGISTRATION_FLAG_KMD | 671 FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type); 672 673 if (xe_exec_queue_is_parallel(q)) { 674 u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); 675 struct iosys_map map = xe_lrc_parallel_map(lrc); 676 677 info.wq_desc_lo = lower_32_bits(ggtt_addr + 678 offsetof(struct guc_submit_parallel_scratch, wq_desc)); 679 info.wq_desc_hi = upper_32_bits(ggtt_addr + 680 offsetof(struct guc_submit_parallel_scratch, wq_desc)); 681 info.wq_base_lo = lower_32_bits(ggtt_addr + 682 offsetof(struct guc_submit_parallel_scratch, wq[0])); 683 info.wq_base_hi = upper_32_bits(ggtt_addr + 684 offsetof(struct guc_submit_parallel_scratch, wq[0])); 685 info.wq_size = WQ_SIZE; 686 687 q->guc->wqi_head = 0; 688 q->guc->wqi_tail = 0; 689 xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); 690 parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); 691 } 692 693 /* 694 * We must keep a reference for LR engines if engine is registered with 695 * the GuC as jobs signal immediately and can't destroy an engine if the 696 * GuC has a reference to it. 697 */ 698 if (xe_exec_queue_is_lr(q)) 699 xe_exec_queue_get(q); 700 701 set_exec_queue_registered(q); 702 trace_xe_exec_queue_register(q); 703 if (xe_exec_queue_is_parallel(q)) 704 __register_mlrc_exec_queue(guc, q, &info); 705 else 706 __register_exec_queue(guc, &info); 707 init_policies(guc, q); 708} 709 710static u32 wq_space_until_wrap(struct xe_exec_queue *q) 711{ 712 return (WQ_SIZE - q->guc->wqi_tail); 713} 714 715static bool vf_recovery(struct xe_guc *guc) 716{ 717 return xe_gt_recovery_pending(guc_to_gt(guc)); 718} 719 720static inline void relaxed_ms_sleep(unsigned int delay_ms) 721{ 722 unsigned long min_us, max_us; 723 724 if (!delay_ms) 725 return; 726 727 if (delay_ms > 20) { 728 msleep(delay_ms); 729 return; 730 } 731 732 min_us = mul_u32_u32(delay_ms, 1000); 733 max_us = min_us + 500; 734 735 usleep_range(min_us, max_us); 736} 737 738static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) 739{ 740 struct xe_guc *guc = exec_queue_to_guc(q); 741 struct xe_device *xe = guc_to_xe(guc); 742 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 743 unsigned int sleep_period_ms = 1, sleep_total_ms = 0; 744 745#define AVAILABLE_SPACE \ 746 CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) 747 if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { 748try_again: 749 q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); 750 if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { 751 if (sleep_total_ms > 2000) { 752 xe_gt_reset_async(q->gt); 753 return -ENODEV; 754 } 755 756 msleep(sleep_period_ms); 757 sleep_total_ms += sleep_period_ms; 758 if (sleep_period_ms < 64) 759 sleep_period_ms <<= 1; 760 goto try_again; 761 } 762 } 763#undef AVAILABLE_SPACE 764 765 return 0; 766} 767 768static int wq_noop_append(struct xe_exec_queue *q) 769{ 770 struct xe_guc *guc = exec_queue_to_guc(q); 771 struct xe_device *xe = guc_to_xe(guc); 772 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 773 u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; 774 775 if (wq_wait_for_space(q, wq_space_until_wrap(q))) 776 return -ENODEV; 777 778 xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw)); 779 780 parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], 781 FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 782 FIELD_PREP(WQ_LEN_MASK, len_dw)); 783 q->guc->wqi_tail = 0; 784 785 return 0; 786} 787 788static void wq_item_append(struct xe_exec_queue *q) 789{ 790 struct xe_guc *guc = exec_queue_to_guc(q); 791 struct xe_device *xe = guc_to_xe(guc); 792 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 793#define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ 794 u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; 795 u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); 796 u32 len_dw = (wqi_size / sizeof(u32)) - 1; 797 int i = 0, j; 798 799 if (wqi_size > wq_space_until_wrap(q)) { 800 if (wq_noop_append(q)) 801 return; 802 } 803 if (wq_wait_for_space(q, wqi_size)) 804 return; 805 806 wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | 807 FIELD_PREP(WQ_LEN_MASK, len_dw); 808 wqi[i++] = xe_lrc_descriptor(q->lrc[0]); 809 wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | 810 FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); 811 wqi[i++] = 0; 812 for (j = 1; j < q->width; ++j) { 813 struct xe_lrc *lrc = q->lrc[j]; 814 815 wqi[i++] = lrc->ring.tail / sizeof(u64); 816 } 817 818 xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32)); 819 820 iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, 821 wq[q->guc->wqi_tail / sizeof(u32)])); 822 xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); 823 q->guc->wqi_tail += wqi_size; 824 xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE); 825 826 xe_device_wmb(xe); 827 828 map = xe_lrc_parallel_map(q->lrc[0]); 829 parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); 830} 831 832#define RESUME_PENDING ~0x0ull 833static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) 834{ 835 struct xe_guc *guc = exec_queue_to_guc(q); 836 struct xe_lrc *lrc = q->lrc[0]; 837 u32 action[3]; 838 u32 g2h_len = 0; 839 u32 num_g2h = 0; 840 int len = 0; 841 bool extra_submit = false; 842 843 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 844 845 if (!job->restore_replay || job->last_replay) { 846 if (xe_exec_queue_is_parallel(q)) 847 wq_item_append(q); 848 else 849 xe_lrc_set_ring_tail(lrc, lrc->ring.tail); 850 job->last_replay = false; 851 } 852 853 if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) 854 return; 855 856 if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { 857 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; 858 action[len++] = q->guc->id; 859 action[len++] = GUC_CONTEXT_ENABLE; 860 g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; 861 num_g2h = 1; 862 if (xe_exec_queue_is_parallel(q)) 863 extra_submit = true; 864 865 q->guc->resume_time = RESUME_PENDING; 866 set_exec_queue_pending_enable(q); 867 set_exec_queue_enabled(q); 868 trace_xe_exec_queue_scheduling_enable(q); 869 } else { 870 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; 871 action[len++] = q->guc->id; 872 trace_xe_exec_queue_submit(q); 873 } 874 875 xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); 876 877 if (extra_submit) { 878 len = 0; 879 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; 880 action[len++] = q->guc->id; 881 trace_xe_exec_queue_submit(q); 882 883 xe_guc_ct_send(&guc->ct, action, len, 0, 0); 884 } 885} 886 887static struct dma_fence * 888guc_exec_queue_run_job(struct drm_sched_job *drm_job) 889{ 890 struct xe_sched_job *job = to_xe_sched_job(drm_job); 891 struct xe_exec_queue *q = job->q; 892 struct xe_guc *guc = exec_queue_to_guc(q); 893 bool lr = xe_exec_queue_is_lr(q), killed_or_banned_or_wedged = 894 exec_queue_killed_or_banned_or_wedged(q); 895 896 xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || 897 exec_queue_banned(q) || exec_queue_suspended(q)); 898 899 trace_xe_sched_job_run(job); 900 901 if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { 902 if (!exec_queue_registered(q)) 903 register_exec_queue(q, GUC_CONTEXT_NORMAL); 904 if (!job->restore_replay) 905 q->ring_ops->emit_job(job); 906 submit_exec_queue(q, job); 907 job->restore_replay = false; 908 } 909 910 /* 911 * We don't care about job-fence ordering in LR VMs because these fences 912 * are never exported; they are used solely to keep jobs on the pending 913 * list. Once a queue enters an error state, there's no need to track 914 * them. 915 */ 916 if (killed_or_banned_or_wedged && lr) 917 xe_sched_job_set_error(job, -ECANCELED); 918 919 return job->fence; 920} 921 922static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) 923{ 924 struct xe_sched_job *job = to_xe_sched_job(drm_job); 925 926 trace_xe_sched_job_free(job); 927 xe_sched_job_put(job); 928} 929 930int xe_guc_read_stopped(struct xe_guc *guc) 931{ 932 return atomic_read(&guc->submission_state.stopped); 933} 934 935#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ 936 u32 action[] = { \ 937 XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ 938 q->guc->id, \ 939 GUC_CONTEXT_##enable_disable, \ 940 } 941 942static void disable_scheduling_deregister(struct xe_guc *guc, 943 struct xe_exec_queue *q) 944{ 945 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); 946 int ret; 947 948 set_min_preemption_timeout(guc, q); 949 smp_rmb(); 950 ret = wait_event_timeout(guc->ct.wq, 951 (!exec_queue_pending_enable(q) && 952 !exec_queue_pending_disable(q)) || 953 xe_guc_read_stopped(guc) || 954 vf_recovery(guc), 955 HZ * 5); 956 if (!ret && !vf_recovery(guc)) { 957 struct xe_gpu_scheduler *sched = &q->guc->sched; 958 959 xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n"); 960 xe_sched_submission_start(sched); 961 xe_gt_reset_async(q->gt); 962 if (!xe_exec_queue_is_lr(q)) 963 xe_sched_tdr_queue_imm(sched); 964 return; 965 } 966 967 clear_exec_queue_enabled(q); 968 set_exec_queue_pending_disable(q); 969 set_exec_queue_destroyed(q); 970 trace_xe_exec_queue_scheduling_disable(q); 971 972 /* 973 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H 974 * handler and we are not allowed to reserved G2H space in handlers. 975 */ 976 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 977 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + 978 G2H_LEN_DW_DEREGISTER_CONTEXT, 2); 979} 980 981static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) 982{ 983 struct xe_guc *guc = exec_queue_to_guc(q); 984 struct xe_device *xe = guc_to_xe(guc); 985 986 /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ 987 wake_up_all(&xe->ufence_wq); 988 989 if (xe_exec_queue_is_lr(q)) 990 queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); 991 else 992 xe_sched_tdr_queue_imm(&q->guc->sched); 993} 994 995/** 996 * xe_guc_submit_wedge() - Wedge GuC submission 997 * @guc: the GuC object 998 * 999 * Save exec queue's registered with GuC state by taking a ref to each queue. 1000 * Register a DRMM handler to drop refs upon driver unload. 1001 */ 1002void xe_guc_submit_wedge(struct xe_guc *guc) 1003{ 1004 struct xe_gt *gt = guc_to_gt(guc); 1005 struct xe_exec_queue *q; 1006 unsigned long index; 1007 int err; 1008 1009 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); 1010 1011 /* 1012 * If device is being wedged even before submission_state is 1013 * initialized, there's nothing to do here. 1014 */ 1015 if (!guc->submission_state.initialized) 1016 return; 1017 1018 err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, 1019 guc_submit_wedged_fini, guc); 1020 if (err) { 1021 xe_gt_err(gt, "Failed to register clean-up on wedged.mode=2; " 1022 "Although device is wedged.\n"); 1023 return; 1024 } 1025 1026 mutex_lock(&guc->submission_state.lock); 1027 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 1028 if (xe_exec_queue_get_unless_zero(q)) 1029 set_exec_queue_wedged(q); 1030 mutex_unlock(&guc->submission_state.lock); 1031} 1032 1033static bool guc_submit_hint_wedged(struct xe_guc *guc) 1034{ 1035 struct xe_device *xe = guc_to_xe(guc); 1036 1037 if (xe->wedged.mode != 2) 1038 return false; 1039 1040 if (xe_device_wedged(xe)) 1041 return true; 1042 1043 xe_device_declare_wedged(xe); 1044 1045 return true; 1046} 1047 1048static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w) 1049{ 1050 struct xe_guc_exec_queue *ge = 1051 container_of(w, struct xe_guc_exec_queue, lr_tdr); 1052 struct xe_exec_queue *q = ge->q; 1053 struct xe_guc *guc = exec_queue_to_guc(q); 1054 struct xe_gpu_scheduler *sched = &ge->sched; 1055 struct xe_sched_job *job; 1056 bool wedged = false; 1057 1058 xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_lr(q)); 1059 1060 if (vf_recovery(guc)) 1061 return; 1062 1063 trace_xe_exec_queue_lr_cleanup(q); 1064 1065 if (!exec_queue_killed(q)) 1066 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); 1067 1068 /* Kill the run_job / process_msg entry points */ 1069 xe_sched_submission_stop(sched); 1070 1071 /* 1072 * Engine state now mostly stable, disable scheduling / deregister if 1073 * needed. This cleanup routine might be called multiple times, where 1074 * the actual async engine deregister drops the final engine ref. 1075 * Calling disable_scheduling_deregister will mark the engine as 1076 * destroyed and fire off the CT requests to disable scheduling / 1077 * deregister, which we only want to do once. We also don't want to mark 1078 * the engine as pending_disable again as this may race with the 1079 * xe_guc_deregister_done_handler() which treats it as an unexpected 1080 * state. 1081 */ 1082 if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { 1083 struct xe_guc *guc = exec_queue_to_guc(q); 1084 int ret; 1085 1086 set_exec_queue_banned(q); 1087 disable_scheduling_deregister(guc, q); 1088 1089 /* 1090 * Must wait for scheduling to be disabled before signalling 1091 * any fences, if GT broken the GT reset code should signal us. 1092 */ 1093 ret = wait_event_timeout(guc->ct.wq, 1094 !exec_queue_pending_disable(q) || 1095 xe_guc_read_stopped(guc) || 1096 vf_recovery(guc), HZ * 5); 1097 if (vf_recovery(guc)) 1098 return; 1099 1100 if (!ret) { 1101 xe_gt_warn(q->gt, "Schedule disable failed to respond, guc_id=%d\n", 1102 q->guc->id); 1103 xe_devcoredump(q, NULL, "Schedule disable failed to respond, guc_id=%d\n", 1104 q->guc->id); 1105 xe_sched_submission_start(sched); 1106 xe_gt_reset_async(q->gt); 1107 return; 1108 } 1109 } 1110 1111 if (!exec_queue_killed(q) && !xe_lrc_ring_is_idle(q->lrc[0])) 1112 xe_devcoredump(q, NULL, "LR job cleanup, guc_id=%d", q->guc->id); 1113 1114 xe_hw_fence_irq_stop(q->fence_irq); 1115 1116 xe_sched_submission_start(sched); 1117 1118 spin_lock(&sched->base.job_list_lock); 1119 list_for_each_entry(job, &sched->base.pending_list, drm.list) 1120 xe_sched_job_set_error(job, -ECANCELED); 1121 spin_unlock(&sched->base.job_list_lock); 1122 1123 xe_hw_fence_irq_start(q->fence_irq); 1124} 1125 1126#define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) 1127 1128static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) 1129{ 1130 struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); 1131 u32 ctx_timestamp, ctx_job_timestamp; 1132 u32 timeout_ms = q->sched_props.job_timeout_ms; 1133 u32 diff; 1134 u64 running_time_ms; 1135 1136 if (!xe_sched_job_started(job)) { 1137 xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started", 1138 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1139 q->guc->id); 1140 1141 return xe_sched_invalidate_job(job, 2); 1142 } 1143 1144 ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(q->lrc[0])); 1145 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); 1146 1147 /* 1148 * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch 1149 * possible overflows with a high timeout. 1150 */ 1151 xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); 1152 1153 diff = ctx_timestamp - ctx_job_timestamp; 1154 1155 /* 1156 * Ensure timeout is within 5% to account for an GuC scheduling latency 1157 */ 1158 running_time_ms = 1159 ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); 1160 1161 xe_gt_dbg(gt, 1162 "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", 1163 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1164 q->guc->id, running_time_ms, timeout_ms, diff); 1165 1166 return running_time_ms >= timeout_ms; 1167} 1168 1169static void enable_scheduling(struct xe_exec_queue *q) 1170{ 1171 MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); 1172 struct xe_guc *guc = exec_queue_to_guc(q); 1173 int ret; 1174 1175 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1176 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1177 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1178 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 1179 1180 set_exec_queue_pending_enable(q); 1181 set_exec_queue_enabled(q); 1182 trace_xe_exec_queue_scheduling_enable(q); 1183 1184 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1185 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); 1186 1187 ret = wait_event_timeout(guc->ct.wq, 1188 !exec_queue_pending_enable(q) || 1189 xe_guc_read_stopped(guc) || 1190 vf_recovery(guc), HZ * 5); 1191 if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) { 1192 xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); 1193 set_exec_queue_banned(q); 1194 xe_gt_reset_async(q->gt); 1195 if (!xe_exec_queue_is_lr(q)) 1196 xe_sched_tdr_queue_imm(&q->guc->sched); 1197 } 1198} 1199 1200static void disable_scheduling(struct xe_exec_queue *q, bool immediate) 1201{ 1202 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); 1203 struct xe_guc *guc = exec_queue_to_guc(q); 1204 1205 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1206 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1207 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1208 1209 if (immediate) 1210 set_min_preemption_timeout(guc, q); 1211 clear_exec_queue_enabled(q); 1212 set_exec_queue_pending_disable(q); 1213 trace_xe_exec_queue_scheduling_disable(q); 1214 1215 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1216 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); 1217} 1218 1219static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) 1220{ 1221 u32 action[] = { 1222 XE_GUC_ACTION_DEREGISTER_CONTEXT, 1223 q->guc->id, 1224 }; 1225 1226 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1227 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1228 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 1229 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1230 1231 set_exec_queue_destroyed(q); 1232 trace_xe_exec_queue_deregister(q); 1233 1234 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1235 G2H_LEN_DW_DEREGISTER_CONTEXT, 1); 1236} 1237 1238static enum drm_gpu_sched_stat 1239guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) 1240{ 1241 struct xe_sched_job *job = to_xe_sched_job(drm_job); 1242 struct xe_sched_job *tmp_job; 1243 struct xe_exec_queue *q = job->q; 1244 struct xe_gpu_scheduler *sched = &q->guc->sched; 1245 struct xe_guc *guc = exec_queue_to_guc(q); 1246 const char *process_name = "no process"; 1247 struct xe_device *xe = guc_to_xe(guc); 1248 unsigned int fw_ref; 1249 int err = -ETIME; 1250 pid_t pid = -1; 1251 int i = 0; 1252 bool wedged = false, skip_timeout_check; 1253 1254 xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_lr(q)); 1255 1256 /* 1257 * TDR has fired before free job worker. Common if exec queue 1258 * immediately closed after last fence signaled. Add back to pending 1259 * list so job can be freed and kick scheduler ensuring free job is not 1260 * lost. 1261 */ 1262 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags) || 1263 vf_recovery(guc)) 1264 return DRM_GPU_SCHED_STAT_NO_HANG; 1265 1266 /* Kill the run_job entry point */ 1267 xe_sched_submission_stop(sched); 1268 1269 /* Must check all state after stopping scheduler */ 1270 skip_timeout_check = exec_queue_reset(q) || 1271 exec_queue_killed_or_banned_or_wedged(q) || 1272 exec_queue_destroyed(q); 1273 1274 /* 1275 * If devcoredump not captured and GuC capture for the job is not ready 1276 * do manual capture first and decide later if we need to use it 1277 */ 1278 if (!exec_queue_killed(q) && !xe->devcoredump.captured && 1279 !xe_guc_capture_get_matching_and_lock(q)) { 1280 /* take force wake before engine register manual capture */ 1281 fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); 1282 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 1283 xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n"); 1284 1285 xe_engine_snapshot_capture_for_queue(q); 1286 1287 xe_force_wake_put(gt_to_fw(q->gt), fw_ref); 1288 } 1289 1290 /* 1291 * XXX: Sampling timeout doesn't work in wedged mode as we have to 1292 * modify scheduling state to read timestamp. We could read the 1293 * timestamp from a register to accumulate current running time but this 1294 * doesn't work for SRIOV. For now assuming timeouts in wedged mode are 1295 * genuine timeouts. 1296 */ 1297 if (!exec_queue_killed(q)) 1298 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); 1299 1300 /* Engine state now stable, disable scheduling to check timestamp */ 1301 if (!wedged && exec_queue_registered(q)) { 1302 int ret; 1303 1304 if (exec_queue_reset(q)) 1305 err = -EIO; 1306 1307 if (!exec_queue_destroyed(q) && xe_uc_fw_is_running(&guc->fw)) { 1308 /* 1309 * Wait for any pending G2H to flush out before 1310 * modifying state 1311 */ 1312 ret = wait_event_timeout(guc->ct.wq, 1313 (!exec_queue_pending_enable(q) && 1314 !exec_queue_pending_disable(q)) || 1315 xe_guc_read_stopped(guc) || 1316 vf_recovery(guc), HZ * 5); 1317 if (vf_recovery(guc)) 1318 goto handle_vf_resume; 1319 if (!ret || xe_guc_read_stopped(guc)) 1320 goto trigger_reset; 1321 1322 /* 1323 * Flag communicates to G2H handler that schedule 1324 * disable originated from a timeout check. The G2H then 1325 * avoid triggering cleanup or deregistering the exec 1326 * queue. 1327 */ 1328 set_exec_queue_check_timeout(q); 1329 disable_scheduling(q, skip_timeout_check); 1330 } 1331 1332 /* 1333 * Must wait for scheduling to be disabled before signalling 1334 * any fences, if GT broken the GT reset code should signal us. 1335 * 1336 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault 1337 * error) messages which can cause the schedule disable to get 1338 * lost. If this occurs, trigger a GT reset to recover. 1339 */ 1340 smp_rmb(); 1341 ret = wait_event_timeout(guc->ct.wq, 1342 !xe_uc_fw_is_running(&guc->fw) || 1343 !exec_queue_pending_disable(q) || 1344 xe_guc_read_stopped(guc) || 1345 vf_recovery(guc), HZ * 5); 1346 if (vf_recovery(guc)) 1347 goto handle_vf_resume; 1348 if (!ret || xe_guc_read_stopped(guc)) { 1349trigger_reset: 1350 if (!ret) 1351 xe_gt_warn(guc_to_gt(guc), 1352 "Schedule disable failed to respond, guc_id=%d", 1353 q->guc->id); 1354 xe_devcoredump(q, job, 1355 "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d", 1356 q->guc->id, ret, xe_guc_read_stopped(guc)); 1357 set_exec_queue_extra_ref(q); 1358 xe_exec_queue_get(q); /* GT reset owns this */ 1359 set_exec_queue_banned(q); 1360 xe_gt_reset_async(q->gt); 1361 xe_sched_tdr_queue_imm(sched); 1362 goto rearm; 1363 } 1364 } 1365 1366 /* 1367 * Check if job is actually timed out, if so restart job execution and TDR 1368 */ 1369 if (!wedged && !skip_timeout_check && !check_timeout(q, job) && 1370 !exec_queue_reset(q) && exec_queue_registered(q)) { 1371 clear_exec_queue_check_timeout(q); 1372 goto sched_enable; 1373 } 1374 1375 if (q->vm && q->vm->xef) { 1376 process_name = q->vm->xef->process_name; 1377 pid = q->vm->xef->pid; 1378 } 1379 1380 if (!exec_queue_killed(q)) 1381 xe_gt_notice(guc_to_gt(guc), 1382 "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", 1383 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1384 q->guc->id, q->flags, process_name, pid); 1385 1386 trace_xe_sched_job_timedout(job); 1387 1388 if (!exec_queue_killed(q)) 1389 xe_devcoredump(q, job, 1390 "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", 1391 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1392 q->guc->id, q->flags); 1393 1394 /* 1395 * Kernel jobs should never fail, nor should VM jobs if they do 1396 * somethings has gone wrong and the GT needs a reset 1397 */ 1398 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, 1399 "Kernel-submitted job timed out\n"); 1400 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), 1401 "VM job timed out on non-killed execqueue\n"); 1402 if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || 1403 (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { 1404 if (!xe_sched_invalidate_job(job, 2)) { 1405 clear_exec_queue_check_timeout(q); 1406 xe_gt_reset_async(q->gt); 1407 goto rearm; 1408 } 1409 } 1410 1411 /* Finish cleaning up exec queue via deregister */ 1412 set_exec_queue_banned(q); 1413 if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { 1414 set_exec_queue_extra_ref(q); 1415 xe_exec_queue_get(q); 1416 __deregister_exec_queue(guc, q); 1417 } 1418 1419 /* Stop fence signaling */ 1420 xe_hw_fence_irq_stop(q->fence_irq); 1421 1422 /* 1423 * Fence state now stable, stop / start scheduler which cleans up any 1424 * fences that are complete 1425 */ 1426 xe_sched_add_pending_job(sched, job); 1427 xe_sched_submission_start(sched); 1428 1429 xe_guc_exec_queue_trigger_cleanup(q); 1430 1431 /* Mark all outstanding jobs as bad, thus completing them */ 1432 spin_lock(&sched->base.job_list_lock); 1433 list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list) 1434 xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED); 1435 spin_unlock(&sched->base.job_list_lock); 1436 1437 /* Start fence signaling */ 1438 xe_hw_fence_irq_start(q->fence_irq); 1439 1440 return DRM_GPU_SCHED_STAT_RESET; 1441 1442sched_enable: 1443 set_exec_queue_pending_tdr_exit(q); 1444 enable_scheduling(q); 1445rearm: 1446 /* 1447 * XXX: Ideally want to adjust timeout based on current execution time 1448 * but there is not currently an easy way to do in DRM scheduler. With 1449 * some thought, do this in a follow up. 1450 */ 1451 xe_sched_submission_start(sched); 1452handle_vf_resume: 1453 return DRM_GPU_SCHED_STAT_NO_HANG; 1454} 1455 1456static void guc_exec_queue_fini(struct xe_exec_queue *q) 1457{ 1458 struct xe_guc_exec_queue *ge = q->guc; 1459 struct xe_guc *guc = exec_queue_to_guc(q); 1460 1461 release_guc_id(guc, q); 1462 xe_sched_entity_fini(&ge->entity); 1463 xe_sched_fini(&ge->sched); 1464 1465 /* 1466 * RCU free due sched being exported via DRM scheduler fences 1467 * (timeline name). 1468 */ 1469 kfree_rcu(ge, rcu); 1470} 1471 1472static void __guc_exec_queue_destroy_async(struct work_struct *w) 1473{ 1474 struct xe_guc_exec_queue *ge = 1475 container_of(w, struct xe_guc_exec_queue, destroy_async); 1476 struct xe_exec_queue *q = ge->q; 1477 struct xe_guc *guc = exec_queue_to_guc(q); 1478 1479 xe_pm_runtime_get(guc_to_xe(guc)); 1480 trace_xe_exec_queue_destroy(q); 1481 1482 if (xe_exec_queue_is_lr(q)) 1483 cancel_work_sync(&ge->lr_tdr); 1484 /* Confirm no work left behind accessing device structures */ 1485 cancel_delayed_work_sync(&ge->sched.base.work_tdr); 1486 1487 xe_exec_queue_fini(q); 1488 1489 xe_pm_runtime_put(guc_to_xe(guc)); 1490} 1491 1492static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) 1493{ 1494 struct xe_guc *guc = exec_queue_to_guc(q); 1495 struct xe_device *xe = guc_to_xe(guc); 1496 1497 INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async); 1498 1499 /* We must block on kernel engines so slabs are empty on driver unload */ 1500 if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) 1501 __guc_exec_queue_destroy_async(&q->guc->destroy_async); 1502 else 1503 queue_work(xe->destroy_wq, &q->guc->destroy_async); 1504} 1505 1506static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q) 1507{ 1508 /* 1509 * Might be done from within the GPU scheduler, need to do async as we 1510 * fini the scheduler when the engine is fini'd, the scheduler can't 1511 * complete fini within itself (circular dependency). Async resolves 1512 * this we and don't really care when everything is fini'd, just that it 1513 * is. 1514 */ 1515 guc_exec_queue_destroy_async(q); 1516} 1517 1518static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) 1519{ 1520 struct xe_exec_queue *q = msg->private_data; 1521 struct xe_guc *guc = exec_queue_to_guc(q); 1522 1523 xe_gt_assert(guc_to_gt(guc), !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); 1524 trace_xe_exec_queue_cleanup_entity(q); 1525 1526 /* 1527 * Expected state transitions for cleanup: 1528 * - If the exec queue is registered and GuC firmware is running, we must first 1529 * disable scheduling and deregister the queue to ensure proper teardown and 1530 * resource release in the GuC, then destroy the exec queue on driver side. 1531 * - If the GuC is already stopped (e.g., during driver unload or GPU reset), 1532 * we cannot expect a response for the deregister request. In this case, 1533 * it is safe to directly destroy the exec queue on driver side, as the GuC 1534 * will not process further requests and all resources must be cleaned up locally. 1535 */ 1536 if (exec_queue_registered(q) && xe_uc_fw_is_running(&guc->fw)) 1537 disable_scheduling_deregister(guc, q); 1538 else 1539 __guc_exec_queue_destroy(guc, q); 1540} 1541 1542static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) 1543{ 1544 return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); 1545} 1546 1547static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) 1548{ 1549 struct xe_exec_queue *q = msg->private_data; 1550 struct xe_guc *guc = exec_queue_to_guc(q); 1551 1552 if (guc_exec_queue_allowed_to_change_state(q)) 1553 init_policies(guc, q); 1554 kfree(msg); 1555} 1556 1557static void __suspend_fence_signal(struct xe_exec_queue *q) 1558{ 1559 struct xe_guc *guc = exec_queue_to_guc(q); 1560 struct xe_device *xe = guc_to_xe(guc); 1561 1562 if (!q->guc->suspend_pending) 1563 return; 1564 1565 WRITE_ONCE(q->guc->suspend_pending, false); 1566 1567 /* 1568 * We use a GuC shared wait queue for VFs because the VF resfix start 1569 * interrupt must be able to wake all instances of suspend_wait. This 1570 * prevents the VF migration worker from being starved during 1571 * scheduling. 1572 */ 1573 if (IS_SRIOV_VF(xe)) 1574 wake_up_all(&guc->ct.wq); 1575 else 1576 wake_up(&q->guc->suspend_wait); 1577} 1578 1579static void suspend_fence_signal(struct xe_exec_queue *q) 1580{ 1581 struct xe_guc *guc = exec_queue_to_guc(q); 1582 1583 xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) || 1584 xe_guc_read_stopped(guc)); 1585 xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending); 1586 1587 __suspend_fence_signal(q); 1588} 1589 1590static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) 1591{ 1592 struct xe_exec_queue *q = msg->private_data; 1593 struct xe_guc *guc = exec_queue_to_guc(q); 1594 1595 if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && 1596 exec_queue_enabled(q)) { 1597 wait_event(guc->ct.wq, vf_recovery(guc) || 1598 ((q->guc->resume_time != RESUME_PENDING || 1599 xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q))); 1600 1601 if (!xe_guc_read_stopped(guc)) { 1602 s64 since_resume_ms = 1603 ktime_ms_delta(ktime_get(), 1604 q->guc->resume_time); 1605 s64 wait_ms = q->vm->preempt.min_run_period_ms - 1606 since_resume_ms; 1607 1608 if (wait_ms > 0 && q->guc->resume_time) 1609 relaxed_ms_sleep(wait_ms); 1610 1611 set_exec_queue_suspended(q); 1612 disable_scheduling(q, false); 1613 } 1614 } else if (q->guc->suspend_pending) { 1615 set_exec_queue_suspended(q); 1616 suspend_fence_signal(q); 1617 } 1618} 1619 1620static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) 1621{ 1622 struct xe_exec_queue *q = msg->private_data; 1623 1624 if (guc_exec_queue_allowed_to_change_state(q)) { 1625 clear_exec_queue_suspended(q); 1626 if (!exec_queue_enabled(q)) { 1627 q->guc->resume_time = RESUME_PENDING; 1628 set_exec_queue_pending_resume(q); 1629 enable_scheduling(q); 1630 } 1631 } else { 1632 clear_exec_queue_suspended(q); 1633 } 1634} 1635 1636#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ 1637#define SET_SCHED_PROPS 2 1638#define SUSPEND 3 1639#define RESUME 4 1640#define OPCODE_MASK 0xf 1641#define MSG_LOCKED BIT(8) 1642#define MSG_HEAD BIT(9) 1643 1644static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) 1645{ 1646 struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data)); 1647 1648 trace_xe_sched_msg_recv(msg); 1649 1650 switch (msg->opcode) { 1651 case CLEANUP: 1652 __guc_exec_queue_process_msg_cleanup(msg); 1653 break; 1654 case SET_SCHED_PROPS: 1655 __guc_exec_queue_process_msg_set_sched_props(msg); 1656 break; 1657 case SUSPEND: 1658 __guc_exec_queue_process_msg_suspend(msg); 1659 break; 1660 case RESUME: 1661 __guc_exec_queue_process_msg_resume(msg); 1662 break; 1663 default: 1664 XE_WARN_ON("Unknown message type"); 1665 } 1666 1667 xe_pm_runtime_put(xe); 1668} 1669 1670static const struct drm_sched_backend_ops drm_sched_ops = { 1671 .run_job = guc_exec_queue_run_job, 1672 .free_job = guc_exec_queue_free_job, 1673 .timedout_job = guc_exec_queue_timedout_job, 1674}; 1675 1676static const struct xe_sched_backend_ops xe_sched_ops = { 1677 .process_msg = guc_exec_queue_process_msg, 1678}; 1679 1680static int guc_exec_queue_init(struct xe_exec_queue *q) 1681{ 1682 struct xe_gpu_scheduler *sched; 1683 struct xe_guc *guc = exec_queue_to_guc(q); 1684 struct xe_guc_exec_queue *ge; 1685 long timeout; 1686 int err, i; 1687 1688 xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc))); 1689 1690 ge = kzalloc(sizeof(*ge), GFP_KERNEL); 1691 if (!ge) 1692 return -ENOMEM; 1693 1694 q->guc = ge; 1695 ge->q = q; 1696 init_rcu_head(&ge->rcu); 1697 init_waitqueue_head(&ge->suspend_wait); 1698 1699 for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i) 1700 INIT_LIST_HEAD(&ge->static_msgs[i].link); 1701 1702 timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : 1703 msecs_to_jiffies(q->sched_props.job_timeout_ms); 1704 err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, 1705 NULL, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, 1706 timeout, guc_to_gt(guc)->ordered_wq, NULL, 1707 q->name, gt_to_xe(q->gt)->drm.dev); 1708 if (err) 1709 goto err_free; 1710 1711 sched = &ge->sched; 1712 err = xe_sched_entity_init(&ge->entity, sched); 1713 if (err) 1714 goto err_sched; 1715 1716 if (xe_exec_queue_is_lr(q)) 1717 INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup); 1718 1719 mutex_lock(&guc->submission_state.lock); 1720 1721 err = alloc_guc_id(guc, q); 1722 if (err) 1723 goto err_entity; 1724 1725 q->entity = &ge->entity; 1726 1727 if (xe_guc_read_stopped(guc) || vf_recovery(guc)) 1728 xe_sched_stop(sched); 1729 1730 mutex_unlock(&guc->submission_state.lock); 1731 1732 xe_exec_queue_assign_name(q, q->guc->id); 1733 1734 trace_xe_exec_queue_create(q); 1735 1736 return 0; 1737 1738err_entity: 1739 mutex_unlock(&guc->submission_state.lock); 1740 xe_sched_entity_fini(&ge->entity); 1741err_sched: 1742 xe_sched_fini(&ge->sched); 1743err_free: 1744 kfree(ge); 1745 1746 return err; 1747} 1748 1749static void guc_exec_queue_kill(struct xe_exec_queue *q) 1750{ 1751 trace_xe_exec_queue_kill(q); 1752 set_exec_queue_killed(q); 1753 __suspend_fence_signal(q); 1754 xe_guc_exec_queue_trigger_cleanup(q); 1755} 1756 1757static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, 1758 u32 opcode) 1759{ 1760 xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); 1761 1762 INIT_LIST_HEAD(&msg->link); 1763 msg->opcode = opcode & OPCODE_MASK; 1764 msg->private_data = q; 1765 1766 trace_xe_sched_msg_add(msg); 1767 if (opcode & MSG_HEAD) 1768 xe_sched_add_msg_head(&q->guc->sched, msg); 1769 else if (opcode & MSG_LOCKED) 1770 xe_sched_add_msg_locked(&q->guc->sched, msg); 1771 else 1772 xe_sched_add_msg(&q->guc->sched, msg); 1773} 1774 1775static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q, 1776 struct xe_sched_msg *msg, 1777 u32 opcode) 1778{ 1779 if (!list_empty(&msg->link)) 1780 return; 1781 1782 guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED | MSG_HEAD); 1783} 1784 1785static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q, 1786 struct xe_sched_msg *msg, 1787 u32 opcode) 1788{ 1789 if (!list_empty(&msg->link)) 1790 return false; 1791 1792 guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED); 1793 1794 return true; 1795} 1796 1797#define STATIC_MSG_CLEANUP 0 1798#define STATIC_MSG_SUSPEND 1 1799#define STATIC_MSG_RESUME 2 1800static void guc_exec_queue_destroy(struct xe_exec_queue *q) 1801{ 1802 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; 1803 1804 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) 1805 guc_exec_queue_add_msg(q, msg, CLEANUP); 1806 else 1807 __guc_exec_queue_destroy(exec_queue_to_guc(q), q); 1808} 1809 1810static int guc_exec_queue_set_priority(struct xe_exec_queue *q, 1811 enum xe_exec_queue_priority priority) 1812{ 1813 struct xe_sched_msg *msg; 1814 1815 if (q->sched_props.priority == priority || 1816 exec_queue_killed_or_banned_or_wedged(q)) 1817 return 0; 1818 1819 msg = kmalloc(sizeof(*msg), GFP_KERNEL); 1820 if (!msg) 1821 return -ENOMEM; 1822 1823 q->sched_props.priority = priority; 1824 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 1825 1826 return 0; 1827} 1828 1829static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) 1830{ 1831 struct xe_sched_msg *msg; 1832 1833 if (q->sched_props.timeslice_us == timeslice_us || 1834 exec_queue_killed_or_banned_or_wedged(q)) 1835 return 0; 1836 1837 msg = kmalloc(sizeof(*msg), GFP_KERNEL); 1838 if (!msg) 1839 return -ENOMEM; 1840 1841 q->sched_props.timeslice_us = timeslice_us; 1842 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 1843 1844 return 0; 1845} 1846 1847static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, 1848 u32 preempt_timeout_us) 1849{ 1850 struct xe_sched_msg *msg; 1851 1852 if (q->sched_props.preempt_timeout_us == preempt_timeout_us || 1853 exec_queue_killed_or_banned_or_wedged(q)) 1854 return 0; 1855 1856 msg = kmalloc(sizeof(*msg), GFP_KERNEL); 1857 if (!msg) 1858 return -ENOMEM; 1859 1860 q->sched_props.preempt_timeout_us = preempt_timeout_us; 1861 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 1862 1863 return 0; 1864} 1865 1866static int guc_exec_queue_suspend(struct xe_exec_queue *q) 1867{ 1868 struct xe_gpu_scheduler *sched = &q->guc->sched; 1869 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; 1870 1871 if (exec_queue_killed_or_banned_or_wedged(q)) 1872 return -EINVAL; 1873 1874 xe_sched_msg_lock(sched); 1875 if (guc_exec_queue_try_add_msg(q, msg, SUSPEND)) 1876 q->guc->suspend_pending = true; 1877 xe_sched_msg_unlock(sched); 1878 1879 return 0; 1880} 1881 1882static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q) 1883{ 1884 struct xe_guc *guc = exec_queue_to_guc(q); 1885 struct xe_device *xe = guc_to_xe(guc); 1886 int ret; 1887 1888 /* 1889 * Likely don't need to check exec_queue_killed() as we clear 1890 * suspend_pending upon kill but to be paranoid but races in which 1891 * suspend_pending is set after kill also check kill here. 1892 */ 1893#define WAIT_COND \ 1894 (!READ_ONCE(q->guc->suspend_pending) || exec_queue_killed(q) || \ 1895 xe_guc_read_stopped(guc)) 1896 1897retry: 1898 if (IS_SRIOV_VF(xe)) 1899 ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND || 1900 vf_recovery(guc), 1901 HZ * 5); 1902 else 1903 ret = wait_event_interruptible_timeout(q->guc->suspend_wait, 1904 WAIT_COND, HZ * 5); 1905 1906 if (vf_recovery(guc) && !xe_device_wedged((guc_to_xe(guc)))) 1907 return -EAGAIN; 1908 1909 if (!ret) { 1910 xe_gt_warn(guc_to_gt(guc), 1911 "Suspend fence, guc_id=%d, failed to respond", 1912 q->guc->id); 1913 /* XXX: Trigger GT reset? */ 1914 return -ETIME; 1915 } else if (IS_SRIOV_VF(xe) && !WAIT_COND) { 1916 /* Corner case on RESFIX DONE where vf_recovery() changes */ 1917 goto retry; 1918 } 1919 1920#undef WAIT_COND 1921 1922 return ret < 0 ? ret : 0; 1923} 1924 1925static void guc_exec_queue_resume(struct xe_exec_queue *q) 1926{ 1927 struct xe_gpu_scheduler *sched = &q->guc->sched; 1928 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; 1929 struct xe_guc *guc = exec_queue_to_guc(q); 1930 1931 xe_gt_assert(guc_to_gt(guc), !q->guc->suspend_pending); 1932 1933 xe_sched_msg_lock(sched); 1934 guc_exec_queue_try_add_msg(q, msg, RESUME); 1935 xe_sched_msg_unlock(sched); 1936} 1937 1938static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) 1939{ 1940 return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); 1941} 1942 1943/* 1944 * All of these functions are an abstraction layer which other parts of Xe can 1945 * use to trap into the GuC backend. All of these functions, aside from init, 1946 * really shouldn't do much other than trap into the DRM scheduler which 1947 * synchronizes these operations. 1948 */ 1949static const struct xe_exec_queue_ops guc_exec_queue_ops = { 1950 .init = guc_exec_queue_init, 1951 .kill = guc_exec_queue_kill, 1952 .fini = guc_exec_queue_fini, 1953 .destroy = guc_exec_queue_destroy, 1954 .set_priority = guc_exec_queue_set_priority, 1955 .set_timeslice = guc_exec_queue_set_timeslice, 1956 .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, 1957 .suspend = guc_exec_queue_suspend, 1958 .suspend_wait = guc_exec_queue_suspend_wait, 1959 .resume = guc_exec_queue_resume, 1960 .reset_status = guc_exec_queue_reset_status, 1961}; 1962 1963static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) 1964{ 1965 struct xe_gpu_scheduler *sched = &q->guc->sched; 1966 1967 /* Stop scheduling + flush any DRM scheduler operations */ 1968 xe_sched_submission_stop(sched); 1969 1970 /* Clean up lost G2H + reset engine state */ 1971 if (exec_queue_registered(q)) { 1972 if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) 1973 xe_exec_queue_put(q); 1974 else if (exec_queue_destroyed(q)) 1975 __guc_exec_queue_destroy(guc, q); 1976 } 1977 if (q->guc->suspend_pending) { 1978 set_exec_queue_suspended(q); 1979 suspend_fence_signal(q); 1980 } 1981 atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | 1982 EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | 1983 EXEC_QUEUE_STATE_SUSPENDED, 1984 &q->guc->state); 1985 q->guc->resume_time = 0; 1986 trace_xe_exec_queue_stop(q); 1987 1988 /* 1989 * Ban any engine (aside from kernel and engines used for VM ops) with a 1990 * started but not complete job or if a job has gone through a GT reset 1991 * more than twice. 1992 */ 1993 if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { 1994 struct xe_sched_job *job = xe_sched_first_pending_job(sched); 1995 bool ban = false; 1996 1997 if (job) { 1998 if ((xe_sched_job_started(job) && 1999 !xe_sched_job_completed(job)) || 2000 xe_sched_invalidate_job(job, 2)) { 2001 trace_xe_sched_job_ban(job); 2002 ban = true; 2003 } 2004 } else if (xe_exec_queue_is_lr(q) && 2005 !xe_lrc_ring_is_idle(q->lrc[0])) { 2006 ban = true; 2007 } 2008 2009 if (ban) { 2010 set_exec_queue_banned(q); 2011 xe_guc_exec_queue_trigger_cleanup(q); 2012 } 2013 } 2014} 2015 2016int xe_guc_submit_reset_prepare(struct xe_guc *guc) 2017{ 2018 int ret; 2019 2020 if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc))) 2021 return 0; 2022 2023 if (!guc->submission_state.initialized) 2024 return 0; 2025 2026 /* 2027 * Using an atomic here rather than submission_state.lock as this 2028 * function can be called while holding the CT lock (engine reset 2029 * failure). submission_state.lock needs the CT lock to resubmit jobs. 2030 * Atomic is not ideal, but it works to prevent against concurrent reset 2031 * and releasing any TDRs waiting on guc->submission_state.stopped. 2032 */ 2033 ret = atomic_fetch_or(1, &guc->submission_state.stopped); 2034 smp_wmb(); 2035 wake_up_all(&guc->ct.wq); 2036 2037 return ret; 2038} 2039 2040void xe_guc_submit_reset_wait(struct xe_guc *guc) 2041{ 2042 wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || 2043 !xe_guc_read_stopped(guc)); 2044} 2045 2046void xe_guc_submit_stop(struct xe_guc *guc) 2047{ 2048 struct xe_exec_queue *q; 2049 unsigned long index; 2050 2051 xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); 2052 2053 mutex_lock(&guc->submission_state.lock); 2054 2055 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2056 /* Prevent redundant attempts to stop parallel queues */ 2057 if (q->guc->id != index) 2058 continue; 2059 2060 guc_exec_queue_stop(guc, q); 2061 } 2062 2063 mutex_unlock(&guc->submission_state.lock); 2064 2065 /* 2066 * No one can enter the backend at this point, aside from new engine 2067 * creation which is protected by guc->submission_state.lock. 2068 */ 2069 2070} 2071 2072static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc, 2073 struct xe_exec_queue *q) 2074{ 2075 bool pending_enable, pending_disable, pending_resume; 2076 2077 pending_enable = exec_queue_pending_enable(q); 2078 pending_resume = exec_queue_pending_resume(q); 2079 2080 if (pending_enable && pending_resume) { 2081 q->guc->needs_resume = true; 2082 xe_gt_dbg(guc_to_gt(guc), "Replay RESUME - guc_id=%d", 2083 q->guc->id); 2084 } 2085 2086 if (pending_enable && !pending_resume && 2087 !exec_queue_pending_tdr_exit(q)) { 2088 clear_exec_queue_registered(q); 2089 if (xe_exec_queue_is_lr(q)) 2090 xe_exec_queue_put(q); 2091 xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER - guc_id=%d", 2092 q->guc->id); 2093 } 2094 2095 if (pending_enable) { 2096 clear_exec_queue_enabled(q); 2097 clear_exec_queue_pending_resume(q); 2098 clear_exec_queue_pending_tdr_exit(q); 2099 clear_exec_queue_pending_enable(q); 2100 xe_gt_dbg(guc_to_gt(guc), "Replay ENABLE - guc_id=%d", 2101 q->guc->id); 2102 } 2103 2104 if (exec_queue_destroyed(q) && exec_queue_registered(q)) { 2105 clear_exec_queue_destroyed(q); 2106 if (exec_queue_extra_ref(q)) 2107 xe_exec_queue_put(q); 2108 else 2109 q->guc->needs_cleanup = true; 2110 clear_exec_queue_extra_ref(q); 2111 xe_gt_dbg(guc_to_gt(guc), "Replay CLEANUP - guc_id=%d", 2112 q->guc->id); 2113 } 2114 2115 pending_disable = exec_queue_pending_disable(q); 2116 2117 if (pending_disable && exec_queue_suspended(q)) { 2118 clear_exec_queue_suspended(q); 2119 q->guc->needs_suspend = true; 2120 xe_gt_dbg(guc_to_gt(guc), "Replay SUSPEND - guc_id=%d", 2121 q->guc->id); 2122 } 2123 2124 if (pending_disable) { 2125 if (!pending_enable) 2126 set_exec_queue_enabled(q); 2127 clear_exec_queue_pending_disable(q); 2128 clear_exec_queue_check_timeout(q); 2129 xe_gt_dbg(guc_to_gt(guc), "Replay DISABLE - guc_id=%d", 2130 q->guc->id); 2131 } 2132 2133 q->guc->resume_time = 0; 2134} 2135 2136static void lrc_parallel_clear(struct xe_lrc *lrc) 2137{ 2138 struct xe_device *xe = gt_to_xe(lrc->gt); 2139 struct iosys_map map = xe_lrc_parallel_map(lrc); 2140 int i; 2141 2142 for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) 2143 parallel_write(xe, map, wq[i], 2144 FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 2145 FIELD_PREP(WQ_LEN_MASK, 0)); 2146} 2147 2148/* 2149 * This function is quite complex but only real way to ensure no state is lost 2150 * during VF resume flows. The function scans the queue state, make adjustments 2151 * as needed, and queues jobs / messages which replayed upon unpause. 2152 */ 2153static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) 2154{ 2155 struct xe_gpu_scheduler *sched = &q->guc->sched; 2156 struct xe_sched_job *job; 2157 int i; 2158 2159 lockdep_assert_held(&guc->submission_state.lock); 2160 2161 /* Stop scheduling + flush any DRM scheduler operations */ 2162 xe_sched_submission_stop(sched); 2163 if (xe_exec_queue_is_lr(q)) 2164 cancel_work_sync(&q->guc->lr_tdr); 2165 else 2166 cancel_delayed_work_sync(&sched->base.work_tdr); 2167 2168 guc_exec_queue_revert_pending_state_change(guc, q); 2169 2170 if (xe_exec_queue_is_parallel(q)) { 2171 /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ 2172 struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); 2173 2174 /* 2175 * NOP existing WQ commands that may contain stale GGTT 2176 * addresses. These will be replayed upon unpause. The hardware 2177 * seems to get confused if the WQ head/tail pointers are 2178 * adjusted. 2179 */ 2180 if (lrc) 2181 lrc_parallel_clear(lrc); 2182 } 2183 2184 job = xe_sched_first_pending_job(sched); 2185 if (job) { 2186 job->restore_replay = true; 2187 2188 /* 2189 * Adjust software tail so jobs submitted overwrite previous 2190 * position in ring buffer with new GGTT addresses. 2191 */ 2192 for (i = 0; i < q->width; ++i) 2193 q->lrc[i]->ring.tail = job->ptrs[i].head; 2194 } 2195} 2196 2197/** 2198 * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. 2199 * @guc: the &xe_guc struct instance whose scheduler is to be disabled 2200 */ 2201void xe_guc_submit_pause(struct xe_guc *guc) 2202{ 2203 struct xe_exec_queue *q; 2204 unsigned long index; 2205 2206 xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); 2207 2208 mutex_lock(&guc->submission_state.lock); 2209 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2210 /* Prevent redundant attempts to stop parallel queues */ 2211 if (q->guc->id != index) 2212 continue; 2213 2214 guc_exec_queue_pause(guc, q); 2215 } 2216 mutex_unlock(&guc->submission_state.lock); 2217} 2218 2219static void guc_exec_queue_start(struct xe_exec_queue *q) 2220{ 2221 struct xe_gpu_scheduler *sched = &q->guc->sched; 2222 2223 if (!exec_queue_killed_or_banned_or_wedged(q)) { 2224 struct xe_sched_job *job = xe_sched_first_pending_job(sched); 2225 int i; 2226 2227 trace_xe_exec_queue_resubmit(q); 2228 if (job) { 2229 for (i = 0; i < q->width; ++i) { 2230 /* 2231 * The GuC context is unregistered at this point 2232 * time, adjusting software ring tail ensures 2233 * jobs are rewritten in original placement, 2234 * adjusting LRC tail ensures the newly loaded 2235 * GuC / contexts only view the LRC tail 2236 * increasing as jobs are written out. 2237 */ 2238 q->lrc[i]->ring.tail = job->ptrs[i].head; 2239 xe_lrc_set_ring_tail(q->lrc[i], 2240 xe_lrc_ring_head(q->lrc[i])); 2241 } 2242 } 2243 xe_sched_resubmit_jobs(sched); 2244 } 2245 2246 xe_sched_submission_start(sched); 2247 xe_sched_submission_resume_tdr(sched); 2248} 2249 2250int xe_guc_submit_start(struct xe_guc *guc) 2251{ 2252 struct xe_exec_queue *q; 2253 unsigned long index; 2254 2255 xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); 2256 2257 mutex_lock(&guc->submission_state.lock); 2258 atomic_dec(&guc->submission_state.stopped); 2259 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2260 /* Prevent redundant attempts to start parallel queues */ 2261 if (q->guc->id != index) 2262 continue; 2263 2264 guc_exec_queue_start(q); 2265 } 2266 mutex_unlock(&guc->submission_state.lock); 2267 2268 wake_up_all(&guc->ct.wq); 2269 2270 return 0; 2271} 2272 2273static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, 2274 struct xe_exec_queue *q) 2275{ 2276 struct xe_gpu_scheduler *sched = &q->guc->sched; 2277 struct xe_sched_job *job = NULL, *__job; 2278 bool restore_replay = false; 2279 2280 list_for_each_entry(__job, &sched->base.pending_list, drm.list) { 2281 job = __job; 2282 restore_replay |= job->restore_replay; 2283 if (restore_replay) { 2284 xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", 2285 q->guc->id, xe_sched_job_seqno(job)); 2286 2287 q->ring_ops->emit_job(job); 2288 job->restore_replay = true; 2289 } 2290 } 2291 2292 if (job) 2293 job->last_replay = true; 2294} 2295 2296/** 2297 * xe_guc_submit_unpause_prepare - Prepare unpause submission tasks on given GuC. 2298 * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause 2299 */ 2300void xe_guc_submit_unpause_prepare(struct xe_guc *guc) 2301{ 2302 struct xe_exec_queue *q; 2303 unsigned long index; 2304 2305 xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); 2306 2307 mutex_lock(&guc->submission_state.lock); 2308 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2309 /* Prevent redundant attempts to stop parallel queues */ 2310 if (q->guc->id != index) 2311 continue; 2312 2313 guc_exec_queue_unpause_prepare(guc, q); 2314 } 2315 mutex_unlock(&guc->submission_state.lock); 2316} 2317 2318static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q) 2319{ 2320 struct xe_gpu_scheduler *sched = &q->guc->sched; 2321 struct xe_sched_msg *msg; 2322 2323 if (q->guc->needs_cleanup) { 2324 msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; 2325 2326 guc_exec_queue_add_msg(q, msg, CLEANUP); 2327 q->guc->needs_cleanup = false; 2328 } 2329 2330 if (q->guc->needs_suspend) { 2331 msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; 2332 2333 xe_sched_msg_lock(sched); 2334 guc_exec_queue_try_add_msg_head(q, msg, SUSPEND); 2335 xe_sched_msg_unlock(sched); 2336 2337 q->guc->needs_suspend = false; 2338 } 2339 2340 /* 2341 * The resume must be in the message queue before the suspend as it is 2342 * not possible for a resume to be issued if a suspend pending is, but 2343 * the inverse is possible. 2344 */ 2345 if (q->guc->needs_resume) { 2346 msg = q->guc->static_msgs + STATIC_MSG_RESUME; 2347 2348 xe_sched_msg_lock(sched); 2349 guc_exec_queue_try_add_msg_head(q, msg, RESUME); 2350 xe_sched_msg_unlock(sched); 2351 2352 q->guc->needs_resume = false; 2353 } 2354} 2355 2356static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q) 2357{ 2358 struct xe_gpu_scheduler *sched = &q->guc->sched; 2359 bool needs_tdr = exec_queue_killed_or_banned_or_wedged(q); 2360 2361 lockdep_assert_held(&guc->submission_state.lock); 2362 2363 xe_sched_resubmit_jobs(sched); 2364 guc_exec_queue_replay_pending_state_change(q); 2365 xe_sched_submission_start(sched); 2366 if (needs_tdr) 2367 xe_guc_exec_queue_trigger_cleanup(q); 2368 xe_sched_submission_resume_tdr(sched); 2369} 2370 2371/** 2372 * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. 2373 * @guc: the &xe_guc struct instance whose scheduler is to be enabled 2374 */ 2375void xe_guc_submit_unpause(struct xe_guc *guc) 2376{ 2377 struct xe_exec_queue *q; 2378 unsigned long index; 2379 2380 mutex_lock(&guc->submission_state.lock); 2381 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2382 /* 2383 * Prevent redundant attempts to stop parallel queues, or queues 2384 * created after resfix done. 2385 */ 2386 if (q->guc->id != index || 2387 !READ_ONCE(q->guc->sched.base.pause_submit)) 2388 continue; 2389 2390 guc_exec_queue_unpause(guc, q); 2391 } 2392 mutex_unlock(&guc->submission_state.lock); 2393} 2394 2395/** 2396 * xe_guc_submit_pause_abort - Abort all paused submission task on given GuC. 2397 * @guc: the &xe_guc struct instance whose scheduler is to be aborted 2398 */ 2399void xe_guc_submit_pause_abort(struct xe_guc *guc) 2400{ 2401 struct xe_exec_queue *q; 2402 unsigned long index; 2403 2404 mutex_lock(&guc->submission_state.lock); 2405 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2406 struct xe_gpu_scheduler *sched = &q->guc->sched; 2407 2408 /* Prevent redundant attempts to stop parallel queues */ 2409 if (q->guc->id != index) 2410 continue; 2411 2412 xe_sched_submission_start(sched); 2413 guc_exec_queue_kill(q); 2414 } 2415 mutex_unlock(&guc->submission_state.lock); 2416} 2417 2418static struct xe_exec_queue * 2419g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) 2420{ 2421 struct xe_gt *gt = guc_to_gt(guc); 2422 struct xe_exec_queue *q; 2423 2424 if (unlikely(guc_id >= GUC_ID_MAX)) { 2425 xe_gt_err(gt, "Invalid guc_id %u\n", guc_id); 2426 return NULL; 2427 } 2428 2429 q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); 2430 if (unlikely(!q)) { 2431 xe_gt_err(gt, "No exec queue found for guc_id %u\n", guc_id); 2432 return NULL; 2433 } 2434 2435 xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id); 2436 xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width)); 2437 2438 return q; 2439} 2440 2441static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) 2442{ 2443 u32 action[] = { 2444 XE_GUC_ACTION_DEREGISTER_CONTEXT, 2445 q->guc->id, 2446 }; 2447 2448 xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); 2449 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 2450 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 2451 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 2452 2453 trace_xe_exec_queue_deregister(q); 2454 2455 xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); 2456} 2457 2458static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, 2459 u32 runnable_state) 2460{ 2461 trace_xe_exec_queue_scheduling_done(q); 2462 2463 if (runnable_state == 1) { 2464 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); 2465 2466 q->guc->resume_time = ktime_get(); 2467 clear_exec_queue_pending_resume(q); 2468 clear_exec_queue_pending_tdr_exit(q); 2469 clear_exec_queue_pending_enable(q); 2470 smp_wmb(); 2471 wake_up_all(&guc->ct.wq); 2472 } else { 2473 bool check_timeout = exec_queue_check_timeout(q); 2474 2475 xe_gt_assert(guc_to_gt(guc), runnable_state == 0); 2476 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); 2477 2478 if (q->guc->suspend_pending) { 2479 suspend_fence_signal(q); 2480 clear_exec_queue_pending_disable(q); 2481 } else { 2482 if (exec_queue_banned(q) || check_timeout) { 2483 smp_wmb(); 2484 wake_up_all(&guc->ct.wq); 2485 } 2486 if (!check_timeout && exec_queue_destroyed(q)) { 2487 /* 2488 * Make sure to clear the pending_disable only 2489 * after sampling the destroyed state. We want 2490 * to ensure we don't trigger the unregister too 2491 * early with something intending to only 2492 * disable scheduling. The caller doing the 2493 * destroy must wait for an ongoing 2494 * pending_disable before marking as destroyed. 2495 */ 2496 clear_exec_queue_pending_disable(q); 2497 deregister_exec_queue(guc, q); 2498 } else { 2499 clear_exec_queue_pending_disable(q); 2500 } 2501 } 2502 } 2503} 2504 2505int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 2506{ 2507 struct xe_exec_queue *q; 2508 u32 guc_id, runnable_state; 2509 2510 if (unlikely(len < 2)) 2511 return -EPROTO; 2512 2513 guc_id = msg[0]; 2514 runnable_state = msg[1]; 2515 2516 q = g2h_exec_queue_lookup(guc, guc_id); 2517 if (unlikely(!q)) 2518 return -EPROTO; 2519 2520 if (unlikely(!exec_queue_pending_enable(q) && 2521 !exec_queue_pending_disable(q))) { 2522 xe_gt_err(guc_to_gt(guc), 2523 "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", 2524 atomic_read(&q->guc->state), q->guc->id, 2525 runnable_state); 2526 return -EPROTO; 2527 } 2528 2529 handle_sched_done(guc, q, runnable_state); 2530 2531 return 0; 2532} 2533 2534static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) 2535{ 2536 trace_xe_exec_queue_deregister_done(q); 2537 2538 clear_exec_queue_registered(q); 2539 2540 if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) 2541 xe_exec_queue_put(q); 2542 else 2543 __guc_exec_queue_destroy(guc, q); 2544} 2545 2546int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 2547{ 2548 struct xe_exec_queue *q; 2549 u32 guc_id; 2550 2551 if (unlikely(len < 1)) 2552 return -EPROTO; 2553 2554 guc_id = msg[0]; 2555 2556 q = g2h_exec_queue_lookup(guc, guc_id); 2557 if (unlikely(!q)) 2558 return -EPROTO; 2559 2560 if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || 2561 exec_queue_pending_enable(q) || exec_queue_enabled(q)) { 2562 xe_gt_err(guc_to_gt(guc), 2563 "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", 2564 atomic_read(&q->guc->state), q->guc->id); 2565 return -EPROTO; 2566 } 2567 2568 handle_deregister_done(guc, q); 2569 2570 return 0; 2571} 2572 2573int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) 2574{ 2575 struct xe_gt *gt = guc_to_gt(guc); 2576 struct xe_exec_queue *q; 2577 u32 guc_id; 2578 2579 if (unlikely(len < 1)) 2580 return -EPROTO; 2581 2582 guc_id = msg[0]; 2583 2584 q = g2h_exec_queue_lookup(guc, guc_id); 2585 if (unlikely(!q)) 2586 return -EPROTO; 2587 2588 xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d", 2589 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 2590 2591 trace_xe_exec_queue_reset(q); 2592 2593 /* 2594 * A banned engine is a NOP at this point (came from 2595 * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel 2596 * jobs by setting timeout of the job to the minimum value kicking 2597 * guc_exec_queue_timedout_job. 2598 */ 2599 set_exec_queue_reset(q); 2600 if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) 2601 xe_guc_exec_queue_trigger_cleanup(q); 2602 2603 return 0; 2604} 2605 2606/* 2607 * xe_guc_error_capture_handler - Handler of GuC captured message 2608 * @guc: The GuC object 2609 * @msg: Point to the message 2610 * @len: The message length 2611 * 2612 * When GuC captured data is ready, GuC will send message 2613 * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be 2614 * called 1st to check status before process the data comes with the message. 2615 * 2616 * Returns: error code. 0 if success 2617 */ 2618int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len) 2619{ 2620 u32 status; 2621 2622 if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN)) 2623 return -EPROTO; 2624 2625 status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK; 2626 if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE) 2627 xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space"); 2628 2629 xe_guc_capture_process(guc); 2630 2631 return 0; 2632} 2633 2634int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, 2635 u32 len) 2636{ 2637 struct xe_gt *gt = guc_to_gt(guc); 2638 struct xe_exec_queue *q; 2639 u32 guc_id; 2640 u32 type = XE_GUC_CAT_ERR_TYPE_INVALID; 2641 2642 if (unlikely(!len || len > 2)) 2643 return -EPROTO; 2644 2645 guc_id = msg[0]; 2646 2647 if (len == 2) 2648 type = msg[1]; 2649 2650 if (guc_id == GUC_ID_UNKNOWN) { 2651 /* 2652 * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF 2653 * context. In such case only PF will be notified about that fault. 2654 */ 2655 xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n"); 2656 return 0; 2657 } 2658 2659 q = g2h_exec_queue_lookup(guc, guc_id); 2660 if (unlikely(!q)) 2661 return -EPROTO; 2662 2663 /* 2664 * The type is HW-defined and changes based on platform, so we don't 2665 * decode it in the kernel and only check if it is valid. 2666 * See bspec 54047 and 72187 for details. 2667 */ 2668 if (type != XE_GUC_CAT_ERR_TYPE_INVALID) 2669 xe_gt_dbg(gt, 2670 "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d", 2671 type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 2672 else 2673 xe_gt_dbg(gt, 2674 "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d", 2675 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 2676 2677 trace_xe_exec_queue_memory_cat_error(q); 2678 2679 /* Treat the same as engine reset */ 2680 set_exec_queue_reset(q); 2681 if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) 2682 xe_guc_exec_queue_trigger_cleanup(q); 2683 2684 return 0; 2685} 2686 2687int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) 2688{ 2689 struct xe_gt *gt = guc_to_gt(guc); 2690 u8 guc_class, instance; 2691 u32 reason; 2692 2693 if (unlikely(len != 3)) 2694 return -EPROTO; 2695 2696 guc_class = msg[0]; 2697 instance = msg[1]; 2698 reason = msg[2]; 2699 2700 /* Unexpected failure of a hardware feature, log an actual error */ 2701 xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X", 2702 guc_class, instance, reason); 2703 2704 xe_gt_reset_async(gt); 2705 2706 return 0; 2707} 2708 2709static void 2710guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, 2711 struct xe_guc_submit_exec_queue_snapshot *snapshot) 2712{ 2713 struct xe_guc *guc = exec_queue_to_guc(q); 2714 struct xe_device *xe = guc_to_xe(guc); 2715 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 2716 int i; 2717 2718 snapshot->guc.wqi_head = q->guc->wqi_head; 2719 snapshot->guc.wqi_tail = q->guc->wqi_tail; 2720 snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); 2721 snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); 2722 snapshot->parallel.wq_desc.status = parallel_read(xe, map, 2723 wq_desc.wq_status); 2724 2725 if (snapshot->parallel.wq_desc.head != 2726 snapshot->parallel.wq_desc.tail) { 2727 for (i = snapshot->parallel.wq_desc.head; 2728 i != snapshot->parallel.wq_desc.tail; 2729 i = (i + sizeof(u32)) % WQ_SIZE) 2730 snapshot->parallel.wq[i / sizeof(u32)] = 2731 parallel_read(xe, map, wq[i / sizeof(u32)]); 2732 } 2733} 2734 2735static void 2736guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, 2737 struct drm_printer *p) 2738{ 2739 int i; 2740 2741 drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", 2742 snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); 2743 drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", 2744 snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); 2745 drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); 2746 2747 if (snapshot->parallel.wq_desc.head != 2748 snapshot->parallel.wq_desc.tail) { 2749 for (i = snapshot->parallel.wq_desc.head; 2750 i != snapshot->parallel.wq_desc.tail; 2751 i = (i + sizeof(u32)) % WQ_SIZE) 2752 drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), 2753 snapshot->parallel.wq[i / sizeof(u32)]); 2754 } 2755} 2756 2757/** 2758 * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. 2759 * @q: faulty exec queue 2760 * 2761 * This can be printed out in a later stage like during dev_coredump 2762 * analysis. 2763 * 2764 * Returns: a GuC Submit Engine snapshot object that must be freed by the 2765 * caller, using `xe_guc_exec_queue_snapshot_free`. 2766 */ 2767struct xe_guc_submit_exec_queue_snapshot * 2768xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) 2769{ 2770 struct xe_gpu_scheduler *sched = &q->guc->sched; 2771 struct xe_guc_submit_exec_queue_snapshot *snapshot; 2772 int i; 2773 2774 snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); 2775 2776 if (!snapshot) 2777 return NULL; 2778 2779 snapshot->guc.id = q->guc->id; 2780 memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); 2781 snapshot->class = q->class; 2782 snapshot->logical_mask = q->logical_mask; 2783 snapshot->width = q->width; 2784 snapshot->refcount = kref_read(&q->refcount); 2785 snapshot->sched_timeout = sched->base.timeout; 2786 snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; 2787 snapshot->sched_props.preempt_timeout_us = 2788 q->sched_props.preempt_timeout_us; 2789 2790 snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *), 2791 GFP_ATOMIC); 2792 2793 if (snapshot->lrc) { 2794 for (i = 0; i < q->width; ++i) { 2795 struct xe_lrc *lrc = q->lrc[i]; 2796 2797 snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); 2798 } 2799 } 2800 2801 snapshot->schedule_state = atomic_read(&q->guc->state); 2802 snapshot->exec_queue_flags = q->flags; 2803 2804 snapshot->parallel_execution = xe_exec_queue_is_parallel(q); 2805 if (snapshot->parallel_execution) 2806 guc_exec_queue_wq_snapshot_capture(q, snapshot); 2807 2808 spin_lock(&sched->base.job_list_lock); 2809 snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); 2810 snapshot->pending_list = kmalloc_array(snapshot->pending_list_size, 2811 sizeof(struct pending_list_snapshot), 2812 GFP_ATOMIC); 2813 2814 if (snapshot->pending_list) { 2815 struct xe_sched_job *job_iter; 2816 2817 i = 0; 2818 list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) { 2819 snapshot->pending_list[i].seqno = 2820 xe_sched_job_seqno(job_iter); 2821 snapshot->pending_list[i].fence = 2822 dma_fence_is_signaled(job_iter->fence) ? 1 : 0; 2823 snapshot->pending_list[i].finished = 2824 dma_fence_is_signaled(&job_iter->drm.s_fence->finished) 2825 ? 1 : 0; 2826 i++; 2827 } 2828 } 2829 2830 spin_unlock(&sched->base.job_list_lock); 2831 2832 return snapshot; 2833} 2834 2835/** 2836 * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. 2837 * @snapshot: Previously captured snapshot of job. 2838 * 2839 * This captures some data that requires taking some locks, so it cannot be done in signaling path. 2840 */ 2841void 2842xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) 2843{ 2844 int i; 2845 2846 if (!snapshot || !snapshot->lrc) 2847 return; 2848 2849 for (i = 0; i < snapshot->width; ++i) 2850 xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); 2851} 2852 2853/** 2854 * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. 2855 * @snapshot: GuC Submit Engine snapshot object. 2856 * @p: drm_printer where it will be printed out. 2857 * 2858 * This function prints out a given GuC Submit Engine snapshot object. 2859 */ 2860void 2861xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, 2862 struct drm_printer *p) 2863{ 2864 int i; 2865 2866 if (!snapshot) 2867 return; 2868 2869 drm_printf(p, "GuC ID: %d\n", snapshot->guc.id); 2870 drm_printf(p, "\tName: %s\n", snapshot->name); 2871 drm_printf(p, "\tClass: %d\n", snapshot->class); 2872 drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); 2873 drm_printf(p, "\tWidth: %d\n", snapshot->width); 2874 drm_printf(p, "\tRef: %d\n", snapshot->refcount); 2875 drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); 2876 drm_printf(p, "\tTimeslice: %u (us)\n", 2877 snapshot->sched_props.timeslice_us); 2878 drm_printf(p, "\tPreempt timeout: %u (us)\n", 2879 snapshot->sched_props.preempt_timeout_us); 2880 2881 for (i = 0; snapshot->lrc && i < snapshot->width; ++i) 2882 xe_lrc_snapshot_print(snapshot->lrc[i], p); 2883 2884 drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); 2885 drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); 2886 2887 if (snapshot->parallel_execution) 2888 guc_exec_queue_wq_snapshot_print(snapshot, p); 2889 2890 for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; 2891 i++) 2892 drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", 2893 snapshot->pending_list[i].seqno, 2894 snapshot->pending_list[i].fence, 2895 snapshot->pending_list[i].finished); 2896} 2897 2898/** 2899 * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given 2900 * snapshot. 2901 * @snapshot: GuC Submit Engine snapshot object. 2902 * 2903 * This function free all the memory that needed to be allocated at capture 2904 * time. 2905 */ 2906void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) 2907{ 2908 int i; 2909 2910 if (!snapshot) 2911 return; 2912 2913 if (snapshot->lrc) { 2914 for (i = 0; i < snapshot->width; i++) 2915 xe_lrc_snapshot_free(snapshot->lrc[i]); 2916 kfree(snapshot->lrc); 2917 } 2918 kfree(snapshot->pending_list); 2919 kfree(snapshot); 2920} 2921 2922static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) 2923{ 2924 struct xe_guc_submit_exec_queue_snapshot *snapshot; 2925 2926 snapshot = xe_guc_exec_queue_snapshot_capture(q); 2927 xe_guc_exec_queue_snapshot_print(snapshot, p); 2928 xe_guc_exec_queue_snapshot_free(snapshot); 2929} 2930 2931/** 2932 * xe_guc_register_vf_exec_queue - Register exec queue for a given context type. 2933 * @q: Execution queue 2934 * @ctx_type: Type of the context 2935 * 2936 * This function registers the execution queue with the guc. Special context 2937 * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE 2938 * are only applicable for IGPU and in the VF. 2939 * Submits the execution queue to GUC after registering it. 2940 * 2941 * Returns - None. 2942 */ 2943void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type) 2944{ 2945 struct xe_guc *guc = exec_queue_to_guc(q); 2946 struct xe_device *xe = guc_to_xe(guc); 2947 struct xe_gt *gt = guc_to_gt(guc); 2948 2949 xe_gt_assert(gt, IS_SRIOV_VF(xe)); 2950 xe_gt_assert(gt, !IS_DGFX(xe)); 2951 xe_gt_assert(gt, ctx_type == GUC_CONTEXT_COMPRESSION_SAVE || 2952 ctx_type == GUC_CONTEXT_COMPRESSION_RESTORE); 2953 xe_gt_assert(gt, GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 23, 0)); 2954 2955 register_exec_queue(q, ctx_type); 2956 enable_scheduling(q); 2957} 2958 2959/** 2960 * xe_guc_submit_print - GuC Submit Print. 2961 * @guc: GuC. 2962 * @p: drm_printer where it will be printed out. 2963 * 2964 * This function capture and prints snapshots of **all** GuC Engines. 2965 */ 2966void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) 2967{ 2968 struct xe_exec_queue *q; 2969 unsigned long index; 2970 2971 if (!xe_device_uc_enabled(guc_to_xe(guc))) 2972 return; 2973 2974 mutex_lock(&guc->submission_state.lock); 2975 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 2976 guc_exec_queue_print(q, p); 2977 mutex_unlock(&guc->submission_state.lock); 2978} 2979 2980/** 2981 * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all 2982 * exec queues registered to given GuC. 2983 * @guc: the &xe_guc struct instance 2984 * @scratch: scratch buffer to be used as temporary storage 2985 * 2986 * Returns: zero on success, negative error code on failure. 2987 */ 2988int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) 2989{ 2990 struct xe_exec_queue *q; 2991 unsigned long index; 2992 int err = 0; 2993 2994 mutex_lock(&guc->submission_state.lock); 2995 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2996 /* Prevent redundant attempts to stop parallel queues */ 2997 if (q->guc->id != index) 2998 continue; 2999 3000 err = xe_exec_queue_contexts_hwsp_rebase(q, scratch); 3001 if (err) 3002 break; 3003 } 3004 mutex_unlock(&guc->submission_state.lock); 3005 3006 return err; 3007}
~/
Read-only file preview.