xe_gt.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2022 Intel Corporation
   4 */
   5
   6#include "xe_gt.h"
   7
   8#include <linux/minmax.h>
   9
  10#include <drm/drm_managed.h>
  11#include <uapi/drm/xe_drm.h>
  12
  13#include <generated/xe_wa_oob.h>
  14
  15#include "instructions/xe_alu_commands.h"
  16#include "instructions/xe_gfxpipe_commands.h"
  17#include "instructions/xe_mi_commands.h"
  18#include "regs/xe_engine_regs.h"
  19#include "regs/xe_gt_regs.h"
  20#include "xe_assert.h"
  21#include "xe_bb.h"
  22#include "xe_bo.h"
  23#include "xe_device.h"
  24#include "xe_eu_stall.h"
  25#include "xe_exec_queue.h"
  26#include "xe_execlist.h"
  27#include "xe_force_wake.h"
  28#include "xe_ggtt.h"
  29#include "xe_gsc.h"
  30#include "xe_gt_ccs_mode.h"
  31#include "xe_gt_clock.h"
  32#include "xe_gt_freq.h"
  33#include "xe_gt_idle.h"
  34#include "xe_gt_mcr.h"
  35#include "xe_gt_printk.h"
  36#include "xe_gt_sriov_pf.h"
  37#include "xe_gt_sriov_vf.h"
  38#include "xe_gt_sysfs.h"
  39#include "xe_gt_topology.h"
  40#include "xe_guc_exec_queue_types.h"
  41#include "xe_guc_pc.h"
  42#include "xe_guc_submit.h"
  43#include "xe_hw_fence.h"
  44#include "xe_hw_engine_class_sysfs.h"
  45#include "xe_irq.h"
  46#include "xe_lmtt.h"
  47#include "xe_lrc.h"
  48#include "xe_map.h"
  49#include "xe_migrate.h"
  50#include "xe_mmio.h"
  51#include "xe_pagefault.h"
  52#include "xe_pat.h"
  53#include "xe_pm.h"
  54#include "xe_mocs.h"
  55#include "xe_reg_sr.h"
  56#include "xe_ring_ops.h"
  57#include "xe_sa.h"
  58#include "xe_sched_job.h"
  59#include "xe_sriov.h"
  60#include "xe_tlb_inval.h"
  61#include "xe_tuning.h"
  62#include "xe_uc.h"
  63#include "xe_uc_fw.h"
  64#include "xe_vm.h"
  65#include "xe_wa.h"
  66#include "xe_wopcm.h"
  67
  68struct xe_gt *xe_gt_alloc(struct xe_tile *tile)
  69{
  70	struct xe_device *xe = tile_to_xe(tile);
  71	struct drm_device *drm = &xe->drm;
  72	bool shared_wq = xe->info.needs_shared_vf_gt_wq && tile->primary_gt &&
  73		IS_SRIOV_VF(xe);
  74	struct workqueue_struct *ordered_wq;
  75	struct xe_gt *gt;
  76
  77	gt = drmm_kzalloc(drm, sizeof(*gt), GFP_KERNEL);
  78	if (!gt)
  79		return ERR_PTR(-ENOMEM);
  80
  81	gt->tile = tile;
  82	if (shared_wq && tile->primary_gt->ordered_wq)
  83		ordered_wq = tile->primary_gt->ordered_wq;
  84	else
  85		ordered_wq = drmm_alloc_ordered_workqueue(drm, "gt-ordered-wq",
  86							  WQ_MEM_RECLAIM);
  87	if (IS_ERR(ordered_wq))
  88		return ERR_CAST(ordered_wq);
  89
  90	gt->ordered_wq = ordered_wq;
  91
  92	return gt;
  93}
  94
  95void xe_gt_sanitize(struct xe_gt *gt)
  96{
  97	/*
  98	 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
  99	 * reload
 100	 */
 101	xe_guc_submit_disable(&gt->uc.guc);
 102}
 103
 104static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
 105{
 106	unsigned int fw_ref;
 107	u32 reg;
 108
 109	if (!XE_GT_WA(gt, 16023588340))
 110		return;
 111
 112	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 113	if (!fw_ref)
 114		return;
 115
 116	if (xe_gt_is_main_type(gt)) {
 117		reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
 118		reg |= CG_DIS_CNTLBUS;
 119		xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
 120	}
 121
 122	xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF);
 123	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 124}
 125
 126static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
 127{
 128	unsigned int fw_ref;
 129	u32 reg;
 130
 131	if (!XE_GT_WA(gt, 16023588340))
 132		return;
 133
 134	if (xe_gt_is_media_type(gt))
 135		return;
 136
 137	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 138	if (!fw_ref)
 139		return;
 140
 141	reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
 142	reg &= ~CG_DIS_CNTLBUS;
 143	xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
 144
 145	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 146}
 147
 148static void gt_reset_worker(struct work_struct *w);
 149
 150static int emit_job_sync(struct xe_exec_queue *q, struct xe_bb *bb,
 151			 long timeout_jiffies)
 152{
 153	struct xe_sched_job *job;
 154	struct dma_fence *fence;
 155	long timeout;
 156
 157	job = xe_bb_create_job(q, bb);
 158	if (IS_ERR(job))
 159		return PTR_ERR(job);
 160
 161	xe_sched_job_arm(job);
 162	fence = dma_fence_get(&job->drm.s_fence->finished);
 163	xe_sched_job_push(job);
 164
 165	timeout = dma_fence_wait_timeout(fence, false, timeout_jiffies);
 166	dma_fence_put(fence);
 167	if (timeout < 0)
 168		return timeout;
 169	else if (!timeout)
 170		return -ETIME;
 171
 172	return 0;
 173}
 174
 175static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q)
 176{
 177	struct xe_bb *bb;
 178	int ret;
 179
 180	bb = xe_bb_new(gt, 4, false);
 181	if (IS_ERR(bb))
 182		return PTR_ERR(bb);
 183
 184	ret = emit_job_sync(q, bb, HZ);
 185	xe_bb_free(bb, NULL);
 186
 187	return ret;
 188}
 189
 190/* Dwords required to emit a RMW of a register */
 191#define EMIT_RMW_DW 20
 192
 193static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q)
 194{
 195	struct xe_hw_engine *hwe = q->hwe;
 196	struct xe_reg_sr *sr = &hwe->reg_lrc;
 197	struct xe_reg_sr_entry *entry;
 198	int count_rmw = 0, count_rmw_mcr = 0, count = 0, ret;
 199	unsigned long idx;
 200	struct xe_bb *bb;
 201	size_t bb_len = 0;
 202	u32 *cs;
 203
 204	/* count RMW registers as those will be handled separately */
 205	xa_for_each(&sr->xa, idx, entry) {
 206		if (entry->reg.masked || entry->clr_bits == ~0)
 207			++count;
 208		else if (entry->reg.mcr)
 209			++count_rmw_mcr;
 210		else
 211			++count_rmw;
 212	}
 213
 214	if (count)
 215		bb_len += count * 2 + 1;
 216
 217	/*
 218	 * RMW of MCR registers is the same as a normal RMW, except an
 219	 * additional LRI (3 dwords) is required per register to steer the read
 220	 * to a nom-terminated instance.
 221	 *
 222	 * We could probably shorten the batch slightly by eliding the
 223	 * steering for consecutive MCR registers that have the same
 224	 * group/instance target, but it's not worth the extra complexity to do
 225	 * so.
 226	 */
 227	bb_len += count_rmw * EMIT_RMW_DW;
 228	bb_len += count_rmw_mcr * (EMIT_RMW_DW + 3);
 229
 230	/*
 231	 * After doing all RMW, we need 7 trailing dwords to clean up,
 232	 * plus an additional 3 dwords to reset steering if any of the
 233	 * registers were MCR.
 234	 */
 235	if (count_rmw || count_rmw_mcr)
 236		bb_len += 7 + (count_rmw_mcr ? 3 : 0);
 237
 238	if (hwe->class == XE_ENGINE_CLASS_RENDER)
 239		/*
 240		 * Big enough to emit all of the context's 3DSTATE via
 241		 * xe_lrc_emit_hwe_state_instructions()
 242		 */
 243		bb_len += xe_gt_lrc_size(gt, hwe->class) / sizeof(u32);
 244
 245	xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", hwe->name, bb_len);
 246
 247	bb = xe_bb_new(gt, bb_len, false);
 248	if (IS_ERR(bb))
 249		return PTR_ERR(bb);
 250
 251	cs = bb->cs;
 252
 253	if (count) {
 254		/*
 255		 * Emit single LRI with all non RMW regs: 1 leading dw + 2dw per
 256		 * reg + 1
 257		 */
 258
 259		*cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count);
 260
 261		xa_for_each(&sr->xa, idx, entry) {
 262			struct xe_reg reg = entry->reg;
 263			u32 val;
 264
 265			if (reg.masked)
 266				val = entry->clr_bits << 16;
 267			else if (entry->clr_bits == ~0)
 268				val = 0;
 269			else
 270				continue;
 271
 272			val |= entry->set_bits;
 273
 274			*cs++ = reg.addr;
 275			*cs++ = val;
 276			xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
 277		}
 278	}
 279
 280	if (count_rmw || count_rmw_mcr) {
 281		xa_for_each(&sr->xa, idx, entry) {
 282			if (entry->reg.masked || entry->clr_bits == ~0)
 283				continue;
 284
 285			if (entry->reg.mcr) {
 286				struct xe_reg_mcr reg = { .__reg.raw = entry->reg.raw };
 287				u8 group, instance;
 288
 289				xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance);
 290				*cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1);
 291				*cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(hwe->mmio_base).addr;
 292				*cs++ = SELECTIVE_READ_ADDRESSING |
 293					REG_FIELD_PREP(SELECTIVE_READ_GROUP, group) |
 294					REG_FIELD_PREP(SELECTIVE_READ_INSTANCE, instance);
 295			}
 296
 297			*cs++ = MI_LOAD_REGISTER_REG | MI_LRR_DST_CS_MMIO;
 298			*cs++ = entry->reg.addr;
 299			*cs++ = CS_GPR_REG(0, 0).addr;
 300
 301			*cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) |
 302				MI_LRI_LRM_CS_MMIO;
 303			*cs++ = CS_GPR_REG(0, 1).addr;
 304			*cs++ = entry->clr_bits;
 305			*cs++ = CS_GPR_REG(0, 2).addr;
 306			*cs++ = entry->set_bits;
 307
 308			*cs++ = MI_MATH(8);
 309			*cs++ = CS_ALU_INSTR_LOAD(SRCA, REG0);
 310			*cs++ = CS_ALU_INSTR_LOADINV(SRCB, REG1);
 311			*cs++ = CS_ALU_INSTR_AND;
 312			*cs++ = CS_ALU_INSTR_STORE(REG0, ACCU);
 313			*cs++ = CS_ALU_INSTR_LOAD(SRCA, REG0);
 314			*cs++ = CS_ALU_INSTR_LOAD(SRCB, REG2);
 315			*cs++ = CS_ALU_INSTR_OR;
 316			*cs++ = CS_ALU_INSTR_STORE(REG0, ACCU);
 317
 318			*cs++ = MI_LOAD_REGISTER_REG | MI_LRR_SRC_CS_MMIO;
 319			*cs++ = CS_GPR_REG(0, 0).addr;
 320			*cs++ = entry->reg.addr;
 321
 322			xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x%s\n",
 323				  entry->reg.addr, entry->clr_bits, entry->set_bits,
 324				  entry->reg.mcr ? " (MCR)" : "");
 325		}
 326
 327		/* reset used GPR */
 328		*cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(3) |
 329			MI_LRI_LRM_CS_MMIO;
 330		*cs++ = CS_GPR_REG(0, 0).addr;
 331		*cs++ = 0;
 332		*cs++ = CS_GPR_REG(0, 1).addr;
 333		*cs++ = 0;
 334		*cs++ = CS_GPR_REG(0, 2).addr;
 335		*cs++ = 0;
 336
 337		/* reset steering */
 338		if (count_rmw_mcr) {
 339			*cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1);
 340			*cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(q->hwe->mmio_base).addr;
 341			*cs++ = 0;
 342		}
 343	}
 344
 345	cs = xe_lrc_emit_hwe_state_instructions(q, cs);
 346
 347	bb->len = cs - bb->cs;
 348
 349	ret = emit_job_sync(q, bb, HZ);
 350
 351	xe_bb_free(bb, NULL);
 352
 353	return ret;
 354}
 355
 356int xe_gt_record_default_lrcs(struct xe_gt *gt)
 357{
 358	struct xe_device *xe = gt_to_xe(gt);
 359	struct xe_hw_engine *hwe;
 360	enum xe_hw_engine_id id;
 361	int err = 0;
 362
 363	for_each_hw_engine(hwe, gt, id) {
 364		struct xe_exec_queue *q, *nop_q;
 365		void *default_lrc;
 366
 367		if (gt->default_lrc[hwe->class])
 368			continue;
 369
 370		xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
 371		xe_wa_process_lrc(hwe);
 372		xe_hw_engine_setup_default_lrc_state(hwe);
 373		xe_tuning_process_lrc(hwe);
 374
 375		default_lrc = drmm_kzalloc(&xe->drm,
 376					   xe_gt_lrc_size(gt, hwe->class),
 377					   GFP_KERNEL);
 378		if (!default_lrc)
 379			return -ENOMEM;
 380
 381		q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1,
 382					 hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
 383		if (IS_ERR(q)) {
 384			err = PTR_ERR(q);
 385			xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n",
 386				  hwe->name, q);
 387			return err;
 388		}
 389
 390		/* Prime golden LRC with known good state */
 391		err = emit_wa_job(gt, q);
 392		if (err) {
 393			xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n",
 394				  hwe->name, ERR_PTR(err), q->guc->id);
 395			goto put_exec_queue;
 396		}
 397
 398		nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance),
 399					     1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
 400		if (IS_ERR(nop_q)) {
 401			err = PTR_ERR(nop_q);
 402			xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n",
 403				  hwe->name, nop_q);
 404			goto put_exec_queue;
 405		}
 406
 407		/* Switch to different LRC */
 408		err = emit_nop_job(gt, nop_q);
 409		if (err) {
 410			xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n",
 411				  hwe->name, ERR_PTR(err), nop_q->guc->id);
 412			goto put_nop_q;
 413		}
 414
 415		xe_map_memcpy_from(xe, default_lrc,
 416				   &q->lrc[0]->bo->vmap,
 417				   xe_lrc_pphwsp_offset(q->lrc[0]),
 418				   xe_gt_lrc_size(gt, hwe->class));
 419
 420		gt->default_lrc[hwe->class] = default_lrc;
 421put_nop_q:
 422		xe_exec_queue_put(nop_q);
 423put_exec_queue:
 424		xe_exec_queue_put(q);
 425		if (err)
 426			break;
 427	}
 428
 429	return err;
 430}
 431
 432int xe_gt_init_early(struct xe_gt *gt)
 433{
 434	unsigned int fw_ref;
 435	int err;
 436
 437	if (IS_SRIOV_PF(gt_to_xe(gt))) {
 438		err = xe_gt_sriov_pf_init_early(gt);
 439		if (err)
 440			return err;
 441	}
 442
 443	if (IS_SRIOV_VF(gt_to_xe(gt))) {
 444		err = xe_gt_sriov_vf_init_early(gt);
 445		if (err)
 446			return err;
 447	}
 448
 449	xe_reg_sr_init(&gt->reg_sr, "GT", gt_to_xe(gt));
 450
 451	err = xe_wa_gt_init(gt);
 452	if (err)
 453		return err;
 454
 455	err = xe_tuning_init(gt);
 456	if (err)
 457		return err;
 458
 459	xe_wa_process_gt_oob(gt);
 460
 461	xe_force_wake_init_gt(gt, gt_to_fw(gt));
 462	spin_lock_init(&gt->global_invl_lock);
 463
 464	err = xe_gt_tlb_inval_init_early(gt);
 465	if (err)
 466		return err;
 467
 468	xe_mocs_init_early(gt);
 469
 470	/*
 471	 * Only after this point can GT-specific MMIO operations
 472	 * (including things like communication with the GuC)
 473	 * be performed.
 474	 */
 475	xe_gt_mmio_init(gt);
 476
 477	err = xe_uc_init_noalloc(&gt->uc);
 478	if (err)
 479		return err;
 480
 481	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 482	if (!fw_ref)
 483		return -ETIMEDOUT;
 484
 485	xe_gt_mcr_init_early(gt);
 486	xe_pat_init(gt);
 487	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 488
 489	return 0;
 490}
 491
 492static void dump_pat_on_error(struct xe_gt *gt)
 493{
 494	struct drm_printer p;
 495	char prefix[32];
 496
 497	snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id);
 498	p = drm_dbg_printer(&gt_to_xe(gt)->drm, DRM_UT_DRIVER, prefix);
 499
 500	xe_pat_dump(gt, &p);
 501}
 502
 503static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 504{
 505	unsigned int fw_ref;
 506	int err;
 507
 508	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 509	if (!fw_ref)
 510		return -ETIMEDOUT;
 511
 512	err = xe_uc_init(&gt->uc);
 513	if (err)
 514		goto err_force_wake;
 515
 516	xe_gt_topology_init(gt);
 517	xe_gt_mcr_init(gt);
 518	xe_gt_enable_host_l2_vram(gt);
 519
 520	if (xe_gt_is_main_type(gt)) {
 521		err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
 522		if (err)
 523			goto err_force_wake;
 524		if (IS_SRIOV_PF(gt_to_xe(gt)))
 525			xe_lmtt_init(&gt_to_tile(gt)->sriov.pf.lmtt);
 526	}
 527
 528	/* Enable per hw engine IRQs */
 529	xe_irq_enable_hwe(gt);
 530
 531	/* Rerun MCR init as we now have hw engine list */
 532	xe_gt_mcr_init(gt);
 533
 534	err = xe_hw_engines_init_early(gt);
 535	if (err) {
 536		dump_pat_on_error(gt);
 537		goto err_force_wake;
 538	}
 539
 540	err = xe_hw_engine_class_sysfs_init(gt);
 541	if (err)
 542		goto err_force_wake;
 543
 544	/* Initialize CCS mode sysfs after early initialization of HW engines */
 545	err = xe_gt_ccs_mode_sysfs_init(gt);
 546	if (err)
 547		goto err_force_wake;
 548
 549	/*
 550	 * Stash hardware-reported version.  Since this register does not exist
 551	 * on pre-MTL platforms, reading it there will (correctly) return 0.
 552	 */
 553	gt->info.gmdid = xe_mmio_read32(&gt->mmio, GMD_ID);
 554
 555	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 556	return 0;
 557
 558err_force_wake:
 559	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 560
 561	return err;
 562}
 563
 564static int gt_init_with_all_forcewake(struct xe_gt *gt)
 565{
 566	unsigned int fw_ref;
 567	int err;
 568
 569	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 570	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
 571		err = -ETIMEDOUT;
 572		goto err_force_wake;
 573	}
 574
 575	xe_gt_mcr_set_implicit_defaults(gt);
 576	xe_wa_process_gt(gt);
 577	xe_tuning_process_gt(gt);
 578	xe_reg_sr_apply_mmio(&gt->reg_sr, gt);
 579
 580	err = xe_gt_clock_init(gt);
 581	if (err)
 582		goto err_force_wake;
 583
 584	xe_mocs_init(gt);
 585	err = xe_execlist_init(gt);
 586	if (err)
 587		goto err_force_wake;
 588
 589	err = xe_hw_engines_init(gt);
 590	if (err)
 591		goto err_force_wake;
 592
 593	err = xe_uc_init_post_hwconfig(&gt->uc);
 594	if (err)
 595		goto err_force_wake;
 596
 597	if (xe_gt_is_main_type(gt)) {
 598		/*
 599		 * USM has its only SA pool to non-block behind user operations
 600		 */
 601		if (gt_to_xe(gt)->info.has_usm) {
 602			struct xe_device *xe = gt_to_xe(gt);
 603
 604			gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
 605								IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
 606			if (IS_ERR(gt->usm.bb_pool)) {
 607				err = PTR_ERR(gt->usm.bb_pool);
 608				goto err_force_wake;
 609			}
 610		}
 611	}
 612
 613	if (xe_gt_is_main_type(gt)) {
 614		struct xe_tile *tile = gt_to_tile(gt);
 615
 616		err = xe_migrate_init(tile->migrate);
 617		if (err)
 618			goto err_force_wake;
 619	}
 620
 621	err = xe_uc_load_hw(&gt->uc);
 622	if (err)
 623		goto err_force_wake;
 624
 625	/* Configure default CCS mode of 1 engine with all resources */
 626	if (xe_gt_ccs_mode_enabled(gt)) {
 627		gt->ccs_mode = 1;
 628		xe_gt_apply_ccs_mode(gt);
 629	}
 630
 631	if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_is_main_type(gt))
 632		xe_lmtt_init_hw(&gt_to_tile(gt)->sriov.pf.lmtt);
 633
 634	if (IS_SRIOV_PF(gt_to_xe(gt)))
 635		xe_gt_sriov_pf_init_hw(gt);
 636
 637	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 638
 639	return 0;
 640
 641err_force_wake:
 642	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 643
 644	return err;
 645}
 646
 647static void xe_gt_fini(void *arg)
 648{
 649	struct xe_gt *gt = arg;
 650	int i;
 651
 652	if (disable_work_sync(&gt->reset.worker))
 653		/*
 654		 * If gt_reset_worker was halted from executing, take care of
 655		 * releasing the rpm reference here.
 656		 */
 657		xe_pm_runtime_put(gt_to_xe(gt));
 658
 659	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
 660		xe_hw_fence_irq_finish(&gt->fence_irq[i]);
 661
 662	xe_gt_disable_host_l2_vram(gt);
 663}
 664
 665int xe_gt_init(struct xe_gt *gt)
 666{
 667	int err;
 668	int i;
 669
 670	INIT_WORK(&gt->reset.worker, gt_reset_worker);
 671
 672	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) {
 673		gt->ring_ops[i] = xe_ring_ops_get(gt, i);
 674		xe_hw_fence_irq_init(&gt->fence_irq[i]);
 675	}
 676
 677	err = devm_add_action_or_reset(gt_to_xe(gt)->drm.dev, xe_gt_fini, gt);
 678	if (err)
 679		return err;
 680
 681	err = xe_gt_sysfs_init(gt);
 682	if (err)
 683		return err;
 684
 685	err = gt_init_with_gt_forcewake(gt);
 686	if (err)
 687		return err;
 688
 689	err = xe_gt_idle_init(&gt->gtidle);
 690	if (err)
 691		return err;
 692
 693	err = xe_gt_freq_init(gt);
 694	if (err)
 695		return err;
 696
 697	xe_force_wake_init_engines(gt, gt_to_fw(gt));
 698
 699	err = gt_init_with_all_forcewake(gt);
 700	if (err)
 701		return err;
 702
 703	xe_gt_record_user_engines(gt);
 704
 705	err = xe_eu_stall_init(gt);
 706	if (err)
 707		return err;
 708
 709	if (IS_SRIOV_VF(gt_to_xe(gt))) {
 710		err = xe_gt_sriov_vf_init(gt);
 711		if (err)
 712			return err;
 713	}
 714
 715	return 0;
 716}
 717
 718/**
 719 * xe_gt_mmio_init() - Initialize GT's MMIO access
 720 * @gt: the GT object
 721 *
 722 * Initialize GT's MMIO accessor, which will be used to access registers inside
 723 * this GT.
 724 */
 725void xe_gt_mmio_init(struct xe_gt *gt)
 726{
 727	struct xe_tile *tile = gt_to_tile(gt);
 728	struct xe_device *xe = tile_to_xe(tile);
 729
 730	xe_mmio_init(&gt->mmio, tile, tile->mmio.regs, tile->mmio.regs_size);
 731
 732	if (gt->info.type == XE_GT_TYPE_MEDIA) {
 733		gt->mmio.adj_offset = MEDIA_GT_GSI_OFFSET;
 734		gt->mmio.adj_limit = MEDIA_GT_GSI_LENGTH;
 735	} else {
 736		gt->mmio.adj_offset = 0;
 737		gt->mmio.adj_limit = 0;
 738	}
 739
 740	if (IS_SRIOV_VF(xe))
 741		gt->mmio.sriov_vf_gt = gt;
 742}
 743
 744void xe_gt_record_user_engines(struct xe_gt *gt)
 745{
 746	struct xe_hw_engine *hwe;
 747	enum xe_hw_engine_id id;
 748
 749	gt->user_engines.mask = 0;
 750	memset(gt->user_engines.instances_per_class, 0,
 751	       sizeof(gt->user_engines.instances_per_class));
 752
 753	for_each_hw_engine(hwe, gt, id) {
 754		if (xe_hw_engine_is_reserved(hwe))
 755			continue;
 756
 757		gt->user_engines.mask |= BIT_ULL(id);
 758		gt->user_engines.instances_per_class[hwe->class]++;
 759	}
 760
 761	xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask)
 762		     == gt->info.engine_mask);
 763}
 764
 765static int do_gt_reset(struct xe_gt *gt)
 766{
 767	int err;
 768
 769	if (IS_SRIOV_VF(gt_to_xe(gt)))
 770		return xe_gt_sriov_vf_reset(gt);
 771
 772	xe_gsc_wa_14015076503(gt, true);
 773
 774	xe_mmio_write32(&gt->mmio, GDRST, GRDOM_FULL);
 775	err = xe_mmio_wait32(&gt->mmio, GDRST, GRDOM_FULL, 0, 5000, NULL, false);
 776	if (err)
 777		xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n",
 778			  ERR_PTR(err));
 779
 780	xe_gsc_wa_14015076503(gt, false);
 781
 782	return err;
 783}
 784
 785static int vf_gt_restart(struct xe_gt *gt)
 786{
 787	int err;
 788
 789	err = xe_uc_sanitize_reset(&gt->uc);
 790	if (err)
 791		return err;
 792
 793	err = xe_uc_load_hw(&gt->uc);
 794	if (err)
 795		return err;
 796
 797	err = xe_uc_start(&gt->uc);
 798	if (err)
 799		return err;
 800
 801	return 0;
 802}
 803
 804static int do_gt_restart(struct xe_gt *gt)
 805{
 806	struct xe_hw_engine *hwe;
 807	enum xe_hw_engine_id id;
 808	int err;
 809
 810	if (IS_SRIOV_VF(gt_to_xe(gt)))
 811		return vf_gt_restart(gt);
 812
 813	xe_pat_init(gt);
 814
 815	xe_gt_enable_host_l2_vram(gt);
 816
 817	xe_gt_mcr_set_implicit_defaults(gt);
 818	xe_reg_sr_apply_mmio(&gt->reg_sr, gt);
 819
 820	err = xe_wopcm_init(&gt->uc.wopcm);
 821	if (err)
 822		return err;
 823
 824	for_each_hw_engine(hwe, gt, id)
 825		xe_hw_engine_enable_ring(hwe);
 826
 827	err = xe_uc_sanitize_reset(&gt->uc);
 828	if (err)
 829		return err;
 830
 831	err = xe_uc_load_hw(&gt->uc);
 832	if (err)
 833		return err;
 834
 835	if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_is_main_type(gt))
 836		xe_lmtt_init_hw(&gt_to_tile(gt)->sriov.pf.lmtt);
 837
 838	if (IS_SRIOV_PF(gt_to_xe(gt)))
 839		xe_gt_sriov_pf_init_hw(gt);
 840
 841	xe_mocs_init(gt);
 842
 843	for_each_hw_engine(hwe, gt, id)
 844		xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
 845
 846	/* Get CCS mode in sync between sw/hw */
 847	xe_gt_apply_ccs_mode(gt);
 848
 849	err = xe_uc_start(&gt->uc);
 850	if (err)
 851		return err;
 852
 853	/* Restore GT freq to expected values */
 854	xe_gt_sanitize_freq(gt);
 855
 856	if (IS_SRIOV_PF(gt_to_xe(gt)))
 857		xe_gt_sriov_pf_restart(gt);
 858
 859	return 0;
 860}
 861
 862static void gt_reset_worker(struct work_struct *w)
 863{
 864	struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker);
 865	unsigned int fw_ref;
 866	int err;
 867
 868	if (xe_device_wedged(gt_to_xe(gt)))
 869		goto err_pm_put;
 870
 871	/* We only support GT resets with GuC submission */
 872	if (!xe_device_uc_enabled(gt_to_xe(gt)))
 873		goto err_pm_put;
 874
 875	xe_gt_info(gt, "reset started\n");
 876
 877	if (xe_fault_inject_gt_reset()) {
 878		err = -ECANCELED;
 879		goto err_fail;
 880	}
 881
 882	xe_gt_sanitize(gt);
 883
 884	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 885	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
 886		err = -ETIMEDOUT;
 887		goto err_out;
 888	}
 889
 890	if (IS_SRIOV_PF(gt_to_xe(gt)))
 891		xe_gt_sriov_pf_stop_prepare(gt);
 892
 893	xe_uc_gucrc_disable(&gt->uc);
 894	xe_uc_stop_prepare(&gt->uc);
 895	xe_pagefault_reset(gt_to_xe(gt), gt);
 896
 897	xe_uc_stop(&gt->uc);
 898
 899	xe_tlb_inval_reset(&gt->tlb_inval);
 900
 901	err = do_gt_reset(gt);
 902	if (err)
 903		goto err_out;
 904
 905	err = do_gt_restart(gt);
 906	if (err)
 907		goto err_out;
 908
 909	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 910
 911	/* Pair with get while enqueueing the work in xe_gt_reset_async() */
 912	xe_pm_runtime_put(gt_to_xe(gt));
 913
 914	xe_gt_info(gt, "reset done\n");
 915
 916	return;
 917
 918err_out:
 919	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 920	XE_WARN_ON(xe_uc_start(&gt->uc));
 921
 922err_fail:
 923	xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
 924	xe_device_declare_wedged(gt_to_xe(gt));
 925err_pm_put:
 926	xe_pm_runtime_put(gt_to_xe(gt));
 927}
 928
 929void xe_gt_reset_async(struct xe_gt *gt)
 930{
 931	xe_gt_info(gt, "trying reset from %ps\n", __builtin_return_address(0));
 932
 933	/* Don't do a reset while one is already in flight */
 934	if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(&gt->uc))
 935		return;
 936
 937	xe_gt_info(gt, "reset queued\n");
 938
 939	/* Pair with put in gt_reset_worker() if work is enqueued */
 940	xe_pm_runtime_get_noresume(gt_to_xe(gt));
 941	if (!queue_work(gt->ordered_wq, &gt->reset.worker))
 942		xe_pm_runtime_put(gt_to_xe(gt));
 943}
 944
 945void xe_gt_suspend_prepare(struct xe_gt *gt)
 946{
 947	unsigned int fw_ref;
 948
 949	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 950
 951	xe_uc_suspend_prepare(&gt->uc);
 952
 953	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 954}
 955
 956int xe_gt_suspend(struct xe_gt *gt)
 957{
 958	unsigned int fw_ref;
 959	int err;
 960
 961	xe_gt_dbg(gt, "suspending\n");
 962	xe_gt_sanitize(gt);
 963
 964	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 965	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
 966		goto err_msg;
 967
 968	err = xe_uc_suspend(&gt->uc);
 969	if (err)
 970		goto err_force_wake;
 971
 972	xe_gt_idle_disable_pg(gt);
 973
 974	xe_gt_disable_host_l2_vram(gt);
 975
 976	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 977	xe_gt_dbg(gt, "suspended\n");
 978
 979	return 0;
 980
 981err_msg:
 982	err = -ETIMEDOUT;
 983err_force_wake:
 984	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 985	xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
 986
 987	return err;
 988}
 989
 990void xe_gt_shutdown(struct xe_gt *gt)
 991{
 992	unsigned int fw_ref;
 993
 994	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 995	do_gt_reset(gt);
 996	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 997}
 998
 999/**
1000 * xe_gt_sanitize_freq() - Restore saved frequencies if necessary.
1001 * @gt: the GT object
1002 *
1003 * Called after driver init/GSC load completes to restore GT frequencies if we
1004 * limited them for any WAs.
1005 */
1006int xe_gt_sanitize_freq(struct xe_gt *gt)
1007{
1008	int ret = 0;
1009
1010	if ((!xe_uc_fw_is_available(&gt->uc.gsc.fw) ||
1011	     xe_uc_fw_is_loaded(&gt->uc.gsc.fw) ||
1012	     xe_uc_fw_is_in_error_state(&gt->uc.gsc.fw)) &&
1013	    XE_GT_WA(gt, 22019338487))
1014		ret = xe_guc_pc_restore_stashed_freq(&gt->uc.guc.pc);
1015
1016	return ret;
1017}
1018
1019int xe_gt_resume(struct xe_gt *gt)
1020{
1021	unsigned int fw_ref;
1022	int err;
1023
1024	xe_gt_dbg(gt, "resuming\n");
1025	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
1026	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
1027		goto err_msg;
1028
1029	err = do_gt_restart(gt);
1030	if (err)
1031		goto err_force_wake;
1032
1033	xe_gt_idle_enable_pg(gt);
1034
1035	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1036	xe_gt_dbg(gt, "resumed\n");
1037
1038	return 0;
1039
1040err_msg:
1041	err = -ETIMEDOUT;
1042err_force_wake:
1043	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1044	xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
1045
1046	return err;
1047}
1048
1049struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,
1050				     enum xe_engine_class class,
1051				     u16 instance, bool logical)
1052{
1053	struct xe_hw_engine *hwe;
1054	enum xe_hw_engine_id id;
1055
1056	for_each_hw_engine(hwe, gt, id)
1057		if (hwe->class == class &&
1058		    ((!logical && hwe->instance == instance) ||
1059		    (logical && hwe->logical_instance == instance)))
1060			return hwe;
1061
1062	return NULL;
1063}
1064
1065struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt,
1066							 enum xe_engine_class class)
1067{
1068	struct xe_hw_engine *hwe;
1069	enum xe_hw_engine_id id;
1070
1071	for_each_hw_engine(hwe, gt, id) {
1072		switch (class) {
1073		case XE_ENGINE_CLASS_RENDER:
1074		case XE_ENGINE_CLASS_COMPUTE:
1075			if (hwe->class == XE_ENGINE_CLASS_RENDER ||
1076			    hwe->class == XE_ENGINE_CLASS_COMPUTE)
1077				return hwe;
1078			break;
1079		default:
1080			if (hwe->class == class)
1081				return hwe;
1082		}
1083	}
1084
1085	return NULL;
1086}
1087
1088struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt)
1089{
1090	struct xe_hw_engine *hwe;
1091	enum xe_hw_engine_id id;
1092
1093	for_each_hw_engine(hwe, gt, id)
1094		return hwe;
1095
1096	return NULL;
1097}
1098
1099/**
1100 * xe_gt_declare_wedged() - Declare GT wedged
1101 * @gt: the GT object
1102 *
1103 * Wedge the GT which stops all submission, saves desired debug state, and
1104 * cleans up anything which could timeout.
1105 */
1106void xe_gt_declare_wedged(struct xe_gt *gt)
1107{
1108	xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode);
1109
1110	xe_uc_declare_wedged(&gt->uc);
1111	xe_tlb_inval_reset(&gt->tlb_inval);
1112}