xe_guc_ct.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2022 Intel Corporation
   4 */
   5
   6#include "xe_guc_ct.h"
   7
   8#include <linux/bitfield.h>
   9#include <linux/circ_buf.h>
  10#include <linux/delay.h>
  11#include <linux/fault-inject.h>
  12
  13#include <kunit/static_stub.h>
  14
  15#include <drm/drm_managed.h>
  16
  17#include "abi/guc_actions_abi.h"
  18#include "abi/guc_actions_sriov_abi.h"
  19#include "abi/guc_klvs_abi.h"
  20#include "xe_bo.h"
  21#include "xe_devcoredump.h"
  22#include "xe_device.h"
  23#include "xe_gt.h"
  24#include "xe_gt_printk.h"
  25#include "xe_gt_sriov_pf_control.h"
  26#include "xe_gt_sriov_pf_monitor.h"
  27#include "xe_guc.h"
  28#include "xe_guc_log.h"
  29#include "xe_guc_pagefault.h"
  30#include "xe_guc_relay.h"
  31#include "xe_guc_submit.h"
  32#include "xe_guc_tlb_inval.h"
  33#include "xe_map.h"
  34#include "xe_pm.h"
  35#include "xe_sriov_vf.h"
  36#include "xe_trace_guc.h"
  37
  38static void receive_g2h(struct xe_guc_ct *ct);
  39static void g2h_worker_func(struct work_struct *w);
  40static void safe_mode_worker_func(struct work_struct *w);
  41static void ct_exit_safe_mode(struct xe_guc_ct *ct);
  42static void guc_ct_change_state(struct xe_guc_ct *ct,
  43				enum xe_guc_ct_state state);
  44
  45#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
  46enum {
  47	/* Internal states, not error conditions */
  48	CT_DEAD_STATE_REARM,			/* 0x0001 */
  49	CT_DEAD_STATE_CAPTURE,			/* 0x0002 */
  50
  51	/* Error conditions */
  52	CT_DEAD_SETUP,				/* 0x0004 */
  53	CT_DEAD_H2G_WRITE,			/* 0x0008 */
  54	CT_DEAD_H2G_HAS_ROOM,			/* 0x0010 */
  55	CT_DEAD_G2H_READ,			/* 0x0020 */
  56	CT_DEAD_G2H_RECV,			/* 0x0040 */
  57	CT_DEAD_G2H_RELEASE,			/* 0x0080 */
  58	CT_DEAD_DEADLOCK,			/* 0x0100 */
  59	CT_DEAD_PROCESS_FAILED,			/* 0x0200 */
  60	CT_DEAD_FAST_G2H,			/* 0x0400 */
  61	CT_DEAD_PARSE_G2H_RESPONSE,		/* 0x0800 */
  62	CT_DEAD_PARSE_G2H_UNKNOWN,		/* 0x1000 */
  63	CT_DEAD_PARSE_G2H_ORIGIN,		/* 0x2000 */
  64	CT_DEAD_PARSE_G2H_TYPE,			/* 0x4000 */
  65	CT_DEAD_CRASH,				/* 0x8000 */
  66};
  67
  68static void ct_dead_worker_func(struct work_struct *w);
  69static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code);
  70
  71#define CT_DEAD(ct, ctb, reason_code)		ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
  72#else
  73#define CT_DEAD(ct, ctb, reason)			\
  74	do {						\
  75		struct guc_ctb *_ctb = (ctb);		\
  76		if (_ctb)				\
  77			_ctb->info.broken = true;	\
  78	} while (0)
  79#endif
  80
  81/* Used when a CT send wants to block and / or receive data */
  82struct g2h_fence {
  83	u32 *response_buffer;
  84	u32 seqno;
  85	u32 response_data;
  86	u16 response_len;
  87	u16 error;
  88	u16 hint;
  89	u16 reason;
  90	bool cancel;
  91	bool retry;
  92	bool fail;
  93	bool done;
  94};
  95
  96static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
  97{
  98	memset(g2h_fence, 0, sizeof(*g2h_fence));
  99	g2h_fence->response_buffer = response_buffer;
 100	g2h_fence->seqno = ~0x0;
 101}
 102
 103static void g2h_fence_cancel(struct g2h_fence *g2h_fence)
 104{
 105	g2h_fence->cancel = true;
 106	g2h_fence->fail = true;
 107
 108	/* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */
 109	WRITE_ONCE(g2h_fence->done, true);
 110}
 111
 112static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
 113{
 114	return g2h_fence->seqno == ~0x0;
 115}
 116
 117static struct xe_guc *
 118ct_to_guc(struct xe_guc_ct *ct)
 119{
 120	return container_of(ct, struct xe_guc, ct);
 121}
 122
 123static struct xe_gt *
 124ct_to_gt(struct xe_guc_ct *ct)
 125{
 126	return container_of(ct, struct xe_gt, uc.guc.ct);
 127}
 128
 129static struct xe_device *
 130ct_to_xe(struct xe_guc_ct *ct)
 131{
 132	return gt_to_xe(ct_to_gt(ct));
 133}
 134
 135/**
 136 * DOC: GuC CTB Blob
 137 *
 138 * We allocate single blob to hold both CTB descriptors and buffers:
 139 *
 140 *      +--------+-----------------------------------------------+------+
 141 *      | offset | contents                                      | size |
 142 *      +========+===============================================+======+
 143 *      | 0x0000 | H2G CTB Descriptor (send)                     |      |
 144 *      +--------+-----------------------------------------------+  4K  |
 145 *      | 0x0800 | G2H CTB Descriptor (g2h)                      |      |
 146 *      +--------+-----------------------------------------------+------+
 147 *      | 0x1000 | H2G CT Buffer (send)                          | n*4K |
 148 *      |        |                                               |      |
 149 *      +--------+-----------------------------------------------+------+
 150 *      | 0x1000 | G2H CT Buffer (g2h)                           | m*4K |
 151 *      | + n*4K |                                               |      |
 152 *      +--------+-----------------------------------------------+------+
 153 *
 154 * Size of each ``CT Buffer`` must be multiple of 4K.
 155 * We don't expect too many messages in flight at any time, unless we are
 156 * using the GuC submission. In that case each request requires a minimum
 157 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
 158 * enough space to avoid backpressure on the driver. We increase the size
 159 * of the receive buffer (relative to the send) to ensure a G2H response
 160 * CTB has a landing spot.
 161 *
 162 * In addition to submissions, the G2H buffer needs to be able to hold
 163 * enough space for recoverable page fault notifications. The number of
 164 * page faults is interrupt driven and can be as much as the number of
 165 * compute resources available. However, most of the actual work for these
 166 * is in a separate page fault worker thread. Therefore we only need to
 167 * make sure the queue has enough space to handle all of the submissions
 168 * and responses and an extra buffer for incoming page faults.
 169 */
 170
 171#define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
 172#define CTB_H2G_BUFFER_OFFSET	(CTB_DESC_SIZE * 2)
 173#define CTB_H2G_BUFFER_SIZE	(SZ_4K)
 174#define CTB_G2H_BUFFER_SIZE	(SZ_128K)
 175#define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 2)
 176
 177/**
 178 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
 179 * CT command queue
 180 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future.
 181 *
 182 * Observation is that a 4KiB buffer full of commands takes a little over a
 183 * second to process. Use that to calculate maximum time to process a full CT
 184 * command queue.
 185 *
 186 * Return: Maximum time to process a full CT queue in jiffies.
 187 */
 188long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)
 189{
 190	BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));
 191	return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;
 192}
 193
 194static size_t guc_ct_size(void)
 195{
 196	return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE +
 197		CTB_G2H_BUFFER_SIZE;
 198}
 199
 200static void guc_ct_fini(struct drm_device *drm, void *arg)
 201{
 202	struct xe_guc_ct *ct = arg;
 203
 204#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 205	cancel_work_sync(&ct->dead.worker);
 206#endif
 207	ct_exit_safe_mode(ct);
 208	destroy_workqueue(ct->g2h_wq);
 209	xa_destroy(&ct->fence_lookup);
 210}
 211
 212static void primelockdep(struct xe_guc_ct *ct)
 213{
 214	if (!IS_ENABLED(CONFIG_LOCKDEP))
 215		return;
 216
 217	fs_reclaim_acquire(GFP_KERNEL);
 218	might_lock(&ct->lock);
 219	fs_reclaim_release(GFP_KERNEL);
 220}
 221
 222int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct)
 223{
 224	struct xe_device *xe = ct_to_xe(ct);
 225	struct xe_gt *gt = ct_to_gt(ct);
 226	int err;
 227
 228	xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));
 229
 230	err = drmm_mutex_init(&xe->drm, &ct->lock);
 231	if (err)
 232		return err;
 233
 234	primelockdep(ct);
 235
 236	ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM);
 237	if (!ct->g2h_wq)
 238		return -ENOMEM;
 239
 240	spin_lock_init(&ct->fast_lock);
 241	xa_init(&ct->fence_lookup);
 242	INIT_WORK(&ct->g2h_worker, g2h_worker_func);
 243	INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
 244#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 245	spin_lock_init(&ct->dead.lock);
 246	INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
 247#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
 248	stack_depot_init();
 249#endif
 250#endif
 251	init_waitqueue_head(&ct->wq);
 252	init_waitqueue_head(&ct->g2h_fence_wq);
 253
 254	err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);
 255	if (err)
 256		return err;
 257
 258	xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);
 259	ct->state = XE_GUC_CT_STATE_DISABLED;
 260	return 0;
 261}
 262ALLOW_ERROR_INJECTION(xe_guc_ct_init_noalloc, ERRNO); /* See xe_pci_probe() */
 263
 264static void guc_action_disable_ct(void *arg)
 265{
 266	struct xe_guc_ct *ct = arg;
 267
 268	xe_guc_ct_stop(ct);
 269	guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED);
 270}
 271
 272int xe_guc_ct_init(struct xe_guc_ct *ct)
 273{
 274	struct xe_device *xe = ct_to_xe(ct);
 275	struct xe_gt *gt = ct_to_gt(ct);
 276	struct xe_tile *tile = gt_to_tile(gt);
 277	struct xe_bo *bo;
 278
 279	bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
 280					  XE_BO_FLAG_SYSTEM |
 281					  XE_BO_FLAG_GGTT |
 282					  XE_BO_FLAG_GGTT_INVALIDATE |
 283					  XE_BO_FLAG_PINNED_NORESTORE);
 284	if (IS_ERR(bo))
 285		return PTR_ERR(bo);
 286
 287	ct->bo = bo;
 288
 289	return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct);
 290}
 291ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */
 292
 293/**
 294 * xe_guc_ct_init_post_hwconfig - Reinitialize the GuC CTB in VRAM
 295 * @ct: the &xe_guc_ct
 296 *
 297 * Allocate a new BO in VRAM and free the previous BO that was allocated
 298 * in system memory (SMEM). Applicable only for DGFX products.
 299 *
 300 * Return: 0 on success, or a negative errno on failure.
 301 */
 302int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct)
 303{
 304	struct xe_device *xe = ct_to_xe(ct);
 305	struct xe_gt *gt = ct_to_gt(ct);
 306	struct xe_tile *tile = gt_to_tile(gt);
 307	int ret;
 308
 309	xe_assert(xe, !xe_guc_ct_enabled(ct));
 310
 311	if (IS_DGFX(xe)) {
 312		ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->bo);
 313		if (ret)
 314			return ret;
 315	}
 316
 317	devm_remove_action(xe->drm.dev, guc_action_disable_ct, ct);
 318	return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct);
 319}
 320
 321#define desc_read(xe_, guc_ctb__, field_)			\
 322	xe_map_rd_field(xe_, &guc_ctb__->desc, 0,		\
 323			struct guc_ct_buffer_desc, field_)
 324
 325#define desc_write(xe_, guc_ctb__, field_, val_)		\
 326	xe_map_wr_field(xe_, &guc_ctb__->desc, 0,		\
 327			struct guc_ct_buffer_desc, field_, val_)
 328
 329static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
 330				struct iosys_map *map)
 331{
 332	h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
 333	h2g->info.resv_space = 0;
 334	h2g->info.tail = 0;
 335	h2g->info.head = 0;
 336	h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
 337				     h2g->info.size) -
 338			  h2g->info.resv_space;
 339	h2g->info.broken = false;
 340
 341	h2g->desc = *map;
 342	xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
 343
 344	h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET);
 345}
 346
 347static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
 348				struct iosys_map *map)
 349{
 350	g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
 351	g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
 352	g2h->info.head = 0;
 353	g2h->info.tail = 0;
 354	g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
 355				     g2h->info.size) -
 356			  g2h->info.resv_space;
 357	g2h->info.broken = false;
 358
 359	g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
 360	xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
 361
 362	g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET +
 363					    CTB_H2G_BUFFER_SIZE);
 364}
 365
 366static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
 367{
 368	struct xe_guc *guc = ct_to_guc(ct);
 369	u32 desc_addr, ctb_addr, size;
 370	int err;
 371
 372	desc_addr = xe_bo_ggtt_addr(ct->bo);
 373	ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_H2G_BUFFER_OFFSET;
 374	size = ct->ctbs.h2g.info.size * sizeof(u32);
 375
 376	err = xe_guc_self_cfg64(guc,
 377				GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,
 378				desc_addr);
 379	if (err)
 380		return err;
 381
 382	err = xe_guc_self_cfg64(guc,
 383				GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,
 384				ctb_addr);
 385	if (err)
 386		return err;
 387
 388	return xe_guc_self_cfg32(guc,
 389				 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,
 390				 size);
 391}
 392
 393static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
 394{
 395	struct xe_guc *guc = ct_to_guc(ct);
 396	u32 desc_addr, ctb_addr, size;
 397	int err;
 398
 399	desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
 400	ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_H2G_BUFFER_OFFSET +
 401		CTB_H2G_BUFFER_SIZE;
 402	size = ct->ctbs.g2h.info.size * sizeof(u32);
 403
 404	err = xe_guc_self_cfg64(guc,
 405				GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
 406				desc_addr);
 407	if (err)
 408		return err;
 409
 410	err = xe_guc_self_cfg64(guc,
 411				GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
 412				ctb_addr);
 413	if (err)
 414		return err;
 415
 416	return xe_guc_self_cfg32(guc,
 417				 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
 418				 size);
 419}
 420
 421static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)
 422{
 423	u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
 424		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
 425		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
 426		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
 427			   GUC_ACTION_HOST2GUC_CONTROL_CTB),
 428		FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,
 429			   enable ? GUC_CTB_CONTROL_ENABLE :
 430			   GUC_CTB_CONTROL_DISABLE),
 431	};
 432	int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));
 433
 434	return ret > 0 ? -EPROTO : ret;
 435}
 436
 437static void guc_ct_change_state(struct xe_guc_ct *ct,
 438				enum xe_guc_ct_state state)
 439{
 440	struct xe_gt *gt = ct_to_gt(ct);
 441	struct g2h_fence *g2h_fence;
 442	unsigned long idx;
 443
 444	mutex_lock(&ct->lock);		/* Serialise dequeue_one_g2h() */
 445	spin_lock_irq(&ct->fast_lock);	/* Serialise CT fast-path */
 446
 447	xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||
 448		     state == XE_GUC_CT_STATE_STOPPED);
 449
 450	if (ct->g2h_outstanding)
 451		xe_pm_runtime_put(ct_to_xe(ct));
 452	ct->g2h_outstanding = 0;
 453	ct->state = state;
 454
 455	xe_gt_dbg(gt, "GuC CT communication channel %s\n",
 456		  state == XE_GUC_CT_STATE_STOPPED ? "stopped" :
 457		  str_enabled_disabled(state == XE_GUC_CT_STATE_ENABLED));
 458
 459	spin_unlock_irq(&ct->fast_lock);
 460
 461	/* cancel all in-flight send-recv requests */
 462	xa_for_each(&ct->fence_lookup, idx, g2h_fence)
 463		g2h_fence_cancel(g2h_fence);
 464
 465	/* make sure guc_ct_send_recv() will see g2h_fence changes */
 466	smp_mb();
 467	wake_up_all(&ct->g2h_fence_wq);
 468
 469	/*
 470	 * Lockdep doesn't like this under the fast lock and he destroy only
 471	 * needs to be serialized with the send path which ct lock provides.
 472	 */
 473	xa_destroy(&ct->fence_lookup);
 474
 475	mutex_unlock(&ct->lock);
 476}
 477
 478static bool ct_needs_safe_mode(struct xe_guc_ct *ct)
 479{
 480	return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev));
 481}
 482
 483static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct)
 484{
 485	if (!ct_needs_safe_mode(ct))
 486		return false;
 487
 488	queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10);
 489	return true;
 490}
 491
 492static void safe_mode_worker_func(struct work_struct *w)
 493{
 494	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work);
 495
 496	receive_g2h(ct);
 497
 498	if (!ct_restart_safe_mode_worker(ct))
 499		xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n");
 500}
 501
 502static void ct_enter_safe_mode(struct xe_guc_ct *ct)
 503{
 504	if (ct_restart_safe_mode_worker(ct))
 505		xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n");
 506}
 507
 508static void ct_exit_safe_mode(struct xe_guc_ct *ct)
 509{
 510	if (cancel_delayed_work_sync(&ct->safe_mode_worker))
 511		xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n");
 512}
 513
 514static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register)
 515{
 516	struct xe_device *xe = ct_to_xe(ct);
 517	struct xe_gt *gt = ct_to_gt(ct);
 518	int err;
 519
 520	xe_gt_assert(gt, !xe_guc_ct_enabled(ct));
 521
 522	if (needs_register) {
 523		xe_map_memset(xe, &ct->bo->vmap, 0, 0, xe_bo_size(ct->bo));
 524		guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
 525		guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
 526
 527		err = guc_ct_ctb_h2g_register(ct);
 528		if (err)
 529			goto err_out;
 530
 531		err = guc_ct_ctb_g2h_register(ct);
 532		if (err)
 533			goto err_out;
 534
 535		err = guc_ct_control_toggle(ct, true);
 536		if (err)
 537			goto err_out;
 538	} else {
 539		ct->ctbs.h2g.info.broken = false;
 540		ct->ctbs.g2h.info.broken = false;
 541		/* Skip everything in H2G buffer */
 542		xe_map_memset(xe, &ct->bo->vmap, CTB_H2G_BUFFER_OFFSET, 0,
 543			      CTB_H2G_BUFFER_SIZE);
 544	}
 545
 546	guc_ct_change_state(ct, XE_GUC_CT_STATE_ENABLED);
 547
 548	smp_mb();
 549	wake_up_all(&ct->wq);
 550
 551	if (ct_needs_safe_mode(ct))
 552		ct_enter_safe_mode(ct);
 553
 554#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 555	/*
 556	 * The CT has now been reset so the dumper can be re-armed
 557	 * after any existing dead state has been dumped.
 558	 */
 559	spin_lock_irq(&ct->dead.lock);
 560	if (ct->dead.reason) {
 561		ct->dead.reason |= (1 << CT_DEAD_STATE_REARM);
 562		queue_work(system_unbound_wq, &ct->dead.worker);
 563	}
 564	spin_unlock_irq(&ct->dead.lock);
 565#endif
 566
 567	return 0;
 568
 569err_out:
 570	xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));
 571	CT_DEAD(ct, NULL, SETUP);
 572
 573	return err;
 574}
 575
 576/**
 577 * xe_guc_ct_restart() - Restart GuC CT
 578 * @ct: the &xe_guc_ct
 579 *
 580 * Restart GuC CT to an empty state without issuing a CT register MMIO command.
 581 *
 582 * Return: 0 on success, or a negative errno on failure.
 583 */
 584int xe_guc_ct_restart(struct xe_guc_ct *ct)
 585{
 586	return __xe_guc_ct_start(ct, false);
 587}
 588
 589/**
 590 * xe_guc_ct_enable() - Enable GuC CT
 591 * @ct: the &xe_guc_ct
 592 *
 593 * Enable GuC CT to an empty state and issue a CT register MMIO command.
 594 *
 595 * Return: 0 on success, or a negative errno on failure.
 596 */
 597int xe_guc_ct_enable(struct xe_guc_ct *ct)
 598{
 599	return __xe_guc_ct_start(ct, true);
 600}
 601
 602static void stop_g2h_handler(struct xe_guc_ct *ct)
 603{
 604	cancel_work_sync(&ct->g2h_worker);
 605}
 606
 607/**
 608 * xe_guc_ct_disable - Set GuC to disabled state
 609 * @ct: the &xe_guc_ct
 610 *
 611 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected
 612 * in this transition.
 613 */
 614void xe_guc_ct_disable(struct xe_guc_ct *ct)
 615{
 616	guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED);
 617	ct_exit_safe_mode(ct);
 618	stop_g2h_handler(ct);
 619}
 620
 621/**
 622 * xe_guc_ct_flush_and_stop - Flush and stop all processing of G2H / H2G
 623 * @ct: the &xe_guc_ct
 624 */
 625void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct)
 626{
 627	receive_g2h(ct);
 628	xe_guc_ct_stop(ct);
 629}
 630
 631/**
 632 * xe_guc_ct_stop - Set GuC to stopped state
 633 * @ct: the &xe_guc_ct
 634 *
 635 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h
 636 */
 637void xe_guc_ct_stop(struct xe_guc_ct *ct)
 638{
 639	if (!xe_guc_ct_initialized(ct))
 640		return;
 641
 642	guc_ct_change_state(ct, XE_GUC_CT_STATE_STOPPED);
 643	stop_g2h_handler(ct);
 644}
 645
 646static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
 647{
 648	struct guc_ctb *h2g = &ct->ctbs.h2g;
 649
 650	lockdep_assert_held(&ct->lock);
 651
 652	if (cmd_len > h2g->info.space) {
 653		h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);
 654
 655		if (h2g->info.head > h2g->info.size) {
 656			struct xe_device *xe = ct_to_xe(ct);
 657			u32 desc_status = desc_read(xe, h2g, status);
 658
 659			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
 660
 661			xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n",
 662				  h2g->info.head, h2g->info.size);
 663			CT_DEAD(ct, h2g, H2G_HAS_ROOM);
 664			return false;
 665		}
 666
 667		h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
 668					     h2g->info.size) -
 669				  h2g->info.resv_space;
 670		if (cmd_len > h2g->info.space)
 671			return false;
 672	}
 673
 674	return true;
 675}
 676
 677static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)
 678{
 679	if (!g2h_len)
 680		return true;
 681
 682	lockdep_assert_held(&ct->fast_lock);
 683
 684	return ct->ctbs.g2h.info.space > g2h_len;
 685}
 686
 687static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)
 688{
 689	lockdep_assert_held(&ct->lock);
 690
 691	if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))
 692		return -EBUSY;
 693
 694	return 0;
 695}
 696
 697static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
 698{
 699	lockdep_assert_held(&ct->lock);
 700	ct->ctbs.h2g.info.space -= cmd_len;
 701}
 702
 703static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
 704{
 705	xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
 706	xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||
 707		     (g2h_len && num_g2h));
 708
 709	if (g2h_len) {
 710		lockdep_assert_held(&ct->fast_lock);
 711
 712		if (!ct->g2h_outstanding)
 713			xe_pm_runtime_get_noresume(ct_to_xe(ct));
 714
 715		ct->ctbs.g2h.info.space -= g2h_len;
 716		ct->g2h_outstanding += num_g2h;
 717	}
 718}
 719
 720static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
 721{
 722	bool bad = false;
 723
 724	lockdep_assert_held(&ct->fast_lock);
 725
 726	bad = ct->ctbs.g2h.info.space + g2h_len >
 727		     ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space;
 728	bad |= !ct->g2h_outstanding;
 729
 730	if (bad) {
 731		xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n",
 732			  ct->ctbs.g2h.info.space, g2h_len,
 733			  ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space,
 734			  ct->ctbs.g2h.info.space + g2h_len,
 735			  ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space,
 736			  ct->g2h_outstanding);
 737		CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE);
 738		return;
 739	}
 740
 741	ct->ctbs.g2h.info.space += g2h_len;
 742	if (!--ct->g2h_outstanding)
 743		xe_pm_runtime_put(ct_to_xe(ct));
 744}
 745
 746static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
 747{
 748	spin_lock_irq(&ct->fast_lock);
 749	__g2h_release_space(ct, g2h_len);
 750	spin_unlock_irq(&ct->fast_lock);
 751}
 752
 753#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
 754static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
 755{
 756	unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
 757#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
 758	unsigned long entries[SZ_32];
 759	unsigned int n;
 760
 761	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
 762
 763	/* May be called under spinlock, so avoid sleeping */
 764	ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
 765#endif
 766	ct->fast_req[slot].fence = fence;
 767	ct->fast_req[slot].action = action;
 768}
 769#else
 770static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
 771{
 772}
 773#endif
 774
 775/*
 776 * The CT protocol accepts a 16 bits fence. This field is fully owned by the
 777 * driver, the GuC will just copy it to the reply message. Since we need to
 778 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages,
 779 * we use one bit of the seqno as an indicator for that and a rolling counter
 780 * for the remaining 15 bits.
 781 */
 782#define CT_SEQNO_MASK GENMASK(14, 0)
 783#define CT_SEQNO_UNTRACKED BIT(15)
 784static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence)
 785{
 786	u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK;
 787
 788	if (!is_g2h_fence)
 789		seqno |= CT_SEQNO_UNTRACKED;
 790
 791	return seqno;
 792}
 793
 794#define MAKE_ACTION(type, __action)				\
 795({								\
 796	FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |			\
 797	FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |			\
 798		   GUC_HXG_EVENT_MSG_0_DATA0, __action);	\
 799})
 800
 801static bool vf_action_can_safely_fail(struct xe_device *xe, u32 action)
 802{
 803	/*
 804	 * When resuming a VF, we can't reliably track whether context
 805	 * registration has completed in the GuC state machine. It is harmless
 806	 * to resend the request, as it will fail silently if GUC_HXG_TYPE_EVENT
 807	 * is used. Additionally, if there is an H2G protocol issue on a VF,
 808	 * subsequent H2G messages sent as GUC_HXG_TYPE_FAST_REQUEST will likely
 809	 * fail.
 810	 */
 811	return IS_SRIOV_VF(xe) && xe_sriov_vf_migration_supported(xe) &&
 812		(action == XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC ||
 813		 action == XE_GUC_ACTION_REGISTER_CONTEXT);
 814}
 815
 816#define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
 817
 818static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 819		     u32 ct_fence_value, bool want_response)
 820{
 821	struct xe_device *xe = ct_to_xe(ct);
 822	struct xe_gt *gt = ct_to_gt(ct);
 823	struct guc_ctb *h2g = &ct->ctbs.h2g;
 824	u32 cmd[H2G_CT_HEADERS];
 825	u32 tail = h2g->info.tail;
 826	u32 full_len;
 827	struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
 828							 tail * sizeof(u32));
 829	u32 desc_status;
 830
 831	full_len = len + GUC_CTB_HDR_LEN;
 832
 833	lockdep_assert_held(&ct->lock);
 834	xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
 835
 836	desc_status = desc_read(xe, h2g, status);
 837	if (desc_status) {
 838		xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status);
 839		goto corrupted;
 840	}
 841
 842	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
 843		u32 desc_tail = desc_read(xe, h2g, tail);
 844		u32 desc_head = desc_read(xe, h2g, head);
 845
 846		if (tail != desc_tail) {
 847			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH);
 848			xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail);
 849			goto corrupted;
 850		}
 851
 852		if (tail > h2g->info.size) {
 853			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
 854			xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n",
 855				  tail, h2g->info.size);
 856			goto corrupted;
 857		}
 858
 859		if (desc_head >= h2g->info.size) {
 860			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
 861			xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n",
 862				  desc_head, h2g->info.size);
 863			goto corrupted;
 864		}
 865	}
 866
 867	/* Command will wrap, zero fill (NOPs), return and check credits again */
 868	if (tail + full_len > h2g->info.size) {
 869		xe_map_memset(xe, &map, 0, 0,
 870			      (h2g->info.size - tail) * sizeof(u32));
 871		h2g_reserve_space(ct, (h2g->info.size - tail));
 872		h2g->info.tail = 0;
 873		desc_write(xe, h2g, tail, h2g->info.tail);
 874
 875		return -EAGAIN;
 876	}
 877
 878	/*
 879	 * dw0: CT header (including fence)
 880	 * dw1: HXG header (including action code)
 881	 * dw2+: action data
 882	 */
 883	cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
 884		FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
 885		FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);
 886	if (want_response) {
 887		cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_REQUEST, action[0]);
 888	} else if (vf_action_can_safely_fail(xe, action[0])) {
 889		cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_EVENT, action[0]);
 890	} else {
 891		fast_req_track(ct, ct_fence_value,
 892			       FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));
 893
 894		cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_FAST_REQUEST, action[0]);
 895	}
 896
 897	/* H2G header in cmd[1] replaces action[0] so: */
 898	--len;
 899	++action;
 900
 901	/* Write H2G ensuring visible before descriptor update */
 902	xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
 903	xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
 904	xe_device_wmb(xe);
 905
 906	/* Update local copies */
 907	h2g->info.tail = (tail + full_len) % h2g->info.size;
 908	h2g_reserve_space(ct, full_len);
 909
 910	/* Update descriptor */
 911	desc_write(xe, h2g, tail, h2g->info.tail);
 912
 913	trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,
 914			     desc_read(xe, h2g, head), h2g->info.tail);
 915
 916	return 0;
 917
 918corrupted:
 919	CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE);
 920	return -EPIPE;
 921}
 922
 923static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 924				u32 len, u32 g2h_len, u32 num_g2h,
 925				struct g2h_fence *g2h_fence)
 926{
 927	struct xe_gt *gt = ct_to_gt(ct);
 928	u16 seqno;
 929	int ret;
 930
 931	xe_gt_assert(gt, xe_guc_ct_initialized(ct));
 932	xe_gt_assert(gt, !g2h_len || !g2h_fence);
 933	xe_gt_assert(gt, !num_g2h || !g2h_fence);
 934	xe_gt_assert(gt, !g2h_len || num_g2h);
 935	xe_gt_assert(gt, g2h_len || !num_g2h);
 936	lockdep_assert_held(&ct->lock);
 937
 938	if (unlikely(ct->ctbs.h2g.info.broken)) {
 939		ret = -EPIPE;
 940		goto out;
 941	}
 942
 943	if (ct->state == XE_GUC_CT_STATE_DISABLED) {
 944		ret = -ENODEV;
 945		goto out;
 946	}
 947
 948	if (ct->state == XE_GUC_CT_STATE_STOPPED || xe_gt_recovery_pending(gt)) {
 949		ret = -ECANCELED;
 950		goto out;
 951	}
 952
 953	xe_gt_assert(gt, xe_guc_ct_enabled(ct));
 954
 955	if (g2h_fence) {
 956		g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
 957		num_g2h = 1;
 958
 959		if (g2h_fence_needs_alloc(g2h_fence)) {
 960			g2h_fence->seqno = next_ct_seqno(ct, true);
 961			ret = xa_err(xa_store(&ct->fence_lookup,
 962					      g2h_fence->seqno, g2h_fence,
 963					      GFP_ATOMIC));
 964			if (ret)
 965				goto out;
 966		}
 967
 968		seqno = g2h_fence->seqno;
 969	} else {
 970		seqno = next_ct_seqno(ct, false);
 971	}
 972
 973	if (g2h_len)
 974		spin_lock_irq(&ct->fast_lock);
 975retry:
 976	ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);
 977	if (unlikely(ret))
 978		goto out_unlock;
 979
 980	ret = h2g_write(ct, action, len, seqno, !!g2h_fence);
 981	if (unlikely(ret)) {
 982		if (ret == -EAGAIN)
 983			goto retry;
 984		goto out_unlock;
 985	}
 986
 987	__g2h_reserve_space(ct, g2h_len, num_g2h);
 988	xe_guc_notify(ct_to_guc(ct));
 989out_unlock:
 990	if (g2h_len)
 991		spin_unlock_irq(&ct->fast_lock);
 992out:
 993	return ret;
 994}
 995
 996static void kick_reset(struct xe_guc_ct *ct)
 997{
 998	xe_gt_reset_async(ct_to_gt(ct));
 999}
1000
1001static int dequeue_one_g2h(struct xe_guc_ct *ct);
1002
1003/*
1004 * wait before retry of sending h2g message
1005 * Return: true if ready for retry, false if the wait timeouted
1006 */
1007static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len,
1008				       u32 g2h_len, struct g2h_fence *g2h_fence,
1009				       unsigned int *sleep_period_ms)
1010{
1011	struct xe_device *xe = ct_to_xe(ct);
1012
1013	/*
1014	 * We wait to try to restore credits for about 1 second before bailing.
1015	 * In the case of H2G credits we have no choice but just to wait for the
1016	 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In
1017	 * the case of G2H we process any G2H in the channel, hopefully freeing
1018	 * credits as we consume the G2H messages.
1019	 */
1020	if (!h2g_has_room(ct, len + GUC_CTB_HDR_LEN)) {
1021		struct guc_ctb *h2g = &ct->ctbs.h2g;
1022
1023		if (*sleep_period_ms == 1024)
1024			return false;
1025
1026		trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,
1027						 h2g->info.size,
1028						 h2g->info.space,
1029						 len + GUC_CTB_HDR_LEN);
1030		msleep(*sleep_period_ms);
1031		*sleep_period_ms <<= 1;
1032	} else {
1033		struct xe_device *xe = ct_to_xe(ct);
1034		struct guc_ctb *g2h = &ct->ctbs.g2h;
1035		int ret;
1036
1037		trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head,
1038						 desc_read(xe, g2h, tail),
1039						 g2h->info.size,
1040						 g2h->info.space,
1041						 g2h_fence ?
1042						 GUC_CTB_HXG_MSG_MAX_LEN :
1043						 g2h_len);
1044
1045#define g2h_avail(ct)	\
1046	(desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)
1047		if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||
1048					g2h_avail(ct), HZ))
1049			return false;
1050#undef g2h_avail
1051
1052		ret = dequeue_one_g2h(ct);
1053		if (ret < 0) {
1054			if (ret != -ECANCELED)
1055				xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)",
1056					  ERR_PTR(ret));
1057			return false;
1058		}
1059	}
1060	return true;
1061}
1062
1063static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
1064			      u32 g2h_len, u32 num_g2h,
1065			      struct g2h_fence *g2h_fence)
1066{
1067	struct xe_gt *gt = ct_to_gt(ct);
1068	unsigned int sleep_period_ms = 1;
1069	int ret;
1070
1071	xe_gt_assert(gt, !g2h_len || !g2h_fence);
1072	lockdep_assert_held(&ct->lock);
1073	xe_device_assert_mem_access(ct_to_xe(ct));
1074
1075try_again:
1076	ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
1077				   g2h_fence);
1078
1079	if (unlikely(ret == -EBUSY)) {
1080		if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence,
1081						&sleep_period_ms))
1082			goto broken;
1083		goto try_again;
1084	}
1085
1086	return ret;
1087
1088broken:
1089	xe_gt_err(gt, "No forward process on H2G, reset required\n");
1090	CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK);
1091
1092	return -EDEADLK;
1093}
1094
1095static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
1096		       u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)
1097{
1098	int ret;
1099
1100	xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
1101
1102	mutex_lock(&ct->lock);
1103	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
1104	mutex_unlock(&ct->lock);
1105
1106	return ret;
1107}
1108
1109int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
1110		   u32 g2h_len, u32 num_g2h)
1111{
1112	int ret;
1113
1114	ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);
1115	if (ret == -EDEADLK)
1116		kick_reset(ct);
1117
1118	return ret;
1119}
1120
1121int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
1122			  u32 g2h_len, u32 num_g2h)
1123{
1124	int ret;
1125
1126	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
1127	if (ret == -EDEADLK)
1128		kick_reset(ct);
1129
1130	return ret;
1131}
1132
1133int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
1134{
1135	int ret;
1136
1137	lockdep_assert_held(&ct->lock);
1138
1139	ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
1140	if (ret == -EDEADLK)
1141		kick_reset(ct);
1142
1143	return ret;
1144}
1145
1146/*
1147 * Check if a GT reset is in progress or will occur and if GT reset brought the
1148 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.
1149 */
1150static bool retry_failure(struct xe_guc_ct *ct, int ret)
1151{
1152	if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))
1153		return false;
1154
1155#define ct_alive(ct)	\
1156	(xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \
1157	 !ct->ctbs.g2h.info.broken)
1158	if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))
1159		return false;
1160#undef ct_alive
1161
1162	return true;
1163}
1164
1165#define GUC_SEND_RETRY_LIMIT	50
1166#define GUC_SEND_RETRY_MSLEEP	5
1167
1168static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
1169			    u32 *response_buffer, bool no_fail)
1170{
1171	struct xe_gt *gt = ct_to_gt(ct);
1172	struct g2h_fence g2h_fence;
1173	unsigned int retries = 0;
1174	int ret = 0;
1175
1176	/*
1177	 * We use a fence to implement blocking sends / receiving response data.
1178	 * The seqno of the fence is sent in the H2G, returned in the G2H, and
1179	 * an xarray is used as storage media with the seqno being to key.
1180	 * Fields in the fence hold success, failure, retry status and the
1181	 * response data. Safe to allocate on the stack as the xarray is the
1182	 * only reference and it cannot be present after this function exits.
1183	 */
1184retry:
1185	g2h_fence_init(&g2h_fence, response_buffer);
1186retry_same_fence:
1187	ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);
1188	if (unlikely(ret == -ENOMEM)) {
1189		/* Retry allocation /w GFP_KERNEL */
1190		ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno,
1191				      &g2h_fence, GFP_KERNEL));
1192		if (ret)
1193			return ret;
1194
1195		goto retry_same_fence;
1196	} else if (unlikely(ret)) {
1197		if (ret == -EDEADLK)
1198			kick_reset(ct);
1199
1200		if (no_fail && retry_failure(ct, ret))
1201			goto retry_same_fence;
1202
1203		if (!g2h_fence_needs_alloc(&g2h_fence))
1204			xa_erase(&ct->fence_lookup, g2h_fence.seqno);
1205
1206		return ret;
1207	}
1208
1209	/* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response
1210	 * and g2h_fence_cancel.
1211	 */
1212	ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ);
1213	if (!ret) {
1214		LNL_FLUSH_WORK(&ct->g2h_worker);
1215		if (READ_ONCE(g2h_fence.done)) {
1216			xe_gt_warn(gt, "G2H fence %u, action %04x, done\n",
1217				   g2h_fence.seqno, action[0]);
1218			ret = 1;
1219		}
1220	}
1221
1222	/*
1223	 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on
1224	 * the stack, since we have no clue if it will fire after the timeout before we can erase
1225	 * from the xa. Also we have some dependent loads and stores below for which we need the
1226	 * correct ordering, and we lack the needed barriers.
1227	 */
1228	mutex_lock(&ct->lock);
1229	if (!ret) {
1230		xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s",
1231			  g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done));
1232		xa_erase(&ct->fence_lookup, g2h_fence.seqno);
1233		mutex_unlock(&ct->lock);
1234		return -ETIME;
1235	}
1236
1237	if (g2h_fence.retry) {
1238		xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",
1239			  action[0], g2h_fence.reason);
1240		mutex_unlock(&ct->lock);
1241		if (++retries > GUC_SEND_RETRY_LIMIT) {
1242			xe_gt_err(gt, "H2G action %#x reached retry limit=%u, aborting\n",
1243				  action[0], GUC_SEND_RETRY_LIMIT);
1244			return -ELOOP;
1245		}
1246		msleep(GUC_SEND_RETRY_MSLEEP * retries);
1247		goto retry;
1248	}
1249	if (g2h_fence.fail) {
1250		if (g2h_fence.cancel) {
1251			xe_gt_dbg(gt, "H2G request %#x canceled!\n", action[0]);
1252			ret = -ECANCELED;
1253			goto unlock;
1254		}
1255		xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",
1256			  action[0], g2h_fence.error, g2h_fence.hint);
1257		ret = -EIO;
1258	}
1259
1260	if (ret > 0)
1261		ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data;
1262
1263unlock:
1264	mutex_unlock(&ct->lock);
1265
1266	return ret;
1267}
1268
1269/**
1270 * xe_guc_ct_send_recv - Send and receive HXG to the GuC
1271 * @ct: the &xe_guc_ct
1272 * @action: the dword array with `HXG Request`_ message (can't be NULL)
1273 * @len: length of the `HXG Request`_ message (in dwords, can't be 0)
1274 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL)
1275 *
1276 * Send a `HXG Request`_ message to the GuC over CT communication channel and
1277 * blocks until GuC replies with a `HXG Response`_ message.
1278 *
1279 * For non-blocking communication with GuC use xe_guc_ct_send().
1280 *
1281 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_.
1282 *
1283 * Return: response length (in dwords) if &response_buffer was not NULL, or
1284 *         DATA0 from `HXG Response`_ if &response_buffer was NULL, or
1285 *         a negative error code on failure.
1286 */
1287int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
1288			u32 *response_buffer)
1289{
1290	KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer);
1291	return guc_ct_send_recv(ct, action, len, response_buffer, false);
1292}
1293ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO);
1294
1295int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,
1296				u32 len, u32 *response_buffer)
1297{
1298	return guc_ct_send_recv(ct, action, len, response_buffer, true);
1299}
1300
1301static u32 *msg_to_hxg(u32 *msg)
1302{
1303	return msg + GUC_CTB_MSG_MIN_LEN;
1304}
1305
1306static u32 msg_len_to_hxg_len(u32 len)
1307{
1308	return len - GUC_CTB_MSG_MIN_LEN;
1309}
1310
1311static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
1312{
1313	u32 *hxg = msg_to_hxg(msg);
1314	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1315
1316	lockdep_assert_held(&ct->lock);
1317
1318	switch (action) {
1319	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1320	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1321	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1322	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1323		g2h_release_space(ct, len);
1324	}
1325
1326	return 0;
1327}
1328
1329static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action)
1330{
1331	struct xe_gt *gt = ct_to_gt(ct);
1332
1333	if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED)
1334		xe_gt_err(gt, "GuC Crash dump notification\n");
1335	else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION)
1336		xe_gt_err(gt, "GuC Exception notification\n");
1337	else
1338		xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action);
1339
1340	CT_DEAD(ct, NULL, CRASH);
1341
1342	kick_reset(ct);
1343
1344	return 0;
1345}
1346
1347#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
1348static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
1349{
1350	u16 fence_min = U16_MAX, fence_max = 0;
1351	struct xe_gt *gt = ct_to_gt(ct);
1352	bool found = false;
1353	unsigned int n;
1354#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
1355	char *buf;
1356#endif
1357
1358	lockdep_assert_held(&ct->lock);
1359
1360	for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
1361		if (ct->fast_req[n].fence < fence_min)
1362			fence_min = ct->fast_req[n].fence;
1363		if (ct->fast_req[n].fence > fence_max)
1364			fence_max = ct->fast_req[n].fence;
1365
1366		if (ct->fast_req[n].fence != fence)
1367			continue;
1368		found = true;
1369
1370#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
1371		buf = kmalloc(SZ_4K, GFP_NOWAIT);
1372		if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0))
1373			xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s",
1374				  fence, ct->fast_req[n].action, buf);
1375		else
1376			xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
1377				  fence, ct->fast_req[n].action);
1378		kfree(buf);
1379#else
1380		xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
1381			  fence, ct->fast_req[n].action);
1382#endif
1383		break;
1384	}
1385
1386	if (!found)
1387		xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
1388			   fence, fence_min, fence_max, ct->fence_seqno);
1389}
1390#else
1391static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
1392{
1393}
1394#endif
1395
1396static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
1397{
1398	struct xe_gt *gt =  ct_to_gt(ct);
1399	u32 *hxg = msg_to_hxg(msg);
1400	u32 hxg_len = msg_len_to_hxg_len(len);
1401	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
1402	u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1403	struct g2h_fence *g2h_fence;
1404
1405	lockdep_assert_held(&ct->lock);
1406
1407	/*
1408	 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup.
1409	 * Those messages should never fail, so if we do get an error back it
1410	 * means we're likely doing an illegal operation and the GuC is
1411	 * rejecting it. We have no way to inform the code that submitted the
1412	 * H2G that the message was rejected, so we need to escalate the
1413	 * failure to trigger a reset.
1414	 */
1415	if (fence & CT_SEQNO_UNTRACKED) {
1416		if (type == GUC_HXG_TYPE_RESPONSE_FAILURE)
1417			xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n",
1418				  fence,
1419				  FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]),
1420				  FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]));
1421		else
1422			xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n",
1423				  type, fence);
1424
1425		fast_req_report(ct, fence);
1426
1427		/* FIXME: W/A race in the GuC, will get in firmware soon */
1428		if (xe_gt_recovery_pending(gt))
1429			return 0;
1430
1431		CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE);
1432
1433		return -EPROTO;
1434	}
1435
1436	g2h_fence = xa_erase(&ct->fence_lookup, fence);
1437	if (unlikely(!g2h_fence)) {
1438		/* Don't tear down channel, as send could've timed out */
1439		/* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */
1440		xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence);
1441		g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1442		return 0;
1443	}
1444
1445	xe_gt_assert(gt, fence == g2h_fence->seqno);
1446
1447	if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
1448		g2h_fence->fail = true;
1449		g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);
1450		g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]);
1451	} else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1452		g2h_fence->retry = true;
1453		g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);
1454	} else if (g2h_fence->response_buffer) {
1455		g2h_fence->response_len = hxg_len;
1456		memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));
1457	} else {
1458		g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);
1459	}
1460
1461	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
1462
1463	/* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */
1464	WRITE_ONCE(g2h_fence->done, true);
1465	smp_mb();
1466
1467	wake_up_all(&ct->g2h_fence_wq);
1468
1469	return 0;
1470}
1471
1472static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1473{
1474	struct xe_gt *gt = ct_to_gt(ct);
1475	u32 *hxg = msg_to_hxg(msg);
1476	u32 origin, type;
1477	int ret;
1478
1479	lockdep_assert_held(&ct->lock);
1480
1481	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1482	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1483		xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",
1484			  origin);
1485		CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN);
1486
1487		return -EPROTO;
1488	}
1489
1490	type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1491	switch (type) {
1492	case GUC_HXG_TYPE_EVENT:
1493		ret = parse_g2h_event(ct, msg, len);
1494		break;
1495	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1496	case GUC_HXG_TYPE_RESPONSE_FAILURE:
1497	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1498		ret = parse_g2h_response(ct, msg, len);
1499		break;
1500	default:
1501		xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",
1502			  type);
1503		CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE);
1504
1505		ret = -EOPNOTSUPP;
1506	}
1507
1508	return ret;
1509}
1510
1511static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
1512{
1513	struct xe_guc *guc = ct_to_guc(ct);
1514	struct xe_gt *gt = ct_to_gt(ct);
1515	u32 hxg_len = msg_len_to_hxg_len(len);
1516	u32 *hxg = msg_to_hxg(msg);
1517	u32 action, adj_len;
1518	u32 *payload;
1519	int ret = 0;
1520
1521	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1522		return 0;
1523
1524	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1525	payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN;
1526	adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN;
1527
1528	switch (action) {
1529	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1530		ret = xe_guc_sched_done_handler(guc, payload, adj_len);
1531		break;
1532	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1533		ret = xe_guc_deregister_done_handler(guc, payload, adj_len);
1534		break;
1535	case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1536		ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);
1537		break;
1538	case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1539		ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,
1540							      adj_len);
1541		break;
1542	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1543		/* Selftest only at the moment */
1544		break;
1545	case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1546		ret = xe_guc_error_capture_handler(guc, payload, adj_len);
1547		break;
1548	case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1549		/* FIXME: Handle this */
1550		break;
1551	case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:
1552		ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
1553								 adj_len);
1554		break;
1555	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1556		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1557		break;
1558	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1559		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
1560		break;
1561	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
1562		ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
1563		break;
1564	case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:
1565		ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);
1566		break;
1567	case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:
1568		ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);
1569		break;
1570	case GUC_ACTION_GUC2PF_ADVERSE_EVENT:
1571		ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len);
1572		break;
1573	case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1574	case XE_GUC_ACTION_NOTIFY_EXCEPTION:
1575		ret = guc_crash_process_msg(ct, action);
1576		break;
1577#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1578	case XE_GUC_ACTION_TEST_G2G_RECV:
1579		ret = xe_guc_g2g_test_notification(guc, payload, adj_len);
1580		break;
1581#endif
1582	default:
1583		xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1584	}
1585
1586	if (ret) {
1587		xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n",
1588			  action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg);
1589		CT_DEAD(ct, NULL, PROCESS_FAILED);
1590	}
1591
1592	return 0;
1593}
1594
1595static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
1596{
1597	struct xe_device *xe = ct_to_xe(ct);
1598	struct xe_gt *gt = ct_to_gt(ct);
1599	struct guc_ctb *g2h = &ct->ctbs.g2h;
1600	u32 tail, head, len, desc_status;
1601	s32 avail;
1602	u32 action;
1603	u32 *hxg;
1604
1605	xe_gt_assert(gt, xe_guc_ct_initialized(ct));
1606	lockdep_assert_held(&ct->fast_lock);
1607
1608	if (ct->state == XE_GUC_CT_STATE_DISABLED)
1609		return -ENODEV;
1610
1611	if (ct->state == XE_GUC_CT_STATE_STOPPED)
1612		return -ECANCELED;
1613
1614	if (g2h->info.broken)
1615		return -EPIPE;
1616
1617	xe_gt_assert(gt, xe_guc_ct_enabled(ct));
1618
1619	desc_status = desc_read(xe, g2h, status);
1620	if (desc_status) {
1621		if (desc_status & GUC_CTB_STATUS_DISABLED) {
1622			/*
1623			 * Potentially valid if a CLIENT_RESET request resulted in
1624			 * contexts/engines being reset. But should never happen as
1625			 * no contexts should be active when CLIENT_RESET is sent.
1626			 */
1627			xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n");
1628			desc_status &= ~GUC_CTB_STATUS_DISABLED;
1629		}
1630
1631		if (desc_status) {
1632			xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status);
1633			goto corrupted;
1634		}
1635	}
1636
1637	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
1638		u32 desc_tail = desc_read(xe, g2h, tail);
1639		/*
1640		u32 desc_head = desc_read(xe, g2h, head);
1641
1642		 * info.head and desc_head are updated back-to-back at the end of
1643		 * this function and nowhere else. Hence, they cannot be different
1644		 * unless two g2h_read calls are running concurrently. Which is not
1645		 * possible because it is guarded by ct->fast_lock. And yet, some
1646		 * discrete platforms are regularly hitting this error :(.
1647		 *
1648		 * desc_head rolling backwards shouldn't cause any noticeable
1649		 * problems - just a delay in GuC being allowed to proceed past that
1650		 * point in the queue. So for now, just disable the error until it
1651		 * can be root caused.
1652		 *
1653		if (g2h->info.head != desc_head) {
1654			desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH);
1655			xe_gt_err(gt, "CT read: head was modified %u != %u\n",
1656				  desc_head, g2h->info.head);
1657			goto corrupted;
1658		}
1659		 */
1660
1661		if (g2h->info.head > g2h->info.size) {
1662			desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1663			xe_gt_err(gt, "CT read: head out of range: %u vs %u\n",
1664				  g2h->info.head, g2h->info.size);
1665			goto corrupted;
1666		}
1667
1668		if (desc_tail >= g2h->info.size) {
1669			desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
1670			xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n",
1671				  desc_tail, g2h->info.size);
1672			goto corrupted;
1673		}
1674	}
1675
1676	/* Calculate DW available to read */
1677	tail = desc_read(xe, g2h, tail);
1678	avail = tail - g2h->info.head;
1679	if (unlikely(avail == 0))
1680		return 0;
1681
1682	if (avail < 0)
1683		avail += g2h->info.size;
1684
1685	/* Read header */
1686	xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,
1687			   sizeof(u32));
1688	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
1689	if (len > avail) {
1690		xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",
1691			  avail, len);
1692		goto corrupted;
1693	}
1694
1695	head = (g2h->info.head + 1) % g2h->info.size;
1696	avail = len - 1;
1697
1698	/* Read G2H message */
1699	if (avail + head > g2h->info.size) {
1700		u32 avail_til_wrap = g2h->info.size - head;
1701
1702		xe_map_memcpy_from(xe, msg + 1,
1703				   &g2h->cmds, sizeof(u32) * head,
1704				   avail_til_wrap * sizeof(u32));
1705		xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,
1706				   &g2h->cmds, 0,
1707				   (avail - avail_til_wrap) * sizeof(u32));
1708	} else {
1709		xe_map_memcpy_from(xe, msg + 1,
1710				   &g2h->cmds, sizeof(u32) * head,
1711				   avail * sizeof(u32));
1712	}
1713
1714	hxg = msg_to_hxg(msg);
1715	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1716
1717	if (fast_path) {
1718		if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)
1719			return 0;
1720
1721		switch (action) {
1722		case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1723		case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1724			break;	/* Process these in fast-path */
1725		default:
1726			return 0;
1727		}
1728	}
1729
1730	/* Update local / descriptor header */
1731	g2h->info.head = (head + avail) % g2h->info.size;
1732	desc_write(xe, g2h, head, g2h->info.head);
1733
1734	trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id,
1735			     action, len, g2h->info.head, tail);
1736
1737	return len;
1738
1739corrupted:
1740	CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ);
1741	return -EPROTO;
1742}
1743
1744static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
1745{
1746	struct xe_gt *gt = ct_to_gt(ct);
1747	struct xe_guc *guc = ct_to_guc(ct);
1748	u32 hxg_len = msg_len_to_hxg_len(len);
1749	u32 *hxg = msg_to_hxg(msg);
1750	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1751	u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN;
1752	u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1753	int ret = 0;
1754
1755	switch (action) {
1756	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1757		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1758		break;
1759	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1760		__g2h_release_space(ct, len);
1761		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
1762		break;
1763	default:
1764		xe_gt_warn(gt, "NOT_POSSIBLE");
1765	}
1766
1767	if (ret) {
1768		xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",
1769			  action, ERR_PTR(ret));
1770		CT_DEAD(ct, NULL, FAST_G2H);
1771	}
1772}
1773
1774/**
1775 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler
1776 * @ct: GuC CT object
1777 *
1778 * Anything related to page faults is critical for performance, process these
1779 * critical G2H in the IRQ. This is safe as these handlers either just wake up
1780 * waiters or queue another worker.
1781 */
1782void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
1783{
1784	struct xe_device *xe = ct_to_xe(ct);
1785	bool ongoing;
1786	int len;
1787
1788	ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1789	if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1790		return;
1791
1792	spin_lock(&ct->fast_lock);
1793	do {
1794		len = g2h_read(ct, ct->fast_msg, true);
1795		if (len > 0)
1796			g2h_fast_path(ct, ct->fast_msg, len);
1797	} while (len > 0);
1798	spin_unlock(&ct->fast_lock);
1799
1800	if (ongoing)
1801		xe_pm_runtime_put(xe);
1802}
1803
1804/* Returns less than zero on error, 0 on done, 1 on more available */
1805static int dequeue_one_g2h(struct xe_guc_ct *ct)
1806{
1807	int len;
1808	int ret;
1809
1810	lockdep_assert_held(&ct->lock);
1811
1812	spin_lock_irq(&ct->fast_lock);
1813	len = g2h_read(ct, ct->msg, false);
1814	spin_unlock_irq(&ct->fast_lock);
1815	if (len <= 0)
1816		return len;
1817
1818	ret = parse_g2h_msg(ct, ct->msg, len);
1819	if (unlikely(ret < 0))
1820		return ret;
1821
1822	ret = process_g2h_msg(ct, ct->msg, len);
1823	if (unlikely(ret < 0))
1824		return ret;
1825
1826	return 1;
1827}
1828
1829static void receive_g2h(struct xe_guc_ct *ct)
1830{
1831	bool ongoing;
1832	int ret;
1833
1834	/*
1835	 * Normal users must always hold mem_access.ref around CT calls. However
1836	 * during the runtime pm callbacks we rely on CT to talk to the GuC, but
1837	 * at this stage we can't rely on mem_access.ref and even the
1838	 * callback_task will be different than current.  For such cases we just
1839	 * need to ensure we always process the responses from any blocking
1840	 * ct_send requests or where we otherwise expect some response when
1841	 * initiated from those callbacks (which will need to wait for the below
1842	 * dequeue_one_g2h()).  The dequeue_one_g2h() will gracefully fail if
1843	 * the device has suspended to the point that the CT communication has
1844	 * been disabled.
1845	 *
1846	 * If we are inside the runtime pm callback, we can be the only task
1847	 * still issuing CT requests (since that requires having the
1848	 * mem_access.ref).  It seems like it might in theory be possible to
1849	 * receive unsolicited events from the GuC just as we are
1850	 * suspending-resuming, but those will currently anyway be lost when
1851	 * eventually exiting from suspend, hence no need to wake up the device
1852	 * here. If we ever need something stronger than get_if_ongoing() then
1853	 * we need to be careful with blocking the pm callbacks from getting CT
1854	 * responses, if the worker here is blocked on those callbacks
1855	 * completing, creating a deadlock.
1856	 */
1857	ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));
1858	if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1859		return;
1860
1861	do {
1862		mutex_lock(&ct->lock);
1863		ret = dequeue_one_g2h(ct);
1864		mutex_unlock(&ct->lock);
1865
1866		if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
1867			xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret);
1868			CT_DEAD(ct, NULL, G2H_RECV);
1869			kick_reset(ct);
1870		}
1871	} while (ret == 1);
1872
1873	if (ongoing)
1874		xe_pm_runtime_put(ct_to_xe(ct));
1875}
1876
1877static void g2h_worker_func(struct work_struct *w)
1878{
1879	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
1880
1881	receive_g2h(ct);
1882}
1883
1884static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic,
1885							bool want_ctb)
1886{
1887	struct xe_guc_ct_snapshot *snapshot;
1888
1889	snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL);
1890	if (!snapshot)
1891		return NULL;
1892
1893	if (ct->bo && want_ctb) {
1894		snapshot->ctb_size = xe_bo_size(ct->bo);
1895		snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL);
1896	}
1897
1898	return snapshot;
1899}
1900
1901static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
1902				     struct guc_ctb_snapshot *snapshot)
1903{
1904	xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,
1905			   sizeof(struct guc_ct_buffer_desc));
1906	memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));
1907}
1908
1909static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,
1910				   struct drm_printer *p)
1911{
1912	drm_printf(p, "\tsize: %d\n", snapshot->info.size);
1913	drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);
1914	drm_printf(p, "\thead: %d\n", snapshot->info.head);
1915	drm_printf(p, "\ttail: %d\n", snapshot->info.tail);
1916	drm_printf(p, "\tspace: %d\n", snapshot->info.space);
1917	drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);
1918	drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);
1919	drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);
1920	drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);
1921}
1922
1923static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic,
1924							  bool want_ctb)
1925{
1926	struct xe_device *xe = ct_to_xe(ct);
1927	struct xe_guc_ct_snapshot *snapshot;
1928
1929	snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb);
1930	if (!snapshot) {
1931		xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n");
1932		return NULL;
1933	}
1934
1935	if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {
1936		snapshot->ct_enabled = true;
1937		snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
1938		guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g);
1939		guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h);
1940	}
1941
1942	if (ct->bo && snapshot->ctb)
1943		xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size);
1944
1945	return snapshot;
1946}
1947
1948/**
1949 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.
1950 * @ct: GuC CT object.
1951 *
1952 * This can be printed out in a later stage like during dev_coredump
1953 * analysis. This is safe to be called during atomic context.
1954 *
1955 * Returns: a GuC CT snapshot object that must be freed by the caller
1956 * by using `xe_guc_ct_snapshot_free`.
1957 */
1958struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct)
1959{
1960	return guc_ct_snapshot_capture(ct, true, true);
1961}
1962
1963/**
1964 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.
1965 * @snapshot: GuC CT snapshot object.
1966 * @p: drm_printer where it will be printed out.
1967 *
1968 * This function prints out a given GuC CT snapshot object.
1969 */
1970void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,
1971			      struct drm_printer *p)
1972{
1973	if (!snapshot)
1974		return;
1975
1976	if (snapshot->ct_enabled) {
1977		drm_puts(p, "H2G CTB (all sizes in DW):\n");
1978		guc_ctb_snapshot_print(&snapshot->h2g, p);
1979
1980		drm_puts(p, "G2H CTB (all sizes in DW):\n");
1981		guc_ctb_snapshot_print(&snapshot->g2h, p);
1982		drm_printf(p, "\tg2h outstanding: %d\n",
1983			   snapshot->g2h_outstanding);
1984
1985		if (snapshot->ctb) {
1986			drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size);
1987			xe_print_blob_ascii85(p, "[CTB].data", '\n',
1988					      snapshot->ctb, 0, snapshot->ctb_size);
1989		}
1990	} else {
1991		drm_puts(p, "CT disabled\n");
1992	}
1993}
1994
1995/**
1996 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.
1997 * @snapshot: GuC CT snapshot object.
1998 *
1999 * This function free all the memory that needed to be allocated at capture
2000 * time.
2001 */
2002void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)
2003{
2004	if (!snapshot)
2005		return;
2006
2007	kfree(snapshot->ctb);
2008	kfree(snapshot);
2009}
2010
2011/**
2012 * xe_guc_ct_print - GuC CT Print.
2013 * @ct: GuC CT.
2014 * @p: drm_printer where it will be printed out.
2015 * @want_ctb: Should the full CTB content be dumped (vs just the headers)
2016 *
2017 * This function will quickly capture a snapshot of the CT state
2018 * and immediately print it out.
2019 */
2020void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb)
2021{
2022	struct xe_guc_ct_snapshot *snapshot;
2023
2024	snapshot = guc_ct_snapshot_capture(ct, false, want_ctb);
2025	xe_guc_ct_snapshot_print(snapshot, p);
2026	xe_guc_ct_snapshot_free(snapshot);
2027}
2028
2029#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
2030
2031#ifdef CONFIG_FUNCTION_ERROR_INJECTION
2032/*
2033 * This is a helper function which assists the driver in identifying if a fault
2034 * injection test is currently active, allowing it to reduce unnecessary debug
2035 * output. Typically, the function returns zero, but the fault injection
2036 * framework can alter this to return an error. Since faults are injected
2037 * through this function, it's important to ensure the compiler doesn't optimize
2038 * it into an inline function. To avoid such optimization, the 'noinline'
2039 * attribute is applied. Compiler optimizes the static function defined in the
2040 * header file as an inline function.
2041 */
2042noinline int xe_is_injection_active(void) { return 0; }
2043ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO);
2044#else
2045int xe_is_injection_active(void) { return 0; }
2046#endif
2047
2048static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code)
2049{
2050	struct xe_guc_log_snapshot *snapshot_log;
2051	struct xe_guc_ct_snapshot *snapshot_ct;
2052	struct xe_guc *guc = ct_to_guc(ct);
2053	unsigned long flags;
2054	bool have_capture;
2055
2056	if (ctb)
2057		ctb->info.broken = true;
2058	/*
2059	 * Huge dump is getting generated when injecting error for guc CT/MMIO
2060	 * functions. So, let us suppress the dump when fault is injected.
2061	 */
2062	if (xe_is_injection_active())
2063		return;
2064
2065	/* Ignore further errors after the first dump until a reset */
2066	if (ct->dead.reported)
2067		return;
2068
2069	spin_lock_irqsave(&ct->dead.lock, flags);
2070
2071	/* And only capture one dump at a time */
2072	have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE);
2073	ct->dead.reason |= (1 << reason_code) |
2074			   (1 << CT_DEAD_STATE_CAPTURE);
2075
2076	spin_unlock_irqrestore(&ct->dead.lock, flags);
2077
2078	if (have_capture)
2079		return;
2080
2081	snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true);
2082	snapshot_ct = xe_guc_ct_snapshot_capture((ct));
2083
2084	spin_lock_irqsave(&ct->dead.lock, flags);
2085
2086	if (ct->dead.snapshot_log || ct->dead.snapshot_ct) {
2087		xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n");
2088		xe_guc_log_snapshot_free(snapshot_log);
2089		xe_guc_ct_snapshot_free(snapshot_ct);
2090	} else {
2091		ct->dead.snapshot_log = snapshot_log;
2092		ct->dead.snapshot_ct = snapshot_ct;
2093	}
2094
2095	spin_unlock_irqrestore(&ct->dead.lock, flags);
2096
2097	queue_work(system_unbound_wq, &(ct)->dead.worker);
2098}
2099
2100static void ct_dead_print(struct xe_dead_ct *dead)
2101{
2102	struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead);
2103	struct xe_device *xe = ct_to_xe(ct);
2104	struct xe_gt *gt = ct_to_gt(ct);
2105	static int g_count;
2106	struct drm_printer ip = xe_gt_info_printer(gt);
2107	struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count);
2108
2109	if (!dead->reason) {
2110		xe_gt_err(gt, "CTB is dead for no reason!?\n");
2111		return;
2112	}
2113
2114	/* Can't generate a genuine core dump at this point, so just do the good bits */
2115	drm_puts(&lp, "**** Xe Device Coredump ****\n");
2116	drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason);
2117	xe_device_snapshot_print(xe, &lp);
2118
2119	drm_printf(&lp, "**** GT #%d ****\n", gt->info.id);
2120	drm_printf(&lp, "\tTile: %d\n", gt->tile->id);
2121
2122	drm_puts(&lp, "**** GuC Log ****\n");
2123	xe_guc_log_snapshot_print(dead->snapshot_log, &lp);
2124
2125	drm_puts(&lp, "**** GuC CT ****\n");
2126	xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp);
2127
2128	drm_puts(&lp, "Done.\n");
2129}
2130
2131static void ct_dead_worker_func(struct work_struct *w)
2132{
2133	struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker);
2134
2135	if (!ct->dead.reported) {
2136		ct->dead.reported = true;
2137		ct_dead_print(&ct->dead);
2138	}
2139
2140	spin_lock_irq(&ct->dead.lock);
2141
2142	xe_guc_log_snapshot_free(ct->dead.snapshot_log);
2143	ct->dead.snapshot_log = NULL;
2144	xe_guc_ct_snapshot_free(ct->dead.snapshot_ct);
2145	ct->dead.snapshot_ct = NULL;
2146
2147	if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) {
2148		/* A reset has occurred so re-arm the error reporting */
2149		ct->dead.reason = 0;
2150		ct->dead.reported = false;
2151	}
2152
2153	spin_unlock_irq(&ct->dead.lock);
2154}
2155#endif