intel_guc_ct.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2016-2019 Intel Corporation
   4 */
   5
   6#include <linux/circ_buf.h>
   7#include <linux/ktime.h>
   8#include <linux/string_helpers.h>
   9#include <linux/time64.h>
  10#include <linux/timekeeping.h>
  11
  12#include "i915_drv.h"
  13#include "i915_wait_util.h"
  14#include "intel_guc_ct.h"
  15#include "intel_guc_print.h"
  16
  17#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
  18enum {
  19	CT_DEAD_ALIVE = 0,
  20	CT_DEAD_SETUP,
  21	CT_DEAD_WRITE,
  22	CT_DEAD_DEADLOCK,
  23	CT_DEAD_H2G_HAS_ROOM,
  24	CT_DEAD_READ,
  25	CT_DEAD_PROCESS_FAILED,
  26};
  27
  28static void ct_dead_ct_worker_func(struct work_struct *w);
  29
  30#define CT_DEAD(ct, reason)	\
  31	do { \
  32		if (!(ct)->dead_ct_reported) { \
  33			(ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \
  34			queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \
  35		} \
  36	} while (0)
  37#else
  38#define CT_DEAD(ct, reason)	do { } while (0)
  39#endif
  40
  41static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
  42{
  43	return container_of(ct, struct intel_guc, ct);
  44}
  45
  46#define CT_ERROR(_ct, _fmt, ...) \
  47	guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
  48#ifdef CONFIG_DRM_I915_DEBUG_GUC
  49#define CT_DEBUG(_ct, _fmt, ...) \
  50	guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
  51#else
  52#define CT_DEBUG(...)	do { } while (0)
  53#endif
  54#define CT_PROBE_ERROR(_ct, _fmt, ...) \
  55	guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__)
  56
  57/**
  58 * DOC: CTB Blob
  59 *
  60 * We allocate single blob to hold both CTB descriptors and buffers:
  61 *
  62 *      +--------+-----------------------------------------------+------+
  63 *      | offset | contents                                      | size |
  64 *      +========+===============================================+======+
  65 *      | 0x0000 | H2G `CTB Descriptor`_ (send)                  |      |
  66 *      +--------+-----------------------------------------------+  4K  |
  67 *      | 0x0800 | G2H `CTB Descriptor`_ (recv)                  |      |
  68 *      +--------+-----------------------------------------------+------+
  69 *      | 0x1000 | H2G `CT Buffer`_ (send)                       | n*4K |
  70 *      |        |                                               |      |
  71 *      +--------+-----------------------------------------------+------+
  72 *      | 0x1000 | G2H `CT Buffer`_ (recv)                       | m*4K |
  73 *      | + n*4K |                                               |      |
  74 *      +--------+-----------------------------------------------+------+
  75 *
  76 * Size of each `CT Buffer`_ must be multiple of 4K.
  77 * We don't expect too many messages in flight at any time, unless we are
  78 * using the GuC submission. In that case each request requires a minimum
  79 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
  80 * enough space to avoid backpressure on the driver. We increase the size
  81 * of the receive buffer (relative to the send) to ensure a G2H response
  82 * CTB has a landing spot.
  83 */
  84#define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
  85#define CTB_H2G_BUFFER_SIZE	(SZ_4K)
  86#define CTB_G2H_BUFFER_SIZE	(4 * CTB_H2G_BUFFER_SIZE)
  87#define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 4)
  88
  89struct ct_request {
  90	struct list_head link;
  91	u32 fence;
  92	u32 status;
  93	u32 response_len;
  94	u32 *response_buf;
  95};
  96
  97struct ct_incoming_msg {
  98	struct list_head link;
  99	u32 size;
 100	u32 msg[] __counted_by(size);
 101};
 102
 103enum { CTB_SEND = 0, CTB_RECV = 1 };
 104
 105enum { CTB_OWNER_HOST = 0 };
 106
 107/*
 108 * Some H2G commands involve a synchronous response that the driver needs
 109 * to wait for. In such cases, a timeout is required to prevent the driver
 110 * from waiting forever in the case of an error (either no error response
 111 * is defined in the protocol or something has died and requires a reset).
 112 * The specific command may be defined as having a time bound response but
 113 * the CT is a queue and that time guarantee only starts from the point
 114 * when the command reaches the head of the queue and is processed by GuC.
 115 *
 116 * Ideally there would be a helper to report the progress of a given
 117 * command through the CT. However, that would require a significant
 118 * amount of work in the CT layer. In the meantime, provide a reasonable
 119 * estimation of the worst case latency it should take for the entire
 120 * queue to drain. And therefore, how long a caller should wait before
 121 * giving up on their request. The current estimate is based on empirical
 122 * measurement of a test that fills the buffer with context creation and
 123 * destruction requests as they seem to be the slowest operation.
 124 */
 125long intel_guc_ct_max_queue_time_jiffies(void)
 126{
 127	/*
 128	 * A 4KB buffer full of context destroy commands takes a little
 129	 * over a second to process so bump that to 2s to be super safe.
 130	 */
 131	return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K;
 132}
 133
 134static void ct_receive_tasklet_func(struct tasklet_struct *t);
 135static void ct_incoming_request_worker_func(struct work_struct *w);
 136
 137/**
 138 * intel_guc_ct_init_early - Initialize CT state without requiring device access
 139 * @ct: pointer to CT struct
 140 */
 141void intel_guc_ct_init_early(struct intel_guc_ct *ct)
 142{
 143	spin_lock_init(&ct->ctbs.send.lock);
 144	spin_lock_init(&ct->ctbs.recv.lock);
 145	spin_lock_init(&ct->requests.lock);
 146	INIT_LIST_HEAD(&ct->requests.pending);
 147	INIT_LIST_HEAD(&ct->requests.incoming);
 148#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
 149	INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func);
 150#endif
 151	INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
 152	tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);
 153	init_waitqueue_head(&ct->wq);
 154}
 155
 156static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)
 157{
 158	memset(desc, 0, sizeof(*desc));
 159}
 160
 161static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)
 162{
 163	u32 space;
 164
 165	ctb->broken = false;
 166	ctb->tail = 0;
 167	ctb->head = 0;
 168	space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;
 169	atomic_set(&ctb->space, space);
 170
 171	guc_ct_buffer_desc_init(ctb->desc);
 172}
 173
 174static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
 175			       struct guc_ct_buffer_desc *desc,
 176			       u32 *cmds, u32 size_in_bytes, u32 resv_space)
 177{
 178	GEM_BUG_ON(size_in_bytes % 4);
 179
 180	ctb->desc = desc;
 181	ctb->cmds = cmds;
 182	ctb->size = size_in_bytes / 4;
 183	ctb->resv_space = resv_space / 4;
 184
 185	guc_ct_buffer_reset(ctb);
 186}
 187
 188static int guc_action_control_ctb(struct intel_guc *guc, u32 control)
 189{
 190	u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
 191		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
 192		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
 193		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),
 194		FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),
 195	};
 196	int ret;
 197
 198	GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);
 199
 200	/* CT control must go over MMIO */
 201	ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0);
 202
 203	return ret > 0 ? -EPROTO : ret;
 204}
 205
 206static int ct_control_enable(struct intel_guc_ct *ct, bool enable)
 207{
 208	int err;
 209
 210	err = guc_action_control_ctb(ct_to_guc(ct), enable ?
 211				     GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);
 212	if (unlikely(err))
 213		CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",
 214			       str_enable_disable(enable), ERR_PTR(err));
 215
 216	return err;
 217}
 218
 219static int ct_register_buffer(struct intel_guc_ct *ct, bool send,
 220			      u32 desc_addr, u32 buff_addr, u32 size)
 221{
 222	int err;
 223
 224	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
 225				   GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :
 226				   GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
 227				   desc_addr);
 228	if (unlikely(err))
 229		goto failed;
 230
 231	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
 232				   GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :
 233				   GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
 234				   buff_addr);
 235	if (unlikely(err))
 236		goto failed;
 237
 238	err = intel_guc_self_cfg32(ct_to_guc(ct), send ?
 239				   GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :
 240				   GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
 241				   size);
 242	if (unlikely(err))
 243failed:
 244		CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",
 245			       send ? "SEND" : "RECV", ERR_PTR(err));
 246
 247	return err;
 248}
 249
 250/**
 251 * intel_guc_ct_init - Init buffer-based communication
 252 * @ct: pointer to CT struct
 253 *
 254 * Allocate memory required for buffer-based communication.
 255 *
 256 * Return: 0 on success, a negative errno code on failure.
 257 */
 258int intel_guc_ct_init(struct intel_guc_ct *ct)
 259{
 260	struct intel_guc *guc = ct_to_guc(ct);
 261	struct guc_ct_buffer_desc *desc;
 262	u32 blob_size;
 263	u32 cmds_size;
 264	u32 resv_space;
 265	void *blob;
 266	u32 *cmds;
 267	int err;
 268
 269	err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO);
 270	if (err)
 271		return err;
 272
 273	GEM_BUG_ON(ct->vma);
 274
 275	blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
 276	err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);
 277	if (unlikely(err)) {
 278		CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
 279			       blob_size, ERR_PTR(err));
 280		return err;
 281	}
 282
 283	CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
 284
 285	/* store pointers to desc and cmds for send ctb */
 286	desc = blob;
 287	cmds = blob + 2 * CTB_DESC_SIZE;
 288	cmds_size = CTB_H2G_BUFFER_SIZE;
 289	resv_space = 0;
 290	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",
 291		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
 292		 resv_space);
 293
 294	guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space);
 295
 296	/* store pointers to desc and cmds for recv ctb */
 297	desc = blob + CTB_DESC_SIZE;
 298	cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
 299	cmds_size = CTB_G2H_BUFFER_SIZE;
 300	resv_space = G2H_ROOM_BUFFER_SIZE;
 301	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",
 302		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
 303		 resv_space);
 304
 305	guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space);
 306
 307	return 0;
 308}
 309
 310/**
 311 * intel_guc_ct_fini - Fini buffer-based communication
 312 * @ct: pointer to CT struct
 313 *
 314 * Deallocate memory required for buffer-based communication.
 315 */
 316void intel_guc_ct_fini(struct intel_guc_ct *ct)
 317{
 318	GEM_BUG_ON(ct->enabled);
 319
 320	tasklet_kill(&ct->receive_tasklet);
 321	i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
 322	memset(ct, 0, sizeof(*ct));
 323}
 324
 325/**
 326 * intel_guc_ct_enable - Enable buffer based command transport.
 327 * @ct: pointer to CT struct
 328 *
 329 * Return: 0 on success, a negative errno code on failure.
 330 */
 331int intel_guc_ct_enable(struct intel_guc_ct *ct)
 332{
 333	struct intel_guc *guc = ct_to_guc(ct);
 334	u32 base, desc, cmds, size;
 335	void *blob;
 336	int err;
 337
 338	GEM_BUG_ON(ct->enabled);
 339
 340	/* vma should be already allocated and map'ed */
 341	GEM_BUG_ON(!ct->vma);
 342	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
 343	base = intel_guc_ggtt_offset(guc, ct->vma);
 344
 345	/* blob should start with send descriptor */
 346	blob = __px_vaddr(ct->vma->obj);
 347	GEM_BUG_ON(blob != ct->ctbs.send.desc);
 348
 349	/* (re)initialize descriptors */
 350	guc_ct_buffer_reset(&ct->ctbs.send);
 351	guc_ct_buffer_reset(&ct->ctbs.recv);
 352
 353	/*
 354	 * Register both CT buffers starting with RECV buffer.
 355	 * Descriptors are in first half of the blob.
 356	 */
 357	desc = base + ptrdiff(ct->ctbs.recv.desc, blob);
 358	cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);
 359	size = ct->ctbs.recv.size * 4;
 360	err = ct_register_buffer(ct, false, desc, cmds, size);
 361	if (unlikely(err))
 362		goto err_out;
 363
 364	desc = base + ptrdiff(ct->ctbs.send.desc, blob);
 365	cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);
 366	size = ct->ctbs.send.size * 4;
 367	err = ct_register_buffer(ct, true, desc, cmds, size);
 368	if (unlikely(err))
 369		goto err_out;
 370
 371	err = ct_control_enable(ct, true);
 372	if (unlikely(err))
 373		goto err_out;
 374
 375	ct->enabled = true;
 376	ct->stall_time = KTIME_MAX;
 377#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
 378	ct->dead_ct_reported = false;
 379	ct->dead_ct_reason = CT_DEAD_ALIVE;
 380#endif
 381
 382	return 0;
 383
 384err_out:
 385	CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
 386	CT_DEAD(ct, SETUP);
 387	return err;
 388}
 389
 390/**
 391 * intel_guc_ct_disable - Disable buffer based command transport.
 392 * @ct: pointer to CT struct
 393 */
 394void intel_guc_ct_disable(struct intel_guc_ct *ct)
 395{
 396	struct intel_guc *guc = ct_to_guc(ct);
 397
 398	GEM_BUG_ON(!ct->enabled);
 399
 400	ct->enabled = false;
 401
 402	if (intel_guc_is_fw_running(guc)) {
 403		ct_control_enable(ct, false);
 404	}
 405}
 406
 407#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 408static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action)
 409{
 410	unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found);
 411#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
 412	unsigned long entries[SZ_32];
 413	unsigned int n;
 414
 415	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
 416
 417	/* May be called under spinlock, so avoid sleeping */
 418	ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT);
 419#endif
 420	ct->requests.lost_and_found[lost].fence = fence;
 421	ct->requests.lost_and_found[lost].action = action;
 422}
 423#endif
 424
 425static u32 ct_get_next_fence(struct intel_guc_ct *ct)
 426{
 427	/* For now it's trivial */
 428	return ++ct->requests.last_fence;
 429}
 430
 431static int ct_write(struct intel_guc_ct *ct,
 432		    const u32 *action,
 433		    u32 len /* in dwords */,
 434		    u32 fence, u32 flags)
 435{
 436	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
 437	struct guc_ct_buffer_desc *desc = ctb->desc;
 438	u32 tail = ctb->tail;
 439	u32 size = ctb->size;
 440	u32 header;
 441	u32 hxg;
 442	u32 type;
 443	u32 *cmds = ctb->cmds;
 444	unsigned int i;
 445
 446	if (unlikely(desc->status))
 447		goto corrupted;
 448
 449	GEM_BUG_ON(tail > size);
 450
 451#ifdef CONFIG_DRM_I915_DEBUG_GUC
 452	if (unlikely(tail != READ_ONCE(desc->tail))) {
 453		CT_ERROR(ct, "Tail was modified %u != %u\n",
 454			 desc->tail, tail);
 455		desc->status |= GUC_CTB_STATUS_MISMATCH;
 456		goto corrupted;
 457	}
 458	if (unlikely(READ_ONCE(desc->head) >= size)) {
 459		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
 460			 desc->head, size);
 461		desc->status |= GUC_CTB_STATUS_OVERFLOW;
 462		goto corrupted;
 463	}
 464#endif
 465
 466	/*
 467	 * dw0: CT header (including fence)
 468	 * dw1: HXG header (including action code)
 469	 * dw2+: action data
 470	 */
 471	header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
 472		 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
 473		 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);
 474
 475	type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST :
 476		GUC_HXG_TYPE_REQUEST;
 477	hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |
 478		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION |
 479			   GUC_HXG_REQUEST_MSG_0_DATA0, action[0]);
 480
 481	CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",
 482		 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);
 483
 484	cmds[tail] = header;
 485	tail = (tail + 1) % size;
 486
 487	cmds[tail] = hxg;
 488	tail = (tail + 1) % size;
 489
 490	for (i = 1; i < len; i++) {
 491		cmds[tail] = action[i];
 492		tail = (tail + 1) % size;
 493	}
 494	GEM_BUG_ON(tail > size);
 495
 496#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 497	ct_track_lost_and_found(ct, fence,
 498				FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));
 499#endif
 500
 501	/*
 502	 * make sure H2G buffer update and LRC tail update (if this triggering a
 503	 * submission) are visible before updating the descriptor tail
 504	 */
 505	intel_guc_write_barrier(ct_to_guc(ct));
 506
 507	/* update local copies */
 508	ctb->tail = tail;
 509	GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);
 510	atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space);
 511
 512	/* now update descriptor */
 513	WRITE_ONCE(desc->tail, tail);
 514
 515	return 0;
 516
 517corrupted:
 518	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
 519		 desc->head, desc->tail, desc->status);
 520	CT_DEAD(ct, WRITE);
 521	ctb->broken = true;
 522	return -EPIPE;
 523}
 524
 525/**
 526 * wait_for_ct_request_update - Wait for CT request state update.
 527 * @ct:		pointer to CT
 528 * @req:	pointer to pending request
 529 * @status:	placeholder for status
 530 *
 531 * For each sent request, GuC shall send back CT response message.
 532 * Our message handler will update status of tracked request once
 533 * response message with given fence is received. Wait here and
 534 * check for valid response status value.
 535 *
 536 * Return:
 537 * *	0 response received (status is valid)
 538 * *	-ETIMEDOUT no response within hardcoded timeout
 539 */
 540static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
 541{
 542	int err;
 543	bool ct_enabled;
 544
 545	/*
 546	 * Fast commands should complete in less than 10us, so sample quickly
 547	 * up to that length of time, then switch to a slower sleep-wait loop.
 548	 * No GuC command should ever take longer than 10ms but many GuC
 549	 * commands can be inflight at time, so use a 1s timeout on the slower
 550	 * sleep-wait loop.
 551	 */
 552#define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
 553#define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
 554#define done \
 555	(!(ct_enabled = intel_guc_ct_enabled(ct)) || \
 556	 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
 557	 GUC_HXG_ORIGIN_GUC)
 558	err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
 559	if (err)
 560		err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
 561#undef done
 562	if (!ct_enabled)
 563		err = -ENODEV;
 564
 565	*status = req->status;
 566	return err;
 567}
 568
 569#define GUC_CTB_TIMEOUT_MS	1500
 570static inline bool ct_deadlocked(struct intel_guc_ct *ct)
 571{
 572	long timeout = GUC_CTB_TIMEOUT_MS;
 573	bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout;
 574
 575	if (unlikely(ret)) {
 576		struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;
 577		struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;
 578
 579		CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",
 580			 ktime_ms_delta(ktime_get(), ct->stall_time),
 581			 send->status, recv->status);
 582		CT_ERROR(ct, "H2G Space: %u (Bytes)\n",
 583			 atomic_read(&ct->ctbs.send.space) * 4);
 584		CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);
 585		CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);
 586		CT_ERROR(ct, "G2H Space: %u (Bytes)\n",
 587			 atomic_read(&ct->ctbs.recv.space) * 4);
 588		CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);
 589		CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);
 590
 591		CT_DEAD(ct, DEADLOCK);
 592		ct->ctbs.send.broken = true;
 593	}
 594
 595	return ret;
 596}
 597
 598static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)
 599{
 600	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
 601
 602	/*
 603	 * We leave a certain amount of space in the G2H CTB buffer for
 604	 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)
 605	 */
 606	return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw;
 607}
 608
 609static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
 610{
 611	lockdep_assert_held(&ct->ctbs.send.lock);
 612
 613	GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));
 614
 615	if (g2h_len_dw)
 616		atomic_sub(g2h_len_dw, &ct->ctbs.recv.space);
 617}
 618
 619static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
 620{
 621	atomic_add(g2h_len_dw, &ct->ctbs.recv.space);
 622}
 623
 624static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)
 625{
 626	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
 627	struct guc_ct_buffer_desc *desc = ctb->desc;
 628	u32 head;
 629	u32 space;
 630
 631	if (atomic_read(&ctb->space) >= len_dw)
 632		return true;
 633
 634	head = READ_ONCE(desc->head);
 635	if (unlikely(head > ctb->size)) {
 636		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
 637			 head, ctb->size);
 638		desc->status |= GUC_CTB_STATUS_OVERFLOW;
 639		ctb->broken = true;
 640		CT_DEAD(ct, H2G_HAS_ROOM);
 641		return false;
 642	}
 643
 644	space = CIRC_SPACE(ctb->tail, head, ctb->size);
 645	atomic_set(&ctb->space, space);
 646
 647	return space >= len_dw;
 648}
 649
 650static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)
 651{
 652	bool h2g = h2g_has_room(ct, h2g_dw);
 653	bool g2h = g2h_has_room(ct, g2h_dw);
 654
 655	lockdep_assert_held(&ct->ctbs.send.lock);
 656
 657	if (unlikely(!h2g || !g2h)) {
 658		if (ct->stall_time == KTIME_MAX)
 659			ct->stall_time = ktime_get();
 660
 661		/* Be paranoid and kick G2H tasklet to free credits */
 662		if (!g2h)
 663			tasklet_hi_schedule(&ct->receive_tasklet);
 664
 665		if (unlikely(ct_deadlocked(ct)))
 666			return -EPIPE;
 667		else
 668			return -EBUSY;
 669	}
 670
 671	ct->stall_time = KTIME_MAX;
 672	return 0;
 673}
 674
 675#define G2H_LEN_DW(f) ({ \
 676	typeof(f) f_ = (f); \
 677	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \
 678	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \
 679	GUC_CTB_HXG_MSG_MIN_LEN : 0; \
 680})
 681static int ct_send_nb(struct intel_guc_ct *ct,
 682		      const u32 *action,
 683		      u32 len,
 684		      u32 flags)
 685{
 686	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
 687	unsigned long spin_flags;
 688	u32 g2h_len_dw = G2H_LEN_DW(flags);
 689	u32 fence;
 690	int ret;
 691
 692	spin_lock_irqsave(&ctb->lock, spin_flags);
 693
 694	ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw);
 695	if (unlikely(ret))
 696		goto out;
 697
 698	fence = ct_get_next_fence(ct);
 699	ret = ct_write(ct, action, len, fence, flags);
 700	if (unlikely(ret))
 701		goto out;
 702
 703	g2h_reserve_space(ct, g2h_len_dw);
 704	intel_guc_notify(ct_to_guc(ct));
 705
 706out:
 707	spin_unlock_irqrestore(&ctb->lock, spin_flags);
 708
 709	return ret;
 710}
 711
 712static int ct_send(struct intel_guc_ct *ct,
 713		   const u32 *action,
 714		   u32 len,
 715		   u32 *response_buf,
 716		   u32 response_buf_size,
 717		   u32 *status)
 718{
 719	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
 720	struct ct_request request;
 721	unsigned long flags;
 722	unsigned int sleep_period_ms = 1;
 723	bool send_again;
 724	u32 fence;
 725	int err;
 726
 727	GEM_BUG_ON(!ct->enabled);
 728	GEM_BUG_ON(!len);
 729	GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN);
 730	GEM_BUG_ON(!response_buf && response_buf_size);
 731	might_sleep();
 732
 733resend:
 734	send_again = false;
 735
 736	/*
 737	 * We use a lazy spin wait loop here as we believe that if the CT
 738	 * buffers are sized correctly the flow control condition should be
 739	 * rare. Reserving the maximum size in the G2H credits as we don't know
 740	 * how big the response is going to be.
 741	 */
 742retry:
 743	spin_lock_irqsave(&ctb->lock, flags);
 744	if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||
 745		     !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {
 746		if (ct->stall_time == KTIME_MAX)
 747			ct->stall_time = ktime_get();
 748		spin_unlock_irqrestore(&ctb->lock, flags);
 749
 750		if (unlikely(ct_deadlocked(ct)))
 751			return -EPIPE;
 752
 753		if (msleep_interruptible(sleep_period_ms))
 754			return -EINTR;
 755		sleep_period_ms = sleep_period_ms << 1;
 756
 757		goto retry;
 758	}
 759
 760	ct->stall_time = KTIME_MAX;
 761
 762	fence = ct_get_next_fence(ct);
 763	request.fence = fence;
 764	request.status = 0;
 765	request.response_len = response_buf_size;
 766	request.response_buf = response_buf;
 767
 768	spin_lock(&ct->requests.lock);
 769	list_add_tail(&request.link, &ct->requests.pending);
 770	spin_unlock(&ct->requests.lock);
 771
 772	err = ct_write(ct, action, len, fence, 0);
 773	g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
 774
 775	spin_unlock_irqrestore(&ctb->lock, flags);
 776
 777	if (unlikely(err))
 778		goto unlink;
 779
 780	intel_guc_notify(ct_to_guc(ct));
 781
 782	err = wait_for_ct_request_update(ct, &request, status);
 783	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
 784	if (unlikely(err)) {
 785		if (err == -ENODEV)
 786			/* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
 787			 * In this case, output is debug rather than error info
 788			 */
 789			CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
 790				 action[0], request.fence);
 791		else
 792			CT_ERROR(ct, "No response for request %#x (fence %u)\n",
 793				 action[0], request.fence);
 794		goto unlink;
 795	}
 796
 797	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
 798		CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,
 799			 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));
 800		send_again = true;
 801		goto unlink;
 802	}
 803
 804	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {
 805		err = -EIO;
 806		goto unlink;
 807	}
 808
 809	if (response_buf) {
 810		/* There shall be no data in the status */
 811		WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));
 812		/* Return actual response len */
 813		err = request.response_len;
 814	} else {
 815		/* There shall be no response payload */
 816		WARN_ON(request.response_len);
 817		/* Return data decoded from the status dword */
 818		err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);
 819	}
 820
 821unlink:
 822	spin_lock_irqsave(&ct->requests.lock, flags);
 823	list_del(&request.link);
 824	spin_unlock_irqrestore(&ct->requests.lock, flags);
 825
 826	if (unlikely(send_again))
 827		goto resend;
 828
 829	return err;
 830}
 831
 832/*
 833 * Command Transport (CT) buffer based GuC send function.
 834 */
 835int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
 836		      u32 *response_buf, u32 response_buf_size, u32 flags)
 837{
 838	u32 status = ~0; /* undefined */
 839	int ret;
 840
 841	if (unlikely(!ct->enabled)) {
 842		struct intel_guc *guc = ct_to_guc(ct);
 843		struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
 844
 845		WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);
 846		return -ENODEV;
 847	}
 848
 849	if (unlikely(ct->ctbs.send.broken))
 850		return -EPIPE;
 851
 852	if (flags & INTEL_GUC_CT_SEND_NB)
 853		return ct_send_nb(ct, action, len, flags);
 854
 855	ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
 856	if (unlikely(ret < 0)) {
 857		if (ret != -ENODEV)
 858			CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
 859				 action[0], ERR_PTR(ret), status);
 860	} else if (unlikely(ret)) {
 861		CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
 862			 action[0], ret, ret);
 863	}
 864
 865	return ret;
 866}
 867
 868static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
 869{
 870	struct ct_incoming_msg *msg;
 871
 872	msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);
 873	if (msg)
 874		msg->size = num_dwords;
 875	return msg;
 876}
 877
 878static void ct_free_msg(struct ct_incoming_msg *msg)
 879{
 880	kfree(msg);
 881}
 882
 883/*
 884 * Return: number available remaining dwords to read (0 if empty)
 885 *         or a negative error code on failure
 886 */
 887static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
 888{
 889	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
 890	struct guc_ct_buffer_desc *desc = ctb->desc;
 891	u32 head = ctb->head;
 892	u32 tail = READ_ONCE(desc->tail);
 893	u32 size = ctb->size;
 894	u32 *cmds = ctb->cmds;
 895	s32 available;
 896	unsigned int len;
 897	unsigned int i;
 898	u32 header;
 899
 900	if (unlikely(ctb->broken))
 901		return -EPIPE;
 902
 903	if (unlikely(desc->status)) {
 904		u32 status = desc->status;
 905
 906		if (status & GUC_CTB_STATUS_UNUSED) {
 907			/*
 908			 * Potentially valid if a CLIENT_RESET request resulted in
 909			 * contexts/engines being reset. But should never happen as
 910			 * no contexts should be active when CLIENT_RESET is sent.
 911			 */
 912			CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
 913			status &= ~GUC_CTB_STATUS_UNUSED;
 914		}
 915
 916		if (status)
 917			goto corrupted;
 918	}
 919
 920	GEM_BUG_ON(head > size);
 921
 922#ifdef CONFIG_DRM_I915_DEBUG_GUC
 923	if (unlikely(head != READ_ONCE(desc->head))) {
 924		CT_ERROR(ct, "Head was modified %u != %u\n",
 925			 desc->head, head);
 926		desc->status |= GUC_CTB_STATUS_MISMATCH;
 927		goto corrupted;
 928	}
 929#endif
 930	if (unlikely(tail >= size)) {
 931		CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",
 932			 tail, size);
 933		desc->status |= GUC_CTB_STATUS_OVERFLOW;
 934		goto corrupted;
 935	}
 936
 937	/* tail == head condition indicates empty */
 938	available = tail - head;
 939	if (unlikely(available == 0)) {
 940		*msg = NULL;
 941		return 0;
 942	}
 943
 944	/* beware of buffer wrap case */
 945	if (unlikely(available < 0))
 946		available += size;
 947	CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);
 948	GEM_BUG_ON(available < 0);
 949
 950	header = cmds[head];
 951	head = (head + 1) % size;
 952
 953	/* message len with header */
 954	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;
 955	if (unlikely(len > (u32)available)) {
 956		CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
 957			 4, &header,
 958			 4 * (head + available - 1 > size ?
 959			      size - head : available - 1), &cmds[head],
 960			 4 * (head + available - 1 > size ?
 961			      available - 1 - size + head : 0), &cmds[0]);
 962		desc->status |= GUC_CTB_STATUS_UNDERFLOW;
 963		goto corrupted;
 964	}
 965
 966	*msg = ct_alloc_msg(len);
 967	if (!*msg) {
 968		CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
 969			 4, &header,
 970			 4 * (head + available - 1 > size ?
 971			      size - head : available - 1), &cmds[head],
 972			 4 * (head + available - 1 > size ?
 973			      available - 1 - size + head : 0), &cmds[0]);
 974		return available;
 975	}
 976
 977	(*msg)->msg[0] = header;
 978
 979	for (i = 1; i < len; i++) {
 980		(*msg)->msg[i] = cmds[head];
 981		head = (head + 1) % size;
 982	}
 983	CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
 984
 985	/* update local copies */
 986	ctb->head = head;
 987
 988	/* now update descriptor */
 989	WRITE_ONCE(desc->head, head);
 990
 991	intel_guc_write_barrier(ct_to_guc(ct));
 992
 993	return available - len;
 994
 995corrupted:
 996	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
 997		 desc->head, desc->tail, desc->status);
 998	ctb->broken = true;
 999	CT_DEAD(ct, READ);
1000	return -EPIPE;
1001}
1002
1003#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
1004static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1005{
1006	unsigned int n;
1007	char *buf = NULL;
1008	bool found = false;
1009
1010	lockdep_assert_held(&ct->requests.lock);
1011
1012	for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) {
1013		if (ct->requests.lost_and_found[n].fence != fence)
1014			continue;
1015		found = true;
1016
1017#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
1018		buf = kmalloc(SZ_4K, GFP_NOWAIT);
1019		if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack,
1020					       buf, SZ_4K, 0)) {
1021			CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s",
1022				 fence, ct->requests.lost_and_found[n].action, buf);
1023			break;
1024		}
1025#endif
1026		CT_ERROR(ct, "Fence %u was used by action %#04x\n",
1027			 fence, ct->requests.lost_and_found[n].action);
1028		break;
1029	}
1030	kfree(buf);
1031	return found;
1032}
1033#else
1034static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1035{
1036	return false;
1037}
1038#endif
1039
1040static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
1041{
1042	u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);
1043	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);
1044	const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];
1045	const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];
1046	u32 datalen = len - GUC_HXG_MSG_MIN_LEN;
1047	struct ct_request *req;
1048	unsigned long flags;
1049	bool found = false;
1050	int err = 0;
1051
1052	GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);
1053	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);
1054	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&
1055		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&
1056		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);
1057
1058	CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);
1059
1060	spin_lock_irqsave(&ct->requests.lock, flags);
1061	list_for_each_entry(req, &ct->requests.pending, link) {
1062		if (unlikely(fence != req->fence)) {
1063			CT_DEBUG(ct, "request %u awaits response\n",
1064				 req->fence);
1065			continue;
1066		}
1067		if (unlikely(datalen > req->response_len)) {
1068			CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
1069				 req->fence, datalen, req->response_len);
1070			datalen = min(datalen, req->response_len);
1071			err = -EMSGSIZE;
1072		}
1073		if (datalen)
1074			memcpy(req->response_buf, data, 4 * datalen);
1075		req->response_len = datalen;
1076		WRITE_ONCE(req->status, hxg[0]);
1077		found = true;
1078		break;
1079	}
1080
1081#ifdef CONFIG_DRM_I915_SELFTEST
1082	if (!found && ct_to_guc(ct)->fast_response_selftest) {
1083		CT_DEBUG(ct, "Assuming unsolicited response due to FAST_REQUEST selftest\n");
1084		ct_to_guc(ct)->fast_response_selftest++;
1085		found = true;
1086	}
1087#endif
1088
1089	if (!found) {
1090		CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n",
1091			 len, hxg[0], fence, ct->requests.last_fence);
1092		if (!ct_check_lost_and_found(ct, fence)) {
1093			list_for_each_entry(req, &ct->requests.pending, link)
1094				CT_ERROR(ct, "request %u awaits response\n",
1095					 req->fence);
1096		}
1097		err = -ENOKEY;
1098	}
1099	spin_unlock_irqrestore(&ct->requests.lock, flags);
1100
1101	if (unlikely(err))
1102		return err;
1103
1104	ct_free_msg(response);
1105	return 0;
1106}
1107
1108static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1109{
1110	struct intel_guc *guc = ct_to_guc(ct);
1111	const u32 *hxg;
1112	const u32 *payload;
1113	u32 hxg_len, action, len;
1114	int ret;
1115
1116	hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1117	hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;
1118	payload = &hxg[GUC_HXG_MSG_MIN_LEN];
1119	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1120	len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1121
1122	CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
1123
1124	switch (action) {
1125	case INTEL_GUC_ACTION_DEFAULT:
1126		ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
1127		break;
1128	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1129		ret = intel_guc_deregister_done_process_msg(guc, payload,
1130							    len);
1131		break;
1132	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1133		ret = intel_guc_sched_done_process_msg(guc, payload, len);
1134		break;
1135	case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1136		ret = intel_guc_context_reset_process_msg(guc, payload, len);
1137		break;
1138	case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1139		ret = intel_guc_error_capture_process_msg(guc, payload, len);
1140		if (unlikely(ret))
1141			CT_ERROR(ct, "error capture notification failed %x %*ph\n",
1142				 action, 4 * len, payload);
1143		break;
1144	case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1145		ret = intel_guc_engine_failure_process_msg(guc, payload, len);
1146		break;
1147	case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1148		intel_guc_log_handle_flush_event(&guc->log);
1149		ret = 0;
1150		break;
1151	case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1152	case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
1153		ret = intel_guc_crash_process_msg(guc, action);
1154		break;
1155	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1156		ret = intel_guc_tlb_invalidation_done(guc, payload, len);
1157		break;
1158	default:
1159		ret = -EOPNOTSUPP;
1160		break;
1161	}
1162
1163	if (unlikely(ret)) {
1164		CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
1165			 action, ERR_PTR(ret));
1166		return ret;
1167	}
1168
1169	ct_free_msg(request);
1170	return 0;
1171}
1172
1173static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
1174{
1175	unsigned long flags;
1176	struct ct_incoming_msg *request;
1177	bool done;
1178	int err;
1179
1180	spin_lock_irqsave(&ct->requests.lock, flags);
1181	request = list_first_entry_or_null(&ct->requests.incoming,
1182					   struct ct_incoming_msg, link);
1183	if (request)
1184		list_del(&request->link);
1185	done = !!list_empty(&ct->requests.incoming);
1186	spin_unlock_irqrestore(&ct->requests.lock, flags);
1187
1188	if (!request)
1189		return true;
1190
1191	err = ct_process_request(ct, request);
1192	if (unlikely(err)) {
1193		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1194			 ERR_PTR(err), 4 * request->size, request->msg);
1195		CT_DEAD(ct, PROCESS_FAILED);
1196		ct_free_msg(request);
1197	}
1198
1199	return done;
1200}
1201
1202static void ct_incoming_request_worker_func(struct work_struct *w)
1203{
1204	struct intel_guc_ct *ct =
1205		container_of(w, struct intel_guc_ct, requests.worker);
1206	bool done;
1207
1208	do {
1209		done = ct_process_incoming_requests(ct);
1210	} while (!done);
1211}
1212
1213static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1214{
1215	const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1216	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1217	unsigned long flags;
1218
1219	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);
1220
1221	/*
1222	 * Adjusting the space must be done in IRQ or deadlock can occur as the
1223	 * CTB processing in the below workqueue can send CTBs which creates a
1224	 * circular dependency if the space was returned there.
1225	 */
1226	switch (action) {
1227	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1228	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1229	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1230		g2h_release_space(ct, request->size);
1231	}
1232
1233	/*
1234	 * TLB invalidation responses must be handled immediately as processing
1235	 * of other G2H notifications may be blocked by an invalidation request.
1236	 */
1237	if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE)
1238		return ct_process_request(ct, request);
1239
1240	spin_lock_irqsave(&ct->requests.lock, flags);
1241	list_add_tail(&request->link, &ct->requests.incoming);
1242	spin_unlock_irqrestore(&ct->requests.lock, flags);
1243
1244	queue_work(system_unbound_wq, &ct->requests.worker);
1245	return 0;
1246}
1247
1248static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1249{
1250	u32 origin, type;
1251	u32 *hxg;
1252	int err;
1253
1254	if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))
1255		return -EBADMSG;
1256
1257	hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];
1258
1259	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1260	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1261		err = -EPROTO;
1262		goto failed;
1263	}
1264
1265	type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1266	switch (type) {
1267	case GUC_HXG_TYPE_EVENT:
1268		err = ct_handle_event(ct, msg);
1269		break;
1270	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1271	case GUC_HXG_TYPE_RESPONSE_FAILURE:
1272	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1273		err = ct_handle_response(ct, msg);
1274		break;
1275	default:
1276		err = -EOPNOTSUPP;
1277	}
1278
1279	if (unlikely(err)) {
1280failed:
1281		CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",
1282			 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);
1283	}
1284	return err;
1285}
1286
1287static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1288{
1289	u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);
1290	int err;
1291
1292	if (format == GUC_CTB_FORMAT_HXG)
1293		err = ct_handle_hxg(ct, msg);
1294	else
1295		err = -EOPNOTSUPP;
1296
1297	if (unlikely(err)) {
1298		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1299			 ERR_PTR(err), 4 * msg->size, msg->msg);
1300		ct_free_msg(msg);
1301	}
1302}
1303
1304/*
1305 * Return: number available remaining dwords to read (0 if empty)
1306 *         or a negative error code on failure
1307 */
1308static int ct_receive(struct intel_guc_ct *ct)
1309{
1310	struct ct_incoming_msg *msg = NULL;
1311	unsigned long flags;
1312	int ret;
1313
1314	spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
1315	ret = ct_read(ct, &msg);
1316	spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);
1317	if (ret < 0)
1318		return ret;
1319
1320	if (msg)
1321		ct_handle_msg(ct, msg);
1322
1323	return ret;
1324}
1325
1326static void ct_try_receive_message(struct intel_guc_ct *ct)
1327{
1328	struct intel_guc *guc = ct_to_guc(ct);
1329	int ret;
1330
1331	if (!ct->enabled) {
1332		GEM_WARN_ON(!guc_to_gt(guc)->uc.reset_in_progress);
1333		return;
1334	}
1335
1336	/* When interrupt disabled, message handling is not expected */
1337	if (!guc->interrupts.enabled)
1338		return;
1339
1340	ret = ct_receive(ct);
1341	if (ret > 0)
1342		tasklet_hi_schedule(&ct->receive_tasklet);
1343}
1344
1345static void ct_receive_tasklet_func(struct tasklet_struct *t)
1346{
1347	struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
1348
1349	ct_try_receive_message(ct);
1350}
1351
1352/*
1353 * When we're communicating with the GuC over CT, GuC uses events
1354 * to notify us about new messages being posted on the RECV buffer.
1355 */
1356void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
1357{
1358	if (unlikely(!ct->enabled)) {
1359		WARN(1, "Unexpected GuC event received while CT disabled!\n");
1360		return;
1361	}
1362
1363	ct_try_receive_message(ct);
1364}
1365
1366void intel_guc_ct_print_info(struct intel_guc_ct *ct,
1367			     struct drm_printer *p)
1368{
1369	drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled));
1370
1371	if (!ct->enabled)
1372		return;
1373
1374	drm_printf(p, "H2G Space: %u\n",
1375		   atomic_read(&ct->ctbs.send.space) * 4);
1376	drm_printf(p, "Head: %u\n",
1377		   ct->ctbs.send.desc->head);
1378	drm_printf(p, "Tail: %u\n",
1379		   ct->ctbs.send.desc->tail);
1380	drm_printf(p, "G2H Space: %u\n",
1381		   atomic_read(&ct->ctbs.recv.space) * 4);
1382	drm_printf(p, "Head: %u\n",
1383		   ct->ctbs.recv.desc->head);
1384	drm_printf(p, "Tail: %u\n",
1385		   ct->ctbs.recv.desc->tail);
1386}
1387
1388#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
1389static void ct_dead_ct_worker_func(struct work_struct *w)
1390{
1391	struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker);
1392	struct intel_guc *guc = ct_to_guc(ct);
1393
1394	if (ct->dead_ct_reported)
1395		return;
1396
1397	if (i915_error_injected())
1398		return;
1399
1400	ct->dead_ct_reported = true;
1401
1402	guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);
1403	intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U);
1404}
1405#endif