xe_memirq.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2023 Intel Corporation
  4 */
  5
  6#include <drm/drm_managed.h>
  7
  8#include "regs/xe_guc_regs.h"
  9#include "regs/xe_irq_regs.h"
 10#include "regs/xe_regs.h"
 11
 12#include "xe_assert.h"
 13#include "xe_bo.h"
 14#include "xe_device.h"
 15#include "xe_device_types.h"
 16#include "xe_gt.h"
 17#include "xe_guc.h"
 18#include "xe_hw_engine.h"
 19#include "xe_map.h"
 20#include "xe_memirq.h"
 21#include "xe_tile_printk.h"
 22
 23#define memirq_assert(m, condition)	xe_tile_assert(memirq_to_tile(m), condition)
 24#define memirq_printk(m, _level, _fmt, ...)			\
 25	xe_tile_##_level(memirq_to_tile(m), "MEMIRQ: " _fmt, ##__VA_ARGS__)
 26
 27#ifdef CONFIG_DRM_XE_DEBUG_MEMIRQ
 28#define memirq_debug(m, _fmt, ...)	memirq_printk(m, dbg, _fmt, ##__VA_ARGS__)
 29#else
 30#define memirq_debug(...)
 31#endif
 32
 33#define memirq_err(m, _fmt, ...)	memirq_printk(m, err, _fmt, ##__VA_ARGS__)
 34#define memirq_err_ratelimited(m, _fmt, ...)	\
 35	memirq_printk(m, err_ratelimited, _fmt, ##__VA_ARGS__)
 36
 37static struct xe_tile *memirq_to_tile(struct xe_memirq *memirq)
 38{
 39	return container_of(memirq, struct xe_tile, memirq);
 40}
 41
 42static struct xe_device *memirq_to_xe(struct xe_memirq *memirq)
 43{
 44	return tile_to_xe(memirq_to_tile(memirq));
 45}
 46
 47static const char *guc_name(struct xe_guc *guc)
 48{
 49	return xe_gt_is_media_type(guc_to_gt(guc)) ? "media GuC" : "GuC";
 50}
 51
 52/**
 53 * DOC: Memory Based Interrupts
 54 *
 55 * MMIO register based interrupts infrastructure used for non-virtualized mode
 56 * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently
 57 * to allow delivering interrupts to a large number of Virtual machines or
 58 * containers. Memory based interrupt status reporting provides an efficient
 59 * and scalable infrastructure.
 60 *
 61 * For memory based interrupt status reporting hardware sequence is:
 62 *  * Engine writes the interrupt event to memory
 63 *    (Pointer to memory location is provided by SW. This memory surface must
 64 *    be mapped to system memory and must be marked as un-cacheable (UC) on
 65 *    Graphics IP Caches)
 66 *  * Engine triggers an interrupt to host.
 67 */
 68
 69/**
 70 * DOC: Memory Based Interrupts Page Layout
 71 *
 72 * `Memory Based Interrupts`_ requires three different objects, which are
 73 * called "page" in the specs, even if they aren't page-sized or aligned.
 74 *
 75 * To simplify the code we allocate a single page size object and then use
 76 * offsets to embedded "pages". The address of those "pages" are then
 77 * programmed in the HW via LRI and LRM in the context image.
 78 *
 79 * - _`Interrupt Status Report Page`: this page contains the interrupt
 80 *   status vectors for each unit. Each bit in the interrupt vectors is
 81 *   converted to a byte, with the byte being set to 0xFF when an
 82 *   interrupt is triggered; interrupt vectors are 16b big so each unit
 83 *   gets 16B. One space is reserved for each bit in one of the
 84 *   GT_INTR_DWx registers, so this object needs a total of 1024B.
 85 *   This object needs to be 4KiB aligned.
 86 *
 87 * - _`Interrupt Source Report Page`: this is the equivalent of the
 88 *   GT_INTR_DWx registers, with each bit in those registers being
 89 *   mapped to a byte here. The offsets are the same, just bytes instead
 90 *   of bits. This object needs to be cacheline aligned.
 91 *
 92 * - Interrupt Mask: the HW needs a location to fetch the interrupt
 93 *   mask vector to be used by the LRM in the context, so we just use
 94 *   the next available space in the interrupt page.
 95 *
 96 * ::
 97 *
 98 *   0x0000   +===========+  <== Interrupt Status Report Page
 99 *            |           |
100 *            |           |     ____ +----+----------------+
101 *            |           |    /     |  0 | USER INTERRUPT |
102 *            +-----------+ __/      |  1 |                |
103 *            |  HWE(n)   | __       |    | CTX SWITCH     |
104 *            +-----------+   \      |    | WAIT SEMAPHORE |
105 *            |           |    \____ | 15 |                |
106 *            |           |          +----+----------------+
107 *            |           |
108 *   0x0400   +===========+  <== Interrupt Source Report Page
109 *            |  HWE(0)   |
110 *            |  HWE(1)   |
111 *            |           |
112 *            |  HWE(x)   |
113 *   0x0440   +===========+  <== Interrupt Enable Mask
114 *            |           |
115 *            |           |
116 *            +-----------+
117 *
118 *
119 * MSI-X use case
120 *
121 * When using MSI-X, hw engines report interrupt status and source to engine
122 * instance 0. For this scenario, in order to differentiate between the
123 * engines, we need to pass different status/source pointers in the LRC.
124 *
125 * The requirements on those pointers are:
126 * - Interrupt status should be 4KiB aligned
127 * - Interrupt source should be 64 bytes aligned
128 *
129 * To accommodate this, we duplicate the memirq page layout above -
130 * allocating a page for each engine instance and pass this page in the LRC.
131 * Note that the same page can be reused for different engine types.
132 * For example, an LRC executing on CCS #x will have pointers to page #x,
133 * and an LRC executing on BCS #x will have the same pointers.
134 *
135 * ::
136 *
137 *   0x0000   +==============================+  <== page for instance 0 (BCS0, CCS0, etc.)
138 *            | Interrupt Status Report Page |
139 *   0x0400   +==============================+
140 *            | Interrupt Source Report Page |
141 *   0x0440   +==============================+
142 *            | Interrupt Enable Mask        |
143 *            +==============================+
144 *            | Not used                     |
145 *   0x1000   +==============================+  <== page for instance 1 (BCS1, CCS1, etc.)
146 *            | Interrupt Status Report Page |
147 *   0x1400   +==============================+
148 *            | Interrupt Source Report Page |
149 *   0x1440   +==============================+
150 *            | Not used                     |
151 *   0x2000   +==============================+  <== page for instance 2 (BCS2, CCS2, etc.)
152 *            | ...                          |
153 *            +==============================+
154 *
155 */
156
157static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq)
158{
159	/*
160	 * When the HW engines are configured to use MSI-X,
161	 * they report interrupt status and source to the offset of
162	 * engine instance 0.
163	 */
164	return xe_device_has_msix(memirq_to_xe(memirq));
165}
166
167static int memirq_alloc_pages(struct xe_memirq *memirq)
168{
169	struct xe_device *xe = memirq_to_xe(memirq);
170	struct xe_tile *tile = memirq_to_tile(memirq);
171	size_t bo_size = hw_reports_to_instance_zero(memirq) ?
172		XE_HW_ENGINE_MAX_INSTANCE * SZ_4K : SZ_4K;
173	struct xe_bo *bo;
174	int err;
175
176	BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET(0), SZ_64));
177	BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET(0), SZ_4K));
178
179	bo = xe_managed_bo_create_pin_map(xe, tile, bo_size,
180					  XE_BO_FLAG_SYSTEM |
181					  XE_BO_FLAG_GGTT |
182					  XE_BO_FLAG_GGTT_INVALIDATE |
183					  XE_BO_FLAG_NEEDS_UC |
184					  XE_BO_FLAG_NEEDS_CPU_ACCESS);
185	if (IS_ERR(bo)) {
186		err = PTR_ERR(bo);
187		goto out;
188	}
189
190	memirq_assert(memirq, !xe_bo_is_vram(bo));
191	memirq_assert(memirq, !memirq->bo);
192
193	iosys_map_memset(&bo->vmap, 0, 0, bo_size);
194
195	memirq->bo = bo;
196	memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0));
197	memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET(0));
198	memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET);
199
200	memirq_assert(memirq, !memirq->source.is_iomem);
201	memirq_assert(memirq, !memirq->status.is_iomem);
202	memirq_assert(memirq, !memirq->mask.is_iomem);
203
204	memirq_debug(memirq, "page offsets: bo %#x bo_size %zu source %#x status %#x\n",
205		     xe_bo_ggtt_addr(bo), bo_size, XE_MEMIRQ_SOURCE_OFFSET(0),
206		     XE_MEMIRQ_STATUS_OFFSET(0));
207
208	return 0;
209
210out:
211	memirq_err(memirq, "Failed to allocate memirq page (%pe)\n", ERR_PTR(err));
212	return err;
213}
214
215static void memirq_set_enable(struct xe_memirq *memirq, bool enable)
216{
217	iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0);
218
219	memirq->enabled = enable;
220}
221
222/**
223 * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_.
224 * @memirq: the &xe_memirq to initialize
225 *
226 * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_
227 * used by `Memory Based Interrupts`_.
228 *
229 * These allocations are managed and will be implicitly released on unload.
230 *
231 * If this function fails then the driver won't be able to operate correctly.
232 * If `Memory Based Interrupts`_ are not used this function will return 0.
233 *
234 * Return: 0 on success or a negative error code on failure.
235 */
236int xe_memirq_init(struct xe_memirq *memirq)
237{
238	struct xe_device *xe = memirq_to_xe(memirq);
239	int err;
240
241	if (!xe_device_uses_memirq(xe))
242		return 0;
243
244	err = memirq_alloc_pages(memirq);
245	if (unlikely(err))
246		return err;
247
248	/* we need to start with all irqs enabled */
249	memirq_set_enable(memirq, true);
250
251	return 0;
252}
253
254static u32 __memirq_source_page(struct xe_memirq *memirq, u16 instance)
255{
256	memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE);
257	memirq_assert(memirq, memirq->bo);
258
259	instance = hw_reports_to_instance_zero(memirq) ? instance : 0;
260	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(instance);
261}
262
263/**
264 * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_.
265 * @memirq: the &xe_memirq to query
266 * @hwe: the hw engine for which we want the report page
267 *
268 * Shall be called when `Memory Based Interrupts`_ are used
269 * and xe_memirq_init() didn't fail.
270 *
271 * Return: GGTT's offset of the `Interrupt Source Report Page`_.
272 */
273u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
274{
275	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
276
277	return __memirq_source_page(memirq, hwe->instance);
278}
279
280static u32 __memirq_status_page(struct xe_memirq *memirq, u16 instance)
281{
282	memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE);
283	memirq_assert(memirq, memirq->bo);
284
285	instance = hw_reports_to_instance_zero(memirq) ? instance : 0;
286	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(instance);
287}
288
289/**
290 * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_.
291 * @memirq: the &xe_memirq to query
292 * @hwe: the hw engine for which we want the report page
293 *
294 * Shall be called when `Memory Based Interrupts`_ are used
295 * and xe_memirq_init() didn't fail.
296 *
297 * Return: GGTT's offset of the `Interrupt Status Report Page`_.
298 */
299u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
300{
301	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
302
303	return __memirq_status_page(memirq, hwe->instance);
304}
305
306/**
307 * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask.
308 * @memirq: the &xe_memirq to query
309 *
310 * Shall be called when `Memory Based Interrupts`_ are used
311 * and xe_memirq_init() didn't fail.
312 *
313 * Return: GGTT's offset of the Interrupt Enable Mask.
314 */
315u32 xe_memirq_enable_ptr(struct xe_memirq *memirq)
316{
317	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
318	memirq_assert(memirq, memirq->bo);
319
320	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET;
321}
322
323/**
324 * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_.
325 * @memirq: the &xe_memirq
326 * @guc: the &xe_guc to setup
327 *
328 * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_
329 * to be used by the GuC when `Memory Based Interrupts`_ are required.
330 *
331 * Shall be called when `Memory Based Interrupts`_ are used
332 * and xe_memirq_init() didn't fail.
333 *
334 * Return: 0 on success or a negative error code on failure.
335 */
336int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc)
337{
338	bool is_media = xe_gt_is_media_type(guc_to_gt(guc));
339	u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
340	u32 source, status;
341	int err;
342
343	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
344
345	source = __memirq_source_page(memirq, 0) + offset;
346	status = __memirq_status_page(memirq, 0) + offset * SZ_16;
347
348	err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY,
349				source);
350	if (unlikely(err))
351		goto failed;
352
353	err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY,
354				status);
355	if (unlikely(err))
356		goto failed;
357
358	return 0;
359
360failed:
361	memirq_err(memirq, "Failed to setup report pages in %s (%pe)\n",
362		   guc_name(guc), ERR_PTR(err));
363	return err;
364}
365
366/**
367 * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_.
368 * @memirq: struct xe_memirq
369 *
370 * This is part of the driver IRQ setup flow.
371 *
372 * This function shall only be used on platforms that use
373 * `Memory Based Interrupts`_.
374 */
375void xe_memirq_reset(struct xe_memirq *memirq)
376{
377	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
378
379	if (memirq->bo)
380		memirq_set_enable(memirq, false);
381}
382
383/**
384 * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_.
385 * @memirq: the &xe_memirq
386 *
387 * This is part of the driver IRQ setup flow.
388 *
389 * This function shall only be used on platforms that use
390 * `Memory Based Interrupts`_.
391 */
392void xe_memirq_postinstall(struct xe_memirq *memirq)
393{
394	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
395
396	if (memirq->bo)
397		memirq_set_enable(memirq, true);
398}
399
400static bool __memirq_received(struct xe_memirq *memirq,
401			      struct iosys_map *vector, u16 offset,
402			      const char *name, bool clear)
403{
404	u8 value;
405
406	value = iosys_map_rd(vector, offset, u8);
407	if (value) {
408		if (value != 0xff)
409			memirq_err_ratelimited(memirq,
410					       "Unexpected memirq value %#x from %s at %u\n",
411					       value, name, offset);
412		if (clear)
413			iosys_map_wr(vector, offset, u8, 0x00);
414	}
415
416	return value;
417}
418
419static bool memirq_received_noclear(struct xe_memirq *memirq,
420				    struct iosys_map *vector,
421				    u16 offset, const char *name)
422{
423	return __memirq_received(memirq, vector, offset, name, false);
424}
425
426static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,
427			    u16 offset, const char *name)
428{
429	return __memirq_received(memirq, vector, offset, name, true);
430}
431
432static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status,
433				   struct xe_hw_engine *hwe)
434{
435	memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr);
436
437	if (memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name))
438		xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT);
439}
440
441static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status,
442				struct xe_guc *guc)
443{
444	const char *name = guc_name(guc);
445
446	memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr);
447
448	if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
449		xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
450
451	/*
452	 * This is a software interrupt that must be cleared after it's consumed
453	 * to avoid race conditions where xe_gt_sriov_vf_recovery_pending()
454	 * returns false.
455	 */
456	if (memirq_received_noclear(memirq, status, ilog2(GUC_INTR_SW_INT_0),
457				    name)) {
458		xe_guc_irq_handler(guc, GUC_INTR_SW_INT_0);
459		iosys_map_wr(status, ilog2(GUC_INTR_SW_INT_0), u8, 0x00);
460	}
461}
462
463/**
464 * xe_memirq_hwe_handler - Check and process interrupts for a specific HW engine.
465 * @memirq: the &xe_memirq
466 * @hwe: the hw engine to process
467 *
468 * This function reads and dispatches `Memory Based Interrupts` for the provided HW engine.
469 */
470void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
471{
472	u16 offset = hwe->irq_offset;
473	u16 instance = hw_reports_to_instance_zero(memirq) ? hwe->instance : 0;
474	struct iosys_map src_offset = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
475							    XE_MEMIRQ_SOURCE_OFFSET(instance));
476
477	if (memirq_received(memirq, &src_offset, offset, "SRC")) {
478		struct iosys_map status_offset =
479			IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
480					      XE_MEMIRQ_STATUS_OFFSET(instance) + offset * SZ_16);
481		memirq_dispatch_engine(memirq, &status_offset, hwe);
482	}
483}
484
485/**
486 * xe_memirq_guc_sw_int_0_irq_pending() - SW_INT_0 IRQ is pending
487 * @memirq: the &xe_memirq
488 * @guc: the &xe_guc to check for IRQ
489 *
490 * Return: True if SW_INT_0 IRQ is pending on @guc, False otherwise
491 */
492bool xe_memirq_guc_sw_int_0_irq_pending(struct xe_memirq *memirq, struct xe_guc *guc)
493{
494	struct xe_gt *gt = guc_to_gt(guc);
495	u32 offset = xe_gt_is_media_type(gt) ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
496	struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->status, offset * SZ_16);
497
498	return memirq_received_noclear(memirq, &map, ilog2(GUC_INTR_SW_INT_0),
499				       guc_name(guc));
500}
501
502/**
503 * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.
504 * @memirq: the &xe_memirq
505 *
506 * This function reads and dispatches `Memory Based Interrupts`.
507 */
508void xe_memirq_handler(struct xe_memirq *memirq)
509{
510	struct xe_device *xe = memirq_to_xe(memirq);
511	struct xe_tile *tile = memirq_to_tile(memirq);
512	struct xe_hw_engine *hwe;
513	enum xe_hw_engine_id id;
514	struct iosys_map map;
515	unsigned int gtid;
516	struct xe_gt *gt;
517
518	if (!memirq->bo)
519		return;
520
521	memirq_assert(memirq, !memirq->source.is_iomem);
522	memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr);
523	memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32);
524
525	for_each_gt(gt, xe, gtid) {
526		if (gt->tile != tile)
527			continue;
528
529		for_each_hw_engine(hwe, gt, id)
530			xe_memirq_hwe_handler(memirq, hwe);
531	}
532
533	/* GuC and media GuC (if present) must be checked separately */
534
535	if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) {
536		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);
537		memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc);
538	}
539
540	if (!tile->media_gt)
541		return;
542
543	if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) {
544		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);
545		memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc);
546	}
547}