xe_guc_tlb_inval.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2025 Intel Corporation
  4 */
  5
  6#include "abi/guc_actions_abi.h"
  7
  8#include "xe_device.h"
  9#include "xe_gt_stats.h"
 10#include "xe_gt_types.h"
 11#include "xe_guc.h"
 12#include "xe_guc_ct.h"
 13#include "xe_guc_tlb_inval.h"
 14#include "xe_force_wake.h"
 15#include "xe_mmio.h"
 16#include "xe_tlb_inval.h"
 17
 18#include "regs/xe_guc_regs.h"
 19
 20/*
 21 * XXX: The seqno algorithm relies on TLB invalidation being processed in order
 22 * which they currently are by the GuC, if that changes the algorithm will need
 23 * to be updated.
 24 */
 25
 26static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len)
 27{
 28	struct xe_gt *gt = guc_to_gt(guc);
 29
 30	xe_gt_assert(gt, action[1]);	/* Seqno */
 31
 32	xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1);
 33	return xe_guc_ct_send(&guc->ct, action, len,
 34			      G2H_LEN_DW_TLB_INVALIDATE, 1);
 35}
 36
 37#define MAKE_INVAL_OP(type)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
 38		XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
 39		XE_GUC_TLB_INVAL_FLUSH_CACHE)
 40
 41static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
 42{
 43	struct xe_guc *guc = tlb_inval->private;
 44	u32 action[] = {
 45		XE_GUC_ACTION_TLB_INVALIDATION_ALL,
 46		seqno,
 47		MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
 48	};
 49
 50	return send_tlb_inval(guc, action, ARRAY_SIZE(action));
 51}
 52
 53static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 54{
 55	struct xe_guc *guc = tlb_inval->private;
 56	struct xe_gt *gt = guc_to_gt(guc);
 57	struct xe_device *xe = guc_to_xe(guc);
 58
 59	/*
 60	 * Returning -ECANCELED in this function is squashed at the caller and
 61	 * signals waiters.
 62	 */
 63
 64	if (xe_guc_ct_enabled(&guc->ct) && guc->submission_state.enabled) {
 65		u32 action[] = {
 66			XE_GUC_ACTION_TLB_INVALIDATION,
 67			seqno,
 68			MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
 69		};
 70
 71		return send_tlb_inval(guc, action, ARRAY_SIZE(action));
 72	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
 73		struct xe_mmio *mmio = &gt->mmio;
 74		unsigned int fw_ref;
 75
 76		if (IS_SRIOV_VF(xe))
 77			return -ECANCELED;
 78
 79		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
 80		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
 81			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
 82					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
 83			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0,
 84					PVC_GUC_TLB_INV_DESC0_VALID);
 85		} else {
 86			xe_mmio_write32(mmio, GUC_TLB_INV_CR,
 87					GUC_TLB_INV_CR_INVALIDATE);
 88		}
 89		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 90	}
 91
 92	return -ECANCELED;
 93}
 94
 95/*
 96 * Ensure that roundup_pow_of_two(length) doesn't overflow.
 97 * Note that roundup_pow_of_two() operates on unsigned long,
 98 * not on u64.
 99 */
100#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
101
102static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
103				u64 start, u64 end, u32 asid)
104{
105#define MAX_TLB_INVALIDATION_LEN	7
106	struct xe_guc *guc = tlb_inval->private;
107	struct xe_gt *gt = guc_to_gt(guc);
108	u32 action[MAX_TLB_INVALIDATION_LEN];
109	u64 length = end - start;
110	int len = 0;
111
112	if (guc_to_xe(guc)->info.force_execlist)
113		return -ECANCELED;
114
115	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
116	action[len++] = seqno;
117	if (!gt_to_xe(gt)->info.has_range_tlb_inval ||
118	    length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
119		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
120	} else {
121		u64 orig_start = start;
122		u64 align;
123
124		if (length < SZ_4K)
125			length = SZ_4K;
126
127		/*
128		 * We need to invalidate a higher granularity if start address
129		 * is not aligned to length. When start is not aligned with
130		 * length we need to find the length large enough to create an
131		 * address mask covering the required range.
132		 */
133		align = roundup_pow_of_two(length);
134		start = ALIGN_DOWN(start, align);
135		end = ALIGN(end, align);
136		length = align;
137		while (start + length < end) {
138			length <<= 1;
139			start = ALIGN_DOWN(orig_start, length);
140		}
141
142		/*
143		 * Minimum invalidation size for a 2MB page that the hardware
144		 * expects is 16MB
145		 */
146		if (length >= SZ_2M) {
147			length = max_t(u64, SZ_16M, length);
148			start = ALIGN_DOWN(orig_start, length);
149		}
150
151		xe_gt_assert(gt, length >= SZ_4K);
152		xe_gt_assert(gt, is_power_of_2(length));
153		xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
154						    ilog2(SZ_2M) + 1)));
155		xe_gt_assert(gt, IS_ALIGNED(start, length));
156
157		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
158		action[len++] = asid;
159		action[len++] = lower_32_bits(start);
160		action[len++] = upper_32_bits(start);
161		action[len++] = ilog2(length) - ilog2(SZ_4K);
162	}
163
164	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
165
166	return send_tlb_inval(guc, action, len);
167}
168
169static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval)
170{
171	struct xe_guc *guc = tlb_inval->private;
172
173	return xe_guc_ct_initialized(&guc->ct);
174}
175
176static void tlb_inval_flush(struct xe_tlb_inval *tlb_inval)
177{
178	struct xe_guc *guc = tlb_inval->private;
179
180	LNL_FLUSH_WORK(&guc->ct.g2h_worker);
181}
182
183static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval)
184{
185	struct xe_guc *guc = tlb_inval->private;
186
187	/* this reflects what HW/GuC needs to process TLB inv request */
188	const long hw_tlb_timeout = HZ / 4;
189
190	/* this estimates actual delay caused by the CTB transport */
191	long delay = xe_guc_ct_queue_proc_time_jiffies(&guc->ct);
192
193	return hw_tlb_timeout + 2 * delay;
194}
195
196static const struct xe_tlb_inval_ops guc_tlb_inval_ops = {
197	.all = send_tlb_inval_all,
198	.ggtt = send_tlb_inval_ggtt,
199	.ppgtt = send_tlb_inval_ppgtt,
200	.initialized = tlb_inval_initialized,
201	.flush = tlb_inval_flush,
202	.timeout_delay = tlb_inval_timeout_delay,
203};
204
205/**
206 * xe_guc_tlb_inval_init_early() - Init GuC TLB invalidation early
207 * @guc: GuC object
208 * @tlb_inval: TLB invalidation client
209 *
210 * Initialize GuC TLB invalidation by setting back pointer in TLB invalidation
211 * client to the GuC and setting GuC backend ops.
212 */
213void xe_guc_tlb_inval_init_early(struct xe_guc *guc,
214				 struct xe_tlb_inval *tlb_inval)
215{
216	tlb_inval->private = guc;
217	tlb_inval->ops = &guc_tlb_inval_ops;
218}
219
220/**
221 * xe_guc_tlb_inval_done_handler() - TLB invalidation done handler
222 * @guc: guc
223 * @msg: message indicating TLB invalidation done
224 * @len: length of message
225 *
226 * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any
227 * invalidation fences for seqno. Algorithm for this depends on seqno being
228 * received in-order and asserts this assumption.
229 *
230 * Return: 0 on success, -EPROTO for malformed messages.
231 */
232int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
233{
234	struct xe_gt *gt = guc_to_gt(guc);
235
236	if (unlikely(len != 1))
237		return -EPROTO;
238
239	xe_tlb_inval_done_handler(&gt->tlb_inval, msg[0]);
240
241	return 0;
242}