xe_ring_ops.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2022 Intel Corporation
  4 */
  5
  6#include "xe_ring_ops.h"
  7
  8#include <generated/xe_wa_oob.h>
  9
 10#include "instructions/xe_gpu_commands.h"
 11#include "instructions/xe_mi_commands.h"
 12#include "regs/xe_engine_regs.h"
 13#include "regs/xe_gt_regs.h"
 14#include "regs/xe_lrc_layout.h"
 15#include "xe_exec_queue_types.h"
 16#include "xe_gt.h"
 17#include "xe_lrc.h"
 18#include "xe_macros.h"
 19#include "xe_sched_job.h"
 20#include "xe_sriov.h"
 21#include "xe_vm_types.h"
 22#include "xe_vm.h"
 23#include "xe_wa.h"
 24
 25/*
 26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D
 27 * pipeline (i.e., CCS engines).
 28 */
 29#define PIPE_CONTROL_3D_ENGINE_FLAGS (\
 30		PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
 31		PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
 32		PIPE_CONTROL_TILE_CACHE_FLUSH | \
 33		PIPE_CONTROL_DEPTH_STALL | \
 34		PIPE_CONTROL_STALL_AT_SCOREBOARD | \
 35		PIPE_CONTROL_PSD_SYNC | \
 36		PIPE_CONTROL_AMFS_FLUSH | \
 37		PIPE_CONTROL_VF_CACHE_INVALIDATE | \
 38		PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
 39
 40/* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
 41#define PIPE_CONTROL_3D_ARCH_FLAGS ( \
 42		PIPE_CONTROL_3D_ENGINE_FLAGS | \
 43		PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
 44		PIPE_CONTROL_FLUSH_ENABLE | \
 45		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
 46		PIPE_CONTROL_DC_FLUSH_ENABLE)
 47
 48static u32 preparser_disable(bool state)
 49{
 50	return MI_ARB_CHECK | BIT(8) | state;
 51}
 52
 53static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
 54			      u32 *dw, int i)
 55{
 56	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
 57	dw[i++] = reg.addr + gt->mmio.adj_offset;
 58	dw[i++] = AUX_INV;
 59	dw[i++] = MI_NOOP;
 60
 61	return i;
 62}
 63
 64static int emit_user_interrupt(u32 *dw, int i)
 65{
 66	dw[i++] = MI_USER_INTERRUPT;
 67	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 68	dw[i++] = MI_ARB_CHECK;
 69
 70	return i;
 71}
 72
 73static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
 74{
 75	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
 76	dw[i++] = addr;
 77	dw[i++] = 0;
 78	dw[i++] = value;
 79
 80	return i;
 81}
 82
 83static int emit_flush_dw(u32 *dw, int i)
 84{
 85	dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
 86	dw[i++] = 0;
 87	dw[i++] = 0;
 88	dw[i++] = 0;
 89
 90	return i;
 91}
 92
 93static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)
 94{
 95	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
 96		  flags;
 97	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
 98	dw[i++] = 0;
 99	dw[i++] = value;
100
101	return i;
102}
103
104static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
105{
106	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
107	dw[i++] = lower_32_bits(batch_addr);
108	dw[i++] = upper_32_bits(batch_addr);
109
110	return i;
111}
112
113static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i)
114{
115	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW |
116		  MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0;
117
118	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
119	dw[i++] = 0;
120	dw[i++] = val;
121
122	return i;
123}
124
125static int
126emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
127{
128	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
129	dw[i++] = bit_group_1;
130	dw[i++] = offset;
131	dw[i++] = 0;
132	dw[i++] = value;
133	dw[i++] = 0;
134
135	return i;
136}
137
138static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
139				int i)
140{
141	u32 flags0 = 0;
142	u32 flags1 = PIPE_CONTROL_CS_STALL |
143		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
144		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
145		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
146		PIPE_CONTROL_VF_CACHE_INVALIDATE |
147		PIPE_CONTROL_CONST_CACHE_INVALIDATE |
148		PIPE_CONTROL_STATE_CACHE_INVALIDATE |
149		PIPE_CONTROL_QW_WRITE |
150		PIPE_CONTROL_STORE_DATA_INDEX;
151
152	if (invalidate_tlb)
153		flags1 |= PIPE_CONTROL_TLB_INVALIDATE;
154
155	flags1 &= ~mask_flags;
156
157	if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
158		flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE;
159
160	return emit_pipe_control(dw, i, flags0, flags1,
161				 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
162}
163
164static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
165				       u32 *dw, int i)
166{
167	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
168	dw[i++] = lower_32_bits(addr);
169	dw[i++] = upper_32_bits(addr);
170	dw[i++] = lower_32_bits(value);
171	dw[i++] = upper_32_bits(value);
172
173	return i;
174}
175
176static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
177{
178	struct xe_gt *gt = job->q->gt;
179	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
180	u32 flags;
181
182	if (XE_GT_WA(gt, 14016712196))
183		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
184				      LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
185
186	flags = (PIPE_CONTROL_CS_STALL |
187		 PIPE_CONTROL_TILE_CACHE_FLUSH |
188		 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
189		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
190		 PIPE_CONTROL_DC_FLUSH_ENABLE |
191		 PIPE_CONTROL_FLUSH_ENABLE);
192
193	if (XE_GT_WA(gt, 1409600907))
194		flags |= PIPE_CONTROL_DEPTH_STALL;
195
196	if (lacks_render)
197		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
198	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
199		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
200
201	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
202}
203
204static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
205{
206	if (hwe->class != XE_ENGINE_CLASS_RENDER)
207		return i;
208
209	if (XE_GT_WA(hwe->gt, 16020292621))
210		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
211				      RING_NOPID(hwe->mmio_base).addr, 0);
212
213	return i;
214}
215
216static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
217			      int i)
218{
219	u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
220		    PIPE_CONTROL_QW_WRITE;
221
222	if (!stall_only)
223		flags |= PIPE_CONTROL_FLUSH_ENABLE;
224
225	return emit_pipe_control(dw, i, 0, flags, addr, value);
226}
227
228static u32 get_ppgtt_flag(struct xe_sched_job *job)
229{
230	if (job->q->vm && !job->ggtt)
231		return BIT(8);
232
233	return 0;
234}
235
236static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
237{
238	dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
239	dw[i++] = RING_CTX_TIMESTAMP(0).addr;
240	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
241	dw[i++] = 0;
242
243	return i;
244}
245
246/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
247static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
248				    u64 batch_addr, u32 *head, u32 seqno)
249{
250	u32 dw[MAX_JOB_SIZE_DW], i = 0;
251	u32 ppgtt_flag = get_ppgtt_flag(job);
252	struct xe_gt *gt = job->q->gt;
253
254	*head = lrc->ring.tail;
255
256	i = emit_copy_timestamp(lrc, dw, i);
257
258	if (job->ring_ops_flush_tlb) {
259		dw[i++] = preparser_disable(true);
260		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
261					seqno, MI_INVALIDATE_TLB, dw, i);
262		dw[i++] = preparser_disable(false);
263	} else {
264		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
265					seqno, dw, i);
266	}
267
268	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
269
270	/* Don't preempt fence signaling */
271	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE;
272
273	if (job->user_fence.used) {
274		i = emit_flush_dw(dw, i);
275		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
276						job->user_fence.value,
277						dw, i);
278	}
279
280	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
281
282	i = emit_user_interrupt(dw, i);
283
284	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
285
286	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
287}
288
289static bool has_aux_ccs(struct xe_device *xe)
290{
291	/*
292	 * PVC is a special case that has no compression of either type
293	 * (FlatCCS or AuxCCS).  Also, AuxCCS is no longer used from Xe2
294	 * onward, so any future platforms with no FlatCCS will not have
295	 * AuxCCS either.
296	 */
297	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
298		return false;
299
300	return !xe->info.has_flat_ccs;
301}
302
303static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
304				   u64 batch_addr, u32 *head, u32 seqno)
305{
306	u32 dw[MAX_JOB_SIZE_DW], i = 0;
307	u32 ppgtt_flag = get_ppgtt_flag(job);
308	struct xe_gt *gt = job->q->gt;
309	struct xe_device *xe = gt_to_xe(gt);
310	bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
311
312	*head = lrc->ring.tail;
313
314	i = emit_copy_timestamp(lrc, dw, i);
315
316	dw[i++] = preparser_disable(true);
317
318	/* hsdes: 1809175790 */
319	if (has_aux_ccs(xe)) {
320		if (decode)
321			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
322		else
323			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
324	}
325
326	if (job->ring_ops_flush_tlb)
327		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
328					seqno, MI_INVALIDATE_TLB, dw, i);
329
330	dw[i++] = preparser_disable(false);
331
332	if (!job->ring_ops_flush_tlb)
333		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
334					seqno, dw, i);
335
336	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
337
338	/* Don't preempt fence signaling */
339	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE;
340
341	if (job->user_fence.used) {
342		i = emit_flush_dw(dw, i);
343		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
344						job->user_fence.value,
345						dw, i);
346	}
347
348	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
349
350	i = emit_user_interrupt(dw, i);
351
352	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
353
354	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
355}
356
357static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
358					    struct xe_lrc *lrc,
359					    u64 batch_addr, u32 *head,
360					    u32 seqno)
361{
362	u32 dw[MAX_JOB_SIZE_DW], i = 0;
363	u32 ppgtt_flag = get_ppgtt_flag(job);
364	struct xe_gt *gt = job->q->gt;
365	struct xe_device *xe = gt_to_xe(gt);
366	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
367	u32 mask_flags = 0;
368
369	*head = lrc->ring.tail;
370
371	i = emit_copy_timestamp(lrc, dw, i);
372
373	dw[i++] = preparser_disable(true);
374	if (lacks_render)
375		mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
376	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
377		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
378
379	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
380	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
381
382	/* hsdes: 1809175790 */
383	if (has_aux_ccs(xe))
384		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
385
386	dw[i++] = preparser_disable(false);
387
388	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
389				seqno, dw, i);
390
391	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
392
393	/* Don't preempt fence signaling */
394	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE;
395
396	i = emit_render_cache_flush(job, dw, i);
397
398	if (job->user_fence.used)
399		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
400						job->user_fence.value,
401						dw, i);
402
403	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
404
405	i = emit_user_interrupt(dw, i);
406
407	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
408
409	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
410
411	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
412}
413
414static void emit_migration_job_gen12(struct xe_sched_job *job,
415				     struct xe_lrc *lrc, u32 *head,
416				     u32 seqno)
417{
418	u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc);
419	u32 dw[MAX_JOB_SIZE_DW], i = 0;
420
421	*head = lrc->ring.tail;
422
423	i = emit_copy_timestamp(lrc, dw, i);
424
425	i = emit_store_imm_ggtt(saddr, seqno, dw, i);
426
427	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
428
429	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
430
431	dw[i++] = preparser_disable(true);
432	i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i);
433	dw[i++] = preparser_disable(false);
434
435	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
436
437	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno,
438				job->migrate_flush_flags,
439				dw, i);
440
441	i = emit_user_interrupt(dw, i);
442
443	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
444
445	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
446}
447
448static void emit_job_gen12_gsc(struct xe_sched_job *job)
449{
450	struct xe_gt *gt = job->q->gt;
451
452	xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
453
454	__emit_job_gen12_simple(job, job->q->lrc[0],
455				job->ptrs[0].batch_addr,
456				&job->ptrs[0].head,
457				xe_sched_job_lrc_seqno(job));
458}
459
460static void emit_job_gen12_copy(struct xe_sched_job *job)
461{
462	int i;
463
464	if (xe_sched_job_is_migration(job->q)) {
465		emit_migration_job_gen12(job, job->q->lrc[0],
466					 &job->ptrs[0].head,
467					 xe_sched_job_lrc_seqno(job));
468		return;
469	}
470
471	for (i = 0; i < job->q->width; ++i)
472		__emit_job_gen12_simple(job, job->q->lrc[i],
473					job->ptrs[i].batch_addr,
474					&job->ptrs[i].head,
475					xe_sched_job_lrc_seqno(job));
476}
477
478static void emit_job_gen12_video(struct xe_sched_job *job)
479{
480	int i;
481
482	/* FIXME: Not doing parallel handshake for now */
483	for (i = 0; i < job->q->width; ++i)
484		__emit_job_gen12_video(job, job->q->lrc[i],
485				       job->ptrs[i].batch_addr,
486				       &job->ptrs[i].head,
487				       xe_sched_job_lrc_seqno(job));
488}
489
490static void emit_job_gen12_render_compute(struct xe_sched_job *job)
491{
492	int i;
493
494	for (i = 0; i < job->q->width; ++i)
495		__emit_job_gen12_render_compute(job, job->q->lrc[i],
496						job->ptrs[i].batch_addr,
497						&job->ptrs[i].head,
498						xe_sched_job_lrc_seqno(job));
499}
500
501static const struct xe_ring_ops ring_ops_gen12_gsc = {
502	.emit_job = emit_job_gen12_gsc,
503};
504
505static const struct xe_ring_ops ring_ops_gen12_copy = {
506	.emit_job = emit_job_gen12_copy,
507};
508
509static const struct xe_ring_ops ring_ops_gen12_video = {
510	.emit_job = emit_job_gen12_video,
511};
512
513static const struct xe_ring_ops ring_ops_gen12_render_compute = {
514	.emit_job = emit_job_gen12_render_compute,
515};
516
517const struct xe_ring_ops *
518xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
519{
520	switch (class) {
521	case XE_ENGINE_CLASS_OTHER:
522		return &ring_ops_gen12_gsc;
523	case XE_ENGINE_CLASS_COPY:
524		return &ring_ops_gen12_copy;
525	case XE_ENGINE_CLASS_VIDEO_DECODE:
526	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
527		return &ring_ops_gen12_video;
528	case XE_ENGINE_CLASS_RENDER:
529	case XE_ENGINE_CLASS_COMPUTE:
530		return &ring_ops_gen12_render_compute;
531	default:
532		return NULL;
533	}
534}