gen8_engine_cs.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2014 Intel Corporation
  4 */
  5
  6#include <drm/drm_print.h>
  7
  8#include "gen8_engine_cs.h"
  9#include "intel_engine_regs.h"
 10#include "intel_gpu_commands.h"
 11#include "intel_gt.h"
 12#include "intel_lrc.h"
 13#include "intel_ring.h"
 14
 15int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode)
 16{
 17	bool vf_flush_wa = false, dc_flush_wa = false;
 18	u32 *cs, flags = 0;
 19	int len;
 20
 21	flags |= PIPE_CONTROL_CS_STALL;
 22
 23	if (mode & EMIT_FLUSH) {
 24		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 25		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 26		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 27		flags |= PIPE_CONTROL_FLUSH_ENABLE;
 28	}
 29
 30	if (mode & EMIT_INVALIDATE) {
 31		flags |= PIPE_CONTROL_TLB_INVALIDATE;
 32		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 33		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 34		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 35		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 36		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 37		flags |= PIPE_CONTROL_QW_WRITE;
 38		flags |= PIPE_CONTROL_STORE_DATA_INDEX;
 39
 40		/*
 41		 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
 42		 * pipe control.
 43		 */
 44		if (GRAPHICS_VER(rq->i915) == 9)
 45			vf_flush_wa = true;
 46
 47		/* WaForGAMHang:kbl */
 48		if (IS_KABYLAKE(rq->i915) && IS_GRAPHICS_STEP(rq->i915, 0, STEP_C0))
 49			dc_flush_wa = true;
 50	}
 51
 52	len = 6;
 53
 54	if (vf_flush_wa)
 55		len += 6;
 56
 57	if (dc_flush_wa)
 58		len += 12;
 59
 60	cs = intel_ring_begin(rq, len);
 61	if (IS_ERR(cs))
 62		return PTR_ERR(cs);
 63
 64	if (vf_flush_wa)
 65		cs = gen8_emit_pipe_control(cs, 0, 0);
 66
 67	if (dc_flush_wa)
 68		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,
 69					    0);
 70
 71	cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);
 72
 73	if (dc_flush_wa)
 74		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);
 75
 76	intel_ring_advance(rq, cs);
 77
 78	return 0;
 79}
 80
 81int gen8_emit_flush_xcs(struct i915_request *rq, u32 mode)
 82{
 83	u32 cmd, *cs;
 84
 85	cs = intel_ring_begin(rq, 4);
 86	if (IS_ERR(cs))
 87		return PTR_ERR(cs);
 88
 89	cmd = MI_FLUSH_DW + 1;
 90
 91	/*
 92	 * We always require a command barrier so that subsequent
 93	 * commands, such as breadcrumb interrupts, are strictly ordered
 94	 * wrt the contents of the write cache being flushed to memory
 95	 * (and thus being coherent from the CPU).
 96	 */
 97	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
 98
 99	if (mode & EMIT_INVALIDATE) {
100		cmd |= MI_INVALIDATE_TLB;
101		if (rq->engine->class == VIDEO_DECODE_CLASS)
102			cmd |= MI_INVALIDATE_BSD;
103	}
104
105	*cs++ = cmd;
106	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;
107	*cs++ = 0; /* upper addr */
108	*cs++ = 0; /* value */
109	intel_ring_advance(rq, cs);
110
111	return 0;
112}
113
114int gen11_emit_flush_rcs(struct i915_request *rq, u32 mode)
115{
116	if (mode & EMIT_FLUSH) {
117		u32 *cs;
118		u32 flags = 0;
119
120		flags |= PIPE_CONTROL_CS_STALL;
121
122		flags |= PIPE_CONTROL_TILE_CACHE_FLUSH;
123		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
124		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
125		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
126		flags |= PIPE_CONTROL_FLUSH_ENABLE;
127		flags |= PIPE_CONTROL_QW_WRITE;
128		flags |= PIPE_CONTROL_STORE_DATA_INDEX;
129
130		cs = intel_ring_begin(rq, 6);
131		if (IS_ERR(cs))
132			return PTR_ERR(cs);
133
134		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);
135		intel_ring_advance(rq, cs);
136	}
137
138	if (mode & EMIT_INVALIDATE) {
139		u32 *cs;
140		u32 flags = 0;
141
142		flags |= PIPE_CONTROL_CS_STALL;
143
144		flags |= PIPE_CONTROL_COMMAND_CACHE_INVALIDATE;
145		flags |= PIPE_CONTROL_TLB_INVALIDATE;
146		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
147		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
148		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
149		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
150		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
151		flags |= PIPE_CONTROL_QW_WRITE;
152		flags |= PIPE_CONTROL_STORE_DATA_INDEX;
153
154		cs = intel_ring_begin(rq, 6);
155		if (IS_ERR(cs))
156			return PTR_ERR(cs);
157
158		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);
159		intel_ring_advance(rq, cs);
160	}
161
162	return 0;
163}
164
165static u32 preparser_disable(bool state)
166{
167	return MI_ARB_CHECK | 1 << 8 | state;
168}
169
170static i915_reg_t gen12_get_aux_inv_reg(struct intel_engine_cs *engine)
171{
172	switch (engine->id) {
173	case RCS0:
174		return GEN12_CCS_AUX_INV;
175	case BCS0:
176		return GEN12_BCS0_AUX_INV;
177	case VCS0:
178		return GEN12_VD0_AUX_INV;
179	case VCS2:
180		return GEN12_VD2_AUX_INV;
181	case VECS0:
182		return GEN12_VE0_AUX_INV;
183	case CCS0:
184		return GEN12_CCS0_AUX_INV;
185	default:
186		return INVALID_MMIO_REG;
187	}
188}
189
190static bool gen12_needs_ccs_aux_inv(struct intel_engine_cs *engine)
191{
192	i915_reg_t reg = gen12_get_aux_inv_reg(engine);
193
194	/*
195	 * So far platforms supported by i915 having flat ccs do not require
196	 * AUX invalidation. Check also whether the engine requires it.
197	 */
198	return i915_mmio_reg_valid(reg) && !HAS_FLAT_CCS(engine->i915);
199}
200
201u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs)
202{
203	i915_reg_t inv_reg = gen12_get_aux_inv_reg(engine);
204	u32 gsi_offset = engine->gt->uncore->gsi_offset;
205
206	if (!gen12_needs_ccs_aux_inv(engine))
207		return cs;
208
209	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;
210	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;
211	*cs++ = AUX_INV;
212
213	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |
214		MI_SEMAPHORE_REGISTER_POLL |
215		MI_SEMAPHORE_POLL |
216		MI_SEMAPHORE_SAD_EQ_SDD;
217	*cs++ = 0;
218	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;
219	*cs++ = 0;
220	*cs++ = 0;
221
222	return cs;
223}
224
225static int mtl_dummy_pipe_control(struct i915_request *rq)
226{
227	/* Wa_14016712196 */
228	if (IS_GFX_GT_IP_RANGE(rq->engine->gt, IP_VER(12, 70), IP_VER(12, 74)) ||
229	    IS_DG2(rq->i915)) {
230		u32 *cs;
231
232		/* dummy PIPE_CONTROL + depth flush */
233		cs = intel_ring_begin(rq, 6);
234		if (IS_ERR(cs))
235			return PTR_ERR(cs);
236		cs = gen12_emit_pipe_control(cs,
237					     0,
238					     PIPE_CONTROL_DEPTH_CACHE_FLUSH,
239					     LRC_PPHWSP_SCRATCH_ADDR);
240		intel_ring_advance(rq, cs);
241	}
242
243	return 0;
244}
245
246int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
247{
248	struct intel_engine_cs *engine = rq->engine;
249
250	/*
251	 * On Aux CCS platforms the invalidation of the Aux
252	 * table requires quiescing memory traffic beforehand
253	 */
254	if (mode & EMIT_FLUSH || gen12_needs_ccs_aux_inv(engine)) {
255		u32 bit_group_0 = 0;
256		u32 bit_group_1 = 0;
257		int err;
258		u32 *cs;
259
260		err = mtl_dummy_pipe_control(rq);
261		if (err)
262			return err;
263
264		bit_group_0 |= PIPE_CONTROL0_HDC_PIPELINE_FLUSH;
265
266		/*
267		 * When required, in MTL and beyond platforms we
268		 * need to set the CCS_FLUSH bit in the pipe control
269		 */
270		if (GRAPHICS_VER_FULL(rq->i915) >= IP_VER(12, 70))
271			bit_group_0 |= PIPE_CONTROL_CCS_FLUSH;
272
273		/*
274		 * L3 fabric flush is needed for AUX CCS invalidation
275		 * which happens as part of pipe-control so we can
276		 * ignore PIPE_CONTROL_FLUSH_L3. Also PIPE_CONTROL_FLUSH_L3
277		 * deals with Protected Memory which is not needed for
278		 * AUX CCS invalidation and lead to unwanted side effects.
279		 */
280		if ((mode & EMIT_FLUSH) &&
281		    GRAPHICS_VER_FULL(rq->i915) < IP_VER(12, 70))
282			bit_group_1 |= PIPE_CONTROL_FLUSH_L3;
283
284		bit_group_1 |= PIPE_CONTROL_TILE_CACHE_FLUSH;
285		bit_group_1 |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
286		bit_group_1 |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
287		/* Wa_1409600907:tgl,adl-p */
288		bit_group_1 |= PIPE_CONTROL_DEPTH_STALL;
289		bit_group_1 |= PIPE_CONTROL_DC_FLUSH_ENABLE;
290		bit_group_1 |= PIPE_CONTROL_FLUSH_ENABLE;
291
292		bit_group_1 |= PIPE_CONTROL_STORE_DATA_INDEX;
293		bit_group_1 |= PIPE_CONTROL_QW_WRITE;
294
295		bit_group_1 |= PIPE_CONTROL_CS_STALL;
296
297		if (!HAS_3D_PIPELINE(engine->i915))
298			bit_group_1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
299		else if (engine->class == COMPUTE_CLASS)
300			bit_group_1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
301
302		cs = intel_ring_begin(rq, 6);
303		if (IS_ERR(cs))
304			return PTR_ERR(cs);
305
306		cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1,
307					     LRC_PPHWSP_SCRATCH_ADDR);
308		intel_ring_advance(rq, cs);
309	}
310
311	if (mode & EMIT_INVALIDATE) {
312		u32 flags = 0;
313		u32 *cs, count;
314		int err;
315
316		err = mtl_dummy_pipe_control(rq);
317		if (err)
318			return err;
319
320		flags |= PIPE_CONTROL_COMMAND_CACHE_INVALIDATE;
321		flags |= PIPE_CONTROL_TLB_INVALIDATE;
322		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
323		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
324		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
325		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
326		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
327
328		flags |= PIPE_CONTROL_STORE_DATA_INDEX;
329		flags |= PIPE_CONTROL_QW_WRITE;
330
331		flags |= PIPE_CONTROL_CS_STALL;
332
333		if (!HAS_3D_PIPELINE(engine->i915))
334			flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
335		else if (engine->class == COMPUTE_CLASS)
336			flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
337
338		count = 8;
339		if (gen12_needs_ccs_aux_inv(rq->engine))
340			count += 8;
341
342		cs = intel_ring_begin(rq, count);
343		if (IS_ERR(cs))
344			return PTR_ERR(cs);
345
346		/*
347		 * Prevent the pre-parser from skipping past the TLB
348		 * invalidate and loading a stale page for the batch
349		 * buffer / request payload.
350		 */
351		*cs++ = preparser_disable(true);
352
353		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);
354
355		cs = gen12_emit_aux_table_inv(engine, cs);
356
357		*cs++ = preparser_disable(false);
358		intel_ring_advance(rq, cs);
359	}
360
361	return 0;
362}
363
364int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode)
365{
366	u32 cmd = 4;
367	u32 *cs;
368
369	if (mode & EMIT_INVALIDATE) {
370		cmd += 2;
371
372		if (gen12_needs_ccs_aux_inv(rq->engine))
373			cmd += 8;
374	}
375
376	cs = intel_ring_begin(rq, cmd);
377	if (IS_ERR(cs))
378		return PTR_ERR(cs);
379
380	if (mode & EMIT_INVALIDATE)
381		*cs++ = preparser_disable(true);
382
383	cmd = MI_FLUSH_DW + 1;
384
385	/*
386	 * We always require a command barrier so that subsequent
387	 * commands, such as breadcrumb interrupts, are strictly ordered
388	 * wrt the contents of the write cache being flushed to memory
389	 * (and thus being coherent from the CPU).
390	 */
391	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
392
393	if (mode & EMIT_INVALIDATE) {
394		cmd |= MI_INVALIDATE_TLB;
395		if (rq->engine->class == VIDEO_DECODE_CLASS)
396			cmd |= MI_INVALIDATE_BSD;
397
398		if (gen12_needs_ccs_aux_inv(rq->engine) &&
399		    rq->engine->class == COPY_ENGINE_CLASS)
400			cmd |= MI_FLUSH_DW_CCS;
401	}
402
403	*cs++ = cmd;
404	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;
405	*cs++ = 0; /* upper addr */
406	*cs++ = 0; /* value */
407
408	cs = gen12_emit_aux_table_inv(rq->engine, cs);
409
410	if (mode & EMIT_INVALIDATE)
411		*cs++ = preparser_disable(false);
412
413	intel_ring_advance(rq, cs);
414
415	return 0;
416}
417
418static u32 preempt_address(struct intel_engine_cs *engine)
419{
420	return (i915_ggtt_offset(engine->status_page.vma) +
421		I915_GEM_HWS_PREEMPT_ADDR);
422}
423
424static u32 hwsp_offset(const struct i915_request *rq)
425{
426	const struct intel_timeline *tl;
427
428	/* Before the request is executed, the timeline is fixed */
429	tl = rcu_dereference_protected(rq->timeline,
430				       !i915_request_signaled(rq));
431
432	/* See the comment in i915_request_active_seqno(). */
433	return page_mask_bits(tl->hwsp_offset) + offset_in_page(rq->hwsp_seqno);
434}
435
436int gen8_emit_init_breadcrumb(struct i915_request *rq)
437{
438	u32 *cs;
439
440	GEM_BUG_ON(i915_request_has_initial_breadcrumb(rq));
441	if (!i915_request_timeline(rq)->has_initial_breadcrumb)
442		return 0;
443
444	cs = intel_ring_begin(rq, 6);
445	if (IS_ERR(cs))
446		return PTR_ERR(cs);
447
448	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
449	*cs++ = hwsp_offset(rq);
450	*cs++ = 0;
451	*cs++ = rq->fence.seqno - 1;
452
453	/*
454	 * Check if we have been preempted before we even get started.
455	 *
456	 * After this point i915_request_started() reports true, even if
457	 * we get preempted and so are no longer running.
458	 *
459	 * i915_request_started() is used during preemption processing
460	 * to decide if the request is currently inside the user payload
461	 * or spinning on a kernel semaphore (or earlier). For no-preemption
462	 * requests, we do allow preemption on the semaphore before the user
463	 * payload, but do not allow preemption once the request is started.
464	 *
465	 * i915_request_started() is similarly used during GPU hangs to
466	 * determine if the user's payload was guilty, and if so, the
467	 * request is banned. Before the request is started, it is assumed
468	 * to be unharmed and an innocent victim of another's hang.
469	 */
470	*cs++ = MI_NOOP;
471	*cs++ = MI_ARB_CHECK;
472
473	intel_ring_advance(rq, cs);
474
475	/* Record the updated position of the request's payload */
476	rq->infix = intel_ring_offset(rq, cs);
477
478	__set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags);
479
480	return 0;
481}
482
483static int __xehp_emit_bb_start(struct i915_request *rq,
484				u64 offset, u32 len,
485				const unsigned int flags,
486				u32 arb)
487{
488	struct intel_context *ce = rq->context;
489	u32 wa_offset = lrc_indirect_bb(ce);
490	u32 *cs;
491
492	GEM_BUG_ON(!ce->wa_bb_page);
493
494	cs = intel_ring_begin(rq, 12);
495	if (IS_ERR(cs))
496		return PTR_ERR(cs);
497
498	*cs++ = MI_ARB_ON_OFF | arb;
499
500	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |
501		MI_SRM_LRM_GLOBAL_GTT |
502		MI_LRI_LRM_CS_MMIO;
503	*cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0));
504	*cs++ = wa_offset + DG2_PREDICATE_RESULT_WA;
505	*cs++ = 0;
506
507	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
508		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
509	*cs++ = lower_32_bits(offset);
510	*cs++ = upper_32_bits(offset);
511
512	/* Fixup stray MI_SET_PREDICATE as it prevents us executing the ring */
513	*cs++ = MI_BATCH_BUFFER_START_GEN8;
514	*cs++ = wa_offset + DG2_PREDICATE_RESULT_BB;
515	*cs++ = 0;
516
517	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
518
519	intel_ring_advance(rq, cs);
520
521	return 0;
522}
523
524int xehp_emit_bb_start_noarb(struct i915_request *rq,
525			     u64 offset, u32 len,
526			     const unsigned int flags)
527{
528	return __xehp_emit_bb_start(rq, offset, len, flags, MI_ARB_DISABLE);
529}
530
531int xehp_emit_bb_start(struct i915_request *rq,
532		       u64 offset, u32 len,
533		       const unsigned int flags)
534{
535	return __xehp_emit_bb_start(rq, offset, len, flags, MI_ARB_ENABLE);
536}
537
538int gen8_emit_bb_start_noarb(struct i915_request *rq,
539			     u64 offset, u32 len,
540			     const unsigned int flags)
541{
542	u32 *cs;
543
544	cs = intel_ring_begin(rq, 4);
545	if (IS_ERR(cs))
546		return PTR_ERR(cs);
547
548	/*
549	 * WaDisableCtxRestoreArbitration:bdw,chv
550	 *
551	 * We don't need to perform MI_ARB_ENABLE as often as we do (in
552	 * particular all the gen that do not need the w/a at all!), if we
553	 * took care to make sure that on every switch into this context
554	 * (both ordinary and for preemption) that arbitrartion was enabled
555	 * we would be fine.  However, for gen8 there is another w/a that
556	 * requires us to not preempt inside GPGPU execution, so we keep
557	 * arbitration disabled for gen8 batches. Arbitration will be
558	 * re-enabled before we close the request
559	 * (engine->emit_fini_breadcrumb).
560	 */
561	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
562
563	/* FIXME(BDW+): Address space and security selectors. */
564	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
565		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
566	*cs++ = lower_32_bits(offset);
567	*cs++ = upper_32_bits(offset);
568
569	intel_ring_advance(rq, cs);
570
571	return 0;
572}
573
574int gen8_emit_bb_start(struct i915_request *rq,
575		       u64 offset, u32 len,
576		       const unsigned int flags)
577{
578	u32 *cs;
579
580	if (unlikely(i915_request_has_nopreempt(rq)))
581		return gen8_emit_bb_start_noarb(rq, offset, len, flags);
582
583	cs = intel_ring_begin(rq, 6);
584	if (IS_ERR(cs))
585		return PTR_ERR(cs);
586
587	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
588
589	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
590		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
591	*cs++ = lower_32_bits(offset);
592	*cs++ = upper_32_bits(offset);
593
594	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
595	*cs++ = MI_NOOP;
596
597	intel_ring_advance(rq, cs);
598
599	return 0;
600}
601
602static void assert_request_valid(struct i915_request *rq)
603{
604	struct intel_ring *ring __maybe_unused = rq->ring;
605
606	/* Can we unwind this request without appearing to go forwards? */
607	GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0);
608}
609
610/*
611 * Reserve space for 2 NOOPs at the end of each request to be
612 * used as a workaround for not being allowed to do lite
613 * restore with HEAD==TAIL (WaIdleLiteRestore).
614 */
615static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs)
616{
617	/* Ensure there's always at least one preemption point per-request. */
618	*cs++ = MI_ARB_CHECK;
619	*cs++ = MI_NOOP;
620	rq->wa_tail = intel_ring_offset(rq, cs);
621
622	/* Check that entire request is less than half the ring */
623	assert_request_valid(rq);
624
625	return cs;
626}
627
628static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs)
629{
630	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */
631	*cs++ = MI_SEMAPHORE_WAIT |
632		MI_SEMAPHORE_GLOBAL_GTT |
633		MI_SEMAPHORE_POLL |
634		MI_SEMAPHORE_SAD_EQ_SDD;
635	*cs++ = 0;
636	*cs++ = preempt_address(rq->engine);
637	*cs++ = 0;
638	*cs++ = MI_NOOP;
639
640	return cs;
641}
642
643static __always_inline u32*
644gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)
645{
646	*cs++ = MI_USER_INTERRUPT;
647
648	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
649	if (intel_engine_has_semaphores(rq->engine) &&
650	    !intel_uc_uses_guc_submission(&rq->engine->gt->uc))
651		cs = emit_preempt_busywait(rq, cs);
652
653	rq->tail = intel_ring_offset(rq, cs);
654	assert_ring_tail_valid(rq->ring, rq->tail);
655
656	return gen8_emit_wa_tail(rq, cs);
657}
658
659static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)
660{
661	return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0);
662}
663
664u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)
665{
666	return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs));
667}
668
669u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
670{
671	cs = gen8_emit_pipe_control(cs,
672				    PIPE_CONTROL_CS_STALL |
673				    PIPE_CONTROL_TLB_INVALIDATE |
674				    PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
675				    PIPE_CONTROL_DEPTH_CACHE_FLUSH |
676				    PIPE_CONTROL_DC_FLUSH_ENABLE,
677				    0);
678
679	/* XXX flush+write+CS_STALL all in one upsets gem_concurrent_blt:kbl */
680	cs = gen8_emit_ggtt_write_rcs(cs,
681				      rq->fence.seqno,
682				      hwsp_offset(rq),
683				      PIPE_CONTROL_FLUSH_ENABLE |
684				      PIPE_CONTROL_CS_STALL);
685
686	return gen8_emit_fini_breadcrumb_tail(rq, cs);
687}
688
689u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
690{
691	cs = gen8_emit_pipe_control(cs,
692				    PIPE_CONTROL_CS_STALL |
693				    PIPE_CONTROL_TLB_INVALIDATE |
694				    PIPE_CONTROL_TILE_CACHE_FLUSH |
695				    PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
696				    PIPE_CONTROL_DEPTH_CACHE_FLUSH |
697				    PIPE_CONTROL_DC_FLUSH_ENABLE,
698				    0);
699
700	/*XXX: Look at gen8_emit_fini_breadcrumb_rcs */
701	cs = gen8_emit_ggtt_write_rcs(cs,
702				      rq->fence.seqno,
703				      hwsp_offset(rq),
704				      PIPE_CONTROL_FLUSH_ENABLE |
705				      PIPE_CONTROL_CS_STALL);
706
707	return gen8_emit_fini_breadcrumb_tail(rq, cs);
708}
709
710/*
711 * Note that the CS instruction pre-parser will not stall on the breadcrumb
712 * flush and will continue pre-fetching the instructions after it before the
713 * memory sync is completed. On pre-gen12 HW, the pre-parser will stop at
714 * BB_START/END instructions, so, even though we might pre-fetch the pre-amble
715 * of the next request before the memory has been flushed, we're guaranteed that
716 * we won't access the batch itself too early.
717 * However, on gen12+ the parser can pre-fetch across the BB_START/END commands,
718 * so, if the current request is modifying an instruction in the next request on
719 * the same intel_context, we might pre-fetch and then execute the pre-update
720 * instruction. To avoid this, the users of self-modifying code should either
721 * disable the parser around the code emitting the memory writes, via a new flag
722 * added to MI_ARB_CHECK, or emit the writes from a different intel_context. For
723 * the in-kernel use-cases we've opted to use a separate context, see
724 * reloc_gpu() as an example.
725 * All the above applies only to the instructions themselves. Non-inline data
726 * used by the instructions is not pre-fetched.
727 */
728
729static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)
730{
731	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */
732	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |
733		MI_SEMAPHORE_GLOBAL_GTT |
734		MI_SEMAPHORE_POLL |
735		MI_SEMAPHORE_SAD_EQ_SDD;
736	*cs++ = 0;
737	*cs++ = preempt_address(rq->engine);
738	*cs++ = 0;
739	*cs++ = 0;
740
741	return cs;
742}
743
744/* Wa_14014475959:dg2 */
745/* Wa_16019325821 */
746/* Wa_14019159160 */
747#define HOLD_SWITCHOUT_SEMAPHORE_PPHWSP_OFFSET	0x540
748static u32 hold_switchout_semaphore_offset(struct i915_request *rq)
749{
750	return i915_ggtt_offset(rq->context->state) +
751		(LRC_PPHWSP_PN * PAGE_SIZE) + HOLD_SWITCHOUT_SEMAPHORE_PPHWSP_OFFSET;
752}
753
754/* Wa_14014475959:dg2 */
755/* Wa_16019325821 */
756/* Wa_14019159160 */
757static u32 *hold_switchout_emit_wa_busywait(struct i915_request *rq, u32 *cs)
758{
759	int i;
760
761	*cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL |
762		MI_ATOMIC_MOVE;
763	*cs++ = hold_switchout_semaphore_offset(rq);
764	*cs++ = 0;
765	*cs++ = 1;
766
767	/*
768	 * When MI_ATOMIC_INLINE_DATA set this command must be 11 DW + (1 NOP)
769	 * to align. 4 DWs above + 8 filler DWs here.
770	 */
771	for (i = 0; i < 8; ++i)
772		*cs++ = 0;
773
774	*cs++ = MI_SEMAPHORE_WAIT |
775		MI_SEMAPHORE_GLOBAL_GTT |
776		MI_SEMAPHORE_POLL |
777		MI_SEMAPHORE_SAD_EQ_SDD;
778	*cs++ = 0;
779	*cs++ = hold_switchout_semaphore_offset(rq);
780	*cs++ = 0;
781
782	return cs;
783}
784
785static __always_inline u32*
786gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)
787{
788	*cs++ = MI_USER_INTERRUPT;
789
790	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
791	if (intel_engine_has_semaphores(rq->engine) &&
792	    !intel_uc_uses_guc_submission(&rq->engine->gt->uc))
793		cs = gen12_emit_preempt_busywait(rq, cs);
794
795	/* Wa_14014475959:dg2 */
796	/* Wa_16019325821 */
797	/* Wa_14019159160 */
798	if (intel_engine_uses_wa_hold_switchout(rq->engine))
799		cs = hold_switchout_emit_wa_busywait(rq, cs);
800
801	rq->tail = intel_ring_offset(rq, cs);
802	assert_ring_tail_valid(rq->ring, rq->tail);
803
804	return gen8_emit_wa_tail(rq, cs);
805}
806
807u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)
808{
809	/* XXX Stalling flush before seqno write; post-sync not */
810	cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0));
811	return gen12_emit_fini_breadcrumb_tail(rq, cs);
812}
813
814u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)
815{
816	struct drm_i915_private *i915 = rq->i915;
817	struct intel_gt *gt = rq->engine->gt;
818	u32 flags = (PIPE_CONTROL_CS_STALL |
819		     PIPE_CONTROL_TLB_INVALIDATE |
820		     PIPE_CONTROL_TILE_CACHE_FLUSH |
821		     PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
822		     PIPE_CONTROL_DEPTH_CACHE_FLUSH |
823		     PIPE_CONTROL_DC_FLUSH_ENABLE |
824		     PIPE_CONTROL_FLUSH_ENABLE);
825
826	if (GRAPHICS_VER_FULL(rq->i915) < IP_VER(12, 70))
827		flags |= PIPE_CONTROL_FLUSH_L3;
828
829	/* Wa_14016712196 */
830	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74)) || IS_DG2(i915))
831		/* dummy PIPE_CONTROL + depth flush */
832		cs = gen12_emit_pipe_control(cs, 0,
833					     PIPE_CONTROL_DEPTH_CACHE_FLUSH, 0);
834
835	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
836		/* Wa_1409600907 */
837		flags |= PIPE_CONTROL_DEPTH_STALL;
838
839	if (!HAS_3D_PIPELINE(rq->i915))
840		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
841	else if (rq->engine->class == COMPUTE_CLASS)
842		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
843
844	cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0);
845
846	/*XXX: Look at gen8_emit_fini_breadcrumb_rcs */
847	cs = gen12_emit_ggtt_write_rcs(cs,
848				       rq->fence.seqno,
849				       hwsp_offset(rq),
850				       0,
851				       PIPE_CONTROL_FLUSH_ENABLE |
852				       PIPE_CONTROL_CS_STALL);
853
854	return gen12_emit_fini_breadcrumb_tail(rq, cs);
855}