intel_flipq.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2025 Intel Corporation
  4 */
  5
  6#include <linux/pci.h>
  7
  8#include <drm/drm_print.h>
  9
 10#include "intel_crtc.h"
 11#include "intel_de.h"
 12#include "intel_display_core.h"
 13#include "intel_display_types.h"
 14#include "intel_display_utils.h"
 15#include "intel_dmc.h"
 16#include "intel_dmc_regs.h"
 17#include "intel_dsb.h"
 18#include "intel_flipq.h"
 19#include "intel_step.h"
 20#include "intel_vblank.h"
 21#include "intel_vrr.h"
 22
 23/**
 24 * DOC: DMC Flip Queue
 25 *
 26 * A flip queue is a ring buffer implemented by the pipe DMC firmware.
 27 * The driver inserts entries into the queues to be executed by the
 28 * pipe DMC at a specified presentation timestamp (PTS).
 29 *
 30 * Each pipe DMC provides several queues:
 31 *
 32 * - 1 general queue (two DSB buffers executed per entry)
 33 * - 3 plane queues (one DSB buffer executed per entry)
 34 * - 1 fast queue (deprecated)
 35 */
 36
 37#define for_each_flipq(flipq_id) \
 38	for ((flipq_id) = INTEL_FLIPQ_PLANE_1; (flipq_id) < MAX_INTEL_FLIPQ; (flipq_id)++)
 39
 40static int intel_flipq_offset(enum intel_flipq_id flipq_id)
 41{
 42	switch (flipq_id) {
 43	case INTEL_FLIPQ_PLANE_1:
 44		return 0x008;
 45	case INTEL_FLIPQ_PLANE_2:
 46		return 0x108;
 47	case INTEL_FLIPQ_PLANE_3:
 48		return 0x208;
 49	case INTEL_FLIPQ_GENERAL:
 50		return 0x308;
 51	case INTEL_FLIPQ_FAST:
 52		return 0x3c8;
 53	default:
 54		MISSING_CASE(flipq_id);
 55		return 0;
 56	}
 57}
 58
 59static int intel_flipq_size_dw(enum intel_flipq_id flipq_id)
 60{
 61	switch (flipq_id) {
 62	case INTEL_FLIPQ_PLANE_1:
 63	case INTEL_FLIPQ_PLANE_2:
 64	case INTEL_FLIPQ_PLANE_3:
 65		return 64;
 66	case INTEL_FLIPQ_GENERAL:
 67	case INTEL_FLIPQ_FAST:
 68		return 48;
 69	default:
 70		MISSING_CASE(flipq_id);
 71		return 1;
 72	}
 73}
 74
 75static int intel_flipq_elem_size_dw(enum intel_flipq_id flipq_id)
 76{
 77	switch (flipq_id) {
 78	case INTEL_FLIPQ_PLANE_1:
 79	case INTEL_FLIPQ_PLANE_2:
 80	case INTEL_FLIPQ_PLANE_3:
 81		return 4;
 82	case INTEL_FLIPQ_GENERAL:
 83	case INTEL_FLIPQ_FAST:
 84		return 6;
 85	default:
 86		MISSING_CASE(flipq_id);
 87		return 1;
 88	}
 89}
 90
 91static int intel_flipq_size_entries(enum intel_flipq_id flipq_id)
 92{
 93	return intel_flipq_size_dw(flipq_id) / intel_flipq_elem_size_dw(flipq_id);
 94}
 95
 96static void intel_flipq_crtc_init(struct intel_crtc *crtc)
 97{
 98	struct intel_display *display = to_intel_display(crtc);
 99	enum intel_flipq_id flipq_id;
100
101	for_each_flipq(flipq_id) {
102		struct intel_flipq *flipq = &crtc->flipq[flipq_id];
103
104		flipq->start_mmioaddr = intel_pipedmc_start_mmioaddr(crtc) + intel_flipq_offset(flipq_id);
105		flipq->flipq_id = flipq_id;
106
107		drm_dbg_kms(display->drm, "[CRTC:%d:%s] FQ %d: start 0x%x\n",
108			    crtc->base.base.id, crtc->base.name,
109			    flipq_id, flipq->start_mmioaddr);
110	}
111}
112
113bool intel_flipq_supported(struct intel_display *display)
114{
115	if (!display->params.enable_flipq)
116		return false;
117
118	if (!display->dmc.dmc)
119		return false;
120
121	if (DISPLAY_VER(display) == 20)
122		return true;
123
124	/* DMC firmware expects VRR timing generator to be used */
125	return DISPLAY_VER(display) >= 30 && intel_vrr_always_use_vrr_tg(display);
126}
127
128void intel_flipq_init(struct intel_display *display)
129{
130	struct intel_crtc *crtc;
131
132	intel_dmc_wait_fw_load(display);
133
134	for_each_intel_crtc(display->drm, crtc)
135		intel_flipq_crtc_init(crtc);
136}
137
138static int cdclk_factor(struct intel_display *display)
139{
140	if (DISPLAY_VER(display) >= 30)
141		return 120;
142	else
143		return 280;
144}
145
146int intel_flipq_exec_time_us(struct intel_display *display)
147{
148	return intel_dsb_exec_time_us() +
149		DIV_ROUND_UP(display->cdclk.hw.cdclk * cdclk_factor(display), 540000) +
150		display->sagv.block_time_us;
151}
152
153static int intel_flipq_preempt_timeout_ms(struct intel_display *display)
154{
155	return DIV_ROUND_UP(intel_flipq_exec_time_us(display), 1000);
156}
157
158static void intel_flipq_preempt(struct intel_crtc *crtc, bool preempt)
159{
160	struct intel_display *display = to_intel_display(crtc);
161
162	intel_de_rmw(display, PIPEDMC_FQ_CTRL(crtc->pipe),
163		     PIPEDMC_FQ_CTRL_PREEMPT, preempt ? PIPEDMC_FQ_CTRL_PREEMPT : 0);
164
165	if (preempt &&
166	    intel_de_wait_for_clear_ms(display,
167				       PIPEDMC_FQ_STATUS(crtc->pipe),
168				       PIPEDMC_FQ_STATUS_BUSY,
169				       intel_flipq_preempt_timeout_ms(display)))
170		drm_err(display->drm, "[CRTC:%d:%s] flip queue preempt timeout\n",
171			crtc->base.base.id, crtc->base.name);
172}
173
174static int intel_flipq_current_head(struct intel_crtc *crtc, enum intel_flipq_id flipq_id)
175{
176	struct intel_display *display = to_intel_display(crtc);
177
178	return intel_de_read(display, PIPEDMC_FPQ_CHP(crtc->pipe, flipq_id));
179}
180
181static void intel_flipq_write_tail(struct intel_crtc *crtc)
182{
183	struct intel_display *display = to_intel_display(crtc);
184
185	intel_de_write(display, PIPEDMC_FPQ_ATOMIC_TP(crtc->pipe),
186		       PIPEDMC_FPQ_PLANEQ_3_TP(crtc->flipq[INTEL_FLIPQ_PLANE_3].tail) |
187		       PIPEDMC_FPQ_PLANEQ_2_TP(crtc->flipq[INTEL_FLIPQ_PLANE_2].tail) |
188		       PIPEDMC_FPQ_PLANEQ_1_TP(crtc->flipq[INTEL_FLIPQ_PLANE_1].tail) |
189		       PIPEDMC_FPQ_FASTQ_TP(crtc->flipq[INTEL_FLIPQ_FAST].tail) |
190		       PIPEDMC_FPQ_GENERALQ_TP(crtc->flipq[INTEL_FLIPQ_GENERAL].tail));
191}
192
193static void intel_flipq_sw_dmc_wake(struct intel_crtc *crtc)
194{
195	struct intel_display *display = to_intel_display(crtc);
196
197	intel_de_write(display, PIPEDMC_FPQ_CTL1(crtc->pipe), PIPEDMC_SW_DMC_WAKE);
198}
199
200static int intel_flipq_exec_time_lines(const struct intel_crtc_state *crtc_state)
201{
202	struct intel_display *display = to_intel_display(crtc_state);
203
204	return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode,
205					intel_flipq_exec_time_us(display));
206}
207
208void intel_flipq_dump(struct intel_crtc *crtc,
209		      enum intel_flipq_id flipq_id)
210{
211	struct intel_display *display = to_intel_display(crtc);
212	struct intel_flipq *flipq = &crtc->flipq[flipq_id];
213	u32 tmp;
214
215	drm_dbg_kms(display->drm,
216		    "[CRTC:%d:%s] FQ %d @ 0x%x: ",
217		    crtc->base.base.id, crtc->base.name, flipq_id,
218		    flipq->start_mmioaddr);
219	for (int i = 0 ; i < intel_flipq_size_dw(flipq_id); i++) {
220		printk(KERN_CONT " 0x%08x",
221		       intel_de_read(display, PIPEDMC_FQ_RAM(flipq->start_mmioaddr, i)));
222		if (i % intel_flipq_elem_size_dw(flipq_id) == intel_flipq_elem_size_dw(flipq_id) - 1)
223			printk(KERN_CONT "\n");
224	}
225
226	drm_dbg_kms(display->drm,
227		    "[CRTC:%d:%s] FQ %d: chp=0x%x, hp=0x%x\n",
228		    crtc->base.base.id, crtc->base.name, flipq_id,
229		    intel_de_read(display, PIPEDMC_FPQ_CHP(crtc->pipe, flipq_id)),
230		    intel_de_read(display, PIPEDMC_FPQ_HP(crtc->pipe, flipq_id)));
231
232	drm_dbg_kms(display->drm,
233		    "[CRTC:%d:%s] FQ %d: current head %d\n",
234		    crtc->base.base.id, crtc->base.name, flipq_id,
235		    intel_flipq_current_head(crtc, flipq_id));
236
237	drm_dbg_kms(display->drm,
238		    "[CRTC:%d:%s] flip queue timestamp: 0x%x\n",
239		    crtc->base.base.id, crtc->base.name,
240		    intel_de_read(display, PIPEDMC_FPQ_TS(crtc->pipe)));
241
242	tmp = intel_de_read(display, PIPEDMC_FPQ_ATOMIC_TP(crtc->pipe));
243
244	drm_dbg_kms(display->drm,
245		    "[CRTC:%d:%s] flip queue atomic tails: P3 %d, P2 %d, P1 %d, G %d, F %d\n",
246		    crtc->base.base.id, crtc->base.name,
247		    REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_3_TP_MASK, tmp),
248		    REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_2_TP_MASK, tmp),
249		    REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_1_TP_MASK, tmp),
250		    REG_FIELD_GET(PIPEDMC_FPQ_GENERALQ_TP_MASK, tmp),
251		    REG_FIELD_GET(PIPEDMC_FPQ_FASTQ_TP_MASK, tmp));
252}
253
254void intel_flipq_reset(struct intel_display *display, enum pipe pipe)
255{
256	struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
257	enum intel_flipq_id flipq_id;
258
259	intel_de_write(display, PIPEDMC_FQ_CTRL(pipe), 0);
260
261	intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(pipe), 0);
262	intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(pipe), 0);
263
264	for_each_flipq(flipq_id) {
265		struct intel_flipq *flipq = &crtc->flipq[flipq_id];
266
267		intel_de_write(display, PIPEDMC_FPQ_HP(pipe, flipq_id), 0);
268		intel_de_write(display, PIPEDMC_FPQ_CHP(pipe, flipq_id), 0);
269
270		flipq->tail = 0;
271	}
272
273	intel_de_write(display, PIPEDMC_FPQ_ATOMIC_TP(pipe), 0);
274}
275
276static enum pipedmc_event_id flipq_event_id(struct intel_display *display)
277{
278	if (DISPLAY_VER(display) >= 30)
279		return PIPEDMC_EVENT_FULL_FQ_WAKE_TRIGGER;
280	else
281		return PIPEDMC_EVENT_SCANLINE_INRANGE_FQ_TRIGGER;
282}
283
284void intel_flipq_enable(const struct intel_crtc_state *crtc_state)
285{
286	struct intel_display *display = to_intel_display(crtc_state);
287	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
288	/* FIXME what to do with VRR? */
289	int scanline = intel_mode_vblank_start(&crtc_state->hw.adjusted_mode) -
290		intel_flipq_exec_time_lines(crtc_state);
291
292	if (DISPLAY_VER(display) >= 30) {
293		u32 start_mmioaddr = intel_pipedmc_start_mmioaddr(crtc);
294
295		/* undocumented magic DMC variables */
296		intel_de_write(display, PTL_PIPEDMC_EXEC_TIME_LINES(start_mmioaddr),
297			       intel_flipq_exec_time_lines(crtc_state));
298		intel_de_write(display, PTL_PIPEDMC_END_OF_EXEC_GB(start_mmioaddr),
299			       100);
300	}
301
302	intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(crtc->pipe),
303		       PIPEDMC_SCANLINE_UPPER(scanline));
304	intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(crtc->pipe),
305		       PIPEDMC_SCANLINEINRANGECMP_EN |
306		       PIPEDMC_SCANLINE_LOWER(scanline - 2));
307
308	intel_pipedmc_enable_event(crtc, flipq_event_id(display));
309
310	intel_de_write(display, PIPEDMC_FQ_CTRL(crtc->pipe), PIPEDMC_FQ_CTRL_ENABLE);
311}
312
313void intel_flipq_disable(const struct intel_crtc_state *crtc_state)
314{
315	struct intel_display *display = to_intel_display(crtc_state);
316	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
317
318	intel_flipq_preempt(crtc, true);
319
320	intel_de_write(display, PIPEDMC_FQ_CTRL(crtc->pipe), 0);
321
322	intel_pipedmc_disable_event(crtc, flipq_event_id(display));
323
324	intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(crtc->pipe), 0);
325	intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(crtc->pipe), 0);
326}
327
328static bool assert_flipq_has_room(struct intel_crtc *crtc,
329				  enum intel_flipq_id flipq_id)
330{
331	struct intel_display *display = to_intel_display(crtc);
332	struct intel_flipq *flipq = &crtc->flipq[flipq_id];
333	int head, size = intel_flipq_size_entries(flipq_id);
334
335	head = intel_flipq_current_head(crtc, flipq_id);
336
337	return !drm_WARN(display->drm,
338			 (flipq->tail + size - head) % size >= size - 1,
339			 "[CRTC:%d:%s] FQ %d overflow (head %d, tail %d, size %d)\n",
340			 crtc->base.base.id, crtc->base.name, flipq_id,
341			 head, flipq->tail, size);
342}
343
344static void intel_flipq_write(struct intel_display *display,
345			      struct intel_flipq *flipq, u32 data, int i)
346{
347	intel_de_write(display, PIPEDMC_FQ_RAM(flipq->start_mmioaddr, flipq->tail *
348					       intel_flipq_elem_size_dw(flipq->flipq_id) + i), data);
349}
350
351static void lnl_flipq_add(struct intel_display *display,
352			  struct intel_flipq *flipq,
353			  unsigned int pts,
354			  enum intel_dsb_id dsb_id,
355			  struct intel_dsb *dsb)
356{
357	int i = 0;
358
359	switch (flipq->flipq_id) {
360	case INTEL_FLIPQ_GENERAL:
361		intel_flipq_write(display, flipq, pts, i++);
362		intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
363		intel_flipq_write(display, flipq, LNL_FQ_INTERRUPT |
364				  LNL_FQ_DSB_ID(dsb_id) |
365				  LNL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
366		intel_flipq_write(display, flipq, 0, i++);
367		intel_flipq_write(display, flipq, 0, i++); /* head for second DSB */
368		intel_flipq_write(display, flipq, 0, i++); /* DSB engine + size for second DSB */
369		break;
370	case INTEL_FLIPQ_PLANE_1:
371	case INTEL_FLIPQ_PLANE_2:
372	case INTEL_FLIPQ_PLANE_3:
373		intel_flipq_write(display, flipq, pts, i++);
374		intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
375		intel_flipq_write(display, flipq, LNL_FQ_INTERRUPT |
376				  LNL_FQ_DSB_ID(dsb_id) |
377				  LNL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
378		intel_flipq_write(display, flipq, 0, i++);
379		break;
380	default:
381		MISSING_CASE(flipq->flipq_id);
382		return;
383	}
384}
385
386static void ptl_flipq_add(struct intel_display *display,
387			  struct intel_flipq *flipq,
388			  unsigned int pts,
389			  enum intel_dsb_id dsb_id,
390			  struct intel_dsb *dsb)
391{
392	int i = 0;
393
394	switch (flipq->flipq_id) {
395	case INTEL_FLIPQ_GENERAL:
396		intel_flipq_write(display, flipq, pts, i++);
397		intel_flipq_write(display, flipq, 0, i++);
398		intel_flipq_write(display, flipq, PTL_FQ_INTERRUPT |
399				  PTL_FQ_DSB_ID(dsb_id) |
400				  PTL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
401		intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
402		intel_flipq_write(display, flipq, 0, i++); /* DSB engine + size for second DSB */
403		intel_flipq_write(display, flipq, 0, i++); /* head for second DSB */
404		break;
405	case INTEL_FLIPQ_PLANE_1:
406	case INTEL_FLIPQ_PLANE_2:
407	case INTEL_FLIPQ_PLANE_3:
408		intel_flipq_write(display, flipq, pts, i++);
409		intel_flipq_write(display, flipq, 0, i++);
410		intel_flipq_write(display, flipq, PTL_FQ_INTERRUPT |
411				  PTL_FQ_DSB_ID(dsb_id) |
412				  PTL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
413		intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
414		break;
415	default:
416		MISSING_CASE(flipq->flipq_id);
417		return;
418	}
419}
420
421void intel_flipq_add(struct intel_crtc *crtc,
422		     enum intel_flipq_id flipq_id,
423		     unsigned int pts,
424		     enum intel_dsb_id dsb_id,
425		     struct intel_dsb *dsb)
426{
427	struct intel_display *display = to_intel_display(crtc);
428	struct intel_flipq *flipq = &crtc->flipq[flipq_id];
429
430	if (!assert_flipq_has_room(crtc, flipq_id))
431		return;
432
433	pts += intel_de_read(display, PIPEDMC_FPQ_TS(crtc->pipe));
434
435	intel_flipq_preempt(crtc, true);
436
437	if (DISPLAY_VER(display) >= 30)
438		ptl_flipq_add(display, flipq,  pts, dsb_id, dsb);
439	else
440		lnl_flipq_add(display, flipq,  pts, dsb_id, dsb);
441
442	flipq->tail = (flipq->tail + 1) % intel_flipq_size_entries(flipq->flipq_id);
443	intel_flipq_write_tail(crtc);
444
445	intel_flipq_preempt(crtc, false);
446
447	intel_flipq_sw_dmc_wake(crtc);
448}
449
450/* Wa_18034343758 */
451static bool need_dmc_halt_wa(struct intel_display *display)
452{
453	return DISPLAY_VER(display) == 20 ||
454		(display->platform.pantherlake &&
455		 IS_DISPLAY_STEP(display, STEP_A0, STEP_B0));
456}
457
458void intel_flipq_wait_dmc_halt(struct intel_dsb *dsb, struct intel_crtc *crtc)
459{
460	struct intel_display *display = to_intel_display(crtc);
461
462	if (need_dmc_halt_wa(display))
463		intel_dsb_wait_usec(dsb, 2);
464}
465
466void intel_flipq_unhalt_dmc(struct intel_dsb *dsb, struct intel_crtc *crtc)
467{
468	struct intel_display *display = to_intel_display(crtc);
469
470	if (need_dmc_halt_wa(display))
471		intel_dsb_reg_write(dsb, PIPEDMC_CTL(crtc->pipe), 0);
472}