intel_vrr.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2020 Intel Corporation
  4 *
  5 */
  6
  7#include <drm/drm_print.h>
  8
  9#include "intel_de.h"
 10#include "intel_display_regs.h"
 11#include "intel_display_types.h"
 12#include "intel_dp.h"
 13#include "intel_psr.h"
 14#include "intel_vrr.h"
 15#include "intel_vrr_regs.h"
 16#include "skl_prefill.h"
 17#include "skl_watermark.h"
 18
 19#define FIXED_POINT_PRECISION		100
 20#define CMRR_PRECISION_TOLERANCE	10
 21
 22bool intel_vrr_is_capable(struct intel_connector *connector)
 23{
 24	struct intel_display *display = to_intel_display(connector);
 25	const struct drm_display_info *info = &connector->base.display_info;
 26	struct intel_dp *intel_dp;
 27
 28	if (!HAS_VRR(display))
 29		return false;
 30
 31	/*
 32	 * DP Sink is capable of VRR video timings if
 33	 * Ignore MSA bit is set in DPCD.
 34	 * EDID monitor range also should be atleast 10 for reasonable
 35	 * Adaptive Sync or Variable Refresh Rate end user experience.
 36	 */
 37	switch (connector->base.connector_type) {
 38	case DRM_MODE_CONNECTOR_eDP:
 39		if (!connector->panel.vbt.vrr)
 40			return false;
 41		fallthrough;
 42	case DRM_MODE_CONNECTOR_DisplayPort:
 43		if (connector->mst.dp)
 44			return false;
 45		intel_dp = intel_attached_dp(connector);
 46
 47		if (!drm_dp_sink_can_do_video_without_timing_msa(intel_dp->dpcd))
 48			return false;
 49
 50		break;
 51	default:
 52		return false;
 53	}
 54
 55	return info->monitor_range.max_vfreq - info->monitor_range.min_vfreq > 10;
 56}
 57
 58bool intel_vrr_is_in_range(struct intel_connector *connector, int vrefresh)
 59{
 60	const struct drm_display_info *info = &connector->base.display_info;
 61
 62	return intel_vrr_is_capable(connector) &&
 63		vrefresh >= info->monitor_range.min_vfreq &&
 64		vrefresh <= info->monitor_range.max_vfreq;
 65}
 66
 67bool intel_vrr_possible(const struct intel_crtc_state *crtc_state)
 68{
 69	return crtc_state->vrr.flipline;
 70}
 71
 72void
 73intel_vrr_check_modeset(struct intel_atomic_state *state)
 74{
 75	int i;
 76	struct intel_crtc_state *old_crtc_state, *new_crtc_state;
 77	struct intel_crtc *crtc;
 78
 79	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 80					    new_crtc_state, i) {
 81		if (new_crtc_state->uapi.vrr_enabled !=
 82		    old_crtc_state->uapi.vrr_enabled)
 83			new_crtc_state->uapi.mode_changed = true;
 84	}
 85}
 86
 87static int intel_vrr_extra_vblank_delay(struct intel_display *display)
 88{
 89	/*
 90	 * On ICL/TGL VRR hardware inserts one extra scanline
 91	 * just after vactive, which pushes the vmin decision
 92	 * boundary ahead accordingly, and thus reduces the
 93	 * max guardband length by one scanline.
 94	 */
 95	return DISPLAY_VER(display) < 13 ? 1 : 0;
 96}
 97
 98static int intel_vrr_vmin_flipline_offset(struct intel_display *display)
 99{
100	/*
101	 * ICL/TGL hardware imposes flipline>=vmin+1
102	 *
103	 * We reduce the vmin value to compensate when programming the
104	 * hardware. This approach allows flipline to remain set at the
105	 * original value, and thus the frame will have the desired
106	 * minimum vtotal.
107	 */
108	return DISPLAY_VER(display) < 13 ? 1 : 0;
109}
110
111static int intel_vrr_guardband_to_pipeline_full(const struct intel_crtc_state *crtc_state,
112						int guardband)
113{
114	/* hardware imposes one extra scanline somewhere */
115	return guardband - crtc_state->framestart_delay - 1;
116}
117
118static int intel_vrr_pipeline_full_to_guardband(const struct intel_crtc_state *crtc_state,
119						int pipeline_full)
120{
121	/* hardware imposes one extra scanline somewhere */
122	return pipeline_full + crtc_state->framestart_delay + 1;
123}
124
125/*
126 * Without VRR registers get latched at:
127 *  vblank_start
128 *
129 * With VRR the earliest registers can get latched is:
130 *  intel_vrr_vmin_vblank_start(), which if we want to maintain
131 *  the correct min vtotal is >=vblank_start+1
132 *
133 * The latest point registers can get latched is the vmax decision boundary:
134 *  intel_vrr_vmax_vblank_start()
135 *
136 * Between those two points the vblank exit starts (and hence registers get
137 * latched) ASAP after a push is sent.
138 *
139 * framestart_delay is programmable 1-4.
140 */
141
142int intel_vrr_vmin_vtotal(const struct intel_crtc_state *crtc_state)
143{
144	/* Min vblank actually determined by flipline */
145	return crtc_state->vrr.vmin;
146}
147
148int intel_vrr_vmax_vtotal(const struct intel_crtc_state *crtc_state)
149{
150	return crtc_state->vrr.vmax;
151}
152
153int intel_vrr_vmin_vblank_start(const struct intel_crtc_state *crtc_state)
154{
155	return intel_vrr_vmin_vtotal(crtc_state) - crtc_state->vrr.guardband;
156}
157
158int intel_vrr_vmax_vblank_start(const struct intel_crtc_state *crtc_state)
159{
160	return intel_vrr_vmax_vtotal(crtc_state) - crtc_state->vrr.guardband;
161}
162
163static bool
164is_cmrr_frac_required(struct intel_crtc_state *crtc_state)
165{
166	struct intel_display *display = to_intel_display(crtc_state);
167	int calculated_refresh_k, actual_refresh_k, pixel_clock_per_line;
168	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
169
170	/* Avoid CMRR for now till we have VRR with fixed timings working */
171	if (!HAS_CMRR(display) || true)
172		return false;
173
174	actual_refresh_k =
175		drm_mode_vrefresh(adjusted_mode) * FIXED_POINT_PRECISION;
176	pixel_clock_per_line =
177		adjusted_mode->crtc_clock * 1000 / adjusted_mode->crtc_htotal;
178	calculated_refresh_k =
179		pixel_clock_per_line * FIXED_POINT_PRECISION / adjusted_mode->crtc_vtotal;
180
181	if ((actual_refresh_k - calculated_refresh_k) < CMRR_PRECISION_TOLERANCE)
182		return false;
183
184	return true;
185}
186
187static unsigned int
188cmrr_get_vtotal(struct intel_crtc_state *crtc_state, bool video_mode_required)
189{
190	int multiplier_m = 1, multiplier_n = 1, vtotal, desired_refresh_rate;
191	u64 adjusted_pixel_rate;
192	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
193
194	desired_refresh_rate = drm_mode_vrefresh(adjusted_mode);
195
196	if (video_mode_required) {
197		multiplier_m = 1001;
198		multiplier_n = 1000;
199	}
200
201	crtc_state->cmrr.cmrr_n = mul_u32_u32(desired_refresh_rate * adjusted_mode->crtc_htotal,
202					      multiplier_n);
203	vtotal = DIV_ROUND_UP_ULL(mul_u32_u32(adjusted_mode->crtc_clock * 1000, multiplier_n),
204				  crtc_state->cmrr.cmrr_n);
205	adjusted_pixel_rate = mul_u32_u32(adjusted_mode->crtc_clock * 1000, multiplier_m);
206	crtc_state->cmrr.cmrr_m = do_div(adjusted_pixel_rate, crtc_state->cmrr.cmrr_n);
207
208	return vtotal;
209}
210
211static
212void intel_vrr_compute_cmrr_timings(struct intel_crtc_state *crtc_state)
213{
214	/*
215	 * TODO: Compute precise target refresh rate to determine
216	 * if video_mode_required should be true. Currently set to
217	 * false due to uncertainty about the precise target
218	 * refresh Rate.
219	 */
220	crtc_state->vrr.vmax = cmrr_get_vtotal(crtc_state, false);
221	crtc_state->vrr.vmin = crtc_state->vrr.vmax;
222	crtc_state->vrr.flipline = crtc_state->vrr.vmin;
223
224	crtc_state->cmrr.enable = true;
225	crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
226}
227
228static
229void intel_vrr_compute_vrr_timings(struct intel_crtc_state *crtc_state,
230				   int vmin, int vmax)
231{
232	crtc_state->vrr.vmax = vmax;
233	crtc_state->vrr.vmin = vmin;
234	crtc_state->vrr.flipline = crtc_state->vrr.vmin;
235
236	crtc_state->vrr.enable = true;
237	crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
238}
239
240static
241void intel_vrr_compute_fixed_rr_timings(struct intel_crtc_state *crtc_state)
242{
243	/* For fixed rr,  vmin = vmax = flipline */
244	crtc_state->vrr.vmax = crtc_state->hw.adjusted_mode.crtc_vtotal;
245	crtc_state->vrr.vmin = crtc_state->vrr.vmax;
246	crtc_state->vrr.flipline = crtc_state->vrr.vmin;
247}
248
249static int intel_vrr_hw_value(const struct intel_crtc_state *crtc_state,
250			      int value)
251{
252	struct intel_display *display = to_intel_display(crtc_state);
253
254	/*
255	 * On TGL vmin/vmax/flipline also need to be
256	 * adjusted by the SCL to maintain correct vtotals.
257	 */
258	if (DISPLAY_VER(display) >= 13)
259		return value;
260	else
261		return value - crtc_state->set_context_latency;
262}
263
264/*
265 * For fixed refresh rate mode Vmin, Vmax and Flipline all are set to
266 * Vtotal value.
267 */
268static
269int intel_vrr_fixed_rr_hw_vtotal(const struct intel_crtc_state *crtc_state)
270{
271	return intel_vrr_hw_value(crtc_state, crtc_state->hw.adjusted_mode.crtc_vtotal);
272}
273
274static
275int intel_vrr_fixed_rr_hw_vmax(const struct intel_crtc_state *crtc_state)
276{
277	return intel_vrr_fixed_rr_hw_vtotal(crtc_state);
278}
279
280static
281int intel_vrr_fixed_rr_hw_vmin(const struct intel_crtc_state *crtc_state)
282{
283	struct intel_display *display = to_intel_display(crtc_state);
284
285	return intel_vrr_fixed_rr_hw_vtotal(crtc_state) -
286		intel_vrr_vmin_flipline_offset(display);
287}
288
289static
290int intel_vrr_fixed_rr_hw_flipline(const struct intel_crtc_state *crtc_state)
291{
292	return intel_vrr_fixed_rr_hw_vtotal(crtc_state);
293}
294
295void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state)
296{
297	struct intel_display *display = to_intel_display(crtc_state);
298	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
299
300	if (!intel_vrr_possible(crtc_state))
301		return;
302
303	intel_de_write(display, TRANS_VRR_VMIN(display, cpu_transcoder),
304		       intel_vrr_fixed_rr_hw_vmin(crtc_state) - 1);
305	intel_de_write(display, TRANS_VRR_VMAX(display, cpu_transcoder),
306		       intel_vrr_fixed_rr_hw_vmax(crtc_state) - 1);
307	intel_de_write(display, TRANS_VRR_FLIPLINE(display, cpu_transcoder),
308		       intel_vrr_fixed_rr_hw_flipline(crtc_state) - 1);
309}
310
311static
312int intel_vrr_compute_vmin(struct intel_crtc_state *crtc_state)
313{
314	/*
315	 * To make fixed rr and vrr work seamless the guardband/pipeline full
316	 * should be set such that it satisfies both the fixed and variable
317	 * timings.
318	 * For this set the vmin as crtc_vtotal. With this we never need to
319	 * change anything to do with the guardband.
320	 */
321	return crtc_state->hw.adjusted_mode.crtc_vtotal;
322}
323
324static
325int intel_vrr_compute_vmax(struct intel_connector *connector,
326			   const struct drm_display_mode *adjusted_mode)
327{
328	const struct drm_display_info *info = &connector->base.display_info;
329	int vmax;
330
331	vmax = adjusted_mode->crtc_clock * 1000 /
332		(adjusted_mode->crtc_htotal * info->monitor_range.min_vfreq);
333	vmax = max_t(int, vmax, adjusted_mode->crtc_vtotal);
334
335	return vmax;
336}
337
338void
339intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
340			 struct drm_connector_state *conn_state)
341{
342	struct intel_display *display = to_intel_display(crtc_state);
343	struct intel_connector *connector =
344		to_intel_connector(conn_state->connector);
345	struct intel_dp *intel_dp = intel_attached_dp(connector);
346	bool is_edp = intel_dp_is_edp(intel_dp);
347	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
348	int vmin, vmax;
349
350	if (!HAS_VRR(display))
351		return;
352
353	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
354		return;
355
356	crtc_state->vrr.in_range =
357		intel_vrr_is_in_range(connector, drm_mode_vrefresh(adjusted_mode));
358
359	/*
360	 * Allow fixed refresh rate with VRR Timing Generator.
361	 * For now set the vrr.in_range to 0, to allow fixed_rr but skip actual
362	 * VRR and LRR.
363	 * #TODO For actual VRR with joiner, we need to figure out how to
364	 * correctly sequence transcoder level stuff vs. pipe level stuff
365	 * in the commit.
366	 */
367	if (crtc_state->joiner_pipes)
368		crtc_state->vrr.in_range = false;
369
370	vmin = intel_vrr_compute_vmin(crtc_state);
371
372	if (crtc_state->vrr.in_range) {
373		if (HAS_LRR(display))
374			crtc_state->update_lrr = true;
375		vmax = intel_vrr_compute_vmax(connector, adjusted_mode);
376	} else {
377		vmax = vmin;
378	}
379
380	if (crtc_state->uapi.vrr_enabled && vmin < vmax)
381		intel_vrr_compute_vrr_timings(crtc_state, vmin, vmax);
382	else if (is_cmrr_frac_required(crtc_state) && is_edp)
383		intel_vrr_compute_cmrr_timings(crtc_state);
384	else
385		intel_vrr_compute_fixed_rr_timings(crtc_state);
386
387	if (HAS_AS_SDP(display)) {
388		crtc_state->vrr.vsync_start =
389			(crtc_state->hw.adjusted_mode.crtc_vtotal -
390			 crtc_state->hw.adjusted_mode.crtc_vsync_start);
391		crtc_state->vrr.vsync_end =
392			(crtc_state->hw.adjusted_mode.crtc_vtotal -
393			 crtc_state->hw.adjusted_mode.crtc_vsync_end);
394	}
395}
396
397static int
398intel_vrr_max_hw_guardband(const struct intel_crtc_state *crtc_state)
399{
400	struct intel_display *display = to_intel_display(crtc_state);
401	int max_pipeline_full = REG_FIELD_MAX(VRR_CTL_PIPELINE_FULL_MASK);
402
403	if (DISPLAY_VER(display) >= 13)
404		return REG_FIELD_MAX(XELPD_VRR_CTL_VRR_GUARDBAND_MASK);
405	else
406		return intel_vrr_pipeline_full_to_guardband(crtc_state,
407							    max_pipeline_full);
408}
409
410static int
411intel_vrr_max_vblank_guardband(const struct intel_crtc_state *crtc_state)
412{
413	struct intel_display *display = to_intel_display(crtc_state);
414	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
415
416	return crtc_state->vrr.vmin -
417	       adjusted_mode->crtc_vdisplay -
418	       crtc_state->set_context_latency -
419	       intel_vrr_extra_vblank_delay(display);
420}
421
422static int
423intel_vrr_max_guardband(struct intel_crtc_state *crtc_state)
424{
425	return min(intel_vrr_max_hw_guardband(crtc_state),
426		   intel_vrr_max_vblank_guardband(crtc_state));
427}
428
429static
430int intel_vrr_compute_optimized_guardband(struct intel_crtc_state *crtc_state)
431{
432	struct intel_display *display = to_intel_display(crtc_state);
433	struct skl_prefill_ctx prefill_ctx;
434	int prefill_latency_us;
435	int guardband = 0;
436
437	skl_prefill_init_worst(&prefill_ctx, crtc_state);
438
439	/*
440	 * The SoC power controller runs SAGV mutually exclusive with package C states,
441	 * so the max of package C and SAGV latencies is used to compute the min prefill guardband.
442	 * PM delay = max(sagv_latency, pkgc_max_latency (highest enabled wm level 1 and up))
443	 */
444	prefill_latency_us = max(display->sagv.block_time_us,
445				 skl_watermark_max_latency(display, 1));
446
447	guardband = skl_prefill_min_guardband(&prefill_ctx,
448					      crtc_state,
449					      prefill_latency_us);
450
451	if (intel_crtc_has_dp_encoder(crtc_state)) {
452		guardband = max(guardband, intel_psr_min_guardband(crtc_state));
453		guardband = max(guardband, intel_dp_sdp_min_guardband(crtc_state, true));
454	}
455
456	return guardband;
457}
458
459static bool intel_vrr_use_optimized_guardband(const struct intel_crtc_state *crtc_state)
460{
461	/*
462	 * #TODO: Enable optimized guardband for HDMI
463	 * For HDMI lot of infoframes are transmitted a line or two after vsync.
464	 * Since with optimized guardband the double bufferring point is at delayed vblank,
465	 * we need to ensure that vsync happens after delayed vblank for the HDMI case.
466	 */
467	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
468		return false;
469
470	return true;
471}
472
473void intel_vrr_compute_guardband(struct intel_crtc_state *crtc_state)
474{
475	struct intel_display *display = to_intel_display(crtc_state);
476	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
477	struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
478	int guardband;
479
480	if (!intel_vrr_possible(crtc_state))
481		return;
482
483	if (intel_vrr_use_optimized_guardband(crtc_state))
484		guardband = intel_vrr_compute_optimized_guardband(crtc_state);
485	else
486		guardband = crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay;
487
488	crtc_state->vrr.guardband = min(guardband, intel_vrr_max_guardband(crtc_state));
489
490	if (intel_vrr_always_use_vrr_tg(display)) {
491		adjusted_mode->crtc_vblank_start  =
492			adjusted_mode->crtc_vtotal - crtc_state->vrr.guardband;
493		/*
494		 * pipe_mode has already been derived from the
495		 * original adjusted_mode, keep the two in sync.
496		 */
497		pipe_mode->crtc_vblank_start =
498			adjusted_mode->crtc_vblank_start;
499	}
500
501	if (DISPLAY_VER(display) < 13)
502		crtc_state->vrr.pipeline_full =
503			intel_vrr_guardband_to_pipeline_full(crtc_state,
504							     crtc_state->vrr.guardband);
505}
506
507static u32 trans_vrr_ctl(const struct intel_crtc_state *crtc_state)
508{
509	struct intel_display *display = to_intel_display(crtc_state);
510
511	if (DISPLAY_VER(display) >= 14)
512		return VRR_CTL_FLIP_LINE_EN |
513			XELPD_VRR_CTL_VRR_GUARDBAND(crtc_state->vrr.guardband);
514	else if (DISPLAY_VER(display) >= 13)
515		return VRR_CTL_IGN_MAX_SHIFT | VRR_CTL_FLIP_LINE_EN |
516			XELPD_VRR_CTL_VRR_GUARDBAND(crtc_state->vrr.guardband);
517	else
518		return VRR_CTL_IGN_MAX_SHIFT | VRR_CTL_FLIP_LINE_EN |
519			VRR_CTL_PIPELINE_FULL(crtc_state->vrr.pipeline_full) |
520			VRR_CTL_PIPELINE_FULL_OVERRIDE;
521}
522
523void intel_vrr_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
524{
525	struct intel_display *display = to_intel_display(crtc_state);
526	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
527
528	if (!HAS_VRR(display))
529		return;
530
531	/*
532	 * Bspec says:
533	 * "(note: VRR needs to be programmed after
534	 *  TRANS_DDI_FUNC_CTL and before TRANS_CONF)."
535	 *
536	 * In practice it turns out that ICL can hang if
537	 * TRANS_VRR_VMAX/FLIPLINE are written before
538	 * enabling TRANS_DDI_FUNC_CTL.
539	 */
540	drm_WARN_ON(display->drm,
541		    !(intel_de_read(display, TRANS_DDI_FUNC_CTL(display, cpu_transcoder)) & TRANS_DDI_FUNC_ENABLE));
542
543	/*
544	 * This bit seems to have two meanings depending on the platform:
545	 * TGL: generate VRR "safe window" for DSB vblank waits
546	 * ADL/DG2: make TRANS_SET_CONTEXT_LATENCY effective with VRR
547	 */
548	if (IS_DISPLAY_VER(display, 12, 13))
549		intel_de_rmw(display, CHICKEN_TRANS(display, cpu_transcoder),
550			     0, PIPE_VBLANK_WITH_DELAY);
551
552	if (!intel_vrr_possible(crtc_state)) {
553		intel_de_write(display,
554			       TRANS_VRR_CTL(display, cpu_transcoder), 0);
555		return;
556	}
557
558	if (crtc_state->cmrr.enable) {
559		intel_de_write(display, TRANS_CMRR_M_HI(display, cpu_transcoder),
560			       upper_32_bits(crtc_state->cmrr.cmrr_m));
561		intel_de_write(display, TRANS_CMRR_M_LO(display, cpu_transcoder),
562			       lower_32_bits(crtc_state->cmrr.cmrr_m));
563		intel_de_write(display, TRANS_CMRR_N_HI(display, cpu_transcoder),
564			       upper_32_bits(crtc_state->cmrr.cmrr_n));
565		intel_de_write(display, TRANS_CMRR_N_LO(display, cpu_transcoder),
566			       lower_32_bits(crtc_state->cmrr.cmrr_n));
567	}
568
569	intel_vrr_set_fixed_rr_timings(crtc_state);
570
571	if (!intel_vrr_always_use_vrr_tg(display))
572		intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder),
573			       trans_vrr_ctl(crtc_state));
574
575	if (HAS_AS_SDP(display))
576		intel_de_write(display,
577			       TRANS_VRR_VSYNC(display, cpu_transcoder),
578			       VRR_VSYNC_END(crtc_state->vrr.vsync_end) |
579			       VRR_VSYNC_START(crtc_state->vrr.vsync_start));
580
581	/*
582	 * For BMG and LNL+ onwards the EMP_AS_SDP_TL is used for programming
583	 * double buffering point and transmission line for VRR packets for
584	 * HDMI2.1/DP/eDP/DP->HDMI2.1 PCON.
585	 * Since currently we support VRR only for DP/eDP, so this is programmed
586	 * to for Adaptive Sync SDP to Vsync start.
587	 */
588	if (DISPLAY_VERx100(display) == 1401 || DISPLAY_VER(display) >= 20)
589		intel_de_write(display,
590			       EMP_AS_SDP_TL(display, cpu_transcoder),
591			       EMP_AS_SDP_DB_TL(crtc_state->vrr.vsync_start));
592}
593
594void intel_vrr_send_push(struct intel_dsb *dsb,
595			 const struct intel_crtc_state *crtc_state)
596{
597	struct intel_display *display = to_intel_display(crtc_state);
598	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
599
600	if (!crtc_state->vrr.enable)
601		return;
602
603	if (dsb)
604		intel_dsb_nonpost_start(dsb);
605
606	intel_de_write_dsb(display, dsb,
607			   TRANS_PUSH(display, cpu_transcoder),
608			   TRANS_PUSH_EN | TRANS_PUSH_SEND);
609
610	if (dsb)
611		intel_dsb_nonpost_end(dsb);
612}
613
614void intel_vrr_check_push_sent(struct intel_dsb *dsb,
615			       const struct intel_crtc_state *crtc_state)
616{
617	struct intel_display *display = to_intel_display(crtc_state);
618	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
619	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
620
621	if (!crtc_state->vrr.enable)
622		return;
623
624	/*
625	 * Make sure the push send bit has cleared. This should
626	 * already be the case as long as the caller makes sure
627	 * this is called after the delayed vblank has occurred.
628	 */
629	if (dsb) {
630		int wait_us, count;
631
632		wait_us = 2;
633		count = 1;
634
635		/*
636		 * If the bit hasn't cleared the DSB will
637		 * raise the poll error interrupt.
638		 */
639		intel_dsb_poll(dsb, TRANS_PUSH(display, cpu_transcoder),
640			       TRANS_PUSH_SEND, 0, wait_us, count);
641	} else {
642		if (intel_vrr_is_push_sent(crtc_state))
643			drm_err(display->drm, "[CRTC:%d:%s] VRR push send still pending\n",
644				crtc->base.base.id, crtc->base.name);
645	}
646}
647
648bool intel_vrr_is_push_sent(const struct intel_crtc_state *crtc_state)
649{
650	struct intel_display *display = to_intel_display(crtc_state);
651	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
652
653	if (!crtc_state->vrr.enable)
654		return false;
655
656	return intel_de_read(display, TRANS_PUSH(display, cpu_transcoder)) & TRANS_PUSH_SEND;
657}
658
659bool intel_vrr_always_use_vrr_tg(struct intel_display *display)
660{
661	if (!HAS_VRR(display))
662		return false;
663
664	if (DISPLAY_VER(display) >= 30)
665		return true;
666
667	return false;
668}
669
670static int intel_vrr_hw_vmin(const struct intel_crtc_state *crtc_state)
671{
672	struct intel_display *display = to_intel_display(crtc_state);
673
674	return intel_vrr_hw_value(crtc_state, crtc_state->vrr.vmin) -
675		intel_vrr_vmin_flipline_offset(display);
676}
677
678static int intel_vrr_hw_vmax(const struct intel_crtc_state *crtc_state)
679{
680	return intel_vrr_hw_value(crtc_state, crtc_state->vrr.vmax);
681}
682
683static int intel_vrr_hw_flipline(const struct intel_crtc_state *crtc_state)
684{
685	return intel_vrr_hw_value(crtc_state, crtc_state->vrr.flipline);
686}
687
688static void intel_vrr_set_vrr_timings(const struct intel_crtc_state *crtc_state)
689{
690	struct intel_display *display = to_intel_display(crtc_state);
691	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
692
693	intel_de_write(display, TRANS_VRR_VMIN(display, cpu_transcoder),
694		       intel_vrr_hw_vmin(crtc_state) - 1);
695	intel_de_write(display, TRANS_VRR_VMAX(display, cpu_transcoder),
696		       intel_vrr_hw_vmax(crtc_state) - 1);
697	intel_de_write(display, TRANS_VRR_FLIPLINE(display, cpu_transcoder),
698		       intel_vrr_hw_flipline(crtc_state) - 1);
699}
700
701static void intel_vrr_tg_enable(const struct intel_crtc_state *crtc_state,
702				bool cmrr_enable)
703{
704	struct intel_display *display = to_intel_display(crtc_state);
705	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
706	u32 vrr_ctl;
707
708	intel_de_write(display, TRANS_PUSH(display, cpu_transcoder), TRANS_PUSH_EN);
709
710	vrr_ctl = VRR_CTL_VRR_ENABLE | trans_vrr_ctl(crtc_state);
711
712	/*
713	 * FIXME this might be broken as bspec seems to imply that
714	 * even VRR_CTL_CMRR_ENABLE is armed by TRANS_CMRR_N_HI
715	 * when enabling CMRR (but not when disabling CMRR?).
716	 */
717	if (cmrr_enable)
718		vrr_ctl |= VRR_CTL_CMRR_ENABLE;
719
720	intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder), vrr_ctl);
721}
722
723static void intel_vrr_tg_disable(const struct intel_crtc_state *old_crtc_state)
724{
725	struct intel_display *display = to_intel_display(old_crtc_state);
726	enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
727
728	intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder),
729		       trans_vrr_ctl(old_crtc_state));
730
731	if (intel_de_wait_for_clear_ms(display,
732				       TRANS_VRR_STATUS(display, cpu_transcoder),
733				       VRR_STATUS_VRR_EN_LIVE, 1000))
734		drm_err(display->drm, "Timed out waiting for VRR live status to clear\n");
735
736	intel_de_write(display, TRANS_PUSH(display, cpu_transcoder), 0);
737}
738
739void intel_vrr_enable(const struct intel_crtc_state *crtc_state)
740{
741	struct intel_display *display = to_intel_display(crtc_state);
742
743	if (!crtc_state->vrr.enable)
744		return;
745
746	intel_vrr_set_vrr_timings(crtc_state);
747
748	if (!intel_vrr_always_use_vrr_tg(display))
749		intel_vrr_tg_enable(crtc_state, crtc_state->cmrr.enable);
750}
751
752void intel_vrr_disable(const struct intel_crtc_state *old_crtc_state)
753{
754	struct intel_display *display = to_intel_display(old_crtc_state);
755
756	if (!old_crtc_state->vrr.enable)
757		return;
758
759	if (!intel_vrr_always_use_vrr_tg(display))
760		intel_vrr_tg_disable(old_crtc_state);
761
762	intel_vrr_set_fixed_rr_timings(old_crtc_state);
763}
764
765void intel_vrr_transcoder_enable(const struct intel_crtc_state *crtc_state)
766{
767	struct intel_display *display = to_intel_display(crtc_state);
768
769	intel_vrr_set_transcoder_timings(crtc_state);
770
771	if (!intel_vrr_possible(crtc_state))
772		return;
773
774	if (intel_vrr_always_use_vrr_tg(display))
775		intel_vrr_tg_enable(crtc_state, false);
776}
777
778void intel_vrr_transcoder_disable(const struct intel_crtc_state *old_crtc_state)
779{
780	struct intel_display *display = to_intel_display(old_crtc_state);
781
782	if (!intel_vrr_possible(old_crtc_state))
783		return;
784
785	if (intel_vrr_always_use_vrr_tg(display))
786		intel_vrr_tg_disable(old_crtc_state);
787}
788
789bool intel_vrr_is_fixed_rr(const struct intel_crtc_state *crtc_state)
790{
791	return crtc_state->vrr.flipline &&
792	       crtc_state->vrr.flipline == crtc_state->vrr.vmax &&
793	       crtc_state->vrr.flipline == crtc_state->vrr.vmin;
794}
795
796void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
797{
798	struct intel_display *display = to_intel_display(crtc_state);
799	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
800	u32 trans_vrr_ctl, trans_vrr_vsync;
801	bool vrr_enable;
802
803	trans_vrr_ctl = intel_de_read(display,
804				      TRANS_VRR_CTL(display, cpu_transcoder));
805
806	if (HAS_CMRR(display))
807		crtc_state->cmrr.enable = (trans_vrr_ctl & VRR_CTL_CMRR_ENABLE);
808
809	if (crtc_state->cmrr.enable) {
810		crtc_state->cmrr.cmrr_n =
811			intel_de_read64_2x32(display, TRANS_CMRR_N_LO(display, cpu_transcoder),
812					     TRANS_CMRR_N_HI(display, cpu_transcoder));
813		crtc_state->cmrr.cmrr_m =
814			intel_de_read64_2x32(display, TRANS_CMRR_M_LO(display, cpu_transcoder),
815					     TRANS_CMRR_M_HI(display, cpu_transcoder));
816	}
817
818	if (DISPLAY_VER(display) >= 13) {
819		crtc_state->vrr.guardband =
820			REG_FIELD_GET(XELPD_VRR_CTL_VRR_GUARDBAND_MASK, trans_vrr_ctl);
821	} else {
822		if (trans_vrr_ctl & VRR_CTL_PIPELINE_FULL_OVERRIDE) {
823			crtc_state->vrr.pipeline_full =
824				REG_FIELD_GET(VRR_CTL_PIPELINE_FULL_MASK, trans_vrr_ctl);
825
826			crtc_state->vrr.guardband =
827				intel_vrr_pipeline_full_to_guardband(crtc_state,
828								     crtc_state->vrr.pipeline_full);
829		}
830	}
831
832	if (trans_vrr_ctl & VRR_CTL_FLIP_LINE_EN) {
833		crtc_state->vrr.flipline = intel_de_read(display,
834							 TRANS_VRR_FLIPLINE(display, cpu_transcoder)) + 1;
835		crtc_state->vrr.vmax = intel_de_read(display,
836						     TRANS_VRR_VMAX(display, cpu_transcoder)) + 1;
837		crtc_state->vrr.vmin = intel_de_read(display,
838						     TRANS_VRR_VMIN(display, cpu_transcoder)) + 1;
839
840		if (DISPLAY_VER(display) < 13) {
841			/* undo what intel_vrr_hw_value() does when writing the values */
842			crtc_state->vrr.flipline += crtc_state->set_context_latency;
843			crtc_state->vrr.vmax += crtc_state->set_context_latency;
844			crtc_state->vrr.vmin += crtc_state->set_context_latency;
845
846			crtc_state->vrr.vmin += intel_vrr_vmin_flipline_offset(display);
847		}
848
849		/*
850		 * For platforms that always use VRR Timing Generator, the VTOTAL.Vtotal
851		 * bits are not filled. Since for these platforms TRAN_VMIN is always
852		 * filled with crtc_vtotal, use TRAN_VRR_VMIN to get the vtotal for
853		 * adjusted_mode.
854		 */
855		if (intel_vrr_always_use_vrr_tg(display))
856			crtc_state->hw.adjusted_mode.crtc_vtotal =
857				intel_vrr_vmin_vtotal(crtc_state);
858
859		if (HAS_AS_SDP(display)) {
860			trans_vrr_vsync =
861				intel_de_read(display,
862					      TRANS_VRR_VSYNC(display, cpu_transcoder));
863			crtc_state->vrr.vsync_start =
864				REG_FIELD_GET(VRR_VSYNC_START_MASK, trans_vrr_vsync);
865			crtc_state->vrr.vsync_end =
866				REG_FIELD_GET(VRR_VSYNC_END_MASK, trans_vrr_vsync);
867		}
868	}
869
870	vrr_enable = trans_vrr_ctl & VRR_CTL_VRR_ENABLE;
871
872	if (intel_vrr_always_use_vrr_tg(display))
873		crtc_state->vrr.enable = vrr_enable && !intel_vrr_is_fixed_rr(crtc_state);
874	else
875		crtc_state->vrr.enable = vrr_enable;
876
877	/*
878	 * #TODO: For Both VRR and CMRR the flag I915_MODE_FLAG_VRR is set for mode_flags.
879	 * Since CMRR is currently disabled, set this flag for VRR for now.
880	 * Need to keep this in mind while re-enabling CMRR.
881	 */
882	if (crtc_state->vrr.enable)
883		crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
884
885	/*
886	 * For platforms that always use the VRR timing generator, we overwrite
887	 * crtc_vblank_start with vtotal - guardband to reflect the delayed
888	 * vblank start. This works for both default and optimized guardband values.
889	 * On other platforms, we keep the original value from
890	 * intel_get_transcoder_timings() and apply adjustments only in VRR-specific
891	 * paths as needed.
892	 */
893	if (intel_vrr_always_use_vrr_tg(display))
894		crtc_state->hw.adjusted_mode.crtc_vblank_start =
895			crtc_state->hw.adjusted_mode.crtc_vtotal -
896			crtc_state->vrr.guardband;
897}
898
899int intel_vrr_safe_window_start(const struct intel_crtc_state *crtc_state)
900{
901	struct intel_display *display = to_intel_display(crtc_state);
902
903	if (DISPLAY_VER(display) >= 30)
904		return crtc_state->hw.adjusted_mode.crtc_vdisplay -
905		       crtc_state->set_context_latency;
906	else
907		return crtc_state->hw.adjusted_mode.crtc_vdisplay;
908}
909
910int intel_vrr_vmin_safe_window_end(const struct intel_crtc_state *crtc_state)
911{
912	return intel_vrr_vmin_vblank_start(crtc_state) -
913	       crtc_state->set_context_latency;
914}