intel_crtc.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2020 Intel Corporation
  4 */
  5#include <linux/kernel.h>
  6#include <linux/pm_qos.h>
  7#include <linux/slab.h>
  8
  9#include <drm/drm_atomic_helper.h>
 10#include <drm/drm_fourcc.h>
 11#include <drm/drm_plane.h>
 12#include <drm/drm_print.h>
 13#include <drm/drm_vblank.h>
 14#include <drm/drm_vblank_work.h>
 15
 16#include "i915_drv.h"
 17#include "i915_vgpu.h"
 18#include "i9xx_plane.h"
 19#include "icl_dsi.h"
 20#include "intel_atomic.h"
 21#include "intel_color.h"
 22#include "intel_crtc.h"
 23#include "intel_cursor.h"
 24#include "intel_display_debugfs.h"
 25#include "intel_display_irq.h"
 26#include "intel_display_trace.h"
 27#include "intel_display_types.h"
 28#include "intel_drrs.h"
 29#include "intel_dsi.h"
 30#include "intel_fifo_underrun.h"
 31#include "intel_pipe_crc.h"
 32#include "intel_plane.h"
 33#include "intel_psr.h"
 34#include "intel_sprite.h"
 35#include "intel_vblank.h"
 36#include "intel_vrr.h"
 37#include "skl_universal_plane.h"
 38
 39static void assert_vblank_disabled(struct drm_crtc *crtc)
 40{
 41	struct intel_display *display = to_intel_display(crtc->dev);
 42
 43	if (INTEL_DISPLAY_STATE_WARN(display, drm_crtc_vblank_get(crtc) == 0,
 44				     "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
 45				     crtc->base.id, crtc->name))
 46		drm_crtc_vblank_put(crtc);
 47}
 48
 49struct intel_crtc *intel_first_crtc(struct intel_display *display)
 50{
 51	return to_intel_crtc(drm_crtc_from_index(display->drm, 0));
 52}
 53
 54struct intel_crtc *intel_crtc_for_pipe(struct intel_display *display,
 55				       enum pipe pipe)
 56{
 57	struct intel_crtc *crtc;
 58
 59	for_each_intel_crtc(display->drm, crtc) {
 60		if (crtc->pipe == pipe)
 61			return crtc;
 62	}
 63
 64	return NULL;
 65}
 66
 67void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
 68{
 69	drm_crtc_wait_one_vblank(&crtc->base);
 70}
 71
 72void intel_wait_for_vblank_if_active(struct intel_display *display,
 73				     enum pipe pipe)
 74{
 75	struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
 76
 77	if (crtc->active)
 78		intel_crtc_wait_for_next_vblank(crtc);
 79}
 80
 81u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
 82{
 83	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
 84
 85	if (!crtc->active)
 86		return 0;
 87
 88	if (!vblank->max_vblank_count) {
 89		/* On preempt-rt we cannot take the vblank spinlock since this function is called from tracepoints */
 90		if (IS_ENABLED(CONFIG_PREEMPT_RT))
 91			return (u32)drm_crtc_vblank_count(&crtc->base);
 92		else
 93			return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
 94	}
 95
 96	return crtc->base.funcs->get_vblank_counter(&crtc->base);
 97}
 98
 99u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
100{
101	struct intel_display *display = to_intel_display(crtc_state);
102
103	/*
104	 * From Gen 11, in case of dsi cmd mode, frame counter wouldn't
105	 * have updated at the beginning of TE, if we want to use
106	 * the hw counter, then we would find it updated in only
107	 * the next TE, hence switching to sw counter.
108	 */
109	if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
110				      I915_MODE_FLAG_DSI_USE_TE1))
111		return 0;
112
113	/*
114	 * On i965gm the hardware frame counter reads
115	 * zero when the TV encoder is enabled :(
116	 */
117	if (display->platform.i965gm &&
118	    (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
119		return 0;
120
121	if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
122		return 0xffffffff; /* full 32 bit counter */
123	else if (DISPLAY_VER(display) >= 3)
124		return 0xffffff; /* only 24 bits of frame count */
125	else
126		return 0; /* Gen2 doesn't have a hardware frame counter */
127}
128
129void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
130{
131	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
132
133	crtc->vblank_psr_notify = intel_psr_needs_vblank_notification(crtc_state);
134
135	assert_vblank_disabled(&crtc->base);
136	drm_crtc_set_max_vblank_count(&crtc->base,
137				      intel_crtc_max_vblank_count(crtc_state));
138	drm_crtc_vblank_on(&crtc->base);
139
140	/*
141	 * Should really happen exactly when we enable the pipe
142	 * but we want the frame counters in the trace, and that
143	 * requires vblank support on some platforms/outputs.
144	 */
145	trace_intel_pipe_enable(crtc);
146}
147
148void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
149{
150	struct intel_display *display = to_intel_display(crtc_state);
151	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
152
153	/*
154	 * Should really happen exactly when we disable the pipe
155	 * but we want the frame counters in the trace, and that
156	 * requires vblank support on some platforms/outputs.
157	 */
158	trace_intel_pipe_disable(crtc);
159
160	drm_crtc_vblank_off(&crtc->base);
161	assert_vblank_disabled(&crtc->base);
162
163	crtc->vblank_psr_notify = false;
164
165	flush_work(&display->irq.vblank_notify_work);
166}
167
168struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
169{
170	struct intel_crtc_state *crtc_state;
171
172	crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
173
174	if (crtc_state)
175		intel_crtc_state_reset(crtc_state, crtc);
176
177	return crtc_state;
178}
179
180void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
181			    struct intel_crtc *crtc)
182{
183	memset(crtc_state, 0, sizeof(*crtc_state));
184
185	__drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
186
187	crtc_state->cpu_transcoder = INVALID_TRANSCODER;
188	crtc_state->master_transcoder = INVALID_TRANSCODER;
189	crtc_state->hsw_workaround_pipe = INVALID_PIPE;
190	crtc_state->scaler_state.scaler_id = -1;
191	crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
192	crtc_state->max_link_bpp_x16 = INT_MAX;
193}
194
195static struct intel_crtc *intel_crtc_alloc(void)
196{
197	struct intel_crtc_state *crtc_state;
198	struct intel_crtc *crtc;
199
200	crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
201	if (!crtc)
202		return ERR_PTR(-ENOMEM);
203
204	crtc_state = intel_crtc_state_alloc(crtc);
205	if (!crtc_state) {
206		kfree(crtc);
207		return ERR_PTR(-ENOMEM);
208	}
209
210	crtc->base.state = &crtc_state->uapi;
211	crtc->config = crtc_state;
212
213	return crtc;
214}
215
216static void intel_crtc_free(struct intel_crtc *crtc)
217{
218	intel_crtc_destroy_state(&crtc->base, crtc->base.state);
219	kfree(crtc);
220}
221
222static void intel_crtc_destroy(struct drm_crtc *_crtc)
223{
224	struct intel_crtc *crtc = to_intel_crtc(_crtc);
225
226	cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
227
228	drm_crtc_cleanup(&crtc->base);
229	kfree(crtc);
230}
231
232static int intel_crtc_late_register(struct drm_crtc *crtc)
233{
234	intel_crtc_debugfs_add(to_intel_crtc(crtc));
235	return 0;
236}
237
238#define INTEL_CRTC_FUNCS \
239	.set_config = drm_atomic_helper_set_config, \
240	.destroy = intel_crtc_destroy, \
241	.page_flip = drm_atomic_helper_page_flip, \
242	.atomic_duplicate_state = intel_crtc_duplicate_state, \
243	.atomic_destroy_state = intel_crtc_destroy_state, \
244	.set_crc_source = intel_crtc_set_crc_source, \
245	.verify_crc_source = intel_crtc_verify_crc_source, \
246	.get_crc_sources = intel_crtc_get_crc_sources, \
247	.late_register = intel_crtc_late_register
248
249static const struct drm_crtc_funcs bdw_crtc_funcs = {
250	INTEL_CRTC_FUNCS,
251
252	.get_vblank_counter = g4x_get_vblank_counter,
253	.enable_vblank = bdw_enable_vblank,
254	.disable_vblank = bdw_disable_vblank,
255	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
256};
257
258static const struct drm_crtc_funcs ilk_crtc_funcs = {
259	INTEL_CRTC_FUNCS,
260
261	.get_vblank_counter = g4x_get_vblank_counter,
262	.enable_vblank = ilk_enable_vblank,
263	.disable_vblank = ilk_disable_vblank,
264	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
265};
266
267static const struct drm_crtc_funcs g4x_crtc_funcs = {
268	INTEL_CRTC_FUNCS,
269
270	.get_vblank_counter = g4x_get_vblank_counter,
271	.enable_vblank = i965_enable_vblank,
272	.disable_vblank = i965_disable_vblank,
273	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
274};
275
276static const struct drm_crtc_funcs i965_crtc_funcs = {
277	INTEL_CRTC_FUNCS,
278
279	.get_vblank_counter = i915_get_vblank_counter,
280	.enable_vblank = i965_enable_vblank,
281	.disable_vblank = i965_disable_vblank,
282	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
283};
284
285static const struct drm_crtc_funcs i915gm_crtc_funcs = {
286	INTEL_CRTC_FUNCS,
287
288	.get_vblank_counter = i915_get_vblank_counter,
289	.enable_vblank = i915gm_enable_vblank,
290	.disable_vblank = i915gm_disable_vblank,
291	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
292};
293
294static const struct drm_crtc_funcs i915_crtc_funcs = {
295	INTEL_CRTC_FUNCS,
296
297	.get_vblank_counter = i915_get_vblank_counter,
298	.enable_vblank = i8xx_enable_vblank,
299	.disable_vblank = i8xx_disable_vblank,
300	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
301};
302
303static const struct drm_crtc_funcs i8xx_crtc_funcs = {
304	INTEL_CRTC_FUNCS,
305
306	/* no hw vblank counter */
307	.enable_vblank = i8xx_enable_vblank,
308	.disable_vblank = i8xx_disable_vblank,
309	.get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
310};
311
312int intel_crtc_init(struct intel_display *display, enum pipe pipe)
313{
314	struct intel_plane *primary, *cursor;
315	const struct drm_crtc_funcs *funcs;
316	struct intel_crtc *crtc;
317	int sprite, ret;
318
319	crtc = intel_crtc_alloc();
320	if (IS_ERR(crtc))
321		return PTR_ERR(crtc);
322
323	crtc->pipe = pipe;
324	crtc->num_scalers = DISPLAY_RUNTIME_INFO(display)->num_scalers[pipe];
325
326	if (DISPLAY_VER(display) >= 9)
327		primary = skl_universal_plane_create(display, pipe, PLANE_1);
328	else
329		primary = intel_primary_plane_create(display, pipe);
330	if (IS_ERR(primary)) {
331		ret = PTR_ERR(primary);
332		goto fail;
333	}
334	crtc->plane_ids_mask |= BIT(primary->id);
335
336	intel_init_fifo_underrun_reporting(display, crtc, false);
337
338	for_each_sprite(display, pipe, sprite) {
339		struct intel_plane *plane;
340
341		if (DISPLAY_VER(display) >= 9)
342			plane = skl_universal_plane_create(display, pipe, PLANE_2 + sprite);
343		else
344			plane = intel_sprite_plane_create(display, pipe, sprite);
345		if (IS_ERR(plane)) {
346			ret = PTR_ERR(plane);
347			goto fail;
348		}
349		crtc->plane_ids_mask |= BIT(plane->id);
350	}
351
352	cursor = intel_cursor_plane_create(display, pipe);
353	if (IS_ERR(cursor)) {
354		ret = PTR_ERR(cursor);
355		goto fail;
356	}
357	crtc->plane_ids_mask |= BIT(cursor->id);
358
359	if (HAS_GMCH(display)) {
360		if (display->platform.cherryview ||
361		    display->platform.valleyview ||
362		    display->platform.g4x)
363			funcs = &g4x_crtc_funcs;
364		else if (DISPLAY_VER(display) == 4)
365			funcs = &i965_crtc_funcs;
366		else if (display->platform.i945gm ||
367			 display->platform.i915gm)
368			funcs = &i915gm_crtc_funcs;
369		else if (DISPLAY_VER(display) == 3)
370			funcs = &i915_crtc_funcs;
371		else
372			funcs = &i8xx_crtc_funcs;
373	} else {
374		if (DISPLAY_VER(display) >= 8)
375			funcs = &bdw_crtc_funcs;
376		else
377			funcs = &ilk_crtc_funcs;
378	}
379
380	ret = drm_crtc_init_with_planes(display->drm, &crtc->base,
381					&primary->base, &cursor->base,
382					funcs, "pipe %c", pipe_name(pipe));
383	if (ret)
384		goto fail;
385
386	if (DISPLAY_VER(display) >= 11)
387		drm_crtc_create_scaling_filter_property(&crtc->base,
388						BIT(DRM_SCALING_FILTER_DEFAULT) |
389						BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
390
391	intel_color_crtc_init(crtc);
392	intel_drrs_crtc_init(crtc);
393	intel_crtc_crc_init(crtc);
394
395	cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
396
397	drm_WARN_ON(display->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
398
399	if (HAS_CASF(display))
400		drm_crtc_create_sharpness_strength_property(&crtc->base);
401
402	return 0;
403
404fail:
405	intel_crtc_free(crtc);
406
407	return ret;
408}
409
410int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
411					   struct drm_file *file)
412{
413	struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
414	struct drm_crtc *drm_crtc;
415	struct intel_crtc *crtc;
416
417	drm_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
418	if (!drm_crtc)
419		return -ENOENT;
420
421	crtc = to_intel_crtc(drm_crtc);
422	pipe_from_crtc_id->pipe = crtc->pipe;
423
424	return 0;
425}
426
427static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
428{
429	struct intel_display *display = to_intel_display(crtc_state);
430
431	return crtc_state->hw.active &&
432		!crtc_state->preload_luts &&
433		!intel_crtc_needs_modeset(crtc_state) &&
434		(intel_crtc_needs_color_update(crtc_state) &&
435		 !HAS_DOUBLE_BUFFERED_LUT(display)) &&
436		!intel_color_uses_dsb(crtc_state) &&
437		!crtc_state->use_dsb;
438}
439
440static void intel_crtc_vblank_work(struct kthread_work *base)
441{
442	struct drm_vblank_work *work = to_drm_vblank_work(base);
443	struct intel_crtc_state *crtc_state =
444		container_of(work, typeof(*crtc_state), vblank_work);
445	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
446
447	trace_intel_crtc_vblank_work_start(crtc);
448
449	intel_color_load_luts(crtc_state);
450
451	if (crtc_state->uapi.event) {
452		spin_lock_irq(&crtc->base.dev->event_lock);
453		drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
454		spin_unlock_irq(&crtc->base.dev->event_lock);
455		crtc_state->uapi.event = NULL;
456	}
457
458	trace_intel_crtc_vblank_work_end(crtc);
459}
460
461static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
462{
463	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
464
465	drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
466			     intel_crtc_vblank_work);
467	/*
468	 * Interrupt latency is critical for getting the vblank
469	 * work executed as early as possible during the vblank.
470	 */
471	cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
472}
473
474void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
475{
476	struct intel_crtc_state *crtc_state;
477	struct intel_crtc *crtc;
478	int i;
479
480	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
481		if (!intel_crtc_needs_vblank_work(crtc_state))
482			continue;
483
484		drm_vblank_work_flush(&crtc_state->vblank_work);
485		cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
486					       PM_QOS_DEFAULT_VALUE);
487	}
488}
489
490int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
491			     int usecs)
492{
493	/* paranoia */
494	if (!adjusted_mode->crtc_htotal)
495		return 1;
496
497	return DIV_ROUND_UP_ULL(mul_u32_u32(usecs, adjusted_mode->crtc_clock),
498				1000 * adjusted_mode->crtc_htotal);
499}
500
501int intel_scanlines_to_usecs(const struct drm_display_mode *adjusted_mode,
502			     int scanlines)
503{
504	/* paranoia */
505	if (!adjusted_mode->crtc_clock)
506		return 1;
507
508	return DIV_ROUND_UP_ULL(mul_u32_u32(scanlines, adjusted_mode->crtc_htotal * 1000),
509				adjusted_mode->crtc_clock);
510}
511
512/**
513 * intel_pipe_update_start() - start update of a set of display registers
514 * @state: the atomic state
515 * @crtc: the crtc
516 *
517 * Mark the start of an update to pipe registers that should be updated
518 * atomically regarding vblank. If the next vblank will happens within
519 * the next 100 us, this function waits until the vblank passes.
520 *
521 * After a successful call to this function, interrupts will be disabled
522 * until a subsequent call to intel_pipe_update_end(). That is done to
523 * avoid random delays.
524 */
525void intel_pipe_update_start(struct intel_atomic_state *state,
526			     struct intel_crtc *crtc)
527{
528	struct intel_display *display = to_intel_display(state);
529	const struct intel_crtc_state *old_crtc_state =
530		intel_atomic_get_old_crtc_state(state, crtc);
531	struct intel_crtc_state *new_crtc_state =
532		intel_atomic_get_new_crtc_state(state, crtc);
533	struct intel_vblank_evade_ctx evade;
534	int scanline;
535
536	drm_WARN_ON(display->drm, new_crtc_state->use_dsb);
537
538	intel_psr_lock(new_crtc_state);
539
540	if (new_crtc_state->do_async_flip) {
541		intel_crtc_prepare_vblank_event(new_crtc_state,
542						&crtc->flip_done_event);
543		return;
544	}
545
546	if (intel_crtc_needs_vblank_work(new_crtc_state))
547		intel_crtc_vblank_work_init(new_crtc_state);
548
549	if (state->base.legacy_cursor_update) {
550		struct intel_plane *plane;
551		struct intel_plane_state *old_plane_state, *new_plane_state;
552		int i;
553
554		for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
555						     new_plane_state, i) {
556			if (old_plane_state->uapi.crtc == &crtc->base)
557				intel_plane_init_cursor_vblank_work(old_plane_state,
558								    new_plane_state);
559		}
560	}
561
562	intel_vblank_evade_init(old_crtc_state, new_crtc_state, &evade);
563
564	if (drm_WARN_ON(display->drm, drm_crtc_vblank_get(&crtc->base)))
565		goto irq_disable;
566
567	/*
568	 * Wait for psr to idle out after enabling the VBL interrupts
569	 * VBL interrupts will start the PSR exit and prevent a PSR
570	 * re-entry as well.
571	 */
572	intel_psr_wait_for_idle_locked(new_crtc_state);
573
574	local_irq_disable();
575
576	crtc->debug.min_vbl = evade.min;
577	crtc->debug.max_vbl = evade.max;
578	trace_intel_pipe_update_start(crtc);
579
580	scanline = intel_vblank_evade(&evade);
581
582	drm_crtc_vblank_put(&crtc->base);
583
584	crtc->debug.scanline_start = scanline;
585	crtc->debug.start_vbl_time = ktime_get();
586	crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
587
588	trace_intel_pipe_update_vblank_evaded(crtc);
589	return;
590
591irq_disable:
592	local_irq_disable();
593}
594
595#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
596static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
597{
598	u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
599	unsigned int h;
600
601	h = ilog2(delta >> 9);
602	if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
603		h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
604	crtc->debug.vbl.times[h]++;
605
606	crtc->debug.vbl.sum += delta;
607	if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
608		crtc->debug.vbl.min = delta;
609	if (delta > crtc->debug.vbl.max)
610		crtc->debug.vbl.max = delta;
611
612	if (delta > 1000 * VBLANK_EVASION_TIME_US) {
613		drm_dbg_kms(crtc->base.dev,
614			    "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
615			    pipe_name(crtc->pipe),
616			    div_u64(delta, 1000),
617			    VBLANK_EVASION_TIME_US);
618		crtc->debug.vbl.over++;
619	}
620}
621#else
622static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
623#endif
624
625void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state)
626{
627	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
628	unsigned long irqflags;
629
630	if (!crtc_state->uapi.event)
631		return;
632
633	drm_WARN_ON(crtc->base.dev, drm_crtc_vblank_get(&crtc->base) != 0);
634
635	spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
636	drm_crtc_arm_vblank_event(&crtc->base, crtc_state->uapi.event);
637	spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
638
639	crtc_state->uapi.event = NULL;
640}
641
642void intel_crtc_prepare_vblank_event(struct intel_crtc_state *crtc_state,
643				     struct drm_pending_vblank_event **event)
644{
645	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
646	unsigned long irqflags;
647
648	spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
649	*event = crtc_state->uapi.event;
650	spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
651
652	crtc_state->uapi.event = NULL;
653}
654
655/**
656 * intel_pipe_update_end() - end update of a set of display registers
657 * @state: the atomic state
658 * @crtc: the crtc
659 *
660 * Mark the end of an update started with intel_pipe_update_start(). This
661 * re-enables interrupts and verifies the update was actually completed
662 * before a vblank.
663 */
664void intel_pipe_update_end(struct intel_atomic_state *state,
665			   struct intel_crtc *crtc)
666{
667	struct intel_display *display = to_intel_display(state);
668	struct intel_crtc_state *new_crtc_state =
669		intel_atomic_get_new_crtc_state(state, crtc);
670	enum pipe pipe = crtc->pipe;
671	int scanline_end = intel_get_crtc_scanline(crtc);
672	u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
673	ktime_t end_vbl_time = ktime_get();
674	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
675
676	drm_WARN_ON(display->drm, new_crtc_state->use_dsb);
677
678	if (new_crtc_state->do_async_flip)
679		goto out;
680
681	trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
682
683	/*
684	 * Incase of mipi dsi command mode, we need to set frame update
685	 * request for every commit.
686	 */
687	if (DISPLAY_VER(display) >= 11 &&
688	    intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
689		icl_dsi_frame_update(new_crtc_state);
690
691	/* We're still in the vblank-evade critical section, this can't race.
692	 * Would be slightly nice to just grab the vblank count and arm the
693	 * event outside of the critical section - the spinlock might spin for a
694	 * while ... */
695	if (intel_crtc_needs_vblank_work(new_crtc_state)) {
696		drm_vblank_work_schedule(&new_crtc_state->vblank_work,
697					 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
698					 false);
699	} else {
700		intel_crtc_arm_vblank_event(new_crtc_state);
701	}
702
703	if (state->base.legacy_cursor_update) {
704		struct intel_plane *plane;
705		struct intel_plane_state *old_plane_state;
706		int i;
707
708		for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
709			if (old_plane_state->uapi.crtc == &crtc->base &&
710			    old_plane_state->unpin_work.vblank) {
711				drm_vblank_work_schedule(&old_plane_state->unpin_work,
712							 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
713							 false);
714
715				/* Remove plane from atomic state, cleanup/free is done from vblank worker. */
716				memset(&state->base.planes[i], 0, sizeof(state->base.planes[i]));
717			}
718		}
719	}
720
721	/*
722	 * Send VRR Push to terminate Vblank. If we are already in vblank
723	 * this has to be done _after_ sampling the frame counter, as
724	 * otherwise the push would immediately terminate the vblank and
725	 * the sampled frame counter would correspond to the next frame
726	 * instead of the current frame.
727	 *
728	 * There is a tiny race here (iff vblank evasion failed us) where
729	 * we might sample the frame counter just before vmax vblank start
730	 * but the push would be sent just after it. That would cause the
731	 * push to affect the next frame instead of the current frame,
732	 * which would cause the next frame to terminate already at vmin
733	 * vblank start instead of vmax vblank start.
734	 */
735	if (!state->base.legacy_cursor_update)
736		intel_vrr_send_push(NULL, new_crtc_state);
737
738	local_irq_enable();
739
740	if (intel_vgpu_active(dev_priv))
741		goto out;
742
743	if (crtc->debug.start_vbl_count &&
744	    crtc->debug.start_vbl_count != end_vbl_count) {
745		drm_err(display->drm,
746			"Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
747			pipe_name(pipe), crtc->debug.start_vbl_count,
748			end_vbl_count,
749			ktime_us_delta(end_vbl_time,
750				       crtc->debug.start_vbl_time),
751			crtc->debug.min_vbl, crtc->debug.max_vbl,
752			crtc->debug.scanline_start, scanline_end);
753	}
754
755	dbg_vblank_evade(crtc, end_vbl_time);
756
757out:
758	intel_psr_unlock(new_crtc_state);
759}
760
761bool intel_crtc_enable_changed(const struct intel_crtc_state *old_crtc_state,
762			       const struct intel_crtc_state *new_crtc_state)
763{
764	return old_crtc_state->hw.enable != new_crtc_state->hw.enable;
765}
766
767bool intel_any_crtc_enable_changed(struct intel_atomic_state *state)
768{
769	const struct intel_crtc_state *old_crtc_state, *new_crtc_state;
770	struct intel_crtc *crtc;
771	int i;
772
773	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
774					    new_crtc_state, i) {
775		if (intel_crtc_enable_changed(old_crtc_state, new_crtc_state))
776			return true;
777	}
778
779	return false;
780}
781
782bool intel_crtc_active_changed(const struct intel_crtc_state *old_crtc_state,
783			       const struct intel_crtc_state *new_crtc_state)
784{
785	return old_crtc_state->hw.active != new_crtc_state->hw.active;
786}
787
788bool intel_any_crtc_active_changed(struct intel_atomic_state *state)
789{
790	const struct intel_crtc_state *old_crtc_state, *new_crtc_state;
791	struct intel_crtc *crtc;
792	int i;
793
794	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
795					    new_crtc_state, i) {
796		if (intel_crtc_active_changed(old_crtc_state, new_crtc_state))
797			return true;
798	}
799
800	return false;
801}
802
803unsigned int intel_crtc_bw_num_active_planes(const struct intel_crtc_state *crtc_state)
804{
805	/*
806	 * We assume cursors are small enough
807	 * to not cause bandwidth problems.
808	 */
809	return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
810}
811
812unsigned int intel_crtc_bw_data_rate(const struct intel_crtc_state *crtc_state)
813{
814	struct intel_display *display = to_intel_display(crtc_state);
815	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
816	unsigned int data_rate = 0;
817	enum plane_id plane_id;
818
819	for_each_plane_id_on_crtc(crtc, plane_id) {
820		/*
821		 * We assume cursors are small enough
822		 * to not cause bandwidth problems.
823		 */
824		if (plane_id == PLANE_CURSOR)
825			continue;
826
827		data_rate += crtc_state->data_rate[plane_id];
828
829		if (DISPLAY_VER(display) < 11)
830			data_rate += crtc_state->data_rate_y[plane_id];
831	}
832
833	return data_rate;
834}
835
836/* "Maximum Pipe Read Bandwidth" */
837int intel_crtc_bw_min_cdclk(const struct intel_crtc_state *crtc_state)
838{
839	struct intel_display *display = to_intel_display(crtc_state);
840
841	if (DISPLAY_VER(display) < 12)
842		return 0;
843
844	return DIV_ROUND_UP_ULL(mul_u32_u32(intel_crtc_bw_data_rate(crtc_state), 10), 512);
845}