dpu_crtc.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
   4 * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
   5 * Copyright (C) 2013 Red Hat
   6 * Author: Rob Clark <robdclark@gmail.com>
   7 */
   8
   9#define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
  10#include <linux/sort.h>
  11#include <linux/debugfs.h>
  12#include <linux/ktime.h>
  13#include <linux/bits.h>
  14
  15#include <drm/drm_atomic.h>
  16#include <drm/drm_blend.h>
  17#include <drm/drm_crtc.h>
  18#include <drm/drm_flip_work.h>
  19#include <drm/drm_framebuffer.h>
  20#include <drm/drm_mode.h>
  21#include <drm/drm_probe_helper.h>
  22#include <drm/drm_rect.h>
  23#include <drm/drm_vblank.h>
  24#include <drm/drm_self_refresh_helper.h>
  25
  26#include "dpu_kms.h"
  27#include "dpu_hw_lm.h"
  28#include "dpu_hw_ctl.h"
  29#include "dpu_hw_dspp.h"
  30#include "dpu_crtc.h"
  31#include "dpu_plane.h"
  32#include "dpu_encoder.h"
  33#include "dpu_vbif.h"
  34#include "dpu_core_perf.h"
  35#include "dpu_trace.h"
  36
  37/* layer mixer index on dpu_crtc */
  38#define LEFT_MIXER 0
  39#define RIGHT_MIXER 1
  40
  41/* timeout in ms waiting for frame done */
  42#define DPU_CRTC_FRAME_DONE_TIMEOUT_MS	60
  43
  44#define	CONVERT_S3_15(val) \
  45	(((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0))
  46
  47static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
  48{
  49	struct msm_drm_private *priv = crtc->dev->dev_private;
  50
  51	return to_dpu_kms(priv->kms);
  52}
  53
  54static struct drm_encoder *get_encoder_from_crtc(struct drm_crtc *crtc)
  55{
  56	struct drm_device *dev = crtc->dev;
  57	struct drm_encoder *encoder;
  58
  59	drm_for_each_encoder(encoder, dev)
  60		if (encoder->crtc == crtc)
  61			return encoder;
  62
  63	return NULL;
  64}
  65
  66static enum dpu_crtc_crc_source dpu_crtc_parse_crc_source(const char *src_name)
  67{
  68	if (!src_name ||
  69	    !strcmp(src_name, "none"))
  70		return DPU_CRTC_CRC_SOURCE_NONE;
  71	if (!strcmp(src_name, "auto") ||
  72	    !strcmp(src_name, "lm"))
  73		return DPU_CRTC_CRC_SOURCE_LAYER_MIXER;
  74	if (!strcmp(src_name, "encoder"))
  75		return DPU_CRTC_CRC_SOURCE_ENCODER;
  76
  77	return DPU_CRTC_CRC_SOURCE_INVALID;
  78}
  79
  80static int dpu_crtc_verify_crc_source(struct drm_crtc *crtc,
  81		const char *src_name, size_t *values_cnt)
  82{
  83	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
  84	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
  85
  86	if (source < 0) {
  87		DRM_DEBUG_DRIVER("Invalid source %s for CRTC%d\n", src_name, crtc->index);
  88		return -EINVAL;
  89	}
  90
  91	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER) {
  92		*values_cnt = crtc_state->num_mixers;
  93	} else if (source == DPU_CRTC_CRC_SOURCE_ENCODER) {
  94		struct drm_encoder *drm_enc;
  95
  96		*values_cnt = 0;
  97
  98		drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
  99			*values_cnt += dpu_encoder_get_crc_values_cnt(drm_enc);
 100	}
 101
 102	return 0;
 103}
 104
 105static void dpu_crtc_setup_lm_misr(struct dpu_crtc_state *crtc_state)
 106{
 107	struct dpu_crtc_mixer *m;
 108	int i;
 109
 110	for (i = 0; i < crtc_state->num_mixers; ++i) {
 111		m = &crtc_state->mixers[i];
 112
 113		if (!m->hw_lm || !m->hw_lm->ops.setup_misr)
 114			continue;
 115
 116		/* Calculate MISR over 1 frame */
 117		m->hw_lm->ops.setup_misr(m->hw_lm);
 118	}
 119}
 120
 121static void dpu_crtc_setup_encoder_misr(struct drm_crtc *crtc)
 122{
 123	struct drm_encoder *drm_enc;
 124
 125	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
 126		dpu_encoder_setup_misr(drm_enc);
 127}
 128
 129static int dpu_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 130{
 131	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
 132	enum dpu_crtc_crc_source current_source;
 133	struct dpu_crtc_state *crtc_state;
 134	struct drm_device *drm_dev = crtc->dev;
 135
 136	bool was_enabled;
 137	bool enable = false;
 138	int ret = 0;
 139
 140	if (source < 0) {
 141		DRM_DEBUG_DRIVER("Invalid CRC source %s for CRTC%d\n", src_name, crtc->index);
 142		return -EINVAL;
 143	}
 144
 145	ret = drm_modeset_lock(&crtc->mutex, NULL);
 146
 147	if (ret)
 148		return ret;
 149
 150	enable = (source != DPU_CRTC_CRC_SOURCE_NONE);
 151	crtc_state = to_dpu_crtc_state(crtc->state);
 152
 153	spin_lock_irq(&drm_dev->event_lock);
 154	current_source = crtc_state->crc_source;
 155	spin_unlock_irq(&drm_dev->event_lock);
 156
 157	was_enabled = (current_source != DPU_CRTC_CRC_SOURCE_NONE);
 158
 159	if (!was_enabled && enable) {
 160		ret = drm_crtc_vblank_get(crtc);
 161
 162		if (ret)
 163			goto cleanup;
 164
 165	} else if (was_enabled && !enable) {
 166		drm_crtc_vblank_put(crtc);
 167	}
 168
 169	spin_lock_irq(&drm_dev->event_lock);
 170	crtc_state->crc_source = source;
 171	spin_unlock_irq(&drm_dev->event_lock);
 172
 173	crtc_state->crc_frame_skip_count = 0;
 174
 175	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
 176		dpu_crtc_setup_lm_misr(crtc_state);
 177	else if (source == DPU_CRTC_CRC_SOURCE_ENCODER)
 178		dpu_crtc_setup_encoder_misr(crtc);
 179	else
 180		ret = -EINVAL;
 181
 182cleanup:
 183	drm_modeset_unlock(&crtc->mutex);
 184
 185	return ret;
 186}
 187
 188static u32 dpu_crtc_get_vblank_counter(struct drm_crtc *crtc)
 189{
 190	struct drm_encoder *encoder = get_encoder_from_crtc(crtc);
 191	if (!encoder) {
 192		DRM_ERROR("no encoder found for crtc %d\n", crtc->index);
 193		return 0;
 194	}
 195
 196	return dpu_encoder_get_vsync_count(encoder);
 197}
 198
 199static int dpu_crtc_get_lm_crc(struct drm_crtc *crtc,
 200		struct dpu_crtc_state *crtc_state)
 201{
 202	struct dpu_crtc_mixer *m;
 203	u32 crcs[CRTC_DUAL_MIXERS];
 204
 205	int rc = 0;
 206	int i;
 207
 208	BUILD_BUG_ON(ARRAY_SIZE(crcs) != ARRAY_SIZE(crtc_state->mixers));
 209
 210	for (i = 0; i < crtc_state->num_mixers; ++i) {
 211
 212		m = &crtc_state->mixers[i];
 213
 214		if (!m->hw_lm || !m->hw_lm->ops.collect_misr)
 215			continue;
 216
 217		rc = m->hw_lm->ops.collect_misr(m->hw_lm, &crcs[i]);
 218
 219		if (rc) {
 220			if (rc != -ENODATA)
 221				DRM_DEBUG_DRIVER("MISR read failed\n");
 222			return rc;
 223		}
 224	}
 225
 226	return drm_crtc_add_crc_entry(crtc, true,
 227			drm_crtc_accurate_vblank_count(crtc), crcs);
 228}
 229
 230static int dpu_crtc_get_encoder_crc(struct drm_crtc *crtc)
 231{
 232	struct drm_encoder *drm_enc;
 233	int rc, pos = 0;
 234	u32 crcs[INTF_MAX];
 235
 236	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask) {
 237		rc = dpu_encoder_get_crc(drm_enc, crcs, pos);
 238		if (rc < 0) {
 239			if (rc != -ENODATA)
 240				DRM_DEBUG_DRIVER("MISR read failed\n");
 241
 242			return rc;
 243		}
 244
 245		pos += rc;
 246	}
 247
 248	return drm_crtc_add_crc_entry(crtc, true,
 249			drm_crtc_accurate_vblank_count(crtc), crcs);
 250}
 251
 252static int dpu_crtc_get_crc(struct drm_crtc *crtc)
 253{
 254	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
 255
 256	/* Skip first 2 frames in case of "uncooked" CRCs */
 257	if (crtc_state->crc_frame_skip_count < 2) {
 258		crtc_state->crc_frame_skip_count++;
 259		return 0;
 260	}
 261
 262	if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
 263		return dpu_crtc_get_lm_crc(crtc, crtc_state);
 264	else if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_ENCODER)
 265		return dpu_crtc_get_encoder_crc(crtc);
 266
 267	return -EINVAL;
 268}
 269
 270static bool dpu_crtc_get_scanout_position(struct drm_crtc *crtc,
 271					   bool in_vblank_irq,
 272					   int *vpos, int *hpos,
 273					   ktime_t *stime, ktime_t *etime,
 274					   const struct drm_display_mode *mode)
 275{
 276	unsigned int pipe = crtc->index;
 277	struct drm_encoder *encoder;
 278	int line, vsw, vbp, vactive_start, vactive_end, vfp_end;
 279
 280	encoder = get_encoder_from_crtc(crtc);
 281	if (!encoder) {
 282		DRM_ERROR("no encoder found for crtc %d\n", pipe);
 283		return false;
 284	}
 285
 286	vsw = mode->crtc_vsync_end - mode->crtc_vsync_start;
 287	vbp = mode->crtc_vtotal - mode->crtc_vsync_end;
 288
 289	/*
 290	 * the line counter is 1 at the start of the VSYNC pulse and VTOTAL at
 291	 * the end of VFP. Translate the porch values relative to the line
 292	 * counter positions.
 293	 */
 294
 295	vactive_start = vsw + vbp + 1;
 296	vactive_end = vactive_start + mode->crtc_vdisplay;
 297
 298	/* last scan line before VSYNC */
 299	vfp_end = mode->crtc_vtotal;
 300
 301	if (stime)
 302		*stime = ktime_get();
 303
 304	line = dpu_encoder_get_linecount(encoder);
 305
 306	if (line < vactive_start)
 307		line -= vactive_start;
 308	else if (line > vactive_end)
 309		line = line - vfp_end - vactive_start;
 310	else
 311		line -= vactive_start;
 312
 313	*vpos = line;
 314	*hpos = 0;
 315
 316	if (etime)
 317		*etime = ktime_get();
 318
 319	return true;
 320}
 321
 322static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
 323				      struct dpu_plane_state *pstate,
 324				      const struct msm_format *format,
 325				      const struct dpu_mdss_version *mdss_ver)
 326{
 327	struct dpu_hw_mixer *lm = mixer->hw_lm;
 328	u32 blend_op;
 329	u32 fg_alpha, bg_alpha, max_alpha;
 330
 331	if (mdss_ver->core_major_ver < 12) {
 332		max_alpha = 0xff;
 333		fg_alpha = pstate->base.alpha >> 8;
 334	} else {
 335		max_alpha = 0x3ff;
 336		fg_alpha = pstate->base.alpha >> 6;
 337	}
 338	bg_alpha = max_alpha - fg_alpha;
 339
 340	/* default to opaque blending */
 341	if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE ||
 342	    !format->alpha_enable) {
 343		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
 344			DPU_BLEND_BG_ALPHA_BG_CONST;
 345	} else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) {
 346		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
 347			DPU_BLEND_BG_ALPHA_FG_PIXEL;
 348		if (fg_alpha != max_alpha) {
 349			bg_alpha = fg_alpha;
 350			blend_op |= DPU_BLEND_BG_MOD_ALPHA |
 351				    DPU_BLEND_BG_INV_MOD_ALPHA;
 352		} else {
 353			blend_op |= DPU_BLEND_BG_INV_ALPHA;
 354		}
 355	} else {
 356		/* coverage blending */
 357		blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL |
 358			DPU_BLEND_BG_ALPHA_FG_PIXEL;
 359		if (fg_alpha != max_alpha) {
 360			bg_alpha = fg_alpha;
 361			blend_op |= DPU_BLEND_FG_MOD_ALPHA |
 362				    DPU_BLEND_FG_INV_MOD_ALPHA |
 363				    DPU_BLEND_BG_MOD_ALPHA |
 364				    DPU_BLEND_BG_INV_MOD_ALPHA;
 365		} else {
 366			blend_op |= DPU_BLEND_BG_INV_ALPHA;
 367		}
 368	}
 369
 370	lm->ops.setup_blend_config(lm, pstate->stage,
 371				fg_alpha, bg_alpha, blend_op);
 372
 373	DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n",
 374		  &format->pixel_format, format->alpha_enable, blend_op);
 375}
 376
 377static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
 378{
 379	struct dpu_crtc_state *crtc_state;
 380	int lm_idx;
 381
 382	crtc_state = to_dpu_crtc_state(crtc->state);
 383
 384	for (lm_idx = 0; lm_idx < crtc_state->num_mixers; lm_idx++) {
 385		const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
 386		struct dpu_hw_mixer *hw_lm = crtc_state->mixers[lm_idx].hw_lm;
 387		struct dpu_hw_mixer_cfg cfg;
 388
 389		if (!lm_roi || !drm_rect_visible(lm_roi))
 390			continue;
 391
 392		cfg.out_width = drm_rect_width(lm_roi);
 393		cfg.out_height = drm_rect_height(lm_roi);
 394		cfg.right_mixer = lm_idx & 0x1;
 395		cfg.flags = 0;
 396		hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
 397	}
 398}
 399
 400static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
 401				       struct drm_plane *plane,
 402				       struct dpu_crtc_mixer *mixer,
 403				       u32 lms_in_pair,
 404				       enum dpu_stage stage,
 405				       const struct msm_format *format,
 406				       uint64_t modifier,
 407				       struct dpu_sw_pipe *pipe,
 408				       unsigned int stage_idx,
 409				       struct dpu_hw_stage_cfg *stage_cfg
 410				      )
 411{
 412	u32 lm_idx;
 413	enum dpu_sspp sspp_idx;
 414	struct drm_plane_state *state;
 415
 416	sspp_idx = pipe->sspp->idx;
 417
 418	state = plane->state;
 419
 420	trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
 421				   state, to_dpu_plane_state(state), stage_idx,
 422				   format->pixel_format, pipe,
 423				   modifier);
 424
 425	DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n",
 426			 crtc->base.id,
 427			 stage,
 428			 plane->base.id,
 429			 sspp_idx - SSPP_NONE,
 430			 state->fb ? state->fb->base.id : -1,
 431			 pipe->multirect_index);
 432
 433	stage_cfg->stage[stage][stage_idx] = sspp_idx;
 434	stage_cfg->multirect_index[stage][stage_idx] = pipe->multirect_index;
 435
 436	/* blend config update */
 437	for (lm_idx = 0; lm_idx < lms_in_pair; lm_idx++)
 438		mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, sspp_idx);
 439}
 440
 441static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 442	struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer,
 443	struct dpu_hw_stage_cfg *stage_cfg)
 444{
 445	struct drm_plane *plane;
 446	struct drm_framebuffer *fb;
 447	struct drm_plane_state *state;
 448	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 449	struct dpu_plane_state *pstate = NULL;
 450	const struct msm_format *format;
 451	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
 452	u32 lm_idx, stage, i, pipe_idx, head_pipe_in_stage, lms_in_pair;
 453	bool bg_alpha_enable = false;
 454	DECLARE_BITMAP(active_fetch, SSPP_MAX);
 455	DECLARE_BITMAP(active_pipes, SSPP_MAX);
 456
 457	memset(active_fetch, 0, sizeof(active_fetch));
 458	memset(active_pipes, 0, sizeof(active_pipes));
 459	drm_atomic_crtc_for_each_plane(plane, crtc) {
 460		state = plane->state;
 461		if (!state)
 462			continue;
 463
 464		if (!state->visible)
 465			continue;
 466
 467		pstate = to_dpu_plane_state(state);
 468		fb = state->fb;
 469
 470		format = msm_framebuffer_format(pstate->base.fb);
 471
 472		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
 473			bg_alpha_enable = true;
 474
 475		/* loop pipe per mixer pair with config in stage structure */
 476		for (stage = 0; stage < STAGES_PER_PLANE; stage++) {
 477			head_pipe_in_stage = stage * PIPES_PER_STAGE;
 478			for (i = 0; i < PIPES_PER_STAGE; i++) {
 479				pipe_idx = i + head_pipe_in_stage;
 480				if (!pstate->pipe[pipe_idx].sspp)
 481					continue;
 482				lms_in_pair = min(cstate->num_mixers - (stage * PIPES_PER_STAGE),
 483						  PIPES_PER_STAGE);
 484				set_bit(pstate->pipe[pipe_idx].sspp->idx, active_fetch);
 485				set_bit(pstate->pipe[pipe_idx].sspp->idx, active_pipes);
 486				_dpu_crtc_blend_setup_pipe(crtc, plane,
 487							   &mixer[head_pipe_in_stage],
 488							   lms_in_pair,
 489							   pstate->stage,
 490							   format, fb ? fb->modifier : 0,
 491							   &pstate->pipe[pipe_idx], i,
 492							   &stage_cfg[stage]);
 493			}
 494		}
 495
 496		/* blend config update */
 497		for (lm_idx = 0; lm_idx < cstate->num_mixers; lm_idx++) {
 498			_dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate, format,
 499						  ctl->mdss_ver);
 500
 501			if (bg_alpha_enable && !format->alpha_enable)
 502				mixer[lm_idx].mixer_op_mode = 0;
 503			else
 504				mixer[lm_idx].mixer_op_mode |=
 505						1 << pstate->stage;
 506		}
 507	}
 508
 509	if (ctl->ops.set_active_fetch_pipes)
 510		ctl->ops.set_active_fetch_pipes(ctl, active_fetch);
 511
 512	if (ctl->ops.set_active_pipes)
 513		ctl->ops.set_active_pipes(ctl, active_pipes);
 514
 515	_dpu_crtc_program_lm_output_roi(crtc);
 516}
 517
 518/**
 519 * _dpu_crtc_blend_setup - configure crtc mixers
 520 * @crtc: Pointer to drm crtc structure
 521 */
 522static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
 523{
 524	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 525	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 526	struct dpu_crtc_mixer *mixer = cstate->mixers;
 527	struct dpu_hw_ctl *ctl;
 528	struct dpu_hw_mixer *lm;
 529	struct dpu_hw_stage_cfg stage_cfg[STAGES_PER_PLANE];
 530	DECLARE_BITMAP(active_lms, LM_MAX);
 531	int i;
 532
 533	DRM_DEBUG_ATOMIC("%s\n", dpu_crtc->name);
 534
 535	for (i = 0; i < cstate->num_mixers; i++) {
 536		mixer[i].mixer_op_mode = 0;
 537		if (mixer[i].lm_ctl->ops.clear_all_blendstages)
 538			mixer[i].lm_ctl->ops.clear_all_blendstages(
 539					mixer[i].lm_ctl);
 540		if (mixer[i].lm_ctl->ops.set_active_fetch_pipes)
 541			mixer[i].lm_ctl->ops.set_active_fetch_pipes(mixer[i].lm_ctl, NULL);
 542		if (mixer[i].lm_ctl->ops.set_active_pipes)
 543			mixer[i].lm_ctl->ops.set_active_pipes(mixer[i].lm_ctl, NULL);
 544
 545		if (mixer[i].hw_lm->ops.clear_all_blendstages)
 546			mixer[i].hw_lm->ops.clear_all_blendstages(mixer[i].hw_lm);
 547	}
 548
 549	/* initialize stage cfg */
 550	memset(&stage_cfg, 0, sizeof(stage_cfg));
 551	memset(active_lms, 0, sizeof(active_lms));
 552
 553	_dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, stage_cfg);
 554
 555	for (i = 0; i < cstate->num_mixers; i++) {
 556		ctl = mixer[i].lm_ctl;
 557		lm = mixer[i].hw_lm;
 558
 559		lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
 560
 561		/* stage config flush mask */
 562		ctl->ops.update_pending_flush_mixer(ctl,
 563			mixer[i].hw_lm->idx);
 564
 565		set_bit(lm->idx, active_lms);
 566		if (ctl->ops.set_active_lms)
 567			ctl->ops.set_active_lms(ctl, active_lms);
 568
 569		DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n",
 570			mixer[i].hw_lm->idx - LM_0,
 571			mixer[i].mixer_op_mode,
 572			ctl->idx - CTL_0);
 573
 574		/*
 575		 * call dpu_hw_ctl_setup_blendstage() to blend layers per stage cfg.
 576		 * stage data is shared between PIPES_PER_STAGE pipes.
 577		 */
 578		if (ctl->ops.setup_blendstage)
 579			ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
 580				&stage_cfg[i / PIPES_PER_STAGE]);
 581
 582		if (lm->ops.setup_blendstage)
 583			lm->ops.setup_blendstage(lm, mixer[i].hw_lm->idx,
 584				&stage_cfg[i / PIPES_PER_STAGE]);
 585	}
 586}
 587
 588/**
 589 *  _dpu_crtc_complete_flip - signal pending page_flip events
 590 * Any pending vblank events are added to the vblank_event_list
 591 * so that the next vblank interrupt shall signal them.
 592 * However PAGE_FLIP events are not handled through the vblank_event_list.
 593 * This API signals any pending PAGE_FLIP events requested through
 594 * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
 595 * @crtc: Pointer to drm crtc structure
 596 */
 597static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
 598{
 599	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 600	struct drm_device *dev = crtc->dev;
 601	unsigned long flags;
 602
 603	spin_lock_irqsave(&dev->event_lock, flags);
 604	if (dpu_crtc->event) {
 605		DRM_DEBUG_VBL("%s: send event: %p\n", dpu_crtc->name,
 606			      dpu_crtc->event);
 607		trace_dpu_crtc_complete_flip(DRMID(crtc));
 608		drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
 609		dpu_crtc->event = NULL;
 610	}
 611	spin_unlock_irqrestore(&dev->event_lock, flags);
 612}
 613
 614/**
 615 * dpu_crtc_get_intf_mode - get interface mode of the given crtc
 616 * @crtc: Pointert to crtc
 617 */
 618enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
 619{
 620	struct drm_encoder *encoder;
 621
 622	/*
 623	 * TODO: This function is called from dpu debugfs and as part of atomic
 624	 * check. When called from debugfs, the crtc->mutex must be held to
 625	 * read crtc->state. However reading crtc->state from atomic check isn't
 626	 * allowed (unless you have a good reason, a big comment, and a deep
 627	 * understanding of how the atomic/modeset locks work (<- and this is
 628	 * probably not possible)). So we'll keep the WARN_ON here for now, but
 629	 * really we need to figure out a better way to track our operating mode
 630	 */
 631	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
 632
 633	/* TODO: Returns the first INTF_MODE, could there be multiple values? */
 634	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
 635		return dpu_encoder_get_intf_mode(encoder);
 636
 637	return INTF_MODE_NONE;
 638}
 639
 640/**
 641 * dpu_crtc_vblank_callback - called on vblank irq, issues completion events
 642 * @crtc: Pointer to drm crtc object
 643 */
 644void dpu_crtc_vblank_callback(struct drm_crtc *crtc)
 645{
 646	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 647
 648	/* keep statistics on vblank callback - with auto reset via debugfs */
 649	if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
 650		dpu_crtc->vblank_cb_time = ktime_get();
 651	else
 652		dpu_crtc->vblank_cb_count++;
 653
 654	dpu_crtc_get_crc(crtc);
 655
 656	drm_crtc_handle_vblank(crtc);
 657	trace_dpu_crtc_vblank_cb(DRMID(crtc));
 658}
 659
 660static void dpu_crtc_frame_event_work(struct kthread_work *work)
 661{
 662	struct dpu_crtc_frame_event *fevent = container_of(work,
 663			struct dpu_crtc_frame_event, work);
 664	struct drm_crtc *crtc = fevent->crtc;
 665	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 666	unsigned long flags;
 667	bool frame_done = false;
 668
 669	DPU_ATRACE_BEGIN("crtc_frame_event");
 670
 671	DRM_DEBUG_ATOMIC("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
 672			ktime_to_ns(fevent->ts));
 673
 674	if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
 675				| DPU_ENCODER_FRAME_EVENT_ERROR
 676				| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
 677
 678		if (atomic_read(&dpu_crtc->frame_pending) < 1) {
 679			/* ignore vblank when not pending */
 680		} else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
 681			/* release bandwidth and other resources */
 682			trace_dpu_crtc_frame_event_done(DRMID(crtc),
 683							fevent->event);
 684			dpu_core_perf_crtc_release_bw(crtc);
 685		} else {
 686			trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
 687								fevent->event);
 688		}
 689
 690		if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
 691					| DPU_ENCODER_FRAME_EVENT_ERROR))
 692			frame_done = true;
 693	}
 694
 695	if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
 696		DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
 697				crtc->base.id, ktime_to_ns(fevent->ts));
 698
 699	if (frame_done)
 700		complete_all(&dpu_crtc->frame_done_comp);
 701
 702	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
 703	list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
 704	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
 705	DPU_ATRACE_END("crtc_frame_event");
 706}
 707
 708/**
 709 * dpu_crtc_frame_event_cb - crtc frame event callback API
 710 * @crtc: Pointer to crtc
 711 * @event: Event to process
 712 *
 713 * Encoder may call this for different events from different context - IRQ,
 714 * user thread, commit_thread, etc. Each event should be carefully reviewed and
 715 * should be processed in proper task context to avoid schedulin delay or
 716 * properly manage the irq context's bottom half processing.
 717 */
 718void dpu_crtc_frame_event_cb(struct drm_crtc *crtc, u32 event)
 719{
 720	struct dpu_crtc *dpu_crtc;
 721	struct msm_drm_private *priv;
 722	struct dpu_crtc_frame_event *fevent;
 723	unsigned long flags;
 724	u32 crtc_id;
 725
 726	/* Nothing to do on idle event */
 727	if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
 728		return;
 729
 730	dpu_crtc = to_dpu_crtc(crtc);
 731	priv = crtc->dev->dev_private;
 732	crtc_id = drm_crtc_index(crtc);
 733
 734	trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
 735
 736	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
 737	fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
 738			struct dpu_crtc_frame_event, list);
 739	if (fevent)
 740		list_del_init(&fevent->list);
 741	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
 742
 743	if (!fevent) {
 744		DRM_ERROR_RATELIMITED("crtc%d event %d overflow\n", crtc->base.id, event);
 745		return;
 746	}
 747
 748	fevent->event = event;
 749	fevent->crtc = crtc;
 750	fevent->ts = ktime_get();
 751	kthread_queue_work(priv->kms->event_thread[crtc_id].worker, &fevent->work);
 752}
 753
 754/**
 755 * dpu_crtc_complete_commit - callback signalling completion of current commit
 756 * @crtc: Pointer to drm crtc object
 757 */
 758void dpu_crtc_complete_commit(struct drm_crtc *crtc)
 759{
 760	trace_dpu_crtc_complete_commit(DRMID(crtc));
 761	dpu_core_perf_crtc_update(crtc, 0);
 762	_dpu_crtc_complete_flip(crtc);
 763}
 764
 765static int _dpu_crtc_check_and_setup_lm_bounds(struct drm_crtc *crtc,
 766		struct drm_crtc_state *state)
 767{
 768	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
 769	struct drm_display_mode *adj_mode = &state->adjusted_mode;
 770	u32 crtc_split_width = adj_mode->hdisplay / cstate->num_mixers;
 771	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
 772	int i;
 773
 774	/* if we cannot merge 2 LMs (no 3d mux) better to fail earlier
 775	 * before even checking the width after the split
 776	 */
 777	if (!dpu_kms->catalog->caps->has_3d_merge &&
 778	    adj_mode->hdisplay > dpu_kms->catalog->caps->max_mixer_width)
 779		return -E2BIG;
 780
 781	for (i = 0; i < cstate->num_mixers; i++) {
 782		struct drm_rect *r = &cstate->lm_bounds[i];
 783		r->x1 = crtc_split_width * i;
 784		r->y1 = 0;
 785		r->x2 = r->x1 + crtc_split_width;
 786		r->y2 = adj_mode->vdisplay;
 787
 788		trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
 789
 790		if (drm_rect_width(r) > dpu_kms->catalog->caps->max_mixer_width)
 791			return -E2BIG;
 792	}
 793
 794	return 0;
 795}
 796
 797static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state,
 798		struct dpu_hw_pcc_cfg *cfg)
 799{
 800	struct drm_color_ctm *ctm;
 801
 802	memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg));
 803
 804	ctm = (struct drm_color_ctm *)state->ctm->data;
 805
 806	if (!ctm)
 807		return;
 808
 809	cfg->r.r = CONVERT_S3_15(ctm->matrix[0]);
 810	cfg->g.r = CONVERT_S3_15(ctm->matrix[1]);
 811	cfg->b.r = CONVERT_S3_15(ctm->matrix[2]);
 812
 813	cfg->r.g = CONVERT_S3_15(ctm->matrix[3]);
 814	cfg->g.g = CONVERT_S3_15(ctm->matrix[4]);
 815	cfg->b.g = CONVERT_S3_15(ctm->matrix[5]);
 816
 817	cfg->r.b = CONVERT_S3_15(ctm->matrix[6]);
 818	cfg->g.b = CONVERT_S3_15(ctm->matrix[7]);
 819	cfg->b.b = CONVERT_S3_15(ctm->matrix[8]);
 820}
 821
 822static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
 823{
 824	struct drm_crtc_state *state = crtc->state;
 825	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 826	struct dpu_crtc_mixer *mixer = cstate->mixers;
 827	struct dpu_hw_pcc_cfg cfg;
 828	struct dpu_hw_ctl *ctl;
 829	struct dpu_hw_dspp *dspp;
 830	int i;
 831
 832
 833	if (!state->color_mgmt_changed && !drm_atomic_crtc_needs_modeset(state))
 834		return;
 835
 836	for (i = 0; i < cstate->num_mixers; i++) {
 837		ctl = mixer[i].lm_ctl;
 838		dspp = mixer[i].hw_dspp;
 839
 840		if (!dspp || !dspp->ops.setup_pcc)
 841			continue;
 842
 843		if (!state->ctm) {
 844			dspp->ops.setup_pcc(dspp, NULL);
 845		} else {
 846			_dpu_crtc_get_pcc_coeff(state, &cfg);
 847			dspp->ops.setup_pcc(dspp, &cfg);
 848		}
 849
 850		/* stage config flush mask */
 851		ctl->ops.update_pending_flush_dspp(ctl,
 852			mixer[i].hw_dspp->idx, DPU_DSPP_PCC);
 853	}
 854}
 855
 856static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 857		struct drm_atomic_state *state)
 858{
 859	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 860	struct drm_encoder *encoder;
 861
 862	if (!crtc->state->enable) {
 863		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_begin\n",
 864				crtc->base.id, crtc->state->enable);
 865		return;
 866	}
 867
 868	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
 869
 870	_dpu_crtc_check_and_setup_lm_bounds(crtc, crtc->state);
 871
 872	/* encoder will trigger pending mask now */
 873	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
 874		dpu_encoder_trigger_kickoff_pending(encoder);
 875
 876	/*
 877	 * If no mixers have been allocated in dpu_crtc_atomic_check(),
 878	 * it means we are trying to flush a CRTC whose state is disabled:
 879	 * nothing else needs to be done.
 880	 */
 881	if (unlikely(!cstate->num_mixers))
 882		return;
 883
 884	_dpu_crtc_blend_setup(crtc);
 885
 886	_dpu_crtc_setup_cp_blocks(crtc);
 887
 888	/*
 889	 * PP_DONE irq is only used by command mode for now.
 890	 * It is better to request pending before FLUSH and START trigger
 891	 * to make sure no pp_done irq missed.
 892	 * This is safe because no pp_done will happen before SW trigger
 893	 * in command mode.
 894	 */
 895}
 896
 897static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
 898		struct drm_atomic_state *state)
 899{
 900	struct dpu_crtc *dpu_crtc;
 901	struct drm_device *dev;
 902	struct drm_plane *plane;
 903	struct msm_drm_private *priv;
 904	unsigned long flags;
 905	struct dpu_crtc_state *cstate;
 906
 907	if (!crtc->state->enable) {
 908		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_flush\n",
 909				crtc->base.id, crtc->state->enable);
 910		return;
 911	}
 912
 913	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
 914
 915	dpu_crtc = to_dpu_crtc(crtc);
 916	cstate = to_dpu_crtc_state(crtc->state);
 917	dev = crtc->dev;
 918	priv = dev->dev_private;
 919
 920	if (crtc->index >= ARRAY_SIZE(priv->kms->event_thread)) {
 921		DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
 922		return;
 923	}
 924
 925	WARN_ON(dpu_crtc->event);
 926	spin_lock_irqsave(&dev->event_lock, flags);
 927	dpu_crtc->event = crtc->state->event;
 928	crtc->state->event = NULL;
 929	spin_unlock_irqrestore(&dev->event_lock, flags);
 930
 931	/*
 932	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
 933	 * it means we are trying to flush a CRTC whose state is disabled:
 934	 * nothing else needs to be done.
 935	 */
 936	if (unlikely(!cstate->num_mixers))
 937		return;
 938
 939	/* update performance setting before crtc kickoff */
 940	dpu_core_perf_crtc_update(crtc, 1);
 941
 942	/*
 943	 * Final plane updates: Give each plane a chance to complete all
 944	 *                      required writes/flushing before crtc's "flush
 945	 *                      everything" call below.
 946	 */
 947	drm_atomic_crtc_for_each_plane(plane, crtc) {
 948		if (dpu_crtc->smmu_state.transition_error)
 949			dpu_plane_set_error(plane, true);
 950		dpu_plane_flush(plane);
 951	}
 952
 953	/* Kickoff will be scheduled by outer layer */
 954}
 955
 956/**
 957 * dpu_crtc_destroy_state - state destroy hook
 958 * @crtc: drm CRTC
 959 * @state: CRTC state object to release
 960 */
 961static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
 962		struct drm_crtc_state *state)
 963{
 964	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
 965
 966	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
 967
 968	__drm_atomic_helper_crtc_destroy_state(state);
 969
 970	kfree(cstate);
 971}
 972
 973static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
 974{
 975	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 976	int ret, rc = 0;
 977
 978	if (!atomic_read(&dpu_crtc->frame_pending)) {
 979		DRM_DEBUG_ATOMIC("no frames pending\n");
 980		return 0;
 981	}
 982
 983	DPU_ATRACE_BEGIN("frame done completion wait");
 984	ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
 985			msecs_to_jiffies(DPU_CRTC_FRAME_DONE_TIMEOUT_MS));
 986	if (!ret) {
 987		DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
 988		rc = -ETIMEDOUT;
 989	}
 990	DPU_ATRACE_END("frame done completion wait");
 991
 992	return rc;
 993}
 994
 995static int dpu_crtc_kickoff_clone_mode(struct drm_crtc *crtc)
 996{
 997	struct drm_encoder *encoder;
 998	struct drm_encoder *rt_encoder = NULL, *wb_encoder = NULL;
 999	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1000
1001	/* Find encoder for real time display */
1002	drm_for_each_encoder_mask(encoder, crtc->dev,
1003				  crtc->state->encoder_mask) {
1004		if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
1005			wb_encoder = encoder;
1006		else
1007			rt_encoder = encoder;
1008	}
1009
1010	if (!rt_encoder || !wb_encoder) {
1011		DRM_DEBUG_ATOMIC("real time or wb encoder not found\n");
1012		return -EINVAL;
1013	}
1014
1015	dpu_encoder_prepare_for_kickoff(wb_encoder);
1016	dpu_encoder_prepare_for_kickoff(rt_encoder);
1017
1018	dpu_vbif_clear_errors(dpu_kms);
1019
1020	/*
1021	 * Kickoff real time encoder last as it's the encoder that
1022	 * will do the flush
1023	 */
1024	dpu_encoder_kickoff(wb_encoder);
1025	dpu_encoder_kickoff(rt_encoder);
1026
1027	/* Don't start frame done timers until the kickoffs have finished */
1028	dpu_encoder_start_frame_done_timer(wb_encoder);
1029	dpu_encoder_start_frame_done_timer(rt_encoder);
1030
1031	return 0;
1032}
1033
1034/**
1035 * dpu_crtc_commit_kickoff - trigger kickoff of the commit for this crtc
1036 * @crtc: Pointer to drm crtc object
1037 */
1038void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
1039{
1040	struct drm_encoder *encoder;
1041	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1042	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1043	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1044
1045	/*
1046	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
1047	 * it means we are trying to start a CRTC whose state is disabled:
1048	 * nothing else needs to be done.
1049	 */
1050	if (unlikely(!cstate->num_mixers))
1051		return;
1052
1053	DPU_ATRACE_BEGIN("crtc_commit");
1054
1055	drm_for_each_encoder_mask(encoder, crtc->dev,
1056			crtc->state->encoder_mask) {
1057		if (!dpu_encoder_is_valid_for_commit(encoder)) {
1058			DRM_DEBUG_ATOMIC("invalid FB not kicking off crtc\n");
1059			goto end;
1060		}
1061	}
1062
1063	if (drm_crtc_in_clone_mode(crtc->state)) {
1064		if (dpu_crtc_kickoff_clone_mode(crtc))
1065			goto end;
1066	} else {
1067		/*
1068		 * Encoder will flush/start now, unless it has a tx pending.
1069		 * If so, it may delay and flush at an irq event (e.g. ppdone)
1070		 */
1071		drm_for_each_encoder_mask(encoder, crtc->dev,
1072				crtc->state->encoder_mask)
1073			dpu_encoder_prepare_for_kickoff(encoder);
1074
1075		dpu_vbif_clear_errors(dpu_kms);
1076
1077		drm_for_each_encoder_mask(encoder, crtc->dev,
1078				crtc->state->encoder_mask) {
1079			dpu_encoder_kickoff(encoder);
1080			dpu_encoder_start_frame_done_timer(encoder);
1081		}
1082	}
1083
1084	if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
1085		/* acquire bandwidth and other resources */
1086		DRM_DEBUG_ATOMIC("crtc%d first commit\n", crtc->base.id);
1087	} else
1088		DRM_DEBUG_ATOMIC("crtc%d commit\n", crtc->base.id);
1089
1090	dpu_crtc->play_count++;
1091
1092	reinit_completion(&dpu_crtc->frame_done_comp);
1093
1094end:
1095	DPU_ATRACE_END("crtc_commit");
1096}
1097
1098static void dpu_crtc_reset(struct drm_crtc *crtc)
1099{
1100	struct dpu_crtc_state *cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
1101
1102	if (crtc->state)
1103		dpu_crtc_destroy_state(crtc, crtc->state);
1104
1105	if (cstate)
1106		__drm_atomic_helper_crtc_reset(crtc, &cstate->base);
1107	else
1108		__drm_atomic_helper_crtc_reset(crtc, NULL);
1109}
1110
1111/**
1112 * dpu_crtc_duplicate_state - state duplicate hook
1113 * @crtc: Pointer to drm crtc structure
1114 */
1115static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
1116{
1117	struct dpu_crtc_state *cstate, *old_cstate = to_dpu_crtc_state(crtc->state);
1118
1119	cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
1120	if (!cstate) {
1121		DPU_ERROR("failed to allocate state\n");
1122		return NULL;
1123	}
1124
1125	/* duplicate base helper */
1126	__drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
1127
1128	return &cstate->base;
1129}
1130
1131static void dpu_crtc_atomic_print_state(struct drm_printer *p,
1132					const struct drm_crtc_state *state)
1133{
1134	const struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
1135	int i;
1136
1137	for (i = 0; i < cstate->num_mixers; i++) {
1138		drm_printf(p, "\tlm[%d]=%d\n", i, cstate->mixers[i].hw_lm->idx - LM_0);
1139		drm_printf(p, "\tctl[%d]=%d\n", i, cstate->mixers[i].lm_ctl->idx - CTL_0);
1140		if (cstate->mixers[i].hw_dspp)
1141			drm_printf(p, "\tdspp[%d]=%d\n", i, cstate->mixers[i].hw_dspp->idx - DSPP_0);
1142	}
1143}
1144
1145static void dpu_crtc_disable(struct drm_crtc *crtc,
1146			     struct drm_atomic_state *state)
1147{
1148	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1149									      crtc);
1150	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1151	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1152	struct drm_encoder *encoder;
1153	unsigned long flags;
1154	bool release_bandwidth = false;
1155
1156	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1157
1158	/* If disable is triggered while in self refresh mode,
1159	 * reset the encoder software state so that in enable
1160	 * it won't trigger a warn while assigning crtc.
1161	 */
1162	if (old_crtc_state->self_refresh_active) {
1163		drm_for_each_encoder_mask(encoder, crtc->dev,
1164					old_crtc_state->encoder_mask) {
1165			dpu_encoder_assign_crtc(encoder, NULL);
1166		}
1167		return;
1168	}
1169
1170	/* Disable/save vblank irq handling */
1171	drm_crtc_vblank_off(crtc);
1172
1173	drm_for_each_encoder_mask(encoder, crtc->dev,
1174				  old_crtc_state->encoder_mask) {
1175		/* in video mode, we hold an extra bandwidth reference
1176		 * as we cannot drop bandwidth at frame-done if any
1177		 * crtc is being used in video mode.
1178		 */
1179		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1180			release_bandwidth = true;
1181
1182		/*
1183		 * If disable is triggered during psr active(e.g: screen dim in PSR),
1184		 * we will need encoder->crtc connection to process the device sleep &
1185		 * preserve it during psr sequence.
1186		 */
1187		if (!crtc->state->self_refresh_active)
1188			dpu_encoder_assign_crtc(encoder, NULL);
1189	}
1190
1191	/* wait for frame_event_done completion */
1192	if (_dpu_crtc_wait_for_frame_done(crtc))
1193		DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1194				crtc->base.id,
1195				atomic_read(&dpu_crtc->frame_pending));
1196
1197	trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
1198	dpu_crtc->enabled = false;
1199
1200	if (atomic_read(&dpu_crtc->frame_pending)) {
1201		trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
1202				     atomic_read(&dpu_crtc->frame_pending));
1203		if (release_bandwidth)
1204			dpu_core_perf_crtc_release_bw(crtc);
1205		atomic_set(&dpu_crtc->frame_pending, 0);
1206	}
1207
1208	dpu_core_perf_crtc_update(crtc, 0);
1209
1210	/* disable clk & bw control until clk & bw properties are set */
1211	cstate->bw_control = false;
1212	cstate->bw_split_vote = false;
1213
1214	if (crtc->state->event && !crtc->state->active) {
1215		spin_lock_irqsave(&crtc->dev->event_lock, flags);
1216		drm_crtc_send_vblank_event(crtc, crtc->state->event);
1217		crtc->state->event = NULL;
1218		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
1219	}
1220
1221	pm_runtime_put_sync(crtc->dev->dev);
1222}
1223
1224static void dpu_crtc_enable(struct drm_crtc *crtc,
1225		struct drm_atomic_state *state)
1226{
1227	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1228	struct drm_encoder *encoder;
1229	bool request_bandwidth = false;
1230	struct drm_crtc_state *old_crtc_state;
1231
1232	old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
1233
1234	pm_runtime_get_sync(crtc->dev->dev);
1235
1236	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1237
1238	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
1239		/* in video mode, we hold an extra bandwidth reference
1240		 * as we cannot drop bandwidth at frame-done if any
1241		 * crtc is being used in video mode.
1242		 */
1243		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1244			request_bandwidth = true;
1245	}
1246
1247	if (request_bandwidth)
1248		atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1249
1250	trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
1251	dpu_crtc->enabled = true;
1252
1253	if (!old_crtc_state->self_refresh_active) {
1254		drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1255			dpu_encoder_assign_crtc(encoder, crtc);
1256	}
1257
1258	/* Enable/restore vblank irq handling */
1259	drm_crtc_vblank_on(crtc);
1260}
1261
1262static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
1263{
1264	struct drm_crtc *crtc = cstate->crtc;
1265	struct drm_encoder *encoder;
1266
1267	if (cstate->self_refresh_active)
1268		return true;
1269
1270	drm_for_each_encoder_mask (encoder, crtc->dev, cstate->encoder_mask) {
1271		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_CMD) {
1272			return true;
1273		}
1274	}
1275
1276	return false;
1277}
1278
1279static int dpu_crtc_reassign_planes(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state)
1280{
1281	int total_planes = crtc->dev->mode_config.num_total_plane;
1282	struct drm_atomic_state *state = crtc_state->state;
1283	struct dpu_global_state *global_state;
1284	struct drm_plane_state **states;
1285	struct drm_plane *plane;
1286	int ret;
1287
1288	global_state = dpu_kms_get_global_state(crtc_state->state);
1289	if (IS_ERR(global_state))
1290		return PTR_ERR(global_state);
1291
1292	dpu_rm_release_all_sspp(global_state, crtc);
1293
1294	if (!crtc_state->enable)
1295		return 0;
1296
1297	states = kcalloc(total_planes, sizeof(*states), GFP_KERNEL);
1298	if (!states)
1299		return -ENOMEM;
1300
1301	drm_atomic_crtc_state_for_each_plane(plane, crtc_state) {
1302		struct drm_plane_state *plane_state =
1303			drm_atomic_get_plane_state(state, plane);
1304
1305		if (IS_ERR(plane_state)) {
1306			ret = PTR_ERR(plane_state);
1307			goto done;
1308		}
1309
1310		states[plane_state->normalized_zpos] = plane_state;
1311	}
1312
1313	ret = dpu_assign_plane_resources(global_state, state, crtc, states, total_planes);
1314
1315done:
1316	kfree(states);
1317	return ret;
1318}
1319
1320#define MAX_CHANNELS_PER_CRTC PIPES_PER_PLANE
1321#define MAX_HDISPLAY_SPLIT 1080
1322
1323static struct msm_display_topology dpu_crtc_get_topology(
1324		struct drm_crtc *crtc,
1325		struct dpu_kms *dpu_kms,
1326		struct drm_crtc_state *crtc_state)
1327{
1328	struct drm_display_mode *mode = &crtc_state->adjusted_mode;
1329	struct msm_display_topology topology = {0};
1330	struct drm_encoder *drm_enc;
1331
1332	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc_state->encoder_mask)
1333		dpu_encoder_update_topology(drm_enc, &topology, crtc_state->state,
1334					    &crtc_state->adjusted_mode);
1335
1336	topology.cwb_enabled = drm_crtc_in_clone_mode(crtc_state);
1337
1338	/*
1339	 * Datapath topology selection
1340	 *
1341	 * Dual display
1342	 * 2 LM, 2 INTF ( Split display using 2 interfaces)
1343	 *
1344	 * Single display
1345	 * 1 LM, 1 INTF
1346	 * 2 LM, 1 INTF (stream merge to support high resolution interfaces)
1347	 *
1348	 * If DSC is enabled, use 2 LMs for 2:2:1 topology
1349	 *
1350	 * Add dspps to the reservation requirements if ctm is requested
1351	 *
1352	 * Only hardcode num_lm to 2 for cases where num_intf == 2 and CWB is not
1353	 * enabled. This is because in cases where CWB is enabled, num_intf will
1354	 * count both the WB and real-time phys encoders.
1355	 *
1356	 * For non-DSC CWB usecases, have the num_lm be decided by the
1357	 * (mode->hdisplay > MAX_HDISPLAY_SPLIT) check.
1358	 */
1359
1360	if (topology.num_intf == 2 && !topology.cwb_enabled)
1361		topology.num_lm = 2;
1362	else if (topology.num_dsc == 2)
1363		topology.num_lm = 2;
1364	else if (dpu_kms->catalog->caps->has_3d_merge)
1365		topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1;
1366	else
1367		topology.num_lm = 1;
1368
1369	if (crtc_state->ctm)
1370		topology.num_dspp = topology.num_lm;
1371
1372	return topology;
1373}
1374
1375static int dpu_crtc_assign_resources(struct drm_crtc *crtc,
1376				     struct drm_crtc_state *crtc_state)
1377{
1378	struct dpu_hw_blk *hw_ctl[MAX_CHANNELS_PER_CRTC];
1379	struct dpu_hw_blk *hw_lm[MAX_CHANNELS_PER_CRTC];
1380	struct dpu_hw_blk *hw_dspp[MAX_CHANNELS_PER_CRTC];
1381	int i, num_lm, num_ctl, num_dspp;
1382	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1383	struct dpu_global_state *global_state;
1384	struct dpu_crtc_state *cstate;
1385	struct msm_display_topology topology;
1386	int ret;
1387
1388	/*
1389	 * Release and Allocate resources on every modeset
1390	 */
1391	global_state = dpu_kms_get_global_state(crtc_state->state);
1392	if (IS_ERR(global_state))
1393		return PTR_ERR(global_state);
1394
1395	dpu_rm_release(global_state, crtc);
1396
1397	if (!crtc_state->enable)
1398		return 0;
1399
1400	topology = dpu_crtc_get_topology(crtc, dpu_kms, crtc_state);
1401	ret = dpu_rm_reserve(&dpu_kms->rm, global_state,
1402			     crtc_state->crtc, &topology);
1403	if (ret)
1404		return ret;
1405
1406	cstate = to_dpu_crtc_state(crtc_state);
1407
1408	num_ctl = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1409						crtc_state->crtc,
1410						DPU_HW_BLK_CTL, hw_ctl,
1411						ARRAY_SIZE(hw_ctl));
1412	num_lm = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1413					       crtc_state->crtc,
1414					       DPU_HW_BLK_LM, hw_lm,
1415					       ARRAY_SIZE(hw_lm));
1416	num_dspp = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1417						 crtc_state->crtc,
1418						 DPU_HW_BLK_DSPP, hw_dspp,
1419						 ARRAY_SIZE(hw_dspp));
1420
1421	for (i = 0; i < num_lm; i++) {
1422		int ctl_idx = (i < num_ctl) ? i : (num_ctl-1);
1423
1424		cstate->mixers[i].hw_lm = to_dpu_hw_mixer(hw_lm[i]);
1425		cstate->mixers[i].lm_ctl = to_dpu_hw_ctl(hw_ctl[ctl_idx]);
1426		if (i < num_dspp)
1427			cstate->mixers[i].hw_dspp = to_dpu_hw_dspp(hw_dspp[i]);
1428	}
1429
1430	cstate->num_mixers = num_lm;
1431
1432	return 0;
1433}
1434
1435/**
1436 * dpu_crtc_check_mode_changed: check if full modeset is required
1437 * @old_crtc_state:	Previous CRTC state
1438 * @new_crtc_state:	Corresponding CRTC state to be checked
1439 *
1440 * Check if the changes in the object properties demand full mode set.
1441 */
1442int dpu_crtc_check_mode_changed(struct drm_crtc_state *old_crtc_state,
1443				struct drm_crtc_state *new_crtc_state)
1444{
1445	struct drm_encoder *drm_enc;
1446	struct drm_crtc *crtc = new_crtc_state->crtc;
1447	bool clone_mode_enabled = drm_crtc_in_clone_mode(old_crtc_state);
1448	bool clone_mode_requested = drm_crtc_in_clone_mode(new_crtc_state);
1449
1450	DRM_DEBUG_ATOMIC("%d\n", crtc->base.id);
1451
1452	/* there might be cases where encoder needs a modeset too */
1453	drm_for_each_encoder_mask(drm_enc, crtc->dev, new_crtc_state->encoder_mask) {
1454		if (dpu_encoder_needs_modeset(drm_enc, new_crtc_state->state))
1455			new_crtc_state->mode_changed = true;
1456	}
1457
1458	if ((clone_mode_requested && !clone_mode_enabled) ||
1459	    (!clone_mode_requested && clone_mode_enabled))
1460		new_crtc_state->mode_changed = true;
1461
1462	return 0;
1463}
1464
1465static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
1466		struct drm_atomic_state *state)
1467{
1468	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1469									  crtc);
1470	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1471	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc_state);
1472
1473	const struct drm_plane_state *pstate;
1474	struct drm_plane *plane;
1475
1476	int rc = 0;
1477
1478	bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state);
1479
1480	/* don't reallocate resources if only ACTIVE has beeen changed */
1481	if (crtc_state->mode_changed || crtc_state->connectors_changed) {
1482		rc = dpu_crtc_assign_resources(crtc, crtc_state);
1483		if (rc < 0)
1484			return rc;
1485	}
1486
1487	if (dpu_use_virtual_planes &&
1488	    (crtc_state->planes_changed || crtc_state->zpos_changed)) {
1489		rc = dpu_crtc_reassign_planes(crtc, crtc_state);
1490		if (rc < 0)
1491			return rc;
1492	}
1493
1494	if (!crtc_state->enable || !drm_atomic_crtc_effectively_active(crtc_state)) {
1495		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, active %d, skip atomic_check\n",
1496				crtc->base.id, crtc_state->enable,
1497				crtc_state->active);
1498		memset(&cstate->new_perf, 0, sizeof(cstate->new_perf));
1499		return 0;
1500	}
1501
1502	DRM_DEBUG_ATOMIC("%s: check\n", dpu_crtc->name);
1503
1504	if (cstate->num_mixers) {
1505		rc = _dpu_crtc_check_and_setup_lm_bounds(crtc, crtc_state);
1506		if (rc)
1507			return rc;
1508	}
1509
1510	/* FIXME: move this to dpu_plane_atomic_check? */
1511	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
1512		struct dpu_plane_state *dpu_pstate = to_dpu_plane_state(pstate);
1513
1514		if (IS_ERR_OR_NULL(pstate)) {
1515			rc = PTR_ERR(pstate);
1516			DPU_ERROR("%s: failed to get plane%d state, %d\n",
1517					dpu_crtc->name, plane->base.id, rc);
1518			return rc;
1519		}
1520
1521		if (!pstate->visible)
1522			continue;
1523
1524		dpu_pstate->needs_dirtyfb = needs_dirtyfb;
1525	}
1526
1527	atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1528
1529	rc = dpu_core_perf_crtc_check(crtc, crtc_state);
1530	if (rc) {
1531		DPU_ERROR("crtc%d failed performance check %d\n",
1532				crtc->base.id, rc);
1533		return rc;
1534	}
1535
1536	return 0;
1537}
1538
1539static enum drm_mode_status dpu_crtc_mode_valid(struct drm_crtc *crtc,
1540						const struct drm_display_mode *mode)
1541{
1542	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1543	u64 adjusted_mode_clk;
1544
1545	/* if there is no 3d_mux block we cannot merge LMs so we cannot
1546	 * split the large layer into 2 LMs, filter out such modes
1547	 */
1548	if (!dpu_kms->catalog->caps->has_3d_merge &&
1549	    mode->hdisplay > dpu_kms->catalog->caps->max_mixer_width)
1550		return MODE_BAD_HVALUE;
1551
1552	adjusted_mode_clk = dpu_core_perf_adjusted_mode_clk(mode->clock,
1553							    dpu_kms->perf.perf_cfg);
1554
1555	if (dpu_kms->catalog->caps->has_3d_merge)
1556		adjusted_mode_clk /= 2;
1557
1558	/*
1559	 * The given mode, adjusted for the perf clock factor, should not exceed
1560	 * the max core clock rate
1561	 */
1562	if (dpu_kms->perf.max_core_clk_rate < adjusted_mode_clk * 1000)
1563		return MODE_CLOCK_HIGH;
1564
1565	/*
1566	 * max crtc width is equal to the max mixer width * 2 and max height is 4K
1567	 */
1568	return drm_mode_validate_size(mode,
1569				      2 * dpu_kms->catalog->caps->max_mixer_width,
1570				      4096);
1571}
1572
1573/**
1574 * dpu_crtc_vblank - enable or disable vblanks for this crtc
1575 * @crtc: Pointer to drm crtc object
1576 * @en: true to enable vblanks, false to disable
1577 */
1578int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
1579{
1580	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1581	struct drm_encoder *enc;
1582
1583	trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
1584
1585	/*
1586	 * Normally we would iterate through encoder_mask in crtc state to find
1587	 * attached encoders. In this case, we might be disabling vblank _after_
1588	 * encoder_mask has been cleared.
1589	 *
1590	 * Instead, we "assign" a crtc to the encoder in enable and clear it in
1591	 * disable (which is also after encoder_mask is cleared). So instead of
1592	 * using encoder mask, we'll ask the encoder to toggle itself iff it's
1593	 * currently assigned to our crtc.
1594	 *
1595	 * Note also that this function cannot be called while crtc is disabled
1596	 * since we use drm_crtc_vblank_on/off. So we don't need to worry
1597	 * about the assigned crtcs being inconsistent with the current state
1598	 * (which means no need to worry about modeset locks).
1599	 */
1600	list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
1601		trace_dpu_crtc_vblank_enable(DRMID(crtc), DRMID(enc), en,
1602					     dpu_crtc);
1603
1604		dpu_encoder_toggle_vblank_for_crtc(enc, crtc, en);
1605	}
1606
1607	return 0;
1608}
1609
1610#ifdef CONFIG_DEBUG_FS
1611static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
1612{
1613	struct dpu_crtc *dpu_crtc;
1614	struct dpu_plane_state *pstate = NULL;
1615	struct dpu_crtc_mixer *m;
1616
1617	struct drm_crtc *crtc;
1618	struct drm_plane *plane;
1619	struct drm_display_mode *mode;
1620	struct drm_framebuffer *fb;
1621	struct drm_plane_state *state;
1622	struct dpu_crtc_state *cstate;
1623
1624	int i, out_width;
1625
1626	dpu_crtc = s->private;
1627	crtc = &dpu_crtc->base;
1628
1629	drm_modeset_lock_all(crtc->dev);
1630	cstate = to_dpu_crtc_state(crtc->state);
1631
1632	mode = &crtc->state->adjusted_mode;
1633	out_width = mode->hdisplay / cstate->num_mixers;
1634
1635	seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
1636				mode->hdisplay, mode->vdisplay);
1637
1638	seq_puts(s, "\n");
1639
1640	for (i = 0; i < cstate->num_mixers; ++i) {
1641		m = &cstate->mixers[i];
1642		seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
1643			m->hw_lm->idx - LM_0, m->lm_ctl->idx - CTL_0,
1644			out_width, mode->vdisplay);
1645	}
1646
1647	seq_puts(s, "\n");
1648
1649	drm_atomic_crtc_for_each_plane(plane, crtc) {
1650		pstate = to_dpu_plane_state(plane->state);
1651		state = plane->state;
1652
1653		if (!pstate || !state)
1654			continue;
1655
1656		seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
1657			pstate->stage);
1658
1659		if (plane->state->fb) {
1660			fb = plane->state->fb;
1661
1662			seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
1663				fb->base.id, (char *) &fb->format->format,
1664				fb->width, fb->height);
1665			for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
1666				seq_printf(s, "cpp[%d]:%u ",
1667						i, fb->format->cpp[i]);
1668			seq_puts(s, "\n\t");
1669
1670			seq_printf(s, "modifier:%8llu ", fb->modifier);
1671			seq_puts(s, "\n");
1672
1673			seq_puts(s, "\t");
1674			for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
1675				seq_printf(s, "pitches[%d]:%8u ", i,
1676							fb->pitches[i]);
1677			seq_puts(s, "\n");
1678
1679			seq_puts(s, "\t");
1680			for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
1681				seq_printf(s, "offsets[%d]:%8u ", i,
1682							fb->offsets[i]);
1683			seq_puts(s, "\n");
1684		}
1685
1686		seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
1687			state->src_x, state->src_y, state->src_w, state->src_h);
1688
1689		seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
1690			state->crtc_x, state->crtc_y, state->crtc_w,
1691			state->crtc_h);
1692
1693		for (i = 0; i < PIPES_PER_PLANE; i++) {
1694			if (!pstate->pipe[i].sspp)
1695				continue;
1696			seq_printf(s, "\tsspp[%d]:%s\n",
1697				   i, pstate->pipe[i].sspp->cap->name);
1698			seq_printf(s, "\tmultirect[%d]: mode: %d index: %d\n",
1699				   i, pstate->pipe[i].multirect_mode,
1700				   pstate->pipe[i].multirect_index);
1701		}
1702
1703		seq_puts(s, "\n");
1704	}
1705	if (dpu_crtc->vblank_cb_count) {
1706		ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
1707		s64 diff_ms = ktime_to_ms(diff);
1708		s64 fps = diff_ms ? div_s64(
1709				dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
1710
1711		seq_printf(s,
1712			"vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
1713				fps, dpu_crtc->vblank_cb_count,
1714				ktime_to_ms(diff), dpu_crtc->play_count);
1715
1716		/* reset time & count for next measurement */
1717		dpu_crtc->vblank_cb_count = 0;
1718		dpu_crtc->vblank_cb_time = ktime_set(0, 0);
1719	}
1720
1721	drm_modeset_unlock_all(crtc->dev);
1722
1723	return 0;
1724}
1725
1726DEFINE_SHOW_ATTRIBUTE(_dpu_debugfs_status);
1727
1728static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
1729{
1730	struct drm_crtc *crtc = s->private;
1731	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1732
1733	seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
1734	seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
1735	seq_printf(s, "core_clk_rate: %llu\n",
1736			dpu_crtc->cur_perf.core_clk_rate);
1737	seq_printf(s, "bw_ctl: %uk\n",
1738		   (u32)DIV_ROUND_UP_ULL(dpu_crtc->cur_perf.bw_ctl, 1000));
1739	seq_printf(s, "max_per_pipe_ib: %u\n",
1740				dpu_crtc->cur_perf.max_per_pipe_ib);
1741
1742	return 0;
1743}
1744DEFINE_SHOW_ATTRIBUTE(dpu_crtc_debugfs_state);
1745
1746static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1747{
1748	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1749
1750	debugfs_create_file("status", 0400,
1751			crtc->debugfs_entry,
1752			dpu_crtc, &_dpu_debugfs_status_fops);
1753	debugfs_create_file("state", 0600,
1754			crtc->debugfs_entry,
1755			&dpu_crtc->base,
1756			&dpu_crtc_debugfs_state_fops);
1757
1758	return 0;
1759}
1760#else
1761static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1762{
1763	return 0;
1764}
1765#endif /* CONFIG_DEBUG_FS */
1766
1767static int dpu_crtc_late_register(struct drm_crtc *crtc)
1768{
1769	return _dpu_crtc_init_debugfs(crtc);
1770}
1771
1772static const struct drm_crtc_funcs dpu_crtc_funcs = {
1773	.set_config = drm_atomic_helper_set_config,
1774	.page_flip = drm_atomic_helper_page_flip,
1775	.reset = dpu_crtc_reset,
1776	.atomic_duplicate_state = dpu_crtc_duplicate_state,
1777	.atomic_destroy_state = dpu_crtc_destroy_state,
1778	.atomic_print_state = dpu_crtc_atomic_print_state,
1779	.late_register = dpu_crtc_late_register,
1780	.verify_crc_source = dpu_crtc_verify_crc_source,
1781	.set_crc_source = dpu_crtc_set_crc_source,
1782	.enable_vblank  = msm_crtc_enable_vblank,
1783	.disable_vblank = msm_crtc_disable_vblank,
1784	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
1785	.get_vblank_counter = dpu_crtc_get_vblank_counter,
1786};
1787
1788static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
1789	.atomic_disable = dpu_crtc_disable,
1790	.atomic_enable = dpu_crtc_enable,
1791	.atomic_check = dpu_crtc_atomic_check,
1792	.atomic_begin = dpu_crtc_atomic_begin,
1793	.atomic_flush = dpu_crtc_atomic_flush,
1794	.mode_valid = dpu_crtc_mode_valid,
1795	.get_scanout_position = dpu_crtc_get_scanout_position,
1796};
1797
1798/**
1799 * dpu_crtc_init - create a new crtc object
1800 * @dev: dpu device
1801 * @plane: base plane
1802 * @cursor: cursor plane
1803 * @return: new crtc object or error
1804 *
1805 * initialize CRTC
1806 */
1807struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
1808				struct drm_plane *cursor)
1809{
1810	struct msm_drm_private *priv = dev->dev_private;
1811	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
1812	struct drm_crtc *crtc = NULL;
1813	struct dpu_crtc *dpu_crtc;
1814	int i, ret;
1815
1816	dpu_crtc = drmm_crtc_alloc_with_planes(dev, struct dpu_crtc, base,
1817					       plane, cursor,
1818					       &dpu_crtc_funcs,
1819					       NULL);
1820
1821	if (IS_ERR(dpu_crtc))
1822		return ERR_CAST(dpu_crtc);
1823
1824	crtc = &dpu_crtc->base;
1825	crtc->dev = dev;
1826
1827	spin_lock_init(&dpu_crtc->spin_lock);
1828	atomic_set(&dpu_crtc->frame_pending, 0);
1829
1830	init_completion(&dpu_crtc->frame_done_comp);
1831
1832	INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
1833
1834	for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
1835		INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
1836		list_add(&dpu_crtc->frame_events[i].list,
1837				&dpu_crtc->frame_event_list);
1838		kthread_init_work(&dpu_crtc->frame_events[i].work,
1839				dpu_crtc_frame_event_work);
1840	}
1841
1842	drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
1843
1844	if (dpu_kms->catalog->dspp_count)
1845		drm_crtc_enable_color_mgmt(crtc, 0, true, 0);
1846
1847	/* save user friendly CRTC name for later */
1848	snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
1849
1850	/* initialize event handling */
1851	spin_lock_init(&dpu_crtc->event_lock);
1852
1853	ret = drm_self_refresh_helper_init(crtc);
1854	if (ret) {
1855		DPU_ERROR("Failed to initialize %s with self-refresh helpers %d\n",
1856			crtc->name, ret);
1857		return ERR_PTR(ret);
1858	}
1859
1860	DRM_DEBUG_KMS("%s: successfully initialized crtc\n", dpu_crtc->name);
1861	return crtc;
1862}