rockchip_drm_vop.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) Rockchip Electronics Co., Ltd.
   4 * Author:Mark Yao <mark.yao@rock-chips.com>
   5 */
   6
   7#include <linux/clk.h>
   8#include <linux/component.h>
   9#include <linux/delay.h>
  10#include <linux/iopoll.h>
  11#include <linux/kernel.h>
  12#include <linux/log2.h>
  13#include <linux/module.h>
  14#include <linux/of.h>
  15#include <linux/overflow.h>
  16#include <linux/platform_device.h>
  17#include <linux/pm_runtime.h>
  18#include <linux/reset.h>
  19
  20#include <drm/drm.h>
  21#include <drm/drm_atomic.h>
  22#include <drm/drm_atomic_uapi.h>
  23#include <drm/drm_blend.h>
  24#include <drm/drm_crtc.h>
  25#include <drm/drm_flip_work.h>
  26#include <drm/drm_fourcc.h>
  27#include <drm/drm_framebuffer.h>
  28#include <drm/drm_gem_atomic_helper.h>
  29#include <drm/drm_gem_framebuffer_helper.h>
  30#include <drm/drm_print.h>
  31#include <drm/drm_probe_helper.h>
  32#include <drm/drm_self_refresh_helper.h>
  33#include <drm/drm_vblank.h>
  34
  35#ifdef CONFIG_DRM_ANALOGIX_DP
  36#include <drm/bridge/analogix_dp.h>
  37#endif
  38
  39#include "rockchip_drm_drv.h"
  40#include "rockchip_drm_gem.h"
  41#include "rockchip_drm_fb.h"
  42#include "rockchip_drm_vop.h"
  43#include "rockchip_rgb.h"
  44
  45#define VOP_WIN_SET(vop, win, name, v) \
  46		vop_reg_set(vop, &win->phy->name, win->base, ~0, v, #name)
  47#define VOP_SCL_SET(vop, win, name, v) \
  48		vop_reg_set(vop, &win->phy->scl->name, win->base, ~0, v, #name)
  49#define VOP_SCL_SET_EXT(vop, win, name, v) \
  50		vop_reg_set(vop, &win->phy->scl->ext->name, \
  51			    win->base, ~0, v, #name)
  52
  53#define VOP_WIN_YUV2YUV_SET(vop, win_yuv2yuv, name, v) \
  54	do { \
  55		if (win_yuv2yuv && win_yuv2yuv->name.mask) \
  56			vop_reg_set(vop, &win_yuv2yuv->name, 0, ~0, v, #name); \
  57	} while (0)
  58
  59#define VOP_WIN_YUV2YUV_COEFFICIENT_SET(vop, win_yuv2yuv, name, v) \
  60	do { \
  61		if (win_yuv2yuv && win_yuv2yuv->phy->name.mask) \
  62			vop_reg_set(vop, &win_yuv2yuv->phy->name, win_yuv2yuv->base, ~0, v, #name); \
  63	} while (0)
  64
  65#define VOP_INTR_SET_MASK(vop, name, mask, v) \
  66		vop_reg_set(vop, &vop->data->intr->name, 0, mask, v, #name)
  67
  68#define VOP_REG_SET(vop, group, name, v) \
  69		    vop_reg_set(vop, &vop->data->group->name, 0, ~0, v, #name)
  70
  71#define VOP_HAS_REG(vop, group, name) \
  72		(!!(vop->data->group->name.mask))
  73
  74#define VOP_INTR_SET_TYPE(vop, name, type, v) \
  75	do { \
  76		int i, reg = 0, mask = 0; \
  77		for (i = 0; i < vop->data->intr->nintrs; i++) { \
  78			if (vop->data->intr->intrs[i] & type) { \
  79				reg |= (v) << i; \
  80				mask |= 1 << i; \
  81			} \
  82		} \
  83		VOP_INTR_SET_MASK(vop, name, mask, reg); \
  84	} while (0)
  85#define VOP_INTR_GET_TYPE(vop, name, type) \
  86		vop_get_intr_type(vop, &vop->data->intr->name, type)
  87
  88#define VOP_WIN_GET(vop, win, name) \
  89		vop_read_reg(vop, win->base, &win->phy->name)
  90
  91#define VOP_WIN_HAS_REG(win, name) \
  92	(!!(win->phy->name.mask))
  93
  94#define VOP_WIN_GET_YRGBADDR(vop, win) \
  95		vop_readl(vop, win->base + win->phy->yrgb_mst.offset)
  96
  97#define VOP_WIN_TO_INDEX(vop_win) \
  98	((vop_win) - (vop_win)->vop->win)
  99
 100#define VOP_AFBC_SET(vop, name, v) \
 101	do { \
 102		if ((vop)->data->afbc) \
 103			vop_reg_set((vop), &(vop)->data->afbc->name, \
 104				    0, ~0, v, #name); \
 105	} while (0)
 106
 107#define to_vop(x) container_of(x, struct vop, crtc)
 108#define to_vop_win(x) container_of(x, struct vop_win, base)
 109
 110#define AFBC_FMT_RGB565		0x0
 111#define AFBC_FMT_U8U8U8U8	0x5
 112#define AFBC_FMT_U8U8U8		0x4
 113
 114#define AFBC_TILE_16x16		BIT(4)
 115
 116/*
 117 * The coefficients of the following matrix are all fixed points.
 118 * The format is S2.10 for the 3x3 part of the matrix, and S9.12 for the offsets.
 119 * They are all represented in two's complement.
 120 */
 121static const uint32_t bt601_yuv2rgb[] = {
 122	0x4A8, 0x0,    0x662,
 123	0x4A8, 0x1E6F, 0x1CBF,
 124	0x4A8, 0x812,  0x0,
 125	0x321168, 0x0877CF, 0x2EB127
 126};
 127
 128enum vop_pending {
 129	VOP_PENDING_FB_UNREF,
 130};
 131
 132struct vop_win {
 133	struct drm_plane base;
 134	const struct vop_win_data *data;
 135	const struct vop_win_yuv2yuv_data *yuv2yuv_data;
 136	struct vop *vop;
 137};
 138
 139struct rockchip_rgb;
 140struct vop {
 141	struct drm_crtc crtc;
 142	struct device *dev;
 143	struct drm_device *drm_dev;
 144	bool is_enabled;
 145
 146	struct completion dsp_hold_completion;
 147	unsigned int win_enabled;
 148
 149	/* protected by dev->event_lock */
 150	struct drm_pending_vblank_event *event;
 151
 152	struct drm_flip_work fb_unref_work;
 153	unsigned long pending;
 154
 155	struct completion line_flag_completion;
 156
 157	const struct vop_data *data;
 158
 159	uint32_t *regsbak;
 160	void __iomem *regs;
 161	void __iomem *lut_regs;
 162
 163	/* physical map length of vop register */
 164	uint32_t len;
 165
 166	/* one time only one process allowed to config the register */
 167	spinlock_t reg_lock;
 168	/* lock vop irq reg */
 169	spinlock_t irq_lock;
 170	/* protects crtc enable/disable */
 171	struct mutex vop_lock;
 172
 173	unsigned int irq;
 174
 175	/* vop AHP clk */
 176	struct clk *hclk;
 177	/* vop dclk */
 178	struct clk *dclk;
 179	/* vop share memory frequency */
 180	struct clk *aclk;
 181
 182	/* vop dclk reset */
 183	struct reset_control *dclk_rst;
 184
 185	/* optional internal rgb encoder */
 186	struct rockchip_rgb *rgb;
 187
 188	struct vop_win win[];
 189};
 190
 191static inline uint32_t vop_readl(struct vop *vop, uint32_t offset)
 192{
 193	return readl(vop->regs + offset);
 194}
 195
 196static inline uint32_t vop_read_reg(struct vop *vop, uint32_t base,
 197				    const struct vop_reg *reg)
 198{
 199	return (vop_readl(vop, base + reg->offset) >> reg->shift) & reg->mask;
 200}
 201
 202static void vop_reg_set(struct vop *vop, const struct vop_reg *reg,
 203			uint32_t _offset, uint32_t _mask, uint32_t v,
 204			const char *reg_name)
 205{
 206	int offset, mask, shift;
 207
 208	if (!reg || !reg->mask) {
 209		DRM_DEV_DEBUG(vop->dev, "Warning: not support %s\n", reg_name);
 210		return;
 211	}
 212
 213	offset = reg->offset + _offset;
 214	mask = reg->mask & _mask;
 215	shift = reg->shift;
 216
 217	if (reg->write_mask) {
 218		v = ((v << shift) & 0xffff) | (mask << (shift + 16));
 219	} else {
 220		uint32_t cached_val = vop->regsbak[offset >> 2];
 221
 222		v = (cached_val & ~(mask << shift)) | ((v & mask) << shift);
 223		vop->regsbak[offset >> 2] = v;
 224	}
 225
 226	if (reg->relaxed)
 227		writel_relaxed(v, vop->regs + offset);
 228	else
 229		writel(v, vop->regs + offset);
 230}
 231
 232static inline uint32_t vop_get_intr_type(struct vop *vop,
 233					 const struct vop_reg *reg, int type)
 234{
 235	uint32_t i, ret = 0;
 236	uint32_t regs = vop_read_reg(vop, 0, reg);
 237
 238	for (i = 0; i < vop->data->intr->nintrs; i++) {
 239		if ((type & vop->data->intr->intrs[i]) && (regs & 1 << i))
 240			ret |= vop->data->intr->intrs[i];
 241	}
 242
 243	return ret;
 244}
 245
 246static inline void vop_cfg_done(struct vop *vop)
 247{
 248	VOP_REG_SET(vop, common, cfg_done, 1);
 249}
 250
 251static bool has_rb_swapped(uint32_t version, uint32_t format)
 252{
 253	switch (format) {
 254	case DRM_FORMAT_XBGR8888:
 255	case DRM_FORMAT_ABGR8888:
 256	case DRM_FORMAT_BGR565:
 257		return true;
 258	/*
 259	 * full framework (IP version 3.x) only need rb swapped for RGB888 and
 260	 * little framework (IP version 2.x) only need rb swapped for BGR888,
 261	 * check for 3.x to also only rb swap BGR888 for unknown vop version
 262	 */
 263	case DRM_FORMAT_RGB888:
 264		return VOP_MAJOR(version) == 3;
 265	case DRM_FORMAT_BGR888:
 266		return VOP_MAJOR(version) != 3;
 267	default:
 268		return false;
 269	}
 270}
 271
 272static bool has_uv_swapped(uint32_t format)
 273{
 274	switch (format) {
 275	case DRM_FORMAT_NV21:
 276	case DRM_FORMAT_NV61:
 277	case DRM_FORMAT_NV42:
 278		return true;
 279	default:
 280		return false;
 281	}
 282}
 283
 284static bool is_fmt_10(uint32_t format)
 285{
 286	switch (format) {
 287	case DRM_FORMAT_NV15:
 288	case DRM_FORMAT_NV20:
 289	case DRM_FORMAT_NV30:
 290		return true;
 291	default:
 292		return false;
 293	}
 294}
 295
 296static enum vop_data_format vop_convert_format(uint32_t format)
 297{
 298	switch (format) {
 299	case DRM_FORMAT_XRGB8888:
 300	case DRM_FORMAT_ARGB8888:
 301	case DRM_FORMAT_XBGR8888:
 302	case DRM_FORMAT_ABGR8888:
 303		return VOP_FMT_ARGB8888;
 304	case DRM_FORMAT_RGB888:
 305	case DRM_FORMAT_BGR888:
 306		return VOP_FMT_RGB888;
 307	case DRM_FORMAT_RGB565:
 308	case DRM_FORMAT_BGR565:
 309		return VOP_FMT_RGB565;
 310	case DRM_FORMAT_NV12:
 311	case DRM_FORMAT_NV15:
 312	case DRM_FORMAT_NV21:
 313		return VOP_FMT_YUV420SP;
 314	case DRM_FORMAT_NV16:
 315	case DRM_FORMAT_NV20:
 316	case DRM_FORMAT_NV61:
 317		return VOP_FMT_YUV422SP;
 318	case DRM_FORMAT_NV24:
 319	case DRM_FORMAT_NV30:
 320	case DRM_FORMAT_NV42:
 321		return VOP_FMT_YUV444SP;
 322	default:
 323		DRM_ERROR("unsupported format[%08x]\n", format);
 324		return -EINVAL;
 325	}
 326}
 327
 328static int vop_convert_afbc_format(uint32_t format)
 329{
 330	switch (format) {
 331	case DRM_FORMAT_XRGB8888:
 332	case DRM_FORMAT_ARGB8888:
 333	case DRM_FORMAT_XBGR8888:
 334	case DRM_FORMAT_ABGR8888:
 335		return AFBC_FMT_U8U8U8U8;
 336	case DRM_FORMAT_RGB888:
 337	case DRM_FORMAT_BGR888:
 338		return AFBC_FMT_U8U8U8;
 339	case DRM_FORMAT_RGB565:
 340	case DRM_FORMAT_BGR565:
 341		return AFBC_FMT_RGB565;
 342	default:
 343		DRM_DEBUG_KMS("unsupported AFBC format[%08x]\n", format);
 344		return -EINVAL;
 345	}
 346}
 347
 348static uint16_t scl_vop_cal_scale(enum scale_mode mode, uint32_t src,
 349				  uint32_t dst, bool is_horizontal,
 350				  int vsu_mode, int *vskiplines)
 351{
 352	uint16_t val = 1 << SCL_FT_DEFAULT_FIXPOINT_SHIFT;
 353
 354	if (vskiplines)
 355		*vskiplines = 0;
 356
 357	if (is_horizontal) {
 358		if (mode == SCALE_UP)
 359			val = GET_SCL_FT_BIC(src, dst);
 360		else if (mode == SCALE_DOWN)
 361			val = GET_SCL_FT_BILI_DN(src, dst);
 362	} else {
 363		if (mode == SCALE_UP) {
 364			if (vsu_mode == SCALE_UP_BIL)
 365				val = GET_SCL_FT_BILI_UP(src, dst);
 366			else
 367				val = GET_SCL_FT_BIC(src, dst);
 368		} else if (mode == SCALE_DOWN) {
 369			if (vskiplines) {
 370				*vskiplines = scl_get_vskiplines(src, dst);
 371				val = scl_get_bili_dn_vskip(src, dst,
 372							    *vskiplines);
 373			} else {
 374				val = GET_SCL_FT_BILI_DN(src, dst);
 375			}
 376		}
 377	}
 378
 379	return val;
 380}
 381
 382static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
 383			     uint32_t src_w, uint32_t src_h, uint32_t dst_w,
 384			     uint32_t dst_h, const struct drm_format_info *info)
 385{
 386	uint16_t yrgb_hor_scl_mode, yrgb_ver_scl_mode;
 387	uint16_t cbcr_hor_scl_mode = SCALE_NONE;
 388	uint16_t cbcr_ver_scl_mode = SCALE_NONE;
 389	bool is_yuv = false;
 390	uint16_t cbcr_src_w = src_w / info->hsub;
 391	uint16_t cbcr_src_h = src_h / info->vsub;
 392	uint16_t vsu_mode;
 393	uint16_t lb_mode;
 394	uint32_t val;
 395	int vskiplines;
 396
 397	if (info->is_yuv)
 398		is_yuv = true;
 399
 400	if (dst_w > 4096) {
 401		DRM_DEV_ERROR(vop->dev, "Maximum dst width (4096) exceeded\n");
 402		return;
 403	}
 404
 405	if (!win->phy->scl->ext) {
 406		VOP_SCL_SET(vop, win, scale_yrgb_x,
 407			    scl_cal_scale2(src_w, dst_w));
 408		VOP_SCL_SET(vop, win, scale_yrgb_y,
 409			    scl_cal_scale2(src_h, dst_h));
 410		if (is_yuv) {
 411			VOP_SCL_SET(vop, win, scale_cbcr_x,
 412				    scl_cal_scale2(cbcr_src_w, dst_w));
 413			VOP_SCL_SET(vop, win, scale_cbcr_y,
 414				    scl_cal_scale2(cbcr_src_h, dst_h));
 415		}
 416		return;
 417	}
 418
 419	yrgb_hor_scl_mode = scl_get_scl_mode(src_w, dst_w);
 420	yrgb_ver_scl_mode = scl_get_scl_mode(src_h, dst_h);
 421
 422	if (is_yuv) {
 423		cbcr_hor_scl_mode = scl_get_scl_mode(cbcr_src_w, dst_w);
 424		cbcr_ver_scl_mode = scl_get_scl_mode(cbcr_src_h, dst_h);
 425		if (cbcr_hor_scl_mode == SCALE_DOWN)
 426			lb_mode = scl_vop_cal_lb_mode(dst_w, true);
 427		else
 428			lb_mode = scl_vop_cal_lb_mode(cbcr_src_w, true);
 429	} else {
 430		if (yrgb_hor_scl_mode == SCALE_DOWN)
 431			lb_mode = scl_vop_cal_lb_mode(dst_w, false);
 432		else
 433			lb_mode = scl_vop_cal_lb_mode(src_w, false);
 434	}
 435
 436	VOP_SCL_SET_EXT(vop, win, lb_mode, lb_mode);
 437	if (lb_mode == LB_RGB_3840X2) {
 438		if (yrgb_ver_scl_mode != SCALE_NONE) {
 439			DRM_DEV_ERROR(vop->dev, "not allow yrgb ver scale\n");
 440			return;
 441		}
 442		if (cbcr_ver_scl_mode != SCALE_NONE) {
 443			DRM_DEV_ERROR(vop->dev, "not allow cbcr ver scale\n");
 444			return;
 445		}
 446		vsu_mode = SCALE_UP_BIL;
 447	} else if (lb_mode == LB_RGB_2560X4) {
 448		vsu_mode = SCALE_UP_BIL;
 449	} else {
 450		vsu_mode = SCALE_UP_BIC;
 451	}
 452
 453	val = scl_vop_cal_scale(yrgb_hor_scl_mode, src_w, dst_w,
 454				true, 0, NULL);
 455	VOP_SCL_SET(vop, win, scale_yrgb_x, val);
 456	val = scl_vop_cal_scale(yrgb_ver_scl_mode, src_h, dst_h,
 457				false, vsu_mode, &vskiplines);
 458	VOP_SCL_SET(vop, win, scale_yrgb_y, val);
 459
 460	VOP_SCL_SET_EXT(vop, win, vsd_yrgb_gt4, vskiplines == 4);
 461	VOP_SCL_SET_EXT(vop, win, vsd_yrgb_gt2, vskiplines == 2);
 462
 463	VOP_SCL_SET_EXT(vop, win, yrgb_hor_scl_mode, yrgb_hor_scl_mode);
 464	VOP_SCL_SET_EXT(vop, win, yrgb_ver_scl_mode, yrgb_ver_scl_mode);
 465	VOP_SCL_SET_EXT(vop, win, yrgb_hsd_mode, SCALE_DOWN_BIL);
 466	VOP_SCL_SET_EXT(vop, win, yrgb_vsd_mode, SCALE_DOWN_BIL);
 467	VOP_SCL_SET_EXT(vop, win, yrgb_vsu_mode, vsu_mode);
 468	if (is_yuv) {
 469		val = scl_vop_cal_scale(cbcr_hor_scl_mode, cbcr_src_w,
 470					dst_w, true, 0, NULL);
 471		VOP_SCL_SET(vop, win, scale_cbcr_x, val);
 472		val = scl_vop_cal_scale(cbcr_ver_scl_mode, cbcr_src_h,
 473					dst_h, false, vsu_mode, &vskiplines);
 474		VOP_SCL_SET(vop, win, scale_cbcr_y, val);
 475
 476		VOP_SCL_SET_EXT(vop, win, vsd_cbcr_gt4, vskiplines == 4);
 477		VOP_SCL_SET_EXT(vop, win, vsd_cbcr_gt2, vskiplines == 2);
 478		VOP_SCL_SET_EXT(vop, win, cbcr_hor_scl_mode, cbcr_hor_scl_mode);
 479		VOP_SCL_SET_EXT(vop, win, cbcr_ver_scl_mode, cbcr_ver_scl_mode);
 480		VOP_SCL_SET_EXT(vop, win, cbcr_hsd_mode, SCALE_DOWN_BIL);
 481		VOP_SCL_SET_EXT(vop, win, cbcr_vsd_mode, SCALE_DOWN_BIL);
 482		VOP_SCL_SET_EXT(vop, win, cbcr_vsu_mode, vsu_mode);
 483	}
 484}
 485
 486static void vop_dsp_hold_valid_irq_enable(struct vop *vop)
 487{
 488	unsigned long flags;
 489
 490	if (WARN_ON(!vop->is_enabled))
 491		return;
 492
 493	spin_lock_irqsave(&vop->irq_lock, flags);
 494
 495	VOP_INTR_SET_TYPE(vop, clear, DSP_HOLD_VALID_INTR, 1);
 496	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 1);
 497
 498	spin_unlock_irqrestore(&vop->irq_lock, flags);
 499}
 500
 501static void vop_dsp_hold_valid_irq_disable(struct vop *vop)
 502{
 503	unsigned long flags;
 504
 505	if (WARN_ON(!vop->is_enabled))
 506		return;
 507
 508	spin_lock_irqsave(&vop->irq_lock, flags);
 509
 510	VOP_INTR_SET_TYPE(vop, enable, DSP_HOLD_VALID_INTR, 0);
 511
 512	spin_unlock_irqrestore(&vop->irq_lock, flags);
 513}
 514
 515/*
 516 * (1) each frame starts at the start of the Vsync pulse which is signaled by
 517 *     the "FRAME_SYNC" interrupt.
 518 * (2) the active data region of each frame ends at dsp_vact_end
 519 * (3) we should program this same number (dsp_vact_end) into dsp_line_frag_num,
 520 *      to get "LINE_FLAG" interrupt at the end of the active on screen data.
 521 *
 522 * VOP_INTR_CTRL0.dsp_line_frag_num = VOP_DSP_VACT_ST_END.dsp_vact_end
 523 * Interrupts
 524 * LINE_FLAG -------------------------------+
 525 * FRAME_SYNC ----+                         |
 526 *                |                         |
 527 *                v                         v
 528 *                | Vsync | Vbp |  Vactive  | Vfp |
 529 *                        ^     ^           ^     ^
 530 *                        |     |           |     |
 531 *                        |     |           |     |
 532 * dsp_vs_end ------------+     |           |     |   VOP_DSP_VTOTAL_VS_END
 533 * dsp_vact_start --------------+           |     |   VOP_DSP_VACT_ST_END
 534 * dsp_vact_end ----------------------------+     |   VOP_DSP_VACT_ST_END
 535 * dsp_total -------------------------------------+   VOP_DSP_VTOTAL_VS_END
 536 */
 537static bool vop_line_flag_irq_is_enabled(struct vop *vop)
 538{
 539	uint32_t line_flag_irq;
 540	unsigned long flags;
 541
 542	spin_lock_irqsave(&vop->irq_lock, flags);
 543
 544	line_flag_irq = VOP_INTR_GET_TYPE(vop, enable, LINE_FLAG_INTR);
 545
 546	spin_unlock_irqrestore(&vop->irq_lock, flags);
 547
 548	return !!line_flag_irq;
 549}
 550
 551static void vop_line_flag_irq_enable(struct vop *vop)
 552{
 553	unsigned long flags;
 554
 555	if (WARN_ON(!vop->is_enabled))
 556		return;
 557
 558	spin_lock_irqsave(&vop->irq_lock, flags);
 559
 560	VOP_INTR_SET_TYPE(vop, clear, LINE_FLAG_INTR, 1);
 561	VOP_INTR_SET_TYPE(vop, enable, LINE_FLAG_INTR, 1);
 562
 563	spin_unlock_irqrestore(&vop->irq_lock, flags);
 564}
 565
 566static void vop_line_flag_irq_disable(struct vop *vop)
 567{
 568	unsigned long flags;
 569
 570	if (WARN_ON(!vop->is_enabled))
 571		return;
 572
 573	spin_lock_irqsave(&vop->irq_lock, flags);
 574
 575	VOP_INTR_SET_TYPE(vop, enable, LINE_FLAG_INTR, 0);
 576
 577	spin_unlock_irqrestore(&vop->irq_lock, flags);
 578}
 579
 580static int vop_core_clks_enable(struct vop *vop)
 581{
 582	int ret;
 583
 584	ret = clk_enable(vop->hclk);
 585	if (ret < 0)
 586		return ret;
 587
 588	ret = clk_enable(vop->aclk);
 589	if (ret < 0)
 590		goto err_disable_hclk;
 591
 592	return 0;
 593
 594err_disable_hclk:
 595	clk_disable(vop->hclk);
 596	return ret;
 597}
 598
 599static void vop_core_clks_disable(struct vop *vop)
 600{
 601	clk_disable(vop->aclk);
 602	clk_disable(vop->hclk);
 603}
 604
 605static void vop_win_disable(struct vop *vop, const struct vop_win *vop_win)
 606{
 607	const struct vop_win_data *win = vop_win->data;
 608
 609	if (win->phy->scl && win->phy->scl->ext) {
 610		VOP_SCL_SET_EXT(vop, win, yrgb_hor_scl_mode, SCALE_NONE);
 611		VOP_SCL_SET_EXT(vop, win, yrgb_ver_scl_mode, SCALE_NONE);
 612		VOP_SCL_SET_EXT(vop, win, cbcr_hor_scl_mode, SCALE_NONE);
 613		VOP_SCL_SET_EXT(vop, win, cbcr_ver_scl_mode, SCALE_NONE);
 614	}
 615
 616	VOP_WIN_SET(vop, win, enable, 0);
 617	vop->win_enabled &= ~BIT(VOP_WIN_TO_INDEX(vop_win));
 618}
 619
 620static int vop_enable(struct drm_crtc *crtc, struct drm_crtc_state *old_state)
 621{
 622	struct vop *vop = to_vop(crtc);
 623	int ret, i;
 624
 625	ret = pm_runtime_resume_and_get(vop->dev);
 626	if (ret < 0) {
 627		DRM_DEV_ERROR(vop->dev, "failed to get pm runtime: %d\n", ret);
 628		return ret;
 629	}
 630
 631	ret = vop_core_clks_enable(vop);
 632	if (WARN_ON(ret < 0))
 633		goto err_put_pm_runtime;
 634
 635	ret = clk_enable(vop->dclk);
 636	if (WARN_ON(ret < 0))
 637		goto err_disable_core;
 638
 639	/*
 640	 * Slave iommu shares power, irq and clock with vop.  It was associated
 641	 * automatically with this master device via common driver code.
 642	 * Now that we have enabled the clock we attach it to the shared drm
 643	 * mapping.
 644	 */
 645	ret = rockchip_drm_dma_attach_device(vop->drm_dev, vop->dev);
 646	if (ret) {
 647		DRM_DEV_ERROR(vop->dev,
 648			      "failed to attach dma mapping, %d\n", ret);
 649		goto err_disable_dclk;
 650	}
 651
 652	spin_lock(&vop->reg_lock);
 653	for (i = 0; i < vop->len; i += 4)
 654		writel_relaxed(vop->regsbak[i / 4], vop->regs + i);
 655
 656	/*
 657	 * We need to make sure that all windows are disabled before we
 658	 * enable the crtc. Otherwise we might try to scan from a destroyed
 659	 * buffer later.
 660	 *
 661	 * In the case of enable-after-PSR, we don't need to worry about this
 662	 * case since the buffer is guaranteed to be valid and disabling the
 663	 * window will result in screen glitches on PSR exit.
 664	 */
 665	if (!old_state || !old_state->self_refresh_active) {
 666		for (i = 0; i < vop->data->win_size; i++) {
 667			struct vop_win *vop_win = &vop->win[i];
 668
 669			vop_win_disable(vop, vop_win);
 670		}
 671	}
 672
 673	if (vop->data->afbc) {
 674		struct rockchip_crtc_state *s;
 675		/*
 676		 * Disable AFBC and forget there was a vop window with AFBC
 677		 */
 678		VOP_AFBC_SET(vop, enable, 0);
 679		s = to_rockchip_crtc_state(crtc->state);
 680		s->enable_afbc = false;
 681	}
 682
 683	vop_cfg_done(vop);
 684
 685	spin_unlock(&vop->reg_lock);
 686
 687	/*
 688	 * At here, vop clock & iommu is enable, R/W vop regs would be safe.
 689	 */
 690	vop->is_enabled = true;
 691
 692	spin_lock(&vop->reg_lock);
 693
 694	VOP_REG_SET(vop, common, standby, 1);
 695
 696	spin_unlock(&vop->reg_lock);
 697
 698	drm_crtc_vblank_on(crtc);
 699
 700	return 0;
 701
 702err_disable_dclk:
 703	clk_disable(vop->dclk);
 704err_disable_core:
 705	vop_core_clks_disable(vop);
 706err_put_pm_runtime:
 707	pm_runtime_put_sync(vop->dev);
 708	return ret;
 709}
 710
 711static void rockchip_drm_set_win_enabled(struct drm_crtc *crtc, bool enabled)
 712{
 713        struct vop *vop = to_vop(crtc);
 714        int i;
 715
 716        spin_lock(&vop->reg_lock);
 717
 718        for (i = 0; i < vop->data->win_size; i++) {
 719                struct vop_win *vop_win = &vop->win[i];
 720                const struct vop_win_data *win = vop_win->data;
 721
 722                VOP_WIN_SET(vop, win, enable,
 723                            enabled && (vop->win_enabled & BIT(i)));
 724        }
 725        vop_cfg_done(vop);
 726
 727        spin_unlock(&vop->reg_lock);
 728}
 729
 730static void vop_crtc_atomic_disable(struct drm_crtc *crtc,
 731				    struct drm_atomic_state *state)
 732{
 733	struct vop *vop = to_vop(crtc);
 734
 735	WARN_ON(vop->event);
 736
 737	if (crtc->state->self_refresh_active) {
 738		rockchip_drm_set_win_enabled(crtc, false);
 739		goto out;
 740	}
 741
 742	mutex_lock(&vop->vop_lock);
 743
 744	drm_crtc_vblank_off(crtc);
 745
 746	/*
 747	 * Vop standby will take effect at end of current frame,
 748	 * if dsp hold valid irq happen, it means standby complete.
 749	 *
 750	 * we must wait standby complete when we want to disable aclk,
 751	 * if not, memory bus maybe dead.
 752	 */
 753	reinit_completion(&vop->dsp_hold_completion);
 754	vop_dsp_hold_valid_irq_enable(vop);
 755
 756	spin_lock(&vop->reg_lock);
 757
 758	VOP_REG_SET(vop, common, standby, 1);
 759
 760	spin_unlock(&vop->reg_lock);
 761
 762	if (!wait_for_completion_timeout(&vop->dsp_hold_completion,
 763					 msecs_to_jiffies(200)))
 764		WARN(1, "%s: timed out waiting for DSP hold", crtc->name);
 765
 766	vop_dsp_hold_valid_irq_disable(vop);
 767
 768	vop->is_enabled = false;
 769
 770	/*
 771	 * vop standby complete, so iommu detach is safe.
 772	 */
 773	rockchip_drm_dma_detach_device(vop->drm_dev, vop->dev);
 774
 775	clk_disable(vop->dclk);
 776	vop_core_clks_disable(vop);
 777	pm_runtime_put(vop->dev);
 778
 779	mutex_unlock(&vop->vop_lock);
 780
 781out:
 782	if (crtc->state->event && !crtc->state->active) {
 783		spin_lock_irq(&crtc->dev->event_lock);
 784		drm_crtc_send_vblank_event(crtc, crtc->state->event);
 785		spin_unlock_irq(&crtc->dev->event_lock);
 786
 787		crtc->state->event = NULL;
 788	}
 789}
 790
 791static inline bool rockchip_afbc(u64 modifier)
 792{
 793	return modifier == ROCKCHIP_AFBC_MOD;
 794}
 795
 796static bool rockchip_mod_supported(struct drm_plane *plane,
 797				   u32 format, u64 modifier)
 798{
 799	if (modifier == DRM_FORMAT_MOD_LINEAR)
 800		return true;
 801
 802	if (!rockchip_afbc(modifier)) {
 803		DRM_DEBUG_KMS("Unsupported format modifier 0x%llx\n", modifier);
 804
 805		return false;
 806	}
 807
 808	return vop_convert_afbc_format(format) >= 0;
 809}
 810
 811static int vop_plane_atomic_check(struct drm_plane *plane,
 812			   struct drm_atomic_state *state)
 813{
 814	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 815										 plane);
 816	struct drm_crtc *crtc = new_plane_state->crtc;
 817	struct drm_crtc_state *crtc_state;
 818	struct drm_framebuffer *fb = new_plane_state->fb;
 819	struct vop_win *vop_win = to_vop_win(plane);
 820	const struct vop_win_data *win = vop_win->data;
 821	int ret;
 822	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
 823					DRM_PLANE_NO_SCALING;
 824	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
 825					DRM_PLANE_NO_SCALING;
 826
 827	if (!crtc || WARN_ON(!fb))
 828		return 0;
 829
 830	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 831	if (WARN_ON(!crtc_state))
 832		return -EINVAL;
 833
 834	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
 835						  min_scale, max_scale,
 836						  true, true);
 837	if (ret)
 838		return ret;
 839
 840	if (!new_plane_state->visible)
 841		return 0;
 842
 843	ret = vop_convert_format(fb->format->format);
 844	if (ret < 0)
 845		return ret;
 846
 847	/*
 848	 * Src.x1 can be odd when do clip, but yuv plane start point
 849	 * need align with 2 pixel.
 850	 */
 851	if (fb->format->is_yuv && ((new_plane_state->src.x1 >> 16) % 2)) {
 852		DRM_DEBUG_KMS("Invalid Source: Yuv format not support odd xpos\n");
 853		return -EINVAL;
 854	}
 855
 856	if (fb->format->is_yuv && new_plane_state->rotation & DRM_MODE_REFLECT_Y) {
 857		DRM_DEBUG_KMS("Invalid Source: Yuv format does not support this rotation\n");
 858		return -EINVAL;
 859	}
 860
 861	if (rockchip_afbc(fb->modifier)) {
 862		struct vop *vop = to_vop(crtc);
 863
 864		if (!vop->data->afbc) {
 865			DRM_DEBUG_KMS("vop does not support AFBC\n");
 866			return -EINVAL;
 867		}
 868
 869		ret = vop_convert_afbc_format(fb->format->format);
 870		if (ret < 0)
 871			return ret;
 872
 873		if (new_plane_state->src.x1 || new_plane_state->src.y1) {
 874			DRM_DEBUG_KMS("AFBC does not support offset display, " \
 875				      "xpos=%d, ypos=%d, offset=%d\n",
 876				      new_plane_state->src.x1, new_plane_state->src.y1,
 877				      fb->offsets[0]);
 878			return -EINVAL;
 879		}
 880
 881		if (new_plane_state->rotation && new_plane_state->rotation != DRM_MODE_ROTATE_0) {
 882			DRM_DEBUG_KMS("No rotation support in AFBC, rotation=%d\n",
 883				      new_plane_state->rotation);
 884			return -EINVAL;
 885		}
 886	}
 887
 888	return 0;
 889}
 890
 891static void vop_plane_atomic_disable(struct drm_plane *plane,
 892				     struct drm_atomic_state *state)
 893{
 894	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 895									   plane);
 896	struct vop_win *vop_win = to_vop_win(plane);
 897	struct vop *vop = to_vop(old_state->crtc);
 898
 899	if (!old_state->crtc)
 900		return;
 901
 902	spin_lock(&vop->reg_lock);
 903
 904	vop_win_disable(vop, vop_win);
 905
 906	spin_unlock(&vop->reg_lock);
 907}
 908
 909static void vop_plane_atomic_update(struct drm_plane *plane,
 910		struct drm_atomic_state *state)
 911{
 912	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 913									   plane);
 914	struct drm_crtc *crtc = new_state->crtc;
 915	struct vop_win *vop_win = to_vop_win(plane);
 916	const struct vop_win_data *win = vop_win->data;
 917	const struct vop_win_yuv2yuv_data *win_yuv2yuv = vop_win->yuv2yuv_data;
 918	struct vop *vop = to_vop(new_state->crtc);
 919	struct drm_framebuffer *fb = new_state->fb;
 920	unsigned int actual_w, actual_h;
 921	unsigned int dsp_stx, dsp_sty;
 922	uint32_t act_info, dsp_info, dsp_st;
 923	struct drm_rect *src = &new_state->src;
 924	struct drm_rect *dest = &new_state->dst;
 925	struct drm_gem_object *obj, *uv_obj;
 926	struct rockchip_gem_object *rk_obj, *rk_uv_obj;
 927	unsigned long offset;
 928	dma_addr_t dma_addr;
 929	uint32_t val;
 930	bool rb_swap, uv_swap;
 931	int win_index = VOP_WIN_TO_INDEX(vop_win);
 932	int format;
 933	int is_yuv = fb->format->is_yuv;
 934	int i;
 935
 936	/*
 937	 * can't update plane when vop is disabled.
 938	 */
 939	if (WARN_ON(!crtc))
 940		return;
 941
 942	if (WARN_ON(!vop->is_enabled))
 943		return;
 944
 945	if (!new_state->visible) {
 946		vop_plane_atomic_disable(plane, state);
 947		return;
 948	}
 949
 950	obj = fb->obj[0];
 951	rk_obj = to_rockchip_obj(obj);
 952
 953	actual_w = drm_rect_width(src) >> 16;
 954	actual_h = drm_rect_height(src) >> 16;
 955	act_info = (actual_h - 1) << 16 | ((actual_w - 1) & 0xffff);
 956
 957	dsp_info = (drm_rect_height(dest) - 1) << 16;
 958	dsp_info |= (drm_rect_width(dest) - 1) & 0xffff;
 959
 960	dsp_stx = dest->x1 + crtc->mode.htotal - crtc->mode.hsync_start;
 961	dsp_sty = dest->y1 + crtc->mode.vtotal - crtc->mode.vsync_start;
 962	dsp_st = dsp_sty << 16 | (dsp_stx & 0xffff);
 963
 964	if (fb->format->char_per_block[0])
 965		offset = drm_format_info_min_pitch(fb->format, 0,
 966						   src->x1 >> 16);
 967	else
 968		offset = (src->x1 >> 16) * fb->format->cpp[0];
 969
 970	offset += (src->y1 >> 16) * fb->pitches[0];
 971	dma_addr = rk_obj->dma_addr + offset + fb->offsets[0];
 972
 973	/*
 974	 * For y-mirroring we need to move address
 975	 * to the beginning of the last line.
 976	 */
 977	if (new_state->rotation & DRM_MODE_REFLECT_Y)
 978		dma_addr += (actual_h - 1) * fb->pitches[0];
 979
 980	format = vop_convert_format(fb->format->format);
 981
 982	spin_lock(&vop->reg_lock);
 983
 984	if (rockchip_afbc(fb->modifier)) {
 985		int afbc_format = vop_convert_afbc_format(fb->format->format);
 986
 987		VOP_AFBC_SET(vop, format, afbc_format | AFBC_TILE_16x16);
 988		VOP_AFBC_SET(vop, hreg_block_split, 0);
 989		VOP_AFBC_SET(vop, win_sel, VOP_WIN_TO_INDEX(vop_win));
 990		VOP_AFBC_SET(vop, hdr_ptr, dma_addr);
 991		VOP_AFBC_SET(vop, pic_size, act_info);
 992	}
 993
 994	VOP_WIN_SET(vop, win, format, format);
 995	VOP_WIN_SET(vop, win, fmt_10, is_fmt_10(fb->format->format));
 996	VOP_WIN_SET(vop, win, yrgb_vir, DIV_ROUND_UP(fb->pitches[0], 4));
 997	VOP_WIN_SET(vop, win, yrgb_mst, dma_addr);
 998	VOP_WIN_YUV2YUV_SET(vop, win_yuv2yuv, y2r_en, is_yuv);
 999	VOP_WIN_SET(vop, win, y_mir_en,
1000		    (new_state->rotation & DRM_MODE_REFLECT_Y) ? 1 : 0);
1001	VOP_WIN_SET(vop, win, x_mir_en,
1002		    (new_state->rotation & DRM_MODE_REFLECT_X) ? 1 : 0);
1003
1004	if (is_yuv) {
1005		uv_obj = fb->obj[1];
1006		rk_uv_obj = to_rockchip_obj(uv_obj);
1007
1008		if (fb->format->char_per_block[1])
1009			offset = drm_format_info_min_pitch(fb->format, 1,
1010							   src->x1 >> 16);
1011		else
1012			offset = (src->x1 >> 16) * fb->format->cpp[1];
1013		offset /= fb->format->hsub;
1014		offset += (src->y1 >> 16) * fb->pitches[1] / fb->format->vsub;
1015
1016		dma_addr = rk_uv_obj->dma_addr + offset + fb->offsets[1];
1017		VOP_WIN_SET(vop, win, uv_vir, DIV_ROUND_UP(fb->pitches[1], 4));
1018		VOP_WIN_SET(vop, win, uv_mst, dma_addr);
1019
1020		for (i = 0; i < NUM_YUV2YUV_COEFFICIENTS; i++) {
1021			VOP_WIN_YUV2YUV_COEFFICIENT_SET(vop,
1022							win_yuv2yuv,
1023							y2r_coefficients[i],
1024							bt601_yuv2rgb[i]);
1025		}
1026
1027		uv_swap = has_uv_swapped(fb->format->format);
1028		VOP_WIN_SET(vop, win, uv_swap, uv_swap);
1029	}
1030
1031	if (win->phy->scl)
1032		scl_vop_cal_scl_fac(vop, win, actual_w, actual_h,
1033				    drm_rect_width(dest), drm_rect_height(dest),
1034				    fb->format);
1035
1036	VOP_WIN_SET(vop, win, act_info, act_info);
1037	VOP_WIN_SET(vop, win, dsp_info, dsp_info);
1038	VOP_WIN_SET(vop, win, dsp_st, dsp_st);
1039
1040	rb_swap = has_rb_swapped(vop->data->version, fb->format->format);
1041	VOP_WIN_SET(vop, win, rb_swap, rb_swap);
1042
1043	/*
1044	 * Blending win0 with the background color doesn't seem to work
1045	 * correctly. We only get the background color, no matter the contents
1046	 * of the win0 framebuffer.  However, blending pre-multiplied color
1047	 * with the default opaque black default background color is a no-op,
1048	 * so we can just disable blending to get the correct result.
1049	 */
1050	if (fb->format->has_alpha && win_index > 0) {
1051		VOP_WIN_SET(vop, win, dst_alpha_ctl,
1052			    DST_FACTOR_M0(ALPHA_SRC_INVERSE));
1053		val = SRC_ALPHA_EN(1) | SRC_COLOR_M0(ALPHA_SRC_PRE_MUL) |
1054			SRC_ALPHA_M0(ALPHA_STRAIGHT) |
1055			SRC_BLEND_M0(ALPHA_PER_PIX) |
1056			SRC_ALPHA_CAL_M0(ALPHA_NO_SATURATION) |
1057			SRC_FACTOR_M0(ALPHA_ONE);
1058		VOP_WIN_SET(vop, win, src_alpha_ctl, val);
1059
1060		VOP_WIN_SET(vop, win, alpha_pre_mul, ALPHA_SRC_PRE_MUL);
1061		VOP_WIN_SET(vop, win, alpha_mode, ALPHA_PER_PIX);
1062		VOP_WIN_SET(vop, win, alpha_en, 1);
1063	} else {
1064		VOP_WIN_SET(vop, win, src_alpha_ctl, SRC_ALPHA_EN(0));
1065		VOP_WIN_SET(vop, win, alpha_en, 0);
1066	}
1067
1068	VOP_WIN_SET(vop, win, enable, 1);
1069	vop->win_enabled |= BIT(win_index);
1070	spin_unlock(&vop->reg_lock);
1071}
1072
1073static int vop_plane_atomic_async_check(struct drm_plane *plane,
1074					struct drm_atomic_state *state, bool flip)
1075{
1076	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
1077										 plane);
1078	struct vop_win *vop_win = to_vop_win(plane);
1079	const struct vop_win_data *win = vop_win->data;
1080	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
1081					DRM_PLANE_NO_SCALING;
1082	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
1083					DRM_PLANE_NO_SCALING;
1084	struct drm_crtc_state *crtc_state;
1085
1086	if (plane != new_plane_state->crtc->cursor)
1087		return -EINVAL;
1088
1089	if (!plane->state)
1090		return -EINVAL;
1091
1092	if (!plane->state->fb)
1093		return -EINVAL;
1094
1095	crtc_state = drm_atomic_get_new_crtc_state(state,
1096						   new_plane_state->crtc);
1097
1098	/* Special case for asynchronous cursor updates. */
1099	if (!crtc_state)
1100		crtc_state = plane->crtc->state;
1101
1102	return drm_atomic_helper_check_plane_state(plane->state, crtc_state,
1103						   min_scale, max_scale,
1104						   true, true);
1105}
1106
1107static void vop_plane_atomic_async_update(struct drm_plane *plane,
1108					  struct drm_atomic_state *state)
1109{
1110	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
1111									   plane);
1112	struct vop *vop = to_vop(plane->state->crtc);
1113	struct drm_framebuffer *old_fb = plane->state->fb;
1114
1115	plane->state->crtc_x = new_state->crtc_x;
1116	plane->state->crtc_y = new_state->crtc_y;
1117	plane->state->crtc_h = new_state->crtc_h;
1118	plane->state->crtc_w = new_state->crtc_w;
1119	plane->state->src_x = new_state->src_x;
1120	plane->state->src_y = new_state->src_y;
1121	plane->state->src_h = new_state->src_h;
1122	plane->state->src_w = new_state->src_w;
1123	swap(plane->state->fb, new_state->fb);
1124
1125	if (vop->is_enabled) {
1126		vop_plane_atomic_update(plane, state);
1127		spin_lock(&vop->reg_lock);
1128		vop_cfg_done(vop);
1129		spin_unlock(&vop->reg_lock);
1130
1131		/*
1132		 * A scanout can still be occurring, so we can't drop the
1133		 * reference to the old framebuffer. To solve this we get a
1134		 * reference to old_fb and set a worker to release it later.
1135		 * FIXME: if we perform 500 async_update calls before the
1136		 * vblank, then we can have 500 different framebuffers waiting
1137		 * to be released.
1138		 */
1139		if (old_fb && plane->state->fb != old_fb) {
1140			drm_framebuffer_get(old_fb);
1141			WARN_ON(drm_crtc_vblank_get(plane->state->crtc) != 0);
1142			drm_flip_work_queue(&vop->fb_unref_work, old_fb);
1143			set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
1144		}
1145	}
1146}
1147
1148static const struct drm_plane_helper_funcs plane_helper_funcs = {
1149	.atomic_check = vop_plane_atomic_check,
1150	.atomic_update = vop_plane_atomic_update,
1151	.atomic_disable = vop_plane_atomic_disable,
1152	.atomic_async_check = vop_plane_atomic_async_check,
1153	.atomic_async_update = vop_plane_atomic_async_update,
1154};
1155
1156static const struct drm_plane_funcs vop_plane_funcs = {
1157	.update_plane	= drm_atomic_helper_update_plane,
1158	.disable_plane	= drm_atomic_helper_disable_plane,
1159	.destroy = drm_plane_cleanup,
1160	.reset = drm_atomic_helper_plane_reset,
1161	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
1162	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
1163	.format_mod_supported = rockchip_mod_supported,
1164};
1165
1166static int vop_crtc_enable_vblank(struct drm_crtc *crtc)
1167{
1168	struct vop *vop = to_vop(crtc);
1169	unsigned long flags;
1170
1171	if (WARN_ON(!vop->is_enabled))
1172		return -EPERM;
1173
1174	spin_lock_irqsave(&vop->irq_lock, flags);
1175
1176	VOP_INTR_SET_TYPE(vop, clear, FS_INTR, 1);
1177	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 1);
1178
1179	spin_unlock_irqrestore(&vop->irq_lock, flags);
1180
1181	return 0;
1182}
1183
1184static void vop_crtc_disable_vblank(struct drm_crtc *crtc)
1185{
1186	struct vop *vop = to_vop(crtc);
1187	unsigned long flags;
1188
1189	if (WARN_ON(!vop->is_enabled))
1190		return;
1191
1192	spin_lock_irqsave(&vop->irq_lock, flags);
1193
1194	VOP_INTR_SET_TYPE(vop, enable, FS_INTR, 0);
1195
1196	spin_unlock_irqrestore(&vop->irq_lock, flags);
1197}
1198
1199static enum drm_mode_status vop_crtc_mode_valid(struct drm_crtc *crtc,
1200						const struct drm_display_mode *mode)
1201{
1202	struct vop *vop = to_vop(crtc);
1203
1204	if (vop->data->max_output.width && mode->hdisplay > vop->data->max_output.width)
1205		return MODE_BAD_HVALUE;
1206
1207	return MODE_OK;
1208}
1209
1210static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
1211				const struct drm_display_mode *mode,
1212				struct drm_display_mode *adjusted_mode)
1213{
1214	struct vop *vop = to_vop(crtc);
1215	unsigned long rate;
1216
1217	/*
1218	 * Clock craziness.
1219	 *
1220	 * Key points:
1221	 *
1222	 * - DRM works in kHz.
1223	 * - Clock framework works in Hz.
1224	 * - Rockchip's clock driver picks the clock rate that is the
1225	 *   same _OR LOWER_ than the one requested.
1226	 *
1227	 * Action plan:
1228	 *
1229	 * 1. Try to set the exact rate first, and confirm the clock framework
1230	 *    can provide it.
1231	 *
1232	 * 2. If the clock framework cannot provide the exact rate, we should
1233	 *    add 999 Hz to the requested rate.  That way if the clock we need
1234	 *    is 60000001 Hz (~60 MHz) and DRM tells us to make 60000 kHz then
1235	 *    the clock framework will actually give us the right clock.
1236	 *
1237	 * 3. Get the clock framework to round the rate for us to tell us
1238	 *    what it will actually make.
1239	 *
1240	 * 4. Store the rounded up rate so that we don't need to worry about
1241	 *    this in the actual clk_set_rate().
1242	 */
1243	rate = clk_round_rate(vop->dclk, adjusted_mode->clock * 1000);
1244	if (rate / 1000 != adjusted_mode->clock)
1245		rate = clk_round_rate(vop->dclk,
1246				      adjusted_mode->clock * 1000 + 999);
1247	adjusted_mode->clock = DIV_ROUND_UP(rate, 1000);
1248
1249	return true;
1250}
1251
1252static bool vop_dsp_lut_is_enabled(struct vop *vop)
1253{
1254	return vop_read_reg(vop, 0, &vop->data->common->dsp_lut_en);
1255}
1256
1257static u32 vop_lut_buffer_index(struct vop *vop)
1258{
1259	return vop_read_reg(vop, 0, &vop->data->common->lut_buffer_index);
1260}
1261
1262static void vop_crtc_write_gamma_lut(struct vop *vop, struct drm_crtc *crtc)
1263{
1264	struct drm_color_lut *lut = crtc->state->gamma_lut->data;
1265	unsigned int i, bpc = ilog2(vop->data->lut_size);
1266
1267	for (i = 0; i < crtc->gamma_size; i++) {
1268		u32 word;
1269
1270		word = (drm_color_lut_extract(lut[i].red, bpc) << (2 * bpc)) |
1271		       (drm_color_lut_extract(lut[i].green, bpc) << bpc) |
1272			drm_color_lut_extract(lut[i].blue, bpc);
1273		writel(word, vop->lut_regs + i * 4);
1274	}
1275}
1276
1277static void vop_crtc_gamma_set(struct vop *vop, struct drm_crtc *crtc,
1278			       struct drm_crtc_state *old_state)
1279{
1280	struct drm_crtc_state *state = crtc->state;
1281	unsigned int idle;
1282	u32 lut_idx, old_idx;
1283	int ret;
1284
1285	if (!vop->lut_regs)
1286		return;
1287
1288	if (!state->gamma_lut || !VOP_HAS_REG(vop, common, update_gamma_lut)) {
1289		/*
1290		 * To disable gamma (gamma_lut is null) or to write
1291		 * an update to the LUT, clear dsp_lut_en.
1292		 */
1293		spin_lock(&vop->reg_lock);
1294		VOP_REG_SET(vop, common, dsp_lut_en, 0);
1295		vop_cfg_done(vop);
1296		spin_unlock(&vop->reg_lock);
1297
1298		/*
1299		 * In order to write the LUT to the internal memory,
1300		 * we need to first make sure the dsp_lut_en bit is cleared.
1301		 */
1302		ret = readx_poll_timeout(vop_dsp_lut_is_enabled, vop,
1303					 idle, !idle, 5, 30 * 1000);
1304		if (ret) {
1305			DRM_DEV_ERROR(vop->dev, "display LUT RAM enable timeout!\n");
1306			return;
1307		}
1308
1309		if (!state->gamma_lut)
1310			return;
1311	} else {
1312		/*
1313		 * On RK3399 the gamma LUT can updated without clearing dsp_lut_en,
1314		 * by setting update_gamma_lut then waiting for lut_buffer_index change
1315		 */
1316		old_idx = vop_lut_buffer_index(vop);
1317	}
1318
1319	spin_lock(&vop->reg_lock);
1320	vop_crtc_write_gamma_lut(vop, crtc);
1321	VOP_REG_SET(vop, common, dsp_lut_en, 1);
1322	VOP_REG_SET(vop, common, update_gamma_lut, 1);
1323	vop_cfg_done(vop);
1324	spin_unlock(&vop->reg_lock);
1325
1326	if (VOP_HAS_REG(vop, common, update_gamma_lut)) {
1327		ret = readx_poll_timeout(vop_lut_buffer_index, vop,
1328					 lut_idx, lut_idx != old_idx, 5, 30 * 1000);
1329		if (ret) {
1330			DRM_DEV_ERROR(vop->dev, "gamma LUT update timeout!\n");
1331			return;
1332		}
1333
1334		/*
1335		 * update_gamma_lut is auto cleared by HW, but write 0 to clear the bit
1336		 * in our backup of the regs.
1337		 */
1338		spin_lock(&vop->reg_lock);
1339		VOP_REG_SET(vop, common, update_gamma_lut, 0);
1340		spin_unlock(&vop->reg_lock);
1341	}
1342}
1343
1344static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
1345				  struct drm_atomic_state *state)
1346{
1347	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1348									  crtc);
1349	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1350									      crtc);
1351	struct vop *vop = to_vop(crtc);
1352
1353	/*
1354	 * Only update GAMMA if the 'active' flag is not changed,
1355	 * otherwise it's updated by .atomic_enable.
1356	 */
1357	if (crtc_state->color_mgmt_changed &&
1358	    !crtc_state->active_changed)
1359		vop_crtc_gamma_set(vop, crtc, old_crtc_state);
1360}
1361
1362static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
1363				   struct drm_atomic_state *state)
1364{
1365	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
1366									 crtc);
1367	struct vop *vop = to_vop(crtc);
1368	const struct vop_data *vop_data = vop->data;
1369	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc->state);
1370	struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode;
1371	u16 hsync_len = adjusted_mode->hsync_end - adjusted_mode->hsync_start;
1372	u16 hdisplay = adjusted_mode->hdisplay;
1373	u16 htotal = adjusted_mode->htotal;
1374	u16 hact_st = adjusted_mode->htotal - adjusted_mode->hsync_start;
1375	u16 hact_end = hact_st + hdisplay;
1376	u16 vdisplay = adjusted_mode->vdisplay;
1377	u16 vtotal = adjusted_mode->vtotal;
1378	u16 vsync_len = adjusted_mode->vsync_end - adjusted_mode->vsync_start;
1379	u16 vact_st = adjusted_mode->vtotal - adjusted_mode->vsync_start;
1380	u16 vact_end = vact_st + vdisplay;
1381	uint32_t pin_pol, val;
1382	int dither_bpc = s->output_bpc ? s->output_bpc : 10;
1383	int ret;
1384
1385	if (old_state && old_state->self_refresh_active) {
1386		drm_crtc_vblank_on(crtc);
1387		rockchip_drm_set_win_enabled(crtc, true);
1388		return;
1389	}
1390
1391	mutex_lock(&vop->vop_lock);
1392
1393	WARN_ON(vop->event);
1394
1395	ret = vop_enable(crtc, old_state);
1396	if (ret) {
1397		mutex_unlock(&vop->vop_lock);
1398		DRM_DEV_ERROR(vop->dev, "Failed to enable vop (%d)\n", ret);
1399		return;
1400	}
1401	pin_pol = (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ?
1402		   BIT(HSYNC_POSITIVE) : 0;
1403	pin_pol |= (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ?
1404		   BIT(VSYNC_POSITIVE) : 0;
1405	VOP_REG_SET(vop, output, pin_pol, pin_pol);
1406	VOP_REG_SET(vop, output, mipi_dual_channel_en, 0);
1407
1408	switch (s->output_type) {
1409	case DRM_MODE_CONNECTOR_LVDS:
1410		VOP_REG_SET(vop, output, rgb_dclk_pol, 1);
1411		VOP_REG_SET(vop, output, rgb_pin_pol, pin_pol);
1412		VOP_REG_SET(vop, output, rgb_en, 1);
1413		break;
1414	case DRM_MODE_CONNECTOR_eDP:
1415		VOP_REG_SET(vop, output, edp_dclk_pol, 1);
1416		VOP_REG_SET(vop, output, edp_pin_pol, pin_pol);
1417		VOP_REG_SET(vop, output, edp_en, 1);
1418		break;
1419	case DRM_MODE_CONNECTOR_HDMIA:
1420		VOP_REG_SET(vop, output, hdmi_dclk_pol, 1);
1421		VOP_REG_SET(vop, output, hdmi_pin_pol, pin_pol);
1422		VOP_REG_SET(vop, output, hdmi_en, 1);
1423		break;
1424	case DRM_MODE_CONNECTOR_DSI:
1425		VOP_REG_SET(vop, output, mipi_dclk_pol, 1);
1426		VOP_REG_SET(vop, output, mipi_pin_pol, pin_pol);
1427		VOP_REG_SET(vop, output, mipi_en, 1);
1428		VOP_REG_SET(vop, output, mipi_dual_channel_en,
1429			    !!(s->output_flags & ROCKCHIP_OUTPUT_DSI_DUAL));
1430		break;
1431	case DRM_MODE_CONNECTOR_DisplayPort:
1432		VOP_REG_SET(vop, output, dp_dclk_pol, 0);
1433		VOP_REG_SET(vop, output, dp_pin_pol, pin_pol);
1434		VOP_REG_SET(vop, output, dp_en, 1);
1435		break;
1436	default:
1437		DRM_DEV_ERROR(vop->dev, "unsupported connector_type [%d]\n",
1438			      s->output_type);
1439	}
1440
1441	/*
1442	 * if vop is not support RGB10 output, need force RGB10 to RGB888.
1443	 */
1444	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA &&
1445	    !(vop_data->feature & VOP_FEATURE_OUTPUT_RGB10))
1446		s->output_mode = ROCKCHIP_OUT_MODE_P888;
1447
1448	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA && dither_bpc <= 8)
1449		VOP_REG_SET(vop, common, pre_dither_down, 1);
1450	else
1451		VOP_REG_SET(vop, common, pre_dither_down, 0);
1452
1453	if (dither_bpc == 6) {
1454		VOP_REG_SET(vop, common, dither_down_sel, DITHER_DOWN_ALLEGRO);
1455		VOP_REG_SET(vop, common, dither_down_mode, RGB888_TO_RGB666);
1456		VOP_REG_SET(vop, common, dither_down_en, 1);
1457	} else {
1458		VOP_REG_SET(vop, common, dither_down_en, 0);
1459	}
1460
1461	VOP_REG_SET(vop, common, out_mode, s->output_mode);
1462
1463	VOP_REG_SET(vop, modeset, htotal_pw, (htotal << 16) | hsync_len);
1464	val = hact_st << 16;
1465	val |= hact_end;
1466	VOP_REG_SET(vop, modeset, hact_st_end, val);
1467	VOP_REG_SET(vop, modeset, hpost_st_end, val);
1468
1469	VOP_REG_SET(vop, modeset, vtotal_pw, (vtotal << 16) | vsync_len);
1470	val = vact_st << 16;
1471	val |= vact_end;
1472	VOP_REG_SET(vop, modeset, vact_st_end, val);
1473	VOP_REG_SET(vop, modeset, vpost_st_end, val);
1474
1475	VOP_REG_SET(vop, intr, line_flag_num[0], vact_end);
1476
1477	clk_set_rate(vop->dclk, adjusted_mode->clock * 1000);
1478
1479	VOP_REG_SET(vop, common, standby, 0);
1480	mutex_unlock(&vop->vop_lock);
1481
1482	/*
1483	 * If we have a GAMMA LUT in the state, then let's make sure
1484	 * it's updated. We might be coming out of suspend,
1485	 * which means the LUT internal memory needs to be re-written.
1486	 */
1487	if (crtc->state->gamma_lut)
1488		vop_crtc_gamma_set(vop, crtc, old_state);
1489}
1490
1491static bool vop_fs_irq_is_pending(struct vop *vop)
1492{
1493	return VOP_INTR_GET_TYPE(vop, status, FS_INTR);
1494}
1495
1496static void vop_wait_for_irq_handler(struct vop *vop)
1497{
1498	bool pending;
1499	int ret;
1500
1501	/*
1502	 * Spin until frame start interrupt status bit goes low, which means
1503	 * that interrupt handler was invoked and cleared it. The timeout of
1504	 * 10 msecs is really too long, but it is just a safety measure if
1505	 * something goes really wrong. The wait will only happen in the very
1506	 * unlikely case of a vblank happening exactly at the same time and
1507	 * shouldn't exceed microseconds range.
1508	 */
1509	ret = readx_poll_timeout_atomic(vop_fs_irq_is_pending, vop, pending,
1510					!pending, 0, 10 * 1000);
1511	if (ret)
1512		DRM_DEV_ERROR(vop->dev, "VOP vblank IRQ stuck for 10 ms\n");
1513
1514	synchronize_irq(vop->irq);
1515}
1516
1517static int vop_crtc_atomic_check(struct drm_crtc *crtc,
1518				 struct drm_atomic_state *state)
1519{
1520	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1521									  crtc);
1522	struct vop *vop = to_vop(crtc);
1523	struct drm_plane *plane;
1524	struct drm_plane_state *plane_state;
1525	struct rockchip_crtc_state *s;
1526	int afbc_planes = 0;
1527
1528	if (vop->lut_regs && crtc_state->color_mgmt_changed &&
1529	    crtc_state->gamma_lut) {
1530		unsigned int len;
1531
1532		len = drm_color_lut_size(crtc_state->gamma_lut);
1533		if (len != crtc->gamma_size) {
1534			DRM_DEBUG_KMS("Invalid LUT size; got %d, expected %d\n",
1535				      len, crtc->gamma_size);
1536			return -EINVAL;
1537		}
1538	}
1539
1540	drm_atomic_crtc_state_for_each_plane(plane, crtc_state) {
1541		plane_state =
1542			drm_atomic_get_plane_state(crtc_state->state, plane);
1543		if (IS_ERR(plane_state)) {
1544			DRM_DEBUG_KMS("Cannot get plane state for plane %s\n",
1545				      plane->name);
1546			return PTR_ERR(plane_state);
1547		}
1548
1549		if (drm_is_afbc(plane_state->fb->modifier))
1550			++afbc_planes;
1551	}
1552
1553	if (afbc_planes > 1) {
1554		DRM_DEBUG_KMS("Invalid number of AFBC planes; got %d, expected at most 1\n", afbc_planes);
1555		return -EINVAL;
1556	}
1557
1558	s = to_rockchip_crtc_state(crtc_state);
1559	s->enable_afbc = afbc_planes > 0;
1560
1561	return 0;
1562}
1563
1564static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
1565				  struct drm_atomic_state *state)
1566{
1567	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1568									      crtc);
1569	struct drm_atomic_state *old_state = old_crtc_state->state;
1570	struct drm_plane_state *old_plane_state, *new_plane_state;
1571	struct vop *vop = to_vop(crtc);
1572	struct drm_plane *plane;
1573	struct rockchip_crtc_state *s;
1574	int i;
1575
1576	if (WARN_ON(!vop->is_enabled))
1577		return;
1578
1579	spin_lock(&vop->reg_lock);
1580
1581	/* Enable AFBC if there is some AFBC window, disable otherwise. */
1582	s = to_rockchip_crtc_state(crtc->state);
1583	VOP_AFBC_SET(vop, enable, s->enable_afbc);
1584	vop_cfg_done(vop);
1585
1586	/* Ack the DMA transfer of the previous frame (RK3066). */
1587	if (VOP_HAS_REG(vop, common, dma_stop))
1588		VOP_REG_SET(vop, common, dma_stop, 0);
1589
1590	spin_unlock(&vop->reg_lock);
1591
1592	/*
1593	 * There is a (rather unlikely) possiblity that a vblank interrupt
1594	 * fired before we set the cfg_done bit. To avoid spuriously
1595	 * signalling flip completion we need to wait for it to finish.
1596	 */
1597	vop_wait_for_irq_handler(vop);
1598
1599	spin_lock_irq(&crtc->dev->event_lock);
1600	if (crtc->state->event) {
1601		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
1602		WARN_ON(vop->event);
1603
1604		vop->event = crtc->state->event;
1605		crtc->state->event = NULL;
1606	}
1607	spin_unlock_irq(&crtc->dev->event_lock);
1608
1609	for_each_oldnew_plane_in_state(old_state, plane, old_plane_state,
1610				       new_plane_state, i) {
1611		if (!old_plane_state->fb)
1612			continue;
1613
1614		if (old_plane_state->fb == new_plane_state->fb)
1615			continue;
1616
1617		drm_framebuffer_get(old_plane_state->fb);
1618		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
1619		drm_flip_work_queue(&vop->fb_unref_work, old_plane_state->fb);
1620		set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
1621	}
1622}
1623
1624static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
1625	.mode_valid = vop_crtc_mode_valid,
1626	.mode_fixup = vop_crtc_mode_fixup,
1627	.atomic_check = vop_crtc_atomic_check,
1628	.atomic_begin = vop_crtc_atomic_begin,
1629	.atomic_flush = vop_crtc_atomic_flush,
1630	.atomic_enable = vop_crtc_atomic_enable,
1631	.atomic_disable = vop_crtc_atomic_disable,
1632};
1633
1634static struct drm_crtc_state *vop_crtc_duplicate_state(struct drm_crtc *crtc)
1635{
1636	struct rockchip_crtc_state *rockchip_state;
1637
1638	if (WARN_ON(!crtc->state))
1639		return NULL;
1640
1641	rockchip_state = kmemdup(to_rockchip_crtc_state(crtc->state),
1642				 sizeof(*rockchip_state), GFP_KERNEL);
1643	if (!rockchip_state)
1644		return NULL;
1645
1646	__drm_atomic_helper_crtc_duplicate_state(crtc, &rockchip_state->base);
1647	return &rockchip_state->base;
1648}
1649
1650static void vop_crtc_destroy_state(struct drm_crtc *crtc,
1651				   struct drm_crtc_state *state)
1652{
1653	struct rockchip_crtc_state *s = to_rockchip_crtc_state(state);
1654
1655	__drm_atomic_helper_crtc_destroy_state(&s->base);
1656	kfree(s);
1657}
1658
1659static void vop_crtc_reset(struct drm_crtc *crtc)
1660{
1661	struct rockchip_crtc_state *crtc_state =
1662		kzalloc(sizeof(*crtc_state), GFP_KERNEL);
1663
1664	if (crtc->state)
1665		vop_crtc_destroy_state(crtc, crtc->state);
1666
1667	if (crtc_state)
1668		__drm_atomic_helper_crtc_reset(crtc, &crtc_state->base);
1669	else
1670		__drm_atomic_helper_crtc_reset(crtc, NULL);
1671}
1672
1673#ifdef CONFIG_DRM_ANALOGIX_DP
1674static struct drm_connector *vop_get_edp_connector(struct vop *vop)
1675{
1676	struct drm_connector *connector;
1677	struct drm_connector_list_iter conn_iter;
1678
1679	drm_connector_list_iter_begin(vop->drm_dev, &conn_iter);
1680	drm_for_each_connector_iter(connector, &conn_iter) {
1681		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
1682			drm_connector_list_iter_end(&conn_iter);
1683			return connector;
1684		}
1685	}
1686	drm_connector_list_iter_end(&conn_iter);
1687
1688	return NULL;
1689}
1690
1691static int vop_crtc_set_crc_source(struct drm_crtc *crtc,
1692				   const char *source_name)
1693{
1694	struct vop *vop = to_vop(crtc);
1695	struct drm_connector *connector;
1696	int ret;
1697
1698	connector = vop_get_edp_connector(vop);
1699	if (!connector)
1700		return -EINVAL;
1701
1702	if (source_name && strcmp(source_name, "auto") == 0)
1703		ret = analogix_dp_start_crc(connector);
1704	else if (!source_name)
1705		ret = analogix_dp_stop_crc(connector);
1706	else
1707		ret = -EINVAL;
1708
1709	return ret;
1710}
1711
1712static int
1713vop_crtc_verify_crc_source(struct drm_crtc *crtc, const char *source_name,
1714			   size_t *values_cnt)
1715{
1716	if (source_name && strcmp(source_name, "auto") != 0)
1717		return -EINVAL;
1718
1719	*values_cnt = 3;
1720	return 0;
1721}
1722
1723#else
1724static int vop_crtc_set_crc_source(struct drm_crtc *crtc,
1725				   const char *source_name)
1726{
1727	return -ENODEV;
1728}
1729
1730static int
1731vop_crtc_verify_crc_source(struct drm_crtc *crtc, const char *source_name,
1732			   size_t *values_cnt)
1733{
1734	return -ENODEV;
1735}
1736#endif
1737
1738static const struct drm_crtc_funcs vop_crtc_funcs = {
1739	.set_config = drm_atomic_helper_set_config,
1740	.page_flip = drm_atomic_helper_page_flip,
1741	.destroy = drm_crtc_cleanup,
1742	.reset = vop_crtc_reset,
1743	.atomic_duplicate_state = vop_crtc_duplicate_state,
1744	.atomic_destroy_state = vop_crtc_destroy_state,
1745	.enable_vblank = vop_crtc_enable_vblank,
1746	.disable_vblank = vop_crtc_disable_vblank,
1747	.set_crc_source = vop_crtc_set_crc_source,
1748	.verify_crc_source = vop_crtc_verify_crc_source,
1749};
1750
1751static void vop_fb_unref_worker(struct drm_flip_work *work, void *val)
1752{
1753	struct vop *vop = container_of(work, struct vop, fb_unref_work);
1754	struct drm_framebuffer *fb = val;
1755
1756	drm_crtc_vblank_put(&vop->crtc);
1757	drm_framebuffer_put(fb);
1758}
1759
1760static void vop_handle_vblank(struct vop *vop)
1761{
1762	struct drm_device *drm = vop->drm_dev;
1763	struct drm_crtc *crtc = &vop->crtc;
1764
1765	spin_lock(&drm->event_lock);
1766	if (vop->event) {
1767		drm_crtc_send_vblank_event(crtc, vop->event);
1768		drm_crtc_vblank_put(crtc);
1769		vop->event = NULL;
1770	}
1771	spin_unlock(&drm->event_lock);
1772
1773	if (test_and_clear_bit(VOP_PENDING_FB_UNREF, &vop->pending))
1774		drm_flip_work_commit(&vop->fb_unref_work, system_unbound_wq);
1775}
1776
1777static irqreturn_t vop_isr(int irq, void *data)
1778{
1779	struct vop *vop = data;
1780	struct drm_crtc *crtc = &vop->crtc;
1781	uint32_t active_irqs;
1782	int ret = IRQ_NONE;
1783
1784	/*
1785	 * The irq is shared with the iommu. If the runtime-pm state of the
1786	 * vop-device is disabled the irq has to be targeted at the iommu.
1787	 */
1788	if (!pm_runtime_get_if_in_use(vop->dev))
1789		return IRQ_NONE;
1790
1791	if (vop_core_clks_enable(vop)) {
1792		DRM_DEV_ERROR_RATELIMITED(vop->dev, "couldn't enable clocks\n");
1793		goto out;
1794	}
1795
1796	/*
1797	 * interrupt register has interrupt status, enable and clear bits, we
1798	 * must hold irq_lock to avoid a race with enable/disable_vblank().
1799	*/
1800	spin_lock(&vop->irq_lock);
1801
1802	active_irqs = VOP_INTR_GET_TYPE(vop, status, INTR_MASK);
1803	/* Clear all active interrupt sources */
1804	if (active_irqs)
1805		VOP_INTR_SET_TYPE(vop, clear, active_irqs, 1);
1806
1807	spin_unlock(&vop->irq_lock);
1808
1809	/* This is expected for vop iommu irqs, since the irq is shared */
1810	if (!active_irqs)
1811		goto out_disable;
1812
1813	if (active_irqs & DSP_HOLD_VALID_INTR) {
1814		complete(&vop->dsp_hold_completion);
1815		active_irqs &= ~DSP_HOLD_VALID_INTR;
1816		ret = IRQ_HANDLED;
1817	}
1818
1819	if (active_irqs & LINE_FLAG_INTR) {
1820		complete(&vop->line_flag_completion);
1821		active_irqs &= ~LINE_FLAG_INTR;
1822		ret = IRQ_HANDLED;
1823	}
1824
1825	if (active_irqs & FS_INTR) {
1826		drm_crtc_handle_vblank(crtc);
1827		vop_handle_vblank(vop);
1828		active_irqs &= ~FS_INTR;
1829		ret = IRQ_HANDLED;
1830	}
1831
1832	/* Unhandled irqs are spurious. */
1833	if (active_irqs)
1834		DRM_DEV_ERROR(vop->dev, "Unknown VOP IRQs: %#02x\n",
1835			      active_irqs);
1836
1837out_disable:
1838	vop_core_clks_disable(vop);
1839out:
1840	pm_runtime_put(vop->dev);
1841	return ret;
1842}
1843
1844static void vop_plane_add_properties(struct drm_plane *plane,
1845				     const struct vop_win_data *win_data)
1846{
1847	unsigned int flags = 0;
1848
1849	flags |= VOP_WIN_HAS_REG(win_data, x_mir_en) ? DRM_MODE_REFLECT_X : 0;
1850	flags |= VOP_WIN_HAS_REG(win_data, y_mir_en) ? DRM_MODE_REFLECT_Y : 0;
1851	if (flags)
1852		drm_plane_create_rotation_property(plane, DRM_MODE_ROTATE_0,
1853						   DRM_MODE_ROTATE_0 | flags);
1854}
1855
1856static int vop_create_crtc(struct vop *vop)
1857{
1858	const struct vop_data *vop_data = vop->data;
1859	struct device *dev = vop->dev;
1860	struct drm_device *drm_dev = vop->drm_dev;
1861	struct drm_plane *primary = NULL, *cursor = NULL, *plane, *tmp;
1862	struct drm_crtc *crtc = &vop->crtc;
1863	struct device_node *port;
1864	int ret;
1865	int i;
1866
1867	/*
1868	 * Create drm_plane for primary and cursor planes first, since we need
1869	 * to pass them to drm_crtc_init_with_planes, which sets the
1870	 * "possible_crtcs" to the newly initialized crtc.
1871	 */
1872	for (i = 0; i < vop_data->win_size; i++) {
1873		struct vop_win *vop_win = &vop->win[i];
1874		const struct vop_win_data *win_data = vop_win->data;
1875
1876		if (win_data->type != DRM_PLANE_TYPE_PRIMARY &&
1877		    win_data->type != DRM_PLANE_TYPE_CURSOR)
1878			continue;
1879
1880		ret = drm_universal_plane_init(vop->drm_dev, &vop_win->base,
1881					       0, &vop_plane_funcs,
1882					       win_data->phy->data_formats,
1883					       win_data->phy->nformats,
1884					       win_data->phy->format_modifiers,
1885					       win_data->type, NULL);
1886		if (ret) {
1887			DRM_DEV_ERROR(vop->dev, "failed to init plane %d\n",
1888				      ret);
1889			goto err_cleanup_planes;
1890		}
1891
1892		plane = &vop_win->base;
1893		drm_plane_helper_add(plane, &plane_helper_funcs);
1894		vop_plane_add_properties(plane, win_data);
1895		if (plane->type == DRM_PLANE_TYPE_PRIMARY)
1896			primary = plane;
1897		else if (plane->type == DRM_PLANE_TYPE_CURSOR)
1898			cursor = plane;
1899	}
1900
1901	ret = drm_crtc_init_with_planes(drm_dev, crtc, primary, cursor,
1902					&vop_crtc_funcs, NULL);
1903	if (ret)
1904		goto err_cleanup_planes;
1905
1906	drm_crtc_helper_add(crtc, &vop_crtc_helper_funcs);
1907	if (vop->lut_regs) {
1908		drm_mode_crtc_set_gamma_size(crtc, vop_data->lut_size);
1909		drm_crtc_enable_color_mgmt(crtc, 0, false, vop_data->lut_size);
1910	}
1911
1912	/*
1913	 * Create drm_planes for overlay windows with possible_crtcs restricted
1914	 * to the newly created crtc.
1915	 */
1916	for (i = 0; i < vop_data->win_size; i++) {
1917		struct vop_win *vop_win = &vop->win[i];
1918		const struct vop_win_data *win_data = vop_win->data;
1919		unsigned long possible_crtcs = drm_crtc_mask(crtc);
1920
1921		if (win_data->type != DRM_PLANE_TYPE_OVERLAY)
1922			continue;
1923
1924		ret = drm_universal_plane_init(vop->drm_dev, &vop_win->base,
1925					       possible_crtcs,
1926					       &vop_plane_funcs,
1927					       win_data->phy->data_formats,
1928					       win_data->phy->nformats,
1929					       win_data->phy->format_modifiers,
1930					       win_data->type, NULL);
1931		if (ret) {
1932			DRM_DEV_ERROR(vop->dev, "failed to init overlay %d\n",
1933				      ret);
1934			goto err_cleanup_crtc;
1935		}
1936		drm_plane_helper_add(&vop_win->base, &plane_helper_funcs);
1937		vop_plane_add_properties(&vop_win->base, win_data);
1938	}
1939
1940	port = of_get_child_by_name(dev->of_node, "port");
1941	if (!port) {
1942		DRM_DEV_ERROR(vop->dev, "no port node found in %pOF\n",
1943			      dev->of_node);
1944		ret = -ENOENT;
1945		goto err_cleanup_crtc;
1946	}
1947
1948	drm_flip_work_init(&vop->fb_unref_work, "fb_unref",
1949			   vop_fb_unref_worker);
1950
1951	init_completion(&vop->dsp_hold_completion);
1952	init_completion(&vop->line_flag_completion);
1953	crtc->port = port;
1954
1955	ret = drm_self_refresh_helper_init(crtc);
1956	if (ret)
1957		DRM_DEV_DEBUG_KMS(vop->dev,
1958			"Failed to init %s with SR helpers %d, ignoring\n",
1959			crtc->name, ret);
1960
1961	return 0;
1962
1963err_cleanup_crtc:
1964	drm_crtc_cleanup(crtc);
1965err_cleanup_planes:
1966	list_for_each_entry_safe(plane, tmp, &drm_dev->mode_config.plane_list,
1967				 head)
1968		drm_plane_cleanup(plane);
1969	return ret;
1970}
1971
1972static void vop_destroy_crtc(struct vop *vop)
1973{
1974	struct drm_crtc *crtc = &vop->crtc;
1975	struct drm_device *drm_dev = vop->drm_dev;
1976	struct drm_plane *plane, *tmp;
1977
1978	drm_self_refresh_helper_cleanup(crtc);
1979
1980	of_node_put(crtc->port);
1981
1982	/*
1983	 * We need to cleanup the planes now.  Why?
1984	 *
1985	 * The planes are "&vop->win[i].base".  That means the memory is
1986	 * all part of the big "struct vop" chunk of memory.  That memory
1987	 * was devm allocated and associated with this component.  We need to
1988	 * free it ourselves before vop_unbind() finishes.
1989	 */
1990	list_for_each_entry_safe(plane, tmp, &drm_dev->mode_config.plane_list,
1991				 head)
1992		drm_plane_cleanup(plane);
1993
1994	/*
1995	 * Destroy CRTC after vop_plane_destroy() since vop_disable_plane()
1996	 * references the CRTC.
1997	 */
1998	drm_crtc_cleanup(crtc);
1999	drm_flip_work_cleanup(&vop->fb_unref_work);
2000}
2001
2002static int vop_initial(struct vop *vop)
2003{
2004	struct reset_control *ahb_rst;
2005	int i, ret;
2006
2007	vop->hclk = devm_clk_get(vop->dev, "hclk_vop");
2008	if (IS_ERR(vop->hclk)) {
2009		DRM_DEV_ERROR(vop->dev, "failed to get hclk source\n");
2010		return PTR_ERR(vop->hclk);
2011	}
2012	vop->aclk = devm_clk_get(vop->dev, "aclk_vop");
2013	if (IS_ERR(vop->aclk)) {
2014		DRM_DEV_ERROR(vop->dev, "failed to get aclk source\n");
2015		return PTR_ERR(vop->aclk);
2016	}
2017	vop->dclk = devm_clk_get(vop->dev, "dclk_vop");
2018	if (IS_ERR(vop->dclk)) {
2019		DRM_DEV_ERROR(vop->dev, "failed to get dclk source\n");
2020		return PTR_ERR(vop->dclk);
2021	}
2022
2023	ret = pm_runtime_resume_and_get(vop->dev);
2024	if (ret < 0) {
2025		DRM_DEV_ERROR(vop->dev, "failed to get pm runtime: %d\n", ret);
2026		return ret;
2027	}
2028
2029	ret = clk_prepare(vop->dclk);
2030	if (ret < 0) {
2031		DRM_DEV_ERROR(vop->dev, "failed to prepare dclk\n");
2032		goto err_put_pm_runtime;
2033	}
2034
2035	/* Enable both the hclk and aclk to setup the vop */
2036	ret = clk_prepare_enable(vop->hclk);
2037	if (ret < 0) {
2038		DRM_DEV_ERROR(vop->dev, "failed to prepare/enable hclk\n");
2039		goto err_unprepare_dclk;
2040	}
2041
2042	ret = clk_prepare_enable(vop->aclk);
2043	if (ret < 0) {
2044		DRM_DEV_ERROR(vop->dev, "failed to prepare/enable aclk\n");
2045		goto err_disable_hclk;
2046	}
2047
2048	/*
2049	 * do hclk_reset, reset all vop registers.
2050	 */
2051	ahb_rst = devm_reset_control_get(vop->dev, "ahb");
2052	if (IS_ERR(ahb_rst)) {
2053		DRM_DEV_ERROR(vop->dev, "failed to get ahb reset\n");
2054		ret = PTR_ERR(ahb_rst);
2055		goto err_disable_aclk;
2056	}
2057	reset_control_assert(ahb_rst);
2058	usleep_range(10, 20);
2059	reset_control_deassert(ahb_rst);
2060
2061	VOP_INTR_SET_TYPE(vop, clear, INTR_MASK, 1);
2062	VOP_INTR_SET_TYPE(vop, enable, INTR_MASK, 0);
2063
2064	for (i = 0; i < vop->len; i += sizeof(u32))
2065		vop->regsbak[i / 4] = readl_relaxed(vop->regs + i);
2066
2067	VOP_REG_SET(vop, misc, global_regdone_en, 1);
2068	VOP_REG_SET(vop, common, dsp_blank, 0);
2069
2070	for (i = 0; i < vop->data->win_size; i++) {
2071		struct vop_win *vop_win = &vop->win[i];
2072		const struct vop_win_data *win = vop_win->data;
2073		int channel = i * 2 + 1;
2074
2075		VOP_WIN_SET(vop, win, channel, (channel + 1) << 4 | channel);
2076		vop_win_disable(vop, vop_win);
2077		VOP_WIN_SET(vop, win, gate, 1);
2078	}
2079
2080	vop_cfg_done(vop);
2081
2082	/*
2083	 * do dclk_reset, let all config take affect.
2084	 */
2085	vop->dclk_rst = devm_reset_control_get(vop->dev, "dclk");
2086	if (IS_ERR(vop->dclk_rst)) {
2087		DRM_DEV_ERROR(vop->dev, "failed to get dclk reset\n");
2088		ret = PTR_ERR(vop->dclk_rst);
2089		goto err_disable_aclk;
2090	}
2091	reset_control_assert(vop->dclk_rst);
2092	usleep_range(10, 20);
2093	reset_control_deassert(vop->dclk_rst);
2094
2095	clk_disable(vop->hclk);
2096	clk_disable(vop->aclk);
2097
2098	vop->is_enabled = false;
2099
2100	pm_runtime_put_sync(vop->dev);
2101
2102	return 0;
2103
2104err_disable_aclk:
2105	clk_disable_unprepare(vop->aclk);
2106err_disable_hclk:
2107	clk_disable_unprepare(vop->hclk);
2108err_unprepare_dclk:
2109	clk_unprepare(vop->dclk);
2110err_put_pm_runtime:
2111	pm_runtime_put_sync(vop->dev);
2112	return ret;
2113}
2114
2115/*
2116 * Initialize the vop->win array elements.
2117 */
2118static void vop_win_init(struct vop *vop)
2119{
2120	const struct vop_data *vop_data = vop->data;
2121	unsigned int i;
2122
2123	for (i = 0; i < vop_data->win_size; i++) {
2124		struct vop_win *vop_win = &vop->win[i];
2125		const struct vop_win_data *win_data = &vop_data->win[i];
2126
2127		vop_win->data = win_data;
2128		vop_win->vop = vop;
2129
2130		if (vop_data->win_yuv2yuv)
2131			vop_win->yuv2yuv_data = &vop_data->win_yuv2yuv[i];
2132	}
2133}
2134
2135/**
2136 * rockchip_drm_wait_vact_end
2137 * @crtc: CRTC to enable line flag
2138 * @mstimeout: millisecond for timeout
2139 *
2140 * Wait for vact_end line flag irq or timeout.
2141 *
2142 * Returns:
2143 * Zero on success, negative errno on failure.
2144 */
2145int rockchip_drm_wait_vact_end(struct drm_crtc *crtc, unsigned int mstimeout)
2146{
2147	struct vop *vop = to_vop(crtc);
2148	unsigned long jiffies_left;
2149	int ret = 0;
2150
2151	if (!crtc || !vop->is_enabled)
2152		return -ENODEV;
2153
2154	mutex_lock(&vop->vop_lock);
2155	if (mstimeout <= 0) {
2156		ret = -EINVAL;
2157		goto out;
2158	}
2159
2160	if (vop_line_flag_irq_is_enabled(vop)) {
2161		ret = -EBUSY;
2162		goto out;
2163	}
2164
2165	reinit_completion(&vop->line_flag_completion);
2166	vop_line_flag_irq_enable(vop);
2167
2168	jiffies_left = wait_for_completion_timeout(&vop->line_flag_completion,
2169						   msecs_to_jiffies(mstimeout));
2170	vop_line_flag_irq_disable(vop);
2171
2172	if (jiffies_left == 0) {
2173		DRM_DEV_ERROR(vop->dev, "Timeout waiting for IRQ\n");
2174		ret = -ETIMEDOUT;
2175		goto out;
2176	}
2177
2178out:
2179	mutex_unlock(&vop->vop_lock);
2180	return ret;
2181}
2182EXPORT_SYMBOL(rockchip_drm_wait_vact_end);
2183
2184static int vop_bind(struct device *dev, struct device *master, void *data)
2185{
2186	struct platform_device *pdev = to_platform_device(dev);
2187	const struct vop_data *vop_data;
2188	struct drm_device *drm_dev = data;
2189	struct vop *vop;
2190	struct resource *res;
2191	int ret, irq;
2192
2193	vop_data = of_device_get_match_data(dev);
2194	if (!vop_data)
2195		return -ENODEV;
2196
2197	/* Allocate vop struct and its vop_win array */
2198	vop = devm_kzalloc(dev, struct_size(vop, win, vop_data->win_size),
2199			   GFP_KERNEL);
2200	if (!vop)
2201		return -ENOMEM;
2202
2203	vop->dev = dev;
2204	vop->data = vop_data;
2205	vop->drm_dev = drm_dev;
2206	dev_set_drvdata(dev, vop);
2207
2208	vop_win_init(vop);
2209
2210	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2211	vop->regs = devm_ioremap_resource(dev, res);
2212	if (IS_ERR(vop->regs))
2213		return PTR_ERR(vop->regs);
2214	vop->len = resource_size(res);
2215
2216	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2217	if (res) {
2218		if (vop_data->lut_size != 1024 && vop_data->lut_size != 256) {
2219			DRM_DEV_ERROR(dev, "unsupported gamma LUT size %d\n", vop_data->lut_size);
2220			return -EINVAL;
2221		}
2222		vop->lut_regs = devm_ioremap_resource(dev, res);
2223		if (IS_ERR(vop->lut_regs))
2224			return PTR_ERR(vop->lut_regs);
2225	}
2226
2227	vop->regsbak = devm_kzalloc(dev, vop->len, GFP_KERNEL);
2228	if (!vop->regsbak)
2229		return -ENOMEM;
2230
2231	irq = platform_get_irq(pdev, 0);
2232	if (irq < 0) {
2233		DRM_DEV_ERROR(dev, "cannot find irq for vop\n");
2234		return irq;
2235	}
2236	vop->irq = (unsigned int)irq;
2237
2238	spin_lock_init(&vop->reg_lock);
2239	spin_lock_init(&vop->irq_lock);
2240	mutex_init(&vop->vop_lock);
2241
2242	ret = vop_create_crtc(vop);
2243	if (ret)
2244		return ret;
2245
2246	pm_runtime_enable(&pdev->dev);
2247
2248	ret = vop_initial(vop);
2249	if (ret < 0) {
2250		DRM_DEV_ERROR(&pdev->dev,
2251			      "cannot initial vop dev - err %d\n", ret);
2252		goto err_disable_pm_runtime;
2253	}
2254
2255	ret = devm_request_irq(dev, vop->irq, vop_isr,
2256			       IRQF_SHARED, dev_name(dev), vop);
2257	if (ret)
2258		goto err_disable_pm_runtime;
2259
2260	if (vop->data->feature & VOP_FEATURE_INTERNAL_RGB) {
2261		vop->rgb = rockchip_rgb_init(dev, &vop->crtc, vop->drm_dev, 0);
2262		if (IS_ERR(vop->rgb)) {
2263			ret = PTR_ERR(vop->rgb);
2264			goto err_disable_pm_runtime;
2265		}
2266	}
2267
2268	rockchip_drm_dma_init_device(drm_dev, dev);
2269
2270	return 0;
2271
2272err_disable_pm_runtime:
2273	pm_runtime_disable(&pdev->dev);
2274	vop_destroy_crtc(vop);
2275	return ret;
2276}
2277
2278static void vop_unbind(struct device *dev, struct device *master, void *data)
2279{
2280	struct vop *vop = dev_get_drvdata(dev);
2281
2282	if (vop->rgb)
2283		rockchip_rgb_fini(vop->rgb);
2284
2285	pm_runtime_disable(dev);
2286	vop_destroy_crtc(vop);
2287
2288	clk_unprepare(vop->aclk);
2289	clk_unprepare(vop->hclk);
2290	clk_unprepare(vop->dclk);
2291}
2292
2293const struct component_ops vop_component_ops = {
2294	.bind = vop_bind,
2295	.unbind = vop_unbind,
2296};
2297EXPORT_SYMBOL_GPL(vop_component_ops);