rkvdec.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Rockchip Video Decoder driver
   4 *
   5 * Copyright (C) 2019 Collabora, Ltd.
   6 *
   7 * Based on rkvdec driver by Google LLC. (Tomasz Figa <tfiga@chromium.org>)
   8 * Based on s5p-mfc driver by Samsung Electronics Co., Ltd.
   9 * Copyright (C) 2011 Samsung Electronics Co., Ltd.
  10 */
  11
  12#include <linux/clk.h>
  13#include <linux/interrupt.h>
  14#include <linux/iommu.h>
  15#include <linux/module.h>
  16#include <linux/of.h>
  17#include <linux/of_device.h>
  18#include <linux/platform_device.h>
  19#include <linux/pm.h>
  20#include <linux/pm_runtime.h>
  21#include <linux/slab.h>
  22#include <linux/videodev2.h>
  23#include <linux/workqueue.h>
  24#include <media/v4l2-event.h>
  25#include <media/v4l2-mem2mem.h>
  26#include <media/videobuf2-core.h>
  27#include <media/videobuf2-vmalloc.h>
  28
  29#include "rkvdec.h"
  30#include "rkvdec-regs.h"
  31
  32static bool rkvdec_image_fmt_match(enum rkvdec_image_fmt fmt1,
  33				   enum rkvdec_image_fmt fmt2)
  34{
  35	return fmt1 == fmt2 || fmt2 == RKVDEC_IMG_FMT_ANY ||
  36	       fmt1 == RKVDEC_IMG_FMT_ANY;
  37}
  38
  39static bool rkvdec_image_fmt_changed(struct rkvdec_ctx *ctx,
  40				     enum rkvdec_image_fmt image_fmt)
  41{
  42	if (image_fmt == RKVDEC_IMG_FMT_ANY)
  43		return false;
  44
  45	return ctx->image_fmt != image_fmt;
  46}
  47
  48static u32 rkvdec_enum_decoded_fmt(struct rkvdec_ctx *ctx, int index,
  49				   enum rkvdec_image_fmt image_fmt)
  50{
  51	const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc;
  52	int fmt_idx = -1;
  53	unsigned int i;
  54
  55	if (WARN_ON(!desc))
  56		return 0;
  57
  58	for (i = 0; i < desc->num_decoded_fmts; i++) {
  59		if (!rkvdec_image_fmt_match(desc->decoded_fmts[i].image_fmt,
  60					    image_fmt))
  61			continue;
  62		fmt_idx++;
  63		if (index == fmt_idx)
  64			return desc->decoded_fmts[i].fourcc;
  65	}
  66
  67	return 0;
  68}
  69
  70static bool rkvdec_is_valid_fmt(struct rkvdec_ctx *ctx, u32 fourcc,
  71				enum rkvdec_image_fmt image_fmt)
  72{
  73	const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc;
  74	unsigned int i;
  75
  76	for (i = 0; i < desc->num_decoded_fmts; i++) {
  77		if (rkvdec_image_fmt_match(desc->decoded_fmts[i].image_fmt,
  78					   image_fmt) &&
  79		    desc->decoded_fmts[i].fourcc == fourcc)
  80			return true;
  81	}
  82
  83	return false;
  84}
  85
  86static void rkvdec_fill_decoded_pixfmt(struct rkvdec_ctx *ctx,
  87				       struct v4l2_pix_format_mplane *pix_mp)
  88{
  89	v4l2_fill_pixfmt_mp(pix_mp, pix_mp->pixelformat,
  90			    pix_mp->width, pix_mp->height);
  91	pix_mp->plane_fmt[0].sizeimage += 128 *
  92		DIV_ROUND_UP(pix_mp->width, 16) *
  93		DIV_ROUND_UP(pix_mp->height, 16);
  94}
  95
  96static void rkvdec_reset_fmt(struct rkvdec_ctx *ctx, struct v4l2_format *f,
  97			     u32 fourcc)
  98{
  99	memset(f, 0, sizeof(*f));
 100	f->fmt.pix_mp.pixelformat = fourcc;
 101	f->fmt.pix_mp.field = V4L2_FIELD_NONE;
 102	f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709;
 103	f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
 104	f->fmt.pix_mp.quantization = V4L2_QUANTIZATION_DEFAULT;
 105	f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT;
 106}
 107
 108static void rkvdec_reset_decoded_fmt(struct rkvdec_ctx *ctx)
 109{
 110	struct v4l2_format *f = &ctx->decoded_fmt;
 111	u32 fourcc;
 112
 113	fourcc = rkvdec_enum_decoded_fmt(ctx, 0, ctx->image_fmt);
 114	rkvdec_reset_fmt(ctx, f, fourcc);
 115	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
 116	f->fmt.pix_mp.width = ctx->coded_fmt.fmt.pix_mp.width;
 117	f->fmt.pix_mp.height = ctx->coded_fmt.fmt.pix_mp.height;
 118	rkvdec_fill_decoded_pixfmt(ctx, &f->fmt.pix_mp);
 119}
 120
 121static int rkvdec_try_ctrl(struct v4l2_ctrl *ctrl)
 122{
 123	struct rkvdec_ctx *ctx = container_of(ctrl->handler, struct rkvdec_ctx, ctrl_hdl);
 124	const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc;
 125
 126	if (desc->ops->try_ctrl)
 127		return desc->ops->try_ctrl(ctx, ctrl);
 128
 129	return 0;
 130}
 131
 132static int rkvdec_s_ctrl(struct v4l2_ctrl *ctrl)
 133{
 134	struct rkvdec_ctx *ctx = container_of(ctrl->handler, struct rkvdec_ctx, ctrl_hdl);
 135	const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc;
 136	enum rkvdec_image_fmt image_fmt;
 137	struct vb2_queue *vq;
 138
 139	/* Check if this change requires a capture format reset */
 140	if (!desc->ops->get_image_fmt)
 141		return 0;
 142
 143	image_fmt = desc->ops->get_image_fmt(ctx, ctrl);
 144	if (rkvdec_image_fmt_changed(ctx, image_fmt)) {
 145		vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
 146				     V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
 147		if (vb2_is_busy(vq))
 148			return -EBUSY;
 149
 150		ctx->image_fmt = image_fmt;
 151		rkvdec_reset_decoded_fmt(ctx);
 152	}
 153
 154	return 0;
 155}
 156
 157static const struct v4l2_ctrl_ops rkvdec_ctrl_ops = {
 158	.try_ctrl = rkvdec_try_ctrl,
 159	.s_ctrl = rkvdec_s_ctrl,
 160};
 161
 162static const struct rkvdec_ctrl_desc rkvdec_hevc_ctrl_descs[] = {
 163	{
 164		.cfg.id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS,
 165		.cfg.flags = V4L2_CTRL_FLAG_DYNAMIC_ARRAY,
 166		.cfg.type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS,
 167		.cfg.dims = { 600 },
 168	},
 169	{
 170		.cfg.id = V4L2_CID_STATELESS_HEVC_SPS,
 171		.cfg.ops = &rkvdec_ctrl_ops,
 172	},
 173	{
 174		.cfg.id = V4L2_CID_STATELESS_HEVC_PPS,
 175	},
 176	{
 177		.cfg.id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX,
 178	},
 179	{
 180		.cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS,
 181	},
 182	{
 183		.cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_MODE,
 184		.cfg.min = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED,
 185		.cfg.max = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED,
 186		.cfg.def = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED,
 187	},
 188	{
 189		.cfg.id = V4L2_CID_STATELESS_HEVC_START_CODE,
 190		.cfg.min = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B,
 191		.cfg.def = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B,
 192		.cfg.max = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B,
 193	},
 194	{
 195		.cfg.id = V4L2_CID_MPEG_VIDEO_HEVC_PROFILE,
 196		.cfg.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
 197		.cfg.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10,
 198		.cfg.def = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
 199	},
 200	{
 201		.cfg.id = V4L2_CID_MPEG_VIDEO_HEVC_LEVEL,
 202		.cfg.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
 203		.cfg.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1,
 204	},
 205};
 206
 207static const struct rkvdec_ctrls rkvdec_hevc_ctrls = {
 208	.ctrls = rkvdec_hevc_ctrl_descs,
 209	.num_ctrls = ARRAY_SIZE(rkvdec_hevc_ctrl_descs),
 210};
 211
 212static const struct rkvdec_decoded_fmt_desc rkvdec_hevc_decoded_fmts[] = {
 213	{
 214		.fourcc = V4L2_PIX_FMT_NV12,
 215		.image_fmt = RKVDEC_IMG_FMT_420_8BIT,
 216	},
 217	{
 218		.fourcc = V4L2_PIX_FMT_NV15,
 219		.image_fmt = RKVDEC_IMG_FMT_420_10BIT,
 220	},
 221};
 222
 223static const struct rkvdec_ctrl_desc rkvdec_h264_ctrl_descs[] = {
 224	{
 225		.cfg.id = V4L2_CID_STATELESS_H264_DECODE_PARAMS,
 226	},
 227	{
 228		.cfg.id = V4L2_CID_STATELESS_H264_SPS,
 229		.cfg.ops = &rkvdec_ctrl_ops,
 230	},
 231	{
 232		.cfg.id = V4L2_CID_STATELESS_H264_PPS,
 233	},
 234	{
 235		.cfg.id = V4L2_CID_STATELESS_H264_SCALING_MATRIX,
 236	},
 237	{
 238		.cfg.id = V4L2_CID_STATELESS_H264_DECODE_MODE,
 239		.cfg.min = V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED,
 240		.cfg.max = V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED,
 241		.cfg.def = V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED,
 242	},
 243	{
 244		.cfg.id = V4L2_CID_STATELESS_H264_START_CODE,
 245		.cfg.min = V4L2_STATELESS_H264_START_CODE_ANNEX_B,
 246		.cfg.def = V4L2_STATELESS_H264_START_CODE_ANNEX_B,
 247		.cfg.max = V4L2_STATELESS_H264_START_CODE_ANNEX_B,
 248	},
 249	{
 250		.cfg.id = V4L2_CID_MPEG_VIDEO_H264_PROFILE,
 251		.cfg.min = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE,
 252		.cfg.max = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_422_INTRA,
 253		.cfg.menu_skip_mask =
 254			BIT(V4L2_MPEG_VIDEO_H264_PROFILE_EXTENDED) |
 255			BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_444_PREDICTIVE),
 256		.cfg.def = V4L2_MPEG_VIDEO_H264_PROFILE_MAIN,
 257	},
 258	{
 259		.cfg.id = V4L2_CID_MPEG_VIDEO_H264_LEVEL,
 260		.cfg.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
 261		.cfg.max = V4L2_MPEG_VIDEO_H264_LEVEL_5_1,
 262	},
 263};
 264
 265static const struct rkvdec_ctrls rkvdec_h264_ctrls = {
 266	.ctrls = rkvdec_h264_ctrl_descs,
 267	.num_ctrls = ARRAY_SIZE(rkvdec_h264_ctrl_descs),
 268};
 269
 270static const struct rkvdec_decoded_fmt_desc rkvdec_h264_decoded_fmts[] = {
 271	{
 272		.fourcc = V4L2_PIX_FMT_NV12,
 273		.image_fmt = RKVDEC_IMG_FMT_420_8BIT,
 274	},
 275	{
 276		.fourcc = V4L2_PIX_FMT_NV15,
 277		.image_fmt = RKVDEC_IMG_FMT_420_10BIT,
 278	},
 279	{
 280		.fourcc = V4L2_PIX_FMT_NV16,
 281		.image_fmt = RKVDEC_IMG_FMT_422_8BIT,
 282	},
 283	{
 284		.fourcc = V4L2_PIX_FMT_NV20,
 285		.image_fmt = RKVDEC_IMG_FMT_422_10BIT,
 286	},
 287};
 288
 289static const struct rkvdec_ctrl_desc rkvdec_vp9_ctrl_descs[] = {
 290	{
 291		.cfg.id = V4L2_CID_STATELESS_VP9_FRAME,
 292	},
 293	{
 294		.cfg.id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR,
 295	},
 296	{
 297		.cfg.id = V4L2_CID_MPEG_VIDEO_VP9_PROFILE,
 298		.cfg.min = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
 299		.cfg.max = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
 300		.cfg.def = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
 301	},
 302};
 303
 304static const struct rkvdec_ctrls rkvdec_vp9_ctrls = {
 305	.ctrls = rkvdec_vp9_ctrl_descs,
 306	.num_ctrls = ARRAY_SIZE(rkvdec_vp9_ctrl_descs),
 307};
 308
 309static const struct rkvdec_decoded_fmt_desc rkvdec_vp9_decoded_fmts[] = {
 310	{
 311		.fourcc = V4L2_PIX_FMT_NV12,
 312		.image_fmt = RKVDEC_IMG_FMT_420_8BIT,
 313	},
 314};
 315
 316static const struct rkvdec_coded_fmt_desc rkvdec_coded_fmts[] = {
 317	{
 318		.fourcc = V4L2_PIX_FMT_HEVC_SLICE,
 319		.frmsize = {
 320			.min_width = 64,
 321			.max_width = 4096,
 322			.step_width = 64,
 323			.min_height = 64,
 324			.max_height = 2304,
 325			.step_height = 16,
 326		},
 327		.ctrls = &rkvdec_hevc_ctrls,
 328		.ops = &rkvdec_hevc_fmt_ops,
 329		.num_decoded_fmts = ARRAY_SIZE(rkvdec_hevc_decoded_fmts),
 330		.decoded_fmts = rkvdec_hevc_decoded_fmts,
 331		.capability = RKVDEC_CAPABILITY_HEVC,
 332	},
 333	{
 334		.fourcc = V4L2_PIX_FMT_H264_SLICE,
 335		.frmsize = {
 336			.min_width = 64,
 337			.max_width = 4096,
 338			.step_width = 64,
 339			.min_height = 48,
 340			.max_height = 2560,
 341			.step_height = 16,
 342		},
 343		.ctrls = &rkvdec_h264_ctrls,
 344		.ops = &rkvdec_h264_fmt_ops,
 345		.num_decoded_fmts = ARRAY_SIZE(rkvdec_h264_decoded_fmts),
 346		.decoded_fmts = rkvdec_h264_decoded_fmts,
 347		.subsystem_flags = VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF,
 348		.capability = RKVDEC_CAPABILITY_H264,
 349	},
 350	{
 351		.fourcc = V4L2_PIX_FMT_VP9_FRAME,
 352		.frmsize = {
 353			.min_width = 64,
 354			.max_width = 4096,
 355			.step_width = 64,
 356			.min_height = 64,
 357			.max_height = 2304,
 358			.step_height = 64,
 359		},
 360		.ctrls = &rkvdec_vp9_ctrls,
 361		.ops = &rkvdec_vp9_fmt_ops,
 362		.num_decoded_fmts = ARRAY_SIZE(rkvdec_vp9_decoded_fmts),
 363		.decoded_fmts = rkvdec_vp9_decoded_fmts,
 364		.capability = RKVDEC_CAPABILITY_VP9,
 365	}
 366};
 367
 368static bool rkvdec_is_capable(struct rkvdec_ctx *ctx, unsigned int capability)
 369{
 370	return (ctx->dev->variant->capabilities & capability) == capability;
 371}
 372
 373static const struct rkvdec_coded_fmt_desc *
 374rkvdec_enum_coded_fmt_desc(struct rkvdec_ctx *ctx, int index)
 375{
 376	int fmt_idx = -1;
 377	unsigned int i;
 378
 379	for (i = 0; i < ARRAY_SIZE(rkvdec_coded_fmts); i++) {
 380		if (!rkvdec_is_capable(ctx, rkvdec_coded_fmts[i].capability))
 381			continue;
 382		fmt_idx++;
 383		if (index == fmt_idx)
 384			return &rkvdec_coded_fmts[i];
 385	}
 386
 387	return NULL;
 388}
 389
 390static const struct rkvdec_coded_fmt_desc *
 391rkvdec_find_coded_fmt_desc(struct rkvdec_ctx *ctx, u32 fourcc)
 392{
 393	unsigned int i;
 394
 395	for (i = 0; i < ARRAY_SIZE(rkvdec_coded_fmts); i++) {
 396		if (rkvdec_is_capable(ctx, rkvdec_coded_fmts[i].capability) &&
 397		    rkvdec_coded_fmts[i].fourcc == fourcc)
 398			return &rkvdec_coded_fmts[i];
 399	}
 400
 401	return NULL;
 402}
 403
 404static void rkvdec_reset_coded_fmt(struct rkvdec_ctx *ctx)
 405{
 406	struct v4l2_format *f = &ctx->coded_fmt;
 407
 408	ctx->coded_fmt_desc = rkvdec_enum_coded_fmt_desc(ctx, 0);
 409	rkvdec_reset_fmt(ctx, f, ctx->coded_fmt_desc->fourcc);
 410
 411	f->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
 412	f->fmt.pix_mp.width = ctx->coded_fmt_desc->frmsize.min_width;
 413	f->fmt.pix_mp.height = ctx->coded_fmt_desc->frmsize.min_height;
 414
 415	if (ctx->coded_fmt_desc->ops->adjust_fmt)
 416		ctx->coded_fmt_desc->ops->adjust_fmt(ctx, f);
 417}
 418
 419static int rkvdec_enum_framesizes(struct file *file, void *priv,
 420				  struct v4l2_frmsizeenum *fsize)
 421{
 422	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 423	const struct rkvdec_coded_fmt_desc *desc;
 424
 425	if (fsize->index != 0)
 426		return -EINVAL;
 427
 428	desc = rkvdec_find_coded_fmt_desc(ctx, fsize->pixel_format);
 429	if (!desc)
 430		return -EINVAL;
 431
 432	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
 433	fsize->stepwise.min_width = 1;
 434	fsize->stepwise.max_width = desc->frmsize.max_width;
 435	fsize->stepwise.step_width = 1;
 436	fsize->stepwise.min_height = 1;
 437	fsize->stepwise.max_height = desc->frmsize.max_height;
 438	fsize->stepwise.step_height = 1;
 439
 440	return 0;
 441}
 442
 443static int rkvdec_querycap(struct file *file, void *priv,
 444			   struct v4l2_capability *cap)
 445{
 446	struct rkvdec_dev *rkvdec = video_drvdata(file);
 447	struct video_device *vdev = video_devdata(file);
 448
 449	strscpy(cap->driver, rkvdec->dev->driver->name,
 450		sizeof(cap->driver));
 451	strscpy(cap->card, vdev->name, sizeof(cap->card));
 452	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
 453		 rkvdec->dev->driver->name);
 454	return 0;
 455}
 456
 457static int rkvdec_try_capture_fmt(struct file *file, void *priv,
 458				  struct v4l2_format *f)
 459{
 460	struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
 461	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 462	const struct rkvdec_coded_fmt_desc *coded_desc;
 463
 464	/*
 465	 * The codec context should point to a coded format desc, if the format
 466	 * on the coded end has not been set yet, it should point to the
 467	 * default value.
 468	 */
 469	coded_desc = ctx->coded_fmt_desc;
 470	if (WARN_ON(!coded_desc))
 471		return -EINVAL;
 472
 473	if (!rkvdec_is_valid_fmt(ctx, pix_mp->pixelformat, ctx->image_fmt))
 474		pix_mp->pixelformat = rkvdec_enum_decoded_fmt(ctx, 0,
 475							      ctx->image_fmt);
 476
 477	/* Always apply the frmsize constraint of the coded end. */
 478	pix_mp->width = max(pix_mp->width, ctx->coded_fmt.fmt.pix_mp.width);
 479	pix_mp->height = max(pix_mp->height, ctx->coded_fmt.fmt.pix_mp.height);
 480	v4l2_apply_frmsize_constraints(&pix_mp->width,
 481				       &pix_mp->height,
 482				       &coded_desc->frmsize);
 483
 484	rkvdec_fill_decoded_pixfmt(ctx, pix_mp);
 485	pix_mp->field = V4L2_FIELD_NONE;
 486
 487	return 0;
 488}
 489
 490static int rkvdec_try_output_fmt(struct file *file, void *priv,
 491				 struct v4l2_format *f)
 492{
 493	struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
 494	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 495	const struct rkvdec_coded_fmt_desc *desc;
 496
 497	desc = rkvdec_find_coded_fmt_desc(ctx, pix_mp->pixelformat);
 498	if (!desc) {
 499		desc = rkvdec_enum_coded_fmt_desc(ctx, 0);
 500		pix_mp->pixelformat = desc->fourcc;
 501	}
 502
 503	v4l2_apply_frmsize_constraints(&pix_mp->width,
 504				       &pix_mp->height,
 505				       &desc->frmsize);
 506
 507	pix_mp->field = V4L2_FIELD_NONE;
 508	/* All coded formats are considered single planar for now. */
 509	pix_mp->num_planes = 1;
 510
 511	if (desc->ops->adjust_fmt) {
 512		int ret;
 513
 514		ret = desc->ops->adjust_fmt(ctx, f);
 515		if (ret)
 516			return ret;
 517	}
 518
 519	return 0;
 520}
 521
 522static int rkvdec_s_capture_fmt(struct file *file, void *priv,
 523				struct v4l2_format *f)
 524{
 525	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 526	struct vb2_queue *vq;
 527	int ret;
 528
 529	/* Change not allowed if queue is busy */
 530	vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
 531			     V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
 532	if (vb2_is_busy(vq))
 533		return -EBUSY;
 534
 535	ret = rkvdec_try_capture_fmt(file, priv, f);
 536	if (ret)
 537		return ret;
 538
 539	ctx->decoded_fmt = *f;
 540	return 0;
 541}
 542
 543static int rkvdec_s_output_fmt(struct file *file, void *priv,
 544			       struct v4l2_format *f)
 545{
 546	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 547	struct v4l2_m2m_ctx *m2m_ctx = ctx->fh.m2m_ctx;
 548	const struct rkvdec_coded_fmt_desc *desc;
 549	struct v4l2_format *cap_fmt;
 550	struct vb2_queue *peer_vq, *vq;
 551	int ret;
 552
 553	/*
 554	 * In order to support dynamic resolution change, the decoder admits
 555	 * a resolution change, as long as the pixelformat remains. Can't be
 556	 * done if streaming.
 557	 */
 558	vq = v4l2_m2m_get_vq(m2m_ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
 559	if (vb2_is_streaming(vq) ||
 560	    (vb2_is_busy(vq) &&
 561	     f->fmt.pix_mp.pixelformat != ctx->coded_fmt.fmt.pix_mp.pixelformat))
 562		return -EBUSY;
 563
 564	/*
 565	 * Since format change on the OUTPUT queue will reset the CAPTURE
 566	 * queue, we can't allow doing so when the CAPTURE queue has buffers
 567	 * allocated.
 568	 */
 569	peer_vq = v4l2_m2m_get_vq(m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
 570	if (vb2_is_busy(peer_vq))
 571		return -EBUSY;
 572
 573	ret = rkvdec_try_output_fmt(file, priv, f);
 574	if (ret)
 575		return ret;
 576
 577	desc = rkvdec_find_coded_fmt_desc(ctx, f->fmt.pix_mp.pixelformat);
 578	if (!desc)
 579		return -EINVAL;
 580	ctx->coded_fmt_desc = desc;
 581	ctx->coded_fmt = *f;
 582
 583	/*
 584	 * Current decoded format might have become invalid with newly
 585	 * selected codec, so reset it to default just to be safe and
 586	 * keep internal driver state sane. User is mandated to set
 587	 * the decoded format again after we return, so we don't need
 588	 * anything smarter.
 589	 *
 590	 * Note that this will propagates any size changes to the decoded format.
 591	 */
 592	ctx->image_fmt = RKVDEC_IMG_FMT_ANY;
 593	rkvdec_reset_decoded_fmt(ctx);
 594
 595	/* Propagate colorspace information to capture. */
 596	cap_fmt = &ctx->decoded_fmt;
 597	cap_fmt->fmt.pix_mp.colorspace = f->fmt.pix_mp.colorspace;
 598	cap_fmt->fmt.pix_mp.xfer_func = f->fmt.pix_mp.xfer_func;
 599	cap_fmt->fmt.pix_mp.ycbcr_enc = f->fmt.pix_mp.ycbcr_enc;
 600	cap_fmt->fmt.pix_mp.quantization = f->fmt.pix_mp.quantization;
 601
 602	/* Enable format specific queue features */
 603	vq->subsystem_flags |= desc->subsystem_flags;
 604
 605	return 0;
 606}
 607
 608static int rkvdec_g_output_fmt(struct file *file, void *priv,
 609			       struct v4l2_format *f)
 610{
 611	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 612
 613	*f = ctx->coded_fmt;
 614	return 0;
 615}
 616
 617static int rkvdec_g_capture_fmt(struct file *file, void *priv,
 618				struct v4l2_format *f)
 619{
 620	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 621
 622	*f = ctx->decoded_fmt;
 623	return 0;
 624}
 625
 626static int rkvdec_enum_output_fmt(struct file *file, void *priv,
 627				  struct v4l2_fmtdesc *f)
 628{
 629	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 630	const struct rkvdec_coded_fmt_desc *desc;
 631
 632	desc = rkvdec_enum_coded_fmt_desc(ctx, f->index);
 633	if (!desc)
 634		return -EINVAL;
 635
 636	f->pixelformat = desc->fourcc;
 637	return 0;
 638}
 639
 640static int rkvdec_enum_capture_fmt(struct file *file, void *priv,
 641				   struct v4l2_fmtdesc *f)
 642{
 643	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(file);
 644	u32 fourcc;
 645
 646	fourcc = rkvdec_enum_decoded_fmt(ctx, f->index, ctx->image_fmt);
 647	if (!fourcc)
 648		return -EINVAL;
 649
 650	f->pixelformat = fourcc;
 651	return 0;
 652}
 653
 654static const struct v4l2_ioctl_ops rkvdec_ioctl_ops = {
 655	.vidioc_querycap = rkvdec_querycap,
 656	.vidioc_enum_framesizes = rkvdec_enum_framesizes,
 657
 658	.vidioc_try_fmt_vid_cap_mplane = rkvdec_try_capture_fmt,
 659	.vidioc_try_fmt_vid_out_mplane = rkvdec_try_output_fmt,
 660	.vidioc_s_fmt_vid_out_mplane = rkvdec_s_output_fmt,
 661	.vidioc_s_fmt_vid_cap_mplane = rkvdec_s_capture_fmt,
 662	.vidioc_g_fmt_vid_out_mplane = rkvdec_g_output_fmt,
 663	.vidioc_g_fmt_vid_cap_mplane = rkvdec_g_capture_fmt,
 664	.vidioc_enum_fmt_vid_out = rkvdec_enum_output_fmt,
 665	.vidioc_enum_fmt_vid_cap = rkvdec_enum_capture_fmt,
 666
 667	.vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs,
 668	.vidioc_querybuf = v4l2_m2m_ioctl_querybuf,
 669	.vidioc_qbuf = v4l2_m2m_ioctl_qbuf,
 670	.vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf,
 671	.vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf,
 672	.vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs,
 673	.vidioc_expbuf = v4l2_m2m_ioctl_expbuf,
 674
 675	.vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
 676	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 677
 678	.vidioc_streamon = v4l2_m2m_ioctl_streamon,
 679	.vidioc_streamoff = v4l2_m2m_ioctl_streamoff,
 680
 681	.vidioc_decoder_cmd = v4l2_m2m_ioctl_stateless_decoder_cmd,
 682	.vidioc_try_decoder_cmd = v4l2_m2m_ioctl_stateless_try_decoder_cmd,
 683};
 684
 685static int rkvdec_queue_setup(struct vb2_queue *vq, unsigned int *num_buffers,
 686			      unsigned int *num_planes, unsigned int sizes[],
 687			      struct device *alloc_devs[])
 688{
 689	struct rkvdec_ctx *ctx = vb2_get_drv_priv(vq);
 690	struct v4l2_format *f;
 691	unsigned int i;
 692
 693	if (V4L2_TYPE_IS_OUTPUT(vq->type))
 694		f = &ctx->coded_fmt;
 695	else
 696		f = &ctx->decoded_fmt;
 697
 698	if (*num_planes) {
 699		if (*num_planes != f->fmt.pix_mp.num_planes)
 700			return -EINVAL;
 701
 702		for (i = 0; i < f->fmt.pix_mp.num_planes; i++) {
 703			if (sizes[i] < f->fmt.pix_mp.plane_fmt[i].sizeimage)
 704				return -EINVAL;
 705		}
 706	} else {
 707		*num_planes = f->fmt.pix_mp.num_planes;
 708		for (i = 0; i < f->fmt.pix_mp.num_planes; i++)
 709			sizes[i] = f->fmt.pix_mp.plane_fmt[i].sizeimage;
 710	}
 711
 712	return 0;
 713}
 714
 715static int rkvdec_buf_prepare(struct vb2_buffer *vb)
 716{
 717	struct vb2_queue *vq = vb->vb2_queue;
 718	struct rkvdec_ctx *ctx = vb2_get_drv_priv(vq);
 719	struct v4l2_format *f;
 720	unsigned int i;
 721
 722	if (V4L2_TYPE_IS_OUTPUT(vq->type))
 723		f = &ctx->coded_fmt;
 724	else
 725		f = &ctx->decoded_fmt;
 726
 727	for (i = 0; i < f->fmt.pix_mp.num_planes; ++i) {
 728		u32 sizeimage = f->fmt.pix_mp.plane_fmt[i].sizeimage;
 729
 730		if (vb2_plane_size(vb, i) < sizeimage)
 731			return -EINVAL;
 732	}
 733
 734	/*
 735	 * Buffer's bytesused must be written by driver for CAPTURE buffers.
 736	 * (for OUTPUT buffers, if userspace passes 0 bytesused, v4l2-core sets
 737	 * it to buffer length).
 738	 */
 739	if (V4L2_TYPE_IS_CAPTURE(vq->type))
 740		vb2_set_plane_payload(vb, 0, f->fmt.pix_mp.plane_fmt[0].sizeimage);
 741
 742	return 0;
 743}
 744
 745static void rkvdec_buf_queue(struct vb2_buffer *vb)
 746{
 747	struct rkvdec_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
 748	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 749
 750	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
 751}
 752
 753static int rkvdec_buf_out_validate(struct vb2_buffer *vb)
 754{
 755	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 756
 757	vbuf->field = V4L2_FIELD_NONE;
 758	return 0;
 759}
 760
 761static void rkvdec_buf_request_complete(struct vb2_buffer *vb)
 762{
 763	struct rkvdec_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
 764
 765	v4l2_ctrl_request_complete(vb->req_obj.req, &ctx->ctrl_hdl);
 766}
 767
 768static int rkvdec_start_streaming(struct vb2_queue *q, unsigned int count)
 769{
 770	struct rkvdec_ctx *ctx = vb2_get_drv_priv(q);
 771	const struct rkvdec_coded_fmt_desc *desc;
 772	int ret;
 773
 774	if (V4L2_TYPE_IS_CAPTURE(q->type))
 775		return 0;
 776
 777	desc = ctx->coded_fmt_desc;
 778	if (WARN_ON(!desc))
 779		return -EINVAL;
 780
 781	if (desc->ops->start) {
 782		ret = desc->ops->start(ctx);
 783		if (ret)
 784			return ret;
 785	}
 786
 787	return 0;
 788}
 789
 790static void rkvdec_queue_cleanup(struct vb2_queue *vq, u32 state)
 791{
 792	struct rkvdec_ctx *ctx = vb2_get_drv_priv(vq);
 793
 794	while (true) {
 795		struct vb2_v4l2_buffer *vbuf;
 796
 797		if (V4L2_TYPE_IS_OUTPUT(vq->type))
 798			vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
 799		else
 800			vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
 801
 802		if (!vbuf)
 803			break;
 804
 805		v4l2_ctrl_request_complete(vbuf->vb2_buf.req_obj.req,
 806					   &ctx->ctrl_hdl);
 807		v4l2_m2m_buf_done(vbuf, state);
 808	}
 809}
 810
 811static void rkvdec_stop_streaming(struct vb2_queue *q)
 812{
 813	struct rkvdec_ctx *ctx = vb2_get_drv_priv(q);
 814
 815	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
 816		const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc;
 817
 818		if (WARN_ON(!desc))
 819			return;
 820
 821		if (desc->ops->stop)
 822			desc->ops->stop(ctx);
 823	}
 824
 825	rkvdec_queue_cleanup(q, VB2_BUF_STATE_ERROR);
 826}
 827
 828static const struct vb2_ops rkvdec_queue_ops = {
 829	.queue_setup = rkvdec_queue_setup,
 830	.buf_prepare = rkvdec_buf_prepare,
 831	.buf_queue = rkvdec_buf_queue,
 832	.buf_out_validate = rkvdec_buf_out_validate,
 833	.buf_request_complete = rkvdec_buf_request_complete,
 834	.start_streaming = rkvdec_start_streaming,
 835	.stop_streaming = rkvdec_stop_streaming,
 836};
 837
 838static int rkvdec_request_validate(struct media_request *req)
 839{
 840	unsigned int count;
 841
 842	count = vb2_request_buffer_cnt(req);
 843	if (!count)
 844		return -ENOENT;
 845	else if (count > 1)
 846		return -EINVAL;
 847
 848	return vb2_request_validate(req);
 849}
 850
 851static const struct media_device_ops rkvdec_media_ops = {
 852	.req_validate = rkvdec_request_validate,
 853	.req_queue = v4l2_m2m_request_queue,
 854};
 855
 856static void rkvdec_job_finish_no_pm(struct rkvdec_ctx *ctx,
 857				    enum vb2_buffer_state result)
 858{
 859	if (ctx->coded_fmt_desc->ops->done) {
 860		struct vb2_v4l2_buffer *src_buf, *dst_buf;
 861
 862		src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 863		dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
 864		ctx->coded_fmt_desc->ops->done(ctx, src_buf, dst_buf, result);
 865	}
 866
 867	v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx,
 868					 result);
 869}
 870
 871static void rkvdec_job_finish(struct rkvdec_ctx *ctx,
 872			      enum vb2_buffer_state result)
 873{
 874	struct rkvdec_dev *rkvdec = ctx->dev;
 875
 876	pm_runtime_put_autosuspend(rkvdec->dev);
 877	rkvdec_job_finish_no_pm(ctx, result);
 878}
 879
 880void rkvdec_run_preamble(struct rkvdec_ctx *ctx, struct rkvdec_run *run)
 881{
 882	struct media_request *src_req;
 883
 884	memset(run, 0, sizeof(*run));
 885
 886	run->bufs.src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 887	run->bufs.dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
 888
 889	/* Apply request(s) controls if needed. */
 890	src_req = run->bufs.src->vb2_buf.req_obj.req;
 891	if (src_req)
 892		v4l2_ctrl_request_setup(src_req, &ctx->ctrl_hdl);
 893
 894	v4l2_m2m_buf_copy_metadata(run->bufs.src, run->bufs.dst);
 895}
 896
 897void rkvdec_run_postamble(struct rkvdec_ctx *ctx, struct rkvdec_run *run)
 898{
 899	struct media_request *src_req = run->bufs.src->vb2_buf.req_obj.req;
 900
 901	if (src_req)
 902		v4l2_ctrl_request_complete(src_req, &ctx->ctrl_hdl);
 903}
 904
 905void rkvdec_quirks_disable_qos(struct rkvdec_ctx *ctx)
 906{
 907	struct rkvdec_dev *rkvdec = ctx->dev;
 908	u32 reg;
 909
 910	/* Set undocumented swreg_block_gating_e field */
 911	reg = readl(rkvdec->regs + RKVDEC_REG_QOS_CTRL);
 912	reg &= GENMASK(31, 16);
 913	reg |= 0xEFFF;
 914	writel(reg, rkvdec->regs + RKVDEC_REG_QOS_CTRL);
 915}
 916
 917static void rkvdec_device_run(void *priv)
 918{
 919	struct rkvdec_ctx *ctx = priv;
 920	struct rkvdec_dev *rkvdec = ctx->dev;
 921	const struct rkvdec_coded_fmt_desc *desc = ctx->coded_fmt_desc;
 922	int ret;
 923
 924	if (WARN_ON(!desc))
 925		return;
 926
 927	ret = pm_runtime_resume_and_get(rkvdec->dev);
 928	if (ret < 0) {
 929		rkvdec_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR);
 930		return;
 931	}
 932
 933	ret = desc->ops->run(ctx);
 934	if (ret)
 935		rkvdec_job_finish(ctx, VB2_BUF_STATE_ERROR);
 936}
 937
 938static const struct v4l2_m2m_ops rkvdec_m2m_ops = {
 939	.device_run = rkvdec_device_run,
 940};
 941
 942static int rkvdec_queue_init(void *priv,
 943			     struct vb2_queue *src_vq,
 944			     struct vb2_queue *dst_vq)
 945{
 946	struct rkvdec_ctx *ctx = priv;
 947	struct rkvdec_dev *rkvdec = ctx->dev;
 948	int ret;
 949
 950	src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
 951	src_vq->io_modes = VB2_MMAP | VB2_DMABUF;
 952	src_vq->drv_priv = ctx;
 953	src_vq->ops = &rkvdec_queue_ops;
 954	src_vq->mem_ops = &vb2_dma_contig_memops;
 955
 956	/*
 957	 * Driver does mostly sequential access, so sacrifice TLB efficiency
 958	 * for faster allocation. Also, no CPU access on the source queue,
 959	 * so no kernel mapping needed.
 960	 */
 961	src_vq->dma_attrs = DMA_ATTR_ALLOC_SINGLE_PAGES |
 962			    DMA_ATTR_NO_KERNEL_MAPPING;
 963	src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
 964	src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
 965	src_vq->lock = &rkvdec->vdev_lock;
 966	src_vq->dev = rkvdec->v4l2_dev.dev;
 967	src_vq->supports_requests = true;
 968	src_vq->requires_requests = true;
 969
 970	ret = vb2_queue_init(src_vq);
 971	if (ret)
 972		return ret;
 973
 974	dst_vq->bidirectional = true;
 975	dst_vq->mem_ops = &vb2_dma_contig_memops;
 976	dst_vq->dma_attrs = DMA_ATTR_ALLOC_SINGLE_PAGES |
 977			    DMA_ATTR_NO_KERNEL_MAPPING;
 978	dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
 979	dst_vq->io_modes = VB2_MMAP | VB2_DMABUF;
 980	dst_vq->drv_priv = ctx;
 981	dst_vq->ops = &rkvdec_queue_ops;
 982	dst_vq->buf_struct_size = sizeof(struct rkvdec_decoded_buffer);
 983	dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
 984	dst_vq->lock = &rkvdec->vdev_lock;
 985	dst_vq->dev = rkvdec->v4l2_dev.dev;
 986
 987	return vb2_queue_init(dst_vq);
 988}
 989
 990static int rkvdec_add_ctrls(struct rkvdec_ctx *ctx,
 991			    const struct rkvdec_ctrls *ctrls)
 992{
 993	unsigned int i;
 994
 995	for (i = 0; i < ctrls->num_ctrls; i++) {
 996		const struct v4l2_ctrl_config *cfg = &ctrls->ctrls[i].cfg;
 997
 998		v4l2_ctrl_new_custom(&ctx->ctrl_hdl, cfg, ctx);
 999		if (ctx->ctrl_hdl.error)
1000			return ctx->ctrl_hdl.error;
1001	}
1002
1003	return 0;
1004}
1005
1006static int rkvdec_init_ctrls(struct rkvdec_ctx *ctx)
1007{
1008	unsigned int i, nctrls = 0;
1009	int ret;
1010
1011	for (i = 0; i < ARRAY_SIZE(rkvdec_coded_fmts); i++)
1012		if (rkvdec_is_capable(ctx, rkvdec_coded_fmts[i].capability))
1013			nctrls += rkvdec_coded_fmts[i].ctrls->num_ctrls;
1014
1015	v4l2_ctrl_handler_init(&ctx->ctrl_hdl, nctrls);
1016
1017	for (i = 0; i < ARRAY_SIZE(rkvdec_coded_fmts); i++) {
1018		if (rkvdec_is_capable(ctx, rkvdec_coded_fmts[i].capability)) {
1019			ret = rkvdec_add_ctrls(ctx, rkvdec_coded_fmts[i].ctrls);
1020			if (ret)
1021				goto err_free_handler;
1022		}
1023	}
1024
1025	ret = v4l2_ctrl_handler_setup(&ctx->ctrl_hdl);
1026	if (ret)
1027		goto err_free_handler;
1028
1029	ctx->fh.ctrl_handler = &ctx->ctrl_hdl;
1030	return 0;
1031
1032err_free_handler:
1033	v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
1034	return ret;
1035}
1036
1037static int rkvdec_open(struct file *filp)
1038{
1039	struct rkvdec_dev *rkvdec = video_drvdata(filp);
1040	struct rkvdec_ctx *ctx;
1041	int ret;
1042
1043	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1044	if (!ctx)
1045		return -ENOMEM;
1046
1047	ctx->dev = rkvdec;
1048	rkvdec_reset_coded_fmt(ctx);
1049	rkvdec_reset_decoded_fmt(ctx);
1050	v4l2_fh_init(&ctx->fh, video_devdata(filp));
1051
1052	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(rkvdec->m2m_dev, ctx,
1053					    rkvdec_queue_init);
1054	if (IS_ERR(ctx->fh.m2m_ctx)) {
1055		ret = PTR_ERR(ctx->fh.m2m_ctx);
1056		goto err_free_ctx;
1057	}
1058
1059	ret = rkvdec_init_ctrls(ctx);
1060	if (ret)
1061		goto err_cleanup_m2m_ctx;
1062
1063	v4l2_fh_add(&ctx->fh, filp);
1064
1065	return 0;
1066
1067err_cleanup_m2m_ctx:
1068	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
1069
1070err_free_ctx:
1071	kfree(ctx);
1072	return ret;
1073}
1074
1075static int rkvdec_release(struct file *filp)
1076{
1077	struct rkvdec_ctx *ctx = file_to_rkvdec_ctx(filp);
1078
1079	v4l2_fh_del(&ctx->fh, filp);
1080	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
1081	v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
1082	v4l2_fh_exit(&ctx->fh);
1083	kfree(ctx);
1084
1085	return 0;
1086}
1087
1088static const struct v4l2_file_operations rkvdec_fops = {
1089	.owner = THIS_MODULE,
1090	.open = rkvdec_open,
1091	.release = rkvdec_release,
1092	.poll = v4l2_m2m_fop_poll,
1093	.unlocked_ioctl = video_ioctl2,
1094	.mmap = v4l2_m2m_fop_mmap,
1095};
1096
1097static int rkvdec_v4l2_init(struct rkvdec_dev *rkvdec)
1098{
1099	int ret;
1100
1101	ret = v4l2_device_register(rkvdec->dev, &rkvdec->v4l2_dev);
1102	if (ret) {
1103		dev_err(rkvdec->dev, "Failed to register V4L2 device\n");
1104		return ret;
1105	}
1106
1107	rkvdec->m2m_dev = v4l2_m2m_init(&rkvdec_m2m_ops);
1108	if (IS_ERR(rkvdec->m2m_dev)) {
1109		v4l2_err(&rkvdec->v4l2_dev, "Failed to init mem2mem device\n");
1110		ret = PTR_ERR(rkvdec->m2m_dev);
1111		goto err_unregister_v4l2;
1112	}
1113
1114	rkvdec->mdev.dev = rkvdec->dev;
1115	strscpy(rkvdec->mdev.model, "rkvdec", sizeof(rkvdec->mdev.model));
1116	strscpy(rkvdec->mdev.bus_info, "platform:rkvdec",
1117		sizeof(rkvdec->mdev.bus_info));
1118	media_device_init(&rkvdec->mdev);
1119	rkvdec->mdev.ops = &rkvdec_media_ops;
1120	rkvdec->v4l2_dev.mdev = &rkvdec->mdev;
1121
1122	rkvdec->vdev.lock = &rkvdec->vdev_lock;
1123	rkvdec->vdev.v4l2_dev = &rkvdec->v4l2_dev;
1124	rkvdec->vdev.fops = &rkvdec_fops;
1125	rkvdec->vdev.release = video_device_release_empty;
1126	rkvdec->vdev.vfl_dir = VFL_DIR_M2M;
1127	rkvdec->vdev.device_caps = V4L2_CAP_STREAMING |
1128				   V4L2_CAP_VIDEO_M2M_MPLANE;
1129	rkvdec->vdev.ioctl_ops = &rkvdec_ioctl_ops;
1130	video_set_drvdata(&rkvdec->vdev, rkvdec);
1131	strscpy(rkvdec->vdev.name, "rkvdec", sizeof(rkvdec->vdev.name));
1132
1133	ret = video_register_device(&rkvdec->vdev, VFL_TYPE_VIDEO, -1);
1134	if (ret) {
1135		v4l2_err(&rkvdec->v4l2_dev, "Failed to register video device\n");
1136		goto err_cleanup_mc;
1137	}
1138
1139	ret = v4l2_m2m_register_media_controller(rkvdec->m2m_dev, &rkvdec->vdev,
1140						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
1141	if (ret) {
1142		v4l2_err(&rkvdec->v4l2_dev,
1143			 "Failed to initialize V4L2 M2M media controller\n");
1144		goto err_unregister_vdev;
1145	}
1146
1147	ret = media_device_register(&rkvdec->mdev);
1148	if (ret) {
1149		v4l2_err(&rkvdec->v4l2_dev, "Failed to register media device\n");
1150		goto err_unregister_mc;
1151	}
1152
1153	return 0;
1154
1155err_unregister_mc:
1156	v4l2_m2m_unregister_media_controller(rkvdec->m2m_dev);
1157
1158err_unregister_vdev:
1159	video_unregister_device(&rkvdec->vdev);
1160
1161err_cleanup_mc:
1162	media_device_cleanup(&rkvdec->mdev);
1163	v4l2_m2m_release(rkvdec->m2m_dev);
1164
1165err_unregister_v4l2:
1166	v4l2_device_unregister(&rkvdec->v4l2_dev);
1167	return ret;
1168}
1169
1170static void rkvdec_v4l2_cleanup(struct rkvdec_dev *rkvdec)
1171{
1172	media_device_unregister(&rkvdec->mdev);
1173	v4l2_m2m_unregister_media_controller(rkvdec->m2m_dev);
1174	video_unregister_device(&rkvdec->vdev);
1175	media_device_cleanup(&rkvdec->mdev);
1176	v4l2_m2m_release(rkvdec->m2m_dev);
1177	v4l2_device_unregister(&rkvdec->v4l2_dev);
1178}
1179
1180static void rkvdec_iommu_restore(struct rkvdec_dev *rkvdec)
1181{
1182	if (rkvdec->empty_domain) {
1183		/*
1184		 * To rewrite mapping into the attached IOMMU core, attach a new empty domain that
1185		 * will program an empty table, then detach it to restore the default domain and
1186		 * all cached mappings.
1187		 * This is safely done in this interrupt handler to make sure no memory get mapped
1188		 * through the IOMMU while the empty domain is attached.
1189		 */
1190		iommu_attach_device(rkvdec->empty_domain, rkvdec->dev);
1191		iommu_detach_device(rkvdec->empty_domain, rkvdec->dev);
1192	}
1193}
1194
1195static irqreturn_t rkvdec_irq_handler(int irq, void *priv)
1196{
1197	struct rkvdec_dev *rkvdec = priv;
1198	struct rkvdec_ctx *ctx = v4l2_m2m_get_curr_priv(rkvdec->m2m_dev);
1199	enum vb2_buffer_state state;
1200	u32 status;
1201
1202	status = readl(rkvdec->regs + RKVDEC_REG_INTERRUPT);
1203	writel(0, rkvdec->regs + RKVDEC_REG_INTERRUPT);
1204
1205	if (status & RKVDEC_RDY_STA) {
1206		state = VB2_BUF_STATE_DONE;
1207	} else {
1208		state = VB2_BUF_STATE_ERROR;
1209		if (status & RKVDEC_SOFTRESET_RDY)
1210			rkvdec_iommu_restore(rkvdec);
1211	}
1212
1213	if (cancel_delayed_work(&rkvdec->watchdog_work))
1214		rkvdec_job_finish(ctx, state);
1215
1216	return IRQ_HANDLED;
1217}
1218
1219static void rkvdec_watchdog_func(struct work_struct *work)
1220{
1221	struct rkvdec_dev *rkvdec;
1222	struct rkvdec_ctx *ctx;
1223
1224	rkvdec = container_of(to_delayed_work(work), struct rkvdec_dev,
1225			      watchdog_work);
1226	ctx = v4l2_m2m_get_curr_priv(rkvdec->m2m_dev);
1227	if (ctx) {
1228		dev_err(rkvdec->dev, "Frame processing timed out!\n");
1229		writel(RKVDEC_IRQ_DIS, rkvdec->regs + RKVDEC_REG_INTERRUPT);
1230		writel(0, rkvdec->regs + RKVDEC_REG_SYSCTRL);
1231		rkvdec_job_finish(ctx, VB2_BUF_STATE_ERROR);
1232	}
1233}
1234
1235static const struct rkvdec_variant rk3288_rkvdec_variant = {
1236	.num_regs = 68,
1237	.capabilities = RKVDEC_CAPABILITY_HEVC,
1238};
1239
1240static const struct rkvdec_variant rk3328_rkvdec_variant = {
1241	.num_regs = 109,
1242	.capabilities = RKVDEC_CAPABILITY_HEVC |
1243			RKVDEC_CAPABILITY_H264 |
1244			RKVDEC_CAPABILITY_VP9,
1245	.quirks = RKVDEC_QUIRK_DISABLE_QOS,
1246};
1247
1248static const struct rkvdec_variant rk3399_rkvdec_variant = {
1249	.num_regs = 78,
1250	.capabilities = RKVDEC_CAPABILITY_HEVC |
1251			RKVDEC_CAPABILITY_H264 |
1252			RKVDEC_CAPABILITY_VP9,
1253};
1254
1255static const struct of_device_id of_rkvdec_match[] = {
1256	{
1257		.compatible = "rockchip,rk3288-vdec",
1258		.data = &rk3288_rkvdec_variant,
1259	},
1260	{
1261		.compatible = "rockchip,rk3328-vdec",
1262		.data = &rk3328_rkvdec_variant,
1263	},
1264	{
1265		.compatible = "rockchip,rk3399-vdec",
1266		.data = &rk3399_rkvdec_variant,
1267	},
1268	{ /* sentinel */ }
1269};
1270MODULE_DEVICE_TABLE(of, of_rkvdec_match);
1271
1272static const char * const rkvdec_clk_names[] = {
1273	"axi", "ahb", "cabac", "core"
1274};
1275
1276static int rkvdec_probe(struct platform_device *pdev)
1277{
1278	const struct rkvdec_variant *variant;
1279	struct rkvdec_dev *rkvdec;
1280	unsigned int i;
1281	int ret, irq;
1282
1283	variant = of_device_get_match_data(&pdev->dev);
1284	if (!variant)
1285		return -EINVAL;
1286
1287	rkvdec = devm_kzalloc(&pdev->dev, sizeof(*rkvdec), GFP_KERNEL);
1288	if (!rkvdec)
1289		return -ENOMEM;
1290
1291	platform_set_drvdata(pdev, rkvdec);
1292	rkvdec->dev = &pdev->dev;
1293	rkvdec->variant = variant;
1294	mutex_init(&rkvdec->vdev_lock);
1295	INIT_DELAYED_WORK(&rkvdec->watchdog_work, rkvdec_watchdog_func);
1296
1297	rkvdec->clocks = devm_kcalloc(&pdev->dev, ARRAY_SIZE(rkvdec_clk_names),
1298				      sizeof(*rkvdec->clocks), GFP_KERNEL);
1299	if (!rkvdec->clocks)
1300		return -ENOMEM;
1301
1302	for (i = 0; i < ARRAY_SIZE(rkvdec_clk_names); i++)
1303		rkvdec->clocks[i].id = rkvdec_clk_names[i];
1304
1305	ret = devm_clk_bulk_get(&pdev->dev, ARRAY_SIZE(rkvdec_clk_names),
1306				rkvdec->clocks);
1307	if (ret)
1308		return ret;
1309
1310	rkvdec->regs = devm_platform_ioremap_resource(pdev, 0);
1311	if (IS_ERR(rkvdec->regs))
1312		return PTR_ERR(rkvdec->regs);
1313
1314	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1315	if (ret) {
1316		dev_err(&pdev->dev, "Could not set DMA coherent mask.\n");
1317		return ret;
1318	}
1319
1320	vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32));
1321
1322	irq = platform_get_irq(pdev, 0);
1323	if (irq <= 0)
1324		return -ENXIO;
1325
1326	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
1327					rkvdec_irq_handler, IRQF_ONESHOT,
1328					dev_name(&pdev->dev), rkvdec);
1329	if (ret) {
1330		dev_err(&pdev->dev, "Could not request vdec IRQ\n");
1331		return ret;
1332	}
1333
1334	pm_runtime_set_autosuspend_delay(&pdev->dev, 100);
1335	pm_runtime_use_autosuspend(&pdev->dev);
1336	pm_runtime_enable(&pdev->dev);
1337
1338	ret = rkvdec_v4l2_init(rkvdec);
1339	if (ret)
1340		goto err_disable_runtime_pm;
1341
1342	if (iommu_get_domain_for_dev(&pdev->dev)) {
1343		rkvdec->empty_domain = iommu_paging_domain_alloc(rkvdec->dev);
1344
1345		if (IS_ERR(rkvdec->empty_domain)) {
1346			rkvdec->empty_domain = NULL;
1347			dev_warn(rkvdec->dev, "cannot alloc new empty domain\n");
1348		}
1349	}
1350
1351	return 0;
1352
1353err_disable_runtime_pm:
1354	pm_runtime_dont_use_autosuspend(&pdev->dev);
1355	pm_runtime_disable(&pdev->dev);
1356	return ret;
1357}
1358
1359static void rkvdec_remove(struct platform_device *pdev)
1360{
1361	struct rkvdec_dev *rkvdec = platform_get_drvdata(pdev);
1362
1363	cancel_delayed_work_sync(&rkvdec->watchdog_work);
1364
1365	rkvdec_v4l2_cleanup(rkvdec);
1366	pm_runtime_disable(&pdev->dev);
1367	pm_runtime_dont_use_autosuspend(&pdev->dev);
1368
1369	if (rkvdec->empty_domain)
1370		iommu_domain_free(rkvdec->empty_domain);
1371}
1372
1373#ifdef CONFIG_PM
1374static int rkvdec_runtime_resume(struct device *dev)
1375{
1376	struct rkvdec_dev *rkvdec = dev_get_drvdata(dev);
1377
1378	return clk_bulk_prepare_enable(ARRAY_SIZE(rkvdec_clk_names),
1379				       rkvdec->clocks);
1380}
1381
1382static int rkvdec_runtime_suspend(struct device *dev)
1383{
1384	struct rkvdec_dev *rkvdec = dev_get_drvdata(dev);
1385
1386	clk_bulk_disable_unprepare(ARRAY_SIZE(rkvdec_clk_names),
1387				   rkvdec->clocks);
1388	return 0;
1389}
1390#endif
1391
1392static const struct dev_pm_ops rkvdec_pm_ops = {
1393	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1394				pm_runtime_force_resume)
1395	SET_RUNTIME_PM_OPS(rkvdec_runtime_suspend, rkvdec_runtime_resume, NULL)
1396};
1397
1398static struct platform_driver rkvdec_driver = {
1399	.probe = rkvdec_probe,
1400	.remove = rkvdec_remove,
1401	.driver = {
1402		   .name = "rkvdec",
1403		   .of_match_table = of_rkvdec_match,
1404		   .pm = &rkvdec_pm_ops,
1405	},
1406};
1407module_platform_driver(rkvdec_driver);
1408
1409MODULE_AUTHOR("Boris Brezillon <boris.brezillon@collabora.com>");
1410MODULE_DESCRIPTION("Rockchip Video Decoder driver");
1411MODULE_LICENSE("GPL v2");