iris_vdec.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
  4 */
  5
  6#include <media/v4l2-event.h>
  7#include <media/v4l2-mem2mem.h>
  8
  9#include "iris_buffer.h"
 10#include "iris_common.h"
 11#include "iris_ctrls.h"
 12#include "iris_instance.h"
 13#include "iris_power.h"
 14#include "iris_vdec.h"
 15#include "iris_vpu_buffer.h"
 16
 17#define DEFAULT_CODEC_ALIGNMENT 16
 18
 19int iris_vdec_inst_init(struct iris_inst *inst)
 20{
 21	struct iris_core *core = inst->core;
 22	struct v4l2_format *f;
 23
 24	inst->fmt_src  = kzalloc(sizeof(*inst->fmt_src), GFP_KERNEL);
 25	inst->fmt_dst  = kzalloc(sizeof(*inst->fmt_dst), GFP_KERNEL);
 26
 27	inst->fw_min_count = MIN_BUFFERS;
 28
 29	f = inst->fmt_src;
 30	f->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
 31	f->fmt.pix_mp.width = DEFAULT_WIDTH;
 32	f->fmt.pix_mp.height = DEFAULT_HEIGHT;
 33	f->fmt.pix_mp.pixelformat = V4L2_PIX_FMT_H264;
 34	inst->codec = f->fmt.pix_mp.pixelformat;
 35	f->fmt.pix_mp.num_planes = 1;
 36	f->fmt.pix_mp.plane_fmt[0].bytesperline = 0;
 37	f->fmt.pix_mp.plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_INPUT);
 38	f->fmt.pix_mp.field = V4L2_FIELD_NONE;
 39	inst->buffers[BUF_INPUT].min_count = iris_vpu_buf_count(inst, BUF_INPUT);
 40	inst->buffers[BUF_INPUT].size = f->fmt.pix_mp.plane_fmt[0].sizeimage;
 41
 42	f = inst->fmt_dst;
 43	f->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
 44	f->fmt.pix_mp.pixelformat = V4L2_PIX_FMT_NV12;
 45	f->fmt.pix_mp.width = ALIGN(DEFAULT_WIDTH, 128);
 46	f->fmt.pix_mp.height = ALIGN(DEFAULT_HEIGHT, 32);
 47	f->fmt.pix_mp.num_planes = 1;
 48	f->fmt.pix_mp.plane_fmt[0].bytesperline = ALIGN(DEFAULT_WIDTH, 128);
 49	f->fmt.pix_mp.plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_OUTPUT);
 50	f->fmt.pix_mp.field = V4L2_FIELD_NONE;
 51	f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_DEFAULT;
 52	f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT;
 53	f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
 54	f->fmt.pix_mp.quantization = V4L2_QUANTIZATION_DEFAULT;
 55	inst->buffers[BUF_OUTPUT].min_count = iris_vpu_buf_count(inst, BUF_OUTPUT);
 56	inst->buffers[BUF_OUTPUT].size = f->fmt.pix_mp.plane_fmt[0].sizeimage;
 57
 58	memcpy(&inst->fw_caps[0], &core->inst_fw_caps_dec[0],
 59	       INST_FW_CAP_MAX * sizeof(struct platform_inst_fw_cap));
 60
 61	return iris_ctrls_init(inst);
 62}
 63
 64void iris_vdec_inst_deinit(struct iris_inst *inst)
 65{
 66	kfree(inst->fmt_dst);
 67	kfree(inst->fmt_src);
 68}
 69
 70static const struct iris_fmt iris_vdec_formats_out[] = {
 71	[IRIS_FMT_H264] = {
 72		.pixfmt = V4L2_PIX_FMT_H264,
 73		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
 74	},
 75	[IRIS_FMT_HEVC] = {
 76		.pixfmt = V4L2_PIX_FMT_HEVC,
 77		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
 78	},
 79	[IRIS_FMT_VP9] = {
 80		.pixfmt = V4L2_PIX_FMT_VP9,
 81		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
 82	},
 83};
 84
 85static const struct iris_fmt iris_vdec_formats_cap[] = {
 86	[IRIS_FMT_NV12] = {
 87		.pixfmt = V4L2_PIX_FMT_NV12,
 88		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 89	},
 90	[IRIS_FMT_QC08C] = {
 91		.pixfmt = V4L2_PIX_FMT_QC08C,
 92		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 93	},
 94};
 95
 96static const struct iris_fmt *
 97find_format(struct iris_inst *inst, u32 pixfmt, u32 type)
 98{
 99	const struct iris_fmt *fmt = NULL;
100	unsigned int size = 0;
101	unsigned int i;
102	switch (type) {
103	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
104		fmt = iris_vdec_formats_out;
105		size = ARRAY_SIZE(iris_vdec_formats_out);
106		break;
107	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
108		fmt = iris_vdec_formats_cap;
109		size = ARRAY_SIZE(iris_vdec_formats_cap);
110		break;
111	default:
112		return NULL;
113	}
114
115	for (i = 0; i < size; i++) {
116		if (fmt[i].pixfmt == pixfmt)
117			break;
118	}
119
120	if (i == size || fmt[i].type != type)
121		return NULL;
122
123	return &fmt[i];
124}
125
126static const struct iris_fmt *
127find_format_by_index(struct iris_inst *inst, u32 index, u32 type)
128{
129	const struct iris_fmt *fmt = NULL;
130	unsigned int size = 0;
131
132	switch (type) {
133	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
134		fmt = iris_vdec_formats_out;
135		size = ARRAY_SIZE(iris_vdec_formats_out);
136		break;
137	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
138		fmt = iris_vdec_formats_cap;
139		size = ARRAY_SIZE(iris_vdec_formats_cap);
140		break;
141	default:
142		return NULL;
143	}
144
145	if (index >= size || fmt[index].type != type)
146		return NULL;
147
148	return &fmt[index];
149}
150
151int iris_vdec_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f)
152{
153	const struct iris_fmt *fmt;
154
155	switch (f->type) {
156	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
157		fmt = find_format_by_index(inst, f->index, f->type);
158		if (!fmt)
159			return -EINVAL;
160
161		f->pixelformat = fmt->pixfmt;
162		f->flags = V4L2_FMT_FLAG_COMPRESSED | V4L2_FMT_FLAG_DYN_RESOLUTION;
163		break;
164	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
165		fmt = find_format_by_index(inst, f->index, f->type);
166		if (!fmt)
167			return -EINVAL;
168		f->pixelformat = fmt->pixfmt;
169		break;
170	default:
171		return -EINVAL;
172	}
173
174	return 0;
175}
176
177int iris_vdec_try_fmt(struct iris_inst *inst, struct v4l2_format *f)
178{
179	struct v4l2_pix_format_mplane *pixmp = &f->fmt.pix_mp;
180	struct v4l2_m2m_ctx *m2m_ctx = inst->m2m_ctx;
181	const struct iris_fmt *fmt;
182	struct v4l2_format *f_inst;
183	struct vb2_queue *src_q;
184
185	memset(pixmp->reserved, 0, sizeof(pixmp->reserved));
186	fmt = find_format(inst, pixmp->pixelformat, f->type);
187	switch (f->type) {
188	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
189		if (!fmt) {
190			f_inst = inst->fmt_src;
191			f->fmt.pix_mp.width = f_inst->fmt.pix_mp.width;
192			f->fmt.pix_mp.height = f_inst->fmt.pix_mp.height;
193			f->fmt.pix_mp.pixelformat = f_inst->fmt.pix_mp.pixelformat;
194		}
195		break;
196	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
197		if (!fmt) {
198			f_inst = inst->fmt_dst;
199			f->fmt.pix_mp.pixelformat = f_inst->fmt.pix_mp.pixelformat;
200			f->fmt.pix_mp.width = f_inst->fmt.pix_mp.width;
201			f->fmt.pix_mp.height = f_inst->fmt.pix_mp.height;
202		}
203
204		src_q = v4l2_m2m_get_src_vq(m2m_ctx);
205		if (vb2_is_streaming(src_q)) {
206			f_inst = inst->fmt_src;
207			f->fmt.pix_mp.height = f_inst->fmt.pix_mp.height;
208			f->fmt.pix_mp.width = f_inst->fmt.pix_mp.width;
209		}
210		break;
211	default:
212		return -EINVAL;
213	}
214
215	if (pixmp->field == V4L2_FIELD_ANY)
216		pixmp->field = V4L2_FIELD_NONE;
217
218	pixmp->num_planes = 1;
219
220	return 0;
221}
222
223int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f)
224{
225	struct v4l2_format *fmt, *output_fmt;
226	struct vb2_queue *q;
227	u32 codec_align;
228
229	q = v4l2_m2m_get_vq(inst->m2m_ctx, f->type);
230
231	if (vb2_is_busy(q))
232		return -EBUSY;
233
234	/* Width and height are optional, so fall back to a valid placeholder
235	 * resolution until the real one is decoded from the bitstream.
236	 */
237	if (f->fmt.pix_mp.width == 0 && f->fmt.pix_mp.height == 0) {
238		f->fmt.pix_mp.width = DEFAULT_WIDTH;
239		f->fmt.pix_mp.height = DEFAULT_HEIGHT;
240	}
241
242	iris_vdec_try_fmt(inst, f);
243
244	switch (f->type) {
245	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
246		if (!(find_format(inst, f->fmt.pix_mp.pixelformat, f->type)))
247			return -EINVAL;
248
249		fmt = inst->fmt_src;
250		fmt->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
251		fmt->fmt.pix_mp.pixelformat = f->fmt.pix_mp.pixelformat;
252		inst->codec = fmt->fmt.pix_mp.pixelformat;
253		codec_align = inst->codec == V4L2_PIX_FMT_HEVC ? 32 : 16;
254		fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, codec_align);
255		fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, codec_align);
256		fmt->fmt.pix_mp.num_planes = 1;
257		fmt->fmt.pix_mp.plane_fmt[0].bytesperline = 0;
258		fmt->fmt.pix_mp.plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_INPUT);
259		inst->buffers[BUF_INPUT].min_count = iris_vpu_buf_count(inst, BUF_INPUT);
260		inst->buffers[BUF_INPUT].size = fmt->fmt.pix_mp.plane_fmt[0].sizeimage;
261
262		fmt->fmt.pix_mp.colorspace = f->fmt.pix_mp.colorspace;
263		fmt->fmt.pix_mp.xfer_func = f->fmt.pix_mp.xfer_func;
264		fmt->fmt.pix_mp.ycbcr_enc = f->fmt.pix_mp.ycbcr_enc;
265		fmt->fmt.pix_mp.quantization = f->fmt.pix_mp.quantization;
266
267		output_fmt = inst->fmt_dst;
268		output_fmt->fmt.pix_mp.colorspace = f->fmt.pix_mp.colorspace;
269		output_fmt->fmt.pix_mp.xfer_func = f->fmt.pix_mp.xfer_func;
270		output_fmt->fmt.pix_mp.ycbcr_enc = f->fmt.pix_mp.ycbcr_enc;
271		output_fmt->fmt.pix_mp.quantization = f->fmt.pix_mp.quantization;
272
273		/* Update capture format based on new ip w/h */
274		output_fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, 128);
275		output_fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 32);
276		inst->buffers[BUF_OUTPUT].size = iris_get_buffer_size(inst, BUF_OUTPUT);
277
278		inst->crop.left = 0;
279		inst->crop.top = 0;
280		inst->crop.width = f->fmt.pix_mp.width;
281		inst->crop.height = f->fmt.pix_mp.height;
282		break;
283	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
284		if (!(find_format(inst, f->fmt.pix_mp.pixelformat, f->type)))
285			return -EINVAL;
286
287		fmt = inst->fmt_dst;
288		fmt->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
289		fmt->fmt.pix_mp.pixelformat = f->fmt.pix_mp.pixelformat;
290		fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, 128);
291		fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, 32);
292		fmt->fmt.pix_mp.num_planes = 1;
293		fmt->fmt.pix_mp.plane_fmt[0].bytesperline = ALIGN(f->fmt.pix_mp.width, 128);
294		fmt->fmt.pix_mp.plane_fmt[0].sizeimage = iris_get_buffer_size(inst, BUF_OUTPUT);
295		inst->buffers[BUF_OUTPUT].min_count = iris_vpu_buf_count(inst, BUF_OUTPUT);
296		inst->buffers[BUF_OUTPUT].size = fmt->fmt.pix_mp.plane_fmt[0].sizeimage;
297
298		inst->crop.top = 0;
299		inst->crop.left = 0;
300		inst->crop.width = f->fmt.pix_mp.width;
301		inst->crop.height = f->fmt.pix_mp.height;
302		break;
303	default:
304		return -EINVAL;
305	}
306	memcpy(f, fmt, sizeof(*fmt));
307
308	return 0;
309}
310
311int iris_vdec_validate_format(struct iris_inst *inst, u32 pixelformat)
312{
313	const struct iris_fmt *fmt = NULL;
314
315	fmt = find_format(inst, pixelformat, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
316	if (!fmt) {
317		fmt = find_format(inst, pixelformat, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
318		if (!fmt)
319			return -EINVAL;
320	}
321
322	return 0;
323}
324
325int iris_vdec_subscribe_event(struct iris_inst *inst, const struct v4l2_event_subscription *sub)
326{
327	int ret = 0;
328
329	switch (sub->type) {
330	case V4L2_EVENT_EOS:
331		ret = v4l2_event_subscribe(&inst->fh, sub, 0, NULL);
332		break;
333	case V4L2_EVENT_SOURCE_CHANGE:
334		ret = v4l2_src_change_event_subscribe(&inst->fh, sub);
335		break;
336	case V4L2_EVENT_CTRL:
337		ret = v4l2_ctrl_subscribe_event(&inst->fh, sub);
338		break;
339	default:
340		return -EINVAL;
341	}
342
343	return ret;
344}
345
346void iris_vdec_src_change(struct iris_inst *inst)
347{
348	struct v4l2_m2m_ctx *m2m_ctx = inst->m2m_ctx;
349	struct v4l2_event event = {0};
350	struct vb2_queue *src_q;
351
352	src_q = v4l2_m2m_get_src_vq(m2m_ctx);
353	if (!vb2_is_streaming(src_q))
354		return;
355
356	event.type = V4L2_EVENT_SOURCE_CHANGE;
357	event.u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION;
358	v4l2_event_queue_fh(&inst->fh, &event);
359}
360
361int iris_vdec_streamon_input(struct iris_inst *inst)
362{
363	int ret;
364
365	ret = iris_set_properties(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
366	if (ret)
367		return ret;
368
369	ret = iris_alloc_and_queue_persist_bufs(inst, BUF_PERSIST);
370	if (ret)
371		return ret;
372
373	iris_get_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
374
375	ret = iris_destroy_dequeued_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
376	if (ret)
377		return ret;
378
379	ret = iris_create_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
380	if (ret)
381		return ret;
382
383	ret = iris_queue_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
384	if (ret)
385		return ret;
386
387	return iris_process_streamon_input(inst);
388}
389
390int iris_vdec_streamon_output(struct iris_inst *inst)
391{
392	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
393	int ret;
394
395	ret = hfi_ops->session_set_config_params(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
396	if (ret)
397		return ret;
398
399	iris_get_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
400
401	ret = iris_destroy_dequeued_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
402	if (ret)
403		return ret;
404
405	ret = iris_create_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
406	if (ret)
407		return ret;
408
409	ret = iris_process_streamon_output(inst);
410	if (ret)
411		goto error;
412
413	ret = iris_queue_internal_buffers(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
414	if (ret)
415		goto error;
416
417	return ret;
418
419error:
420	iris_session_streamoff(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
421
422	return ret;
423}
424
425int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
426{
427	struct iris_buffer *buf = to_iris_buffer(vbuf);
428	struct vb2_buffer *vb2 = &vbuf->vb2_buf;
429	struct vb2_queue *q;
430	int ret;
431
432	ret = iris_vb2_buffer_to_driver(vb2, buf);
433	if (ret)
434		return ret;
435
436	if (buf->type == BUF_INPUT)
437		iris_set_ts_metadata(inst, vbuf);
438
439	q = v4l2_m2m_get_vq(inst->m2m_ctx, vb2->type);
440	if (!vb2_is_streaming(q)) {
441		buf->attr |= BUF_ATTR_DEFERRED;
442		return 0;
443	}
444
445	iris_scale_power(inst);
446
447	return iris_queue_buffer(inst, buf);
448}
449
450int iris_vdec_start_cmd(struct iris_inst *inst)
451{
452	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
453	enum iris_inst_sub_state clear_sub_state = 0;
454	struct vb2_queue *dst_vq;
455	int ret;
456
457	dst_vq = v4l2_m2m_get_dst_vq(inst->m2m_ctx);
458
459	if (inst->sub_state & IRIS_INST_SUB_DRC &&
460	    inst->sub_state & IRIS_INST_SUB_DRC_LAST) {
461		vb2_clear_last_buffer_dequeued(dst_vq);
462		clear_sub_state = IRIS_INST_SUB_DRC | IRIS_INST_SUB_DRC_LAST;
463
464		if (inst->sub_state & IRIS_INST_SUB_INPUT_PAUSE) {
465			ret = hfi_ops->session_resume_drc(inst,
466							  V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
467			if (ret)
468				return ret;
469			clear_sub_state |= IRIS_INST_SUB_INPUT_PAUSE;
470		}
471		if (inst->sub_state & IRIS_INST_SUB_OUTPUT_PAUSE) {
472			ret = hfi_ops->session_resume_drc(inst,
473							  V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
474			if (ret)
475				return ret;
476			clear_sub_state |= IRIS_INST_SUB_OUTPUT_PAUSE;
477		}
478	} else if (inst->sub_state & IRIS_INST_SUB_DRAIN &&
479		   inst->sub_state & IRIS_INST_SUB_DRAIN_LAST) {
480		vb2_clear_last_buffer_dequeued(dst_vq);
481		clear_sub_state = IRIS_INST_SUB_DRAIN | IRIS_INST_SUB_DRAIN_LAST;
482		if (inst->sub_state & IRIS_INST_SUB_INPUT_PAUSE) {
483			if (hfi_ops->session_resume_drain) {
484				ret =
485				hfi_ops->session_resume_drain(inst,
486							      V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
487				if (ret)
488					return ret;
489			}
490
491			clear_sub_state |= IRIS_INST_SUB_INPUT_PAUSE;
492		}
493		if (inst->sub_state & IRIS_INST_SUB_OUTPUT_PAUSE) {
494			if (hfi_ops->session_resume_drain) {
495				ret =
496				hfi_ops->session_resume_drain(inst,
497							      V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
498				if (ret)
499					return ret;
500			}
501
502			clear_sub_state |= IRIS_INST_SUB_OUTPUT_PAUSE;
503		}
504	} else {
505		dev_err(inst->core->dev, "start called before receiving last_flag\n");
506		iris_inst_change_state(inst, IRIS_INST_ERROR);
507		return -EBUSY;
508	}
509
510	return iris_inst_change_sub_state(inst, clear_sub_state, 0);
511}
512
513int iris_vdec_stop_cmd(struct iris_inst *inst)
514{
515	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
516	int ret;
517
518	ret = hfi_ops->session_drain(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
519	if (ret)
520		return ret;
521
522	return iris_inst_change_sub_state(inst, 0, IRIS_INST_SUB_DRAIN);
523}