visl-core.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * A virtual stateless decoder device for stateless uAPI development purposes.
  4 *
  5 * This tool's objective is to help the development and testing of userspace
  6 * applications that use the V4L2 stateless API to decode media.
  7 *
  8 * A userspace implementation can use visl to run a decoding loop even when no
  9 * hardware is available or when the kernel uAPI for the codec has not been
 10 * upstreamed yet. This can reveal bugs at an early stage.
 11 *
 12 * This driver can also trace the contents of the V4L2 controls submitted to it.
 13 * It can also dump the contents of the vb2 buffers through a debugfs
 14 * interface. This is in many ways similar to the tracing infrastructure
 15 * available for other popular encode/decode APIs out there and can help develop
 16 * a userspace application by using another (working) one as a reference.
 17 *
 18 * Note that no actual decoding of video frames is performed by visl. The V4L2
 19 * test pattern generator is used to write various debug information to the
 20 * capture buffers instead.
 21 *
 22 * Copyright (C) 2022 Collabora, Ltd.
 23 *
 24 * Based on the vim2m driver, that is:
 25 *
 26 * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
 27 * Pawel Osciak, <pawel@osciak.com>
 28 * Marek Szyprowski, <m.szyprowski@samsung.com>
 29 *
 30 * Based on the vicodec driver, that is:
 31 *
 32 * Copyright 2018 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
 33 *
 34 * Based on the Cedrus VPU driver, that is:
 35 *
 36 * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com>
 37 * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com>
 38 * Copyright (C) 2018 Bootlin
 39 */
 40
 41#include <linux/debugfs.h>
 42#include <linux/module.h>
 43#include <linux/platform_device.h>
 44#include <media/v4l2-ctrls.h>
 45#include <media/v4l2-device.h>
 46#include <media/v4l2-ioctl.h>
 47#include <media/v4l2-mem2mem.h>
 48
 49#include "visl.h"
 50#include "visl-dec.h"
 51#include "visl-debugfs.h"
 52#include "visl-video.h"
 53
 54unsigned int visl_debug;
 55module_param(visl_debug, uint, 0644);
 56MODULE_PARM_DESC(visl_debug, " activates debug info");
 57
 58unsigned int visl_transtime_ms;
 59module_param(visl_transtime_ms, uint, 0644);
 60MODULE_PARM_DESC(visl_transtime_ms, " simulated process time in milliseconds.");
 61
 62/*
 63 * dprintk can be slow through serial. This lets one limit the tracing to a
 64 * particular number of frames
 65 */
 66int visl_dprintk_frame_start = -1;
 67module_param(visl_dprintk_frame_start, int, 0444);
 68MODULE_PARM_DESC(visl_dprintk_frame_start,
 69		 " a frame number to start tracing with dprintk");
 70
 71unsigned int visl_dprintk_nframes;
 72module_param(visl_dprintk_nframes, uint, 0444);
 73MODULE_PARM_DESC(visl_dprintk_nframes,
 74		 " the number of frames to trace with dprintk");
 75
 76bool keep_bitstream_buffers;
 77module_param(keep_bitstream_buffers, bool, 0444);
 78MODULE_PARM_DESC(keep_bitstream_buffers,
 79		 " keep bitstream buffers in debugfs after streaming is stopped");
 80
 81int bitstream_trace_frame_start = -1;
 82module_param(bitstream_trace_frame_start, int, 0444);
 83MODULE_PARM_DESC(bitstream_trace_frame_start,
 84		 " a frame number to start dumping the bitstream through debugfs");
 85
 86unsigned int bitstream_trace_nframes;
 87module_param(bitstream_trace_nframes, uint, 0444);
 88MODULE_PARM_DESC(bitstream_trace_nframes,
 89		 " the number of frames to dump the bitstream through debugfs");
 90
 91bool tpg_verbose;
 92module_param(tpg_verbose, bool, 0644);
 93MODULE_PARM_DESC(tpg_verbose,
 94		 " add more verbose information on the generated output frames");
 95
 96static const struct visl_ctrl_desc visl_fwht_ctrl_descs[] = {
 97	{
 98		.cfg.id = V4L2_CID_STATELESS_FWHT_PARAMS,
 99	},
100};
101
102const struct visl_ctrls visl_fwht_ctrls = {
103	.ctrls = visl_fwht_ctrl_descs,
104	.num_ctrls = ARRAY_SIZE(visl_fwht_ctrl_descs)
105};
106
107static const struct visl_ctrl_desc visl_mpeg2_ctrl_descs[] = {
108	{
109		.cfg.id = V4L2_CID_STATELESS_MPEG2_SEQUENCE,
110	},
111	{
112		.cfg.id = V4L2_CID_STATELESS_MPEG2_PICTURE,
113	},
114	{
115		.cfg.id = V4L2_CID_STATELESS_MPEG2_QUANTISATION,
116	},
117};
118
119const struct visl_ctrls visl_mpeg2_ctrls = {
120	.ctrls = visl_mpeg2_ctrl_descs,
121	.num_ctrls = ARRAY_SIZE(visl_mpeg2_ctrl_descs),
122};
123
124static const struct visl_ctrl_desc visl_vp8_ctrl_descs[] = {
125	{
126		.cfg.id = V4L2_CID_STATELESS_VP8_FRAME,
127	},
128};
129
130const struct visl_ctrls visl_vp8_ctrls = {
131	.ctrls = visl_vp8_ctrl_descs,
132	.num_ctrls = ARRAY_SIZE(visl_vp8_ctrl_descs),
133};
134
135static const struct visl_ctrl_desc visl_vp9_ctrl_descs[] = {
136	{
137		.cfg.id = V4L2_CID_STATELESS_VP9_FRAME,
138	},
139	{
140		.cfg.id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR,
141	},
142};
143
144const struct visl_ctrls visl_vp9_ctrls = {
145	.ctrls = visl_vp9_ctrl_descs,
146	.num_ctrls = ARRAY_SIZE(visl_vp9_ctrl_descs),
147};
148
149static const struct visl_ctrl_desc visl_h264_ctrl_descs[] = {
150	{
151		.cfg.id = V4L2_CID_STATELESS_H264_DECODE_PARAMS,
152	},
153	{
154		.cfg.id = V4L2_CID_STATELESS_H264_SPS,
155	},
156	{
157		.cfg.id = V4L2_CID_STATELESS_H264_PPS,
158	},
159	{
160		.cfg.id = V4L2_CID_STATELESS_H264_SCALING_MATRIX,
161	},
162	{
163		.cfg.id = V4L2_CID_STATELESS_H264_DECODE_MODE,
164		.cfg.min = V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED,
165		.cfg.max = V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED,
166		.cfg.def = V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED,
167	},
168	{
169		.cfg.id = V4L2_CID_STATELESS_H264_START_CODE,
170		.cfg.min = V4L2_STATELESS_H264_START_CODE_NONE,
171		.cfg.max = V4L2_STATELESS_H264_START_CODE_ANNEX_B,
172		.cfg.def = V4L2_STATELESS_H264_START_CODE_NONE,
173	},
174	{
175		.cfg.id = V4L2_CID_STATELESS_H264_SLICE_PARAMS,
176	},
177	{
178		.cfg.id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS,
179	},
180};
181
182const struct visl_ctrls visl_h264_ctrls = {
183	.ctrls = visl_h264_ctrl_descs,
184	.num_ctrls = ARRAY_SIZE(visl_h264_ctrl_descs),
185};
186
187static const struct visl_ctrl_desc visl_hevc_ctrl_descs[] = {
188	{
189		.cfg.id = V4L2_CID_STATELESS_HEVC_SPS,
190	},
191	{
192		.cfg.id = V4L2_CID_STATELESS_HEVC_PPS,
193	},
194	{
195		.cfg.id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS,
196		/* The absolute maximum for level > 6 */
197		.cfg.dims = { 600 },
198	},
199	{
200		.cfg.id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX,
201	},
202	{
203		.cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS,
204	},
205	{
206		.cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_MODE,
207		.cfg.min = V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED,
208		.cfg.max = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED,
209		.cfg.def = V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED,
210	},
211	{
212		.cfg.id = V4L2_CID_STATELESS_HEVC_START_CODE,
213		.cfg.min = V4L2_STATELESS_HEVC_START_CODE_NONE,
214		.cfg.max = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B,
215		.cfg.def = V4L2_STATELESS_HEVC_START_CODE_NONE,
216	},
217	{
218		.cfg.id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS,
219		.cfg.dims = { 256 },
220		.cfg.max = 0xffffffff,
221		.cfg.step = 1,
222	},
223
224};
225
226const struct visl_ctrls visl_hevc_ctrls = {
227	.ctrls = visl_hevc_ctrl_descs,
228	.num_ctrls = ARRAY_SIZE(visl_hevc_ctrl_descs),
229};
230
231static const struct visl_ctrl_desc visl_av1_ctrl_descs[] = {
232	{
233		.cfg.id = V4L2_CID_STATELESS_AV1_FRAME,
234	},
235	{
236		.cfg.id = V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,
237		.cfg.dims = { V4L2_AV1_MAX_TILE_COUNT },
238	},
239	{
240		.cfg.id = V4L2_CID_STATELESS_AV1_SEQUENCE,
241	},
242	{
243		.cfg.id = V4L2_CID_STATELESS_AV1_FILM_GRAIN,
244	},
245};
246
247const struct visl_ctrls visl_av1_ctrls = {
248	.ctrls = visl_av1_ctrl_descs,
249	.num_ctrls = ARRAY_SIZE(visl_av1_ctrl_descs),
250};
251
252struct v4l2_ctrl *visl_find_control(struct visl_ctx *ctx, u32 id)
253{
254	struct v4l2_ctrl_handler *hdl = &ctx->hdl;
255
256	return v4l2_ctrl_find(hdl, id);
257}
258
259void *visl_find_control_data(struct visl_ctx *ctx, u32 id)
260{
261	struct v4l2_ctrl *ctrl;
262
263	ctrl = visl_find_control(ctx, id);
264	if (ctrl)
265		return ctrl->p_cur.p;
266
267	return NULL;
268}
269
270u32 visl_control_num_elems(struct visl_ctx *ctx, u32 id)
271{
272	struct v4l2_ctrl *ctrl;
273
274	ctrl = visl_find_control(ctx, id);
275	if (ctrl)
276		return ctrl->elems;
277
278	return 0;
279}
280
281static void visl_device_release(struct video_device *vdev)
282{
283	struct visl_dev *dev = container_of(vdev, struct visl_dev, vfd);
284
285	v4l2_device_unregister(&dev->v4l2_dev);
286	v4l2_m2m_release(dev->m2m_dev);
287	media_device_cleanup(&dev->mdev);
288	visl_debugfs_deinit(dev);
289	kfree(dev);
290}
291
292#define VISL_CONTROLS_COUNT	ARRAY_SIZE(visl_controls)
293
294static int visl_init_ctrls(struct visl_ctx *ctx)
295{
296	struct visl_dev *dev = ctx->dev;
297	struct v4l2_ctrl_handler *hdl = &ctx->hdl;
298	unsigned int ctrl_cnt = 0;
299	unsigned int i;
300	unsigned int j;
301	const struct visl_ctrls *ctrls;
302
303	for (i = 0; i < num_coded_fmts; i++)
304		ctrl_cnt += visl_coded_fmts[i].ctrls->num_ctrls;
305
306	v4l2_ctrl_handler_init(hdl, ctrl_cnt);
307
308	for (i = 0; i < num_coded_fmts; i++) {
309		ctrls = visl_coded_fmts[i].ctrls;
310		for (j = 0; j < ctrls->num_ctrls; j++)
311			v4l2_ctrl_new_custom(hdl, &ctrls->ctrls[j].cfg, NULL);
312	}
313
314	if (hdl->error) {
315		v4l2_err(&dev->v4l2_dev,
316			 "Failed to initialize control handler\n");
317		v4l2_ctrl_handler_free(hdl);
318		return hdl->error;
319	}
320
321	ctx->fh.ctrl_handler = hdl;
322	v4l2_ctrl_handler_setup(hdl);
323
324	return 0;
325}
326
327static int visl_open(struct file *file)
328{
329	struct visl_dev *dev = video_drvdata(file);
330	struct visl_ctx *ctx = NULL;
331	int rc = 0;
332
333	if (mutex_lock_interruptible(&dev->dev_mutex))
334		return -ERESTARTSYS;
335	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
336	if (!ctx) {
337		rc = -ENOMEM;
338		goto unlock;
339	}
340
341	ctx->tpg_str_buf = kzalloc(TPG_STR_BUF_SZ, GFP_KERNEL);
342
343	v4l2_fh_init(&ctx->fh, video_devdata(file));
344	ctx->dev = dev;
345
346	rc = visl_init_ctrls(ctx);
347	if (rc)
348		goto free_ctx;
349
350	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev, ctx, &visl_queue_init);
351
352	mutex_init(&ctx->vb_mutex);
353
354	if (IS_ERR(ctx->fh.m2m_ctx)) {
355		rc = PTR_ERR(ctx->fh.m2m_ctx);
356		goto free_hdl;
357	}
358
359	rc = visl_set_default_format(ctx);
360	if (rc)
361		goto free_m2m_ctx;
362
363	v4l2_fh_add(&ctx->fh, file);
364
365	dprintk(dev, "Created instance: %p, m2m_ctx: %p\n",
366		ctx, ctx->fh.m2m_ctx);
367
368	mutex_unlock(&dev->dev_mutex);
369	return rc;
370
371free_m2m_ctx:
372	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
373free_hdl:
374	v4l2_ctrl_handler_free(&ctx->hdl);
375	v4l2_fh_exit(&ctx->fh);
376free_ctx:
377	kfree(ctx->tpg_str_buf);
378	kfree(ctx);
379unlock:
380	mutex_unlock(&dev->dev_mutex);
381	return rc;
382}
383
384static int visl_release(struct file *file)
385{
386	struct visl_dev *dev = video_drvdata(file);
387	struct visl_ctx *ctx = visl_file_to_ctx(file);
388
389	dprintk(dev, "Releasing instance %p\n", ctx);
390
391	tpg_free(&ctx->tpg);
392	v4l2_fh_del(&ctx->fh, file);
393	v4l2_fh_exit(&ctx->fh);
394	v4l2_ctrl_handler_free(&ctx->hdl);
395	mutex_lock(&dev->dev_mutex);
396	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
397	mutex_unlock(&dev->dev_mutex);
398
399	kfree(ctx->tpg_str_buf);
400	kfree(ctx);
401
402	return 0;
403}
404
405static const struct v4l2_file_operations visl_fops = {
406	.owner		= THIS_MODULE,
407	.open		= visl_open,
408	.release	= visl_release,
409	.poll		= v4l2_m2m_fop_poll,
410	.unlocked_ioctl	= video_ioctl2,
411	.mmap		= v4l2_m2m_fop_mmap,
412};
413
414static const struct video_device visl_videodev = {
415	.name		= VISL_NAME,
416	.vfl_dir	= VFL_DIR_M2M,
417	.fops		= &visl_fops,
418	.ioctl_ops	= &visl_ioctl_ops,
419	.minor		= -1,
420	.release	= visl_device_release,
421	.device_caps	= V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING,
422};
423
424static const struct v4l2_m2m_ops visl_m2m_ops = {
425	.device_run	= visl_device_run,
426};
427
428static const struct media_device_ops visl_m2m_media_ops = {
429	.req_validate	= visl_request_validate,
430	.req_queue	= v4l2_m2m_request_queue,
431};
432
433static int visl_probe(struct platform_device *pdev)
434{
435	struct visl_dev *dev;
436	struct video_device *vfd;
437	int ret;
438	int rc;
439
440	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
441	if (!dev)
442		return -ENOMEM;
443
444	ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
445	if (ret)
446		goto error_visl_dev;
447
448	mutex_init(&dev->dev_mutex);
449
450	dev->vfd = visl_videodev;
451	vfd = &dev->vfd;
452	vfd->lock = &dev->dev_mutex;
453	vfd->v4l2_dev = &dev->v4l2_dev;
454
455	video_set_drvdata(vfd, dev);
456
457	platform_set_drvdata(pdev, dev);
458
459	dev->m2m_dev = v4l2_m2m_init(&visl_m2m_ops);
460	if (IS_ERR(dev->m2m_dev)) {
461		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem device\n");
462		ret = PTR_ERR(dev->m2m_dev);
463		dev->m2m_dev = NULL;
464		goto error_dev;
465	}
466
467	dev->mdev.dev = &pdev->dev;
468	strscpy(dev->mdev.model, "visl", sizeof(dev->mdev.model));
469	strscpy(dev->mdev.bus_info, "platform:visl",
470		sizeof(dev->mdev.bus_info));
471	media_device_init(&dev->mdev);
472	dev->mdev.ops = &visl_m2m_media_ops;
473	dev->v4l2_dev.mdev = &dev->mdev;
474
475	ret = video_register_device(vfd, VFL_TYPE_VIDEO, -1);
476	if (ret) {
477		v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
478		goto error_m2m;
479	}
480
481	v4l2_info(&dev->v4l2_dev,
482		  "Device registered as /dev/video%d\n", vfd->num);
483
484	ret = v4l2_m2m_register_media_controller(dev->m2m_dev, vfd,
485						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
486	if (ret) {
487		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller\n");
488		goto error_v4l2;
489	}
490
491	ret = media_device_register(&dev->mdev);
492	if (ret) {
493		v4l2_err(&dev->v4l2_dev, "Failed to register mem2mem media device\n");
494		goto error_m2m_mc;
495	}
496
497	rc = visl_debugfs_init(dev);
498	if (rc)
499		dprintk(dev, "visl_debugfs_init failed: %d\n"
500			"Continuing without debugfs support\n", rc);
501
502	return 0;
503
504error_m2m_mc:
505	v4l2_m2m_unregister_media_controller(dev->m2m_dev);
506error_v4l2:
507	video_unregister_device(&dev->vfd);
508	/* visl_device_release called by video_unregister_device to release various objects */
509	return ret;
510error_m2m:
511	v4l2_m2m_release(dev->m2m_dev);
512error_dev:
513	v4l2_device_unregister(&dev->v4l2_dev);
514error_visl_dev:
515	kfree(dev);
516
517	return ret;
518}
519
520static void visl_remove(struct platform_device *pdev)
521{
522	struct visl_dev *dev = platform_get_drvdata(pdev);
523
524	v4l2_info(&dev->v4l2_dev, "Removing " VISL_NAME);
525
526#ifdef CONFIG_MEDIA_CONTROLLER
527	if (media_devnode_is_registered(dev->mdev.devnode)) {
528		media_device_unregister(&dev->mdev);
529		v4l2_m2m_unregister_media_controller(dev->m2m_dev);
530	}
531#endif
532	video_unregister_device(&dev->vfd);
533}
534
535static struct platform_driver visl_pdrv = {
536	.probe		= visl_probe,
537	.remove		= visl_remove,
538	.driver		= {
539		.name	= VISL_NAME,
540	},
541};
542
543static void visl_dev_release(struct device *dev) {}
544
545static struct platform_device visl_pdev = {
546	.name		= VISL_NAME,
547	.dev.release	= visl_dev_release,
548};
549
550static void __exit visl_exit(void)
551{
552	platform_driver_unregister(&visl_pdrv);
553	platform_device_unregister(&visl_pdev);
554}
555
556static int __init visl_init(void)
557{
558	int ret;
559
560	ret = platform_device_register(&visl_pdev);
561	if (ret)
562		return ret;
563
564	ret = platform_driver_register(&visl_pdrv);
565	if (ret)
566		platform_device_unregister(&visl_pdev);
567
568	return ret;
569}
570
571MODULE_DESCRIPTION("Virtual stateless decoder device");
572MODULE_AUTHOR("Daniel Almeida <daniel.almeida@collabora.com>");
573MODULE_LICENSE("GPL");
574
575module_init(visl_init);
576module_exit(visl_exit);