pisp_be.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * PiSP Back End driver.
   4 * Copyright (c) 2021-2024 Raspberry Pi Limited.
   5 *
   6 */
   7#include <linux/clk.h>
   8#include <linux/interrupt.h>
   9#include <linux/io.h>
  10#include <linux/kernel.h>
  11#include <linux/lockdep.h>
  12#include <linux/minmax.h>
  13#include <linux/module.h>
  14#include <linux/platform_device.h>
  15#include <linux/pm_runtime.h>
  16#include <linux/slab.h>
  17#include <media/v4l2-device.h>
  18#include <media/v4l2-ioctl.h>
  19#include <media/videobuf2-dma-contig.h>
  20#include <media/videobuf2-vmalloc.h>
  21
  22#include <uapi/linux/media/raspberrypi/pisp_be_config.h>
  23
  24#include "pisp_be_formats.h"
  25
  26/* Maximum number of config buffers possible */
  27#define PISP_BE_NUM_CONFIG_BUFFERS VB2_MAX_FRAME
  28
  29#define PISPBE_NAME "pispbe"
  30
  31/* Some ISP-BE registers */
  32#define PISP_BE_VERSION_REG		0x0
  33#define PISP_BE_CONTROL_REG		0x4
  34#define PISP_BE_CONTROL_COPY_CONFIG	BIT(1)
  35#define PISP_BE_CONTROL_QUEUE_JOB	BIT(0)
  36#define PISP_BE_CONTROL_NUM_TILES(n)	((n) << 16)
  37#define PISP_BE_TILE_ADDR_LO_REG	0x8
  38#define PISP_BE_TILE_ADDR_HI_REG	0xc
  39#define PISP_BE_STATUS_REG		0x10
  40#define PISP_BE_STATUS_QUEUED		BIT(0)
  41#define PISP_BE_BATCH_STATUS_REG	0x14
  42#define PISP_BE_INTERRUPT_EN_REG	0x18
  43#define PISP_BE_INTERRUPT_STATUS_REG	0x1c
  44#define PISP_BE_AXI_REG			0x20
  45#define PISP_BE_CONFIG_BASE_REG		0x40
  46#define PISP_BE_IO_ADDR_LOW(n)		(PISP_BE_CONFIG_BASE_REG + 8 * (n))
  47#define PISP_BE_IO_ADDR_HIGH(n)		(PISP_BE_IO_ADDR_LOW((n)) + 4)
  48#define PISP_BE_GLOBAL_BAYER_ENABLE	0xb0
  49#define PISP_BE_GLOBAL_RGB_ENABLE	0xb4
  50#define N_HW_ADDRESSES			13
  51#define N_HW_ENABLES			2
  52
  53#define PISP_BE_VERSION_2712		0x02252700
  54#define PISP_BE_VERSION_MINOR_BITS	0xf
  55
  56/*
  57 * This maps our nodes onto the inputs/outputs of the actual PiSP Back End.
  58 * Be wary of the word "OUTPUT" which is used ambiguously here. In a V4L2
  59 * context it means an input to the hardware (source image or metadata).
  60 * Elsewhere it means an output from the hardware.
  61 */
  62enum pispbe_node_ids {
  63	MAIN_INPUT_NODE,
  64	TDN_INPUT_NODE,
  65	STITCH_INPUT_NODE,
  66	OUTPUT0_NODE,
  67	OUTPUT1_NODE,
  68	TDN_OUTPUT_NODE,
  69	STITCH_OUTPUT_NODE,
  70	CONFIG_NODE,
  71	PISPBE_NUM_NODES
  72};
  73
  74struct pispbe_node_description {
  75	const char *ent_name;
  76	enum v4l2_buf_type buf_type;
  77	unsigned int caps;
  78};
  79
  80static const struct pispbe_node_description node_desc[PISPBE_NUM_NODES] = {
  81	/* MAIN_INPUT_NODE */
  82	{
  83		.ent_name = PISPBE_NAME "-input",
  84		.buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
  85		.caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,
  86	},
  87	/* TDN_INPUT_NODE */
  88	{
  89		.ent_name = PISPBE_NAME "-tdn_input",
  90		.buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
  91		.caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,
  92	},
  93	/* STITCH_INPUT_NODE */
  94	{
  95		.ent_name = PISPBE_NAME "-stitch_input",
  96		.buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
  97		.caps = V4L2_CAP_VIDEO_OUTPUT_MPLANE,
  98	},
  99	/* OUTPUT0_NODE */
 100	{
 101		.ent_name = PISPBE_NAME "-output0",
 102		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 103		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
 104	},
 105	/* OUTPUT1_NODE */
 106	{
 107		.ent_name = PISPBE_NAME "-output1",
 108		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 109		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
 110	},
 111	/* TDN_OUTPUT_NODE */
 112	{
 113		.ent_name = PISPBE_NAME "-tdn_output",
 114		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 115		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
 116	},
 117	/* STITCH_OUTPUT_NODE */
 118	{
 119		.ent_name = PISPBE_NAME "-stitch_output",
 120		.buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 121		.caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE,
 122	},
 123	/* CONFIG_NODE */
 124	{
 125		.ent_name = PISPBE_NAME "-config",
 126		.buf_type = V4L2_BUF_TYPE_META_OUTPUT,
 127		.caps = V4L2_CAP_META_OUTPUT,
 128	}
 129};
 130
 131#define NODE_DESC_IS_OUTPUT(desc) ( \
 132	((desc)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \
 133	((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \
 134	((desc)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE))
 135
 136#define NODE_IS_META(node) ( \
 137	((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT))
 138#define NODE_IS_OUTPUT(node) ( \
 139	((node)->buf_type == V4L2_BUF_TYPE_META_OUTPUT) || \
 140	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT) || \
 141	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE))
 142#define NODE_IS_CAPTURE(node) ( \
 143	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) || \
 144	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE))
 145#define NODE_IS_MPLANE(node) ( \
 146	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) || \
 147	((node)->buf_type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE))
 148
 149/*
 150 * Structure to describe a single node /dev/video<N> which represents a single
 151 * input or output queue to the PiSP Back End device.
 152 */
 153struct pispbe_node {
 154	unsigned int id;
 155	int vfl_dir;
 156	enum v4l2_buf_type buf_type;
 157	struct video_device vfd;
 158	struct media_pad pad;
 159	struct media_intf_devnode *intf_devnode;
 160	struct media_link *intf_link;
 161	struct pispbe_dev *pispbe;
 162	/* Video device lock */
 163	struct mutex node_lock;
 164	/* vb2_queue lock */
 165	struct mutex queue_lock;
 166	struct list_head ready_queue;
 167	struct vb2_queue queue;
 168	struct v4l2_format format;
 169	const struct pisp_be_format *pisp_format;
 170};
 171
 172/* For logging only, use the entity name with "pispbe" and separator removed */
 173#define NODE_NAME(node) \
 174		(node_desc[(node)->id].ent_name + sizeof(PISPBE_NAME))
 175
 176/* Records details of the jobs currently running or queued on the h/w. */
 177struct pispbe_job {
 178	bool valid;
 179	/*
 180	 * An array of buffer pointers - remember it's source buffers first,
 181	 * then captures, then metadata last.
 182	 */
 183	struct pispbe_buffer *buf[PISPBE_NUM_NODES];
 184};
 185
 186struct pispbe_hw_enables {
 187	u32 bayer_enables;
 188	u32 rgb_enables;
 189};
 190
 191/* Records a job configuration and memory addresses. */
 192struct pispbe_job_descriptor {
 193	struct list_head queue;
 194	struct pispbe_buffer *buffers[PISPBE_NUM_NODES];
 195	dma_addr_t hw_dma_addrs[N_HW_ADDRESSES];
 196	struct pisp_be_tiles_config *config;
 197	struct pispbe_hw_enables hw_enables;
 198	dma_addr_t tiles;
 199};
 200
 201/*
 202 * Structure representing the entire PiSP Back End device, comprising several
 203 * nodes which share platform resources and a mutex for the actual HW.
 204 */
 205struct pispbe_dev {
 206	struct device *dev;
 207	struct pispbe_dev *pispbe;
 208	struct pisp_be_tiles_config *config;
 209	void __iomem *be_reg_base;
 210	struct clk *clk;
 211	struct v4l2_device v4l2_dev;
 212	struct v4l2_subdev sd;
 213	struct media_device mdev;
 214	struct media_pad pad[PISPBE_NUM_NODES]; /* output pads first */
 215	struct pispbe_node node[PISPBE_NUM_NODES];
 216	dma_addr_t config_dma_addr;
 217	unsigned int sequence;
 218	u32 streaming_map;
 219	struct pispbe_job queued_job, running_job;
 220	/* protects "hw_busy" flag, streaming_map and job_queue */
 221	spinlock_t hw_lock;
 222	bool hw_busy; /* non-zero if a job is queued or is being started */
 223	struct list_head job_queue;
 224	int irq;
 225	u32 hw_version;
 226	u8 done, started;
 227};
 228
 229static u32 pispbe_rd(struct pispbe_dev *pispbe, unsigned int offset)
 230{
 231	return readl(pispbe->be_reg_base + offset);
 232}
 233
 234static void pispbe_wr(struct pispbe_dev *pispbe, unsigned int offset, u32 val)
 235{
 236	writel(val, pispbe->be_reg_base + offset);
 237}
 238
 239/*
 240 * Queue a job to the h/w. If the h/w is idle it will begin immediately.
 241 * Caller must ensure it is "safe to queue", i.e. we don't already have a
 242 * queued, unstarted job.
 243 */
 244static void pispbe_queue_job(struct pispbe_dev *pispbe,
 245			     struct pispbe_job_descriptor *job)
 246{
 247	unsigned int begin, end;
 248
 249	if (pispbe_rd(pispbe, PISP_BE_STATUS_REG) & PISP_BE_STATUS_QUEUED)
 250		dev_err(pispbe->dev, "ERROR: not safe to queue new job!\n");
 251
 252	/*
 253	 * Write configuration to hardware. DMA addresses and enable flags
 254	 * are passed separately, because the driver needs to sanitize them,
 255	 * and we don't want to modify (or be vulnerable to modifications of)
 256	 * the mmap'd buffer.
 257	 */
 258	for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) {
 259		pispbe_wr(pispbe, PISP_BE_IO_ADDR_LOW(u),
 260			  lower_32_bits(job->hw_dma_addrs[u]));
 261		pispbe_wr(pispbe, PISP_BE_IO_ADDR_HIGH(u),
 262			  upper_32_bits(job->hw_dma_addrs[u]));
 263	}
 264	pispbe_wr(pispbe, PISP_BE_GLOBAL_BAYER_ENABLE,
 265		  job->hw_enables.bayer_enables);
 266	pispbe_wr(pispbe, PISP_BE_GLOBAL_RGB_ENABLE,
 267		  job->hw_enables.rgb_enables);
 268
 269	/* Everything else is as supplied by the user. */
 270	begin =	offsetof(struct pisp_be_config, global.bayer_order) /
 271		sizeof(u32);
 272	end = sizeof(struct pisp_be_config) / sizeof(u32);
 273	for (unsigned int u = begin; u < end; u++)
 274		pispbe_wr(pispbe, PISP_BE_CONFIG_BASE_REG + sizeof(u32) * u,
 275			  ((u32 *)job->config)[u]);
 276
 277	/* Read back the addresses -- an error here could be fatal */
 278	for (unsigned int u = 0; u < N_HW_ADDRESSES; ++u) {
 279		unsigned int offset = PISP_BE_IO_ADDR_LOW(u);
 280		u64 along = pispbe_rd(pispbe, offset);
 281
 282		along += ((u64)pispbe_rd(pispbe, offset + 4)) << 32;
 283		if (along != (u64)(job->hw_dma_addrs[u])) {
 284			dev_dbg(pispbe->dev,
 285				"ISP BE config error: check if ISP RAMs enabled?\n");
 286			return;
 287		}
 288	}
 289
 290	/*
 291	 * Write tile pointer to hardware. The IOMMU should prevent
 292	 * out-of-bounds offsets reaching non-ISP buffers.
 293	 */
 294	pispbe_wr(pispbe, PISP_BE_TILE_ADDR_LO_REG, lower_32_bits(job->tiles));
 295	pispbe_wr(pispbe, PISP_BE_TILE_ADDR_HI_REG, upper_32_bits(job->tiles));
 296
 297	/* Enqueue the job */
 298	pispbe_wr(pispbe, PISP_BE_CONTROL_REG,
 299		  PISP_BE_CONTROL_COPY_CONFIG | PISP_BE_CONTROL_QUEUE_JOB |
 300		  PISP_BE_CONTROL_NUM_TILES(job->config->num_tiles));
 301}
 302
 303struct pispbe_buffer {
 304	struct vb2_v4l2_buffer vb;
 305	struct list_head ready_list;
 306	unsigned int config_index;
 307};
 308
 309static int pispbe_get_planes_addr(dma_addr_t addr[3], struct pispbe_buffer *buf,
 310				  struct pispbe_node *node)
 311{
 312	unsigned int num_planes = node->format.fmt.pix_mp.num_planes;
 313	unsigned int plane_factor = 0;
 314	unsigned int size;
 315	unsigned int p;
 316
 317	if (!buf || !node->pisp_format)
 318		return 0;
 319
 320	/*
 321	 * Determine the base plane size. This will not be the same
 322	 * as node->format.fmt.pix_mp.plane_fmt[0].sizeimage for a single
 323	 * plane buffer in an mplane format.
 324	 */
 325	size = node->format.fmt.pix_mp.plane_fmt[0].bytesperline *
 326	       node->format.fmt.pix_mp.height;
 327
 328	for (p = 0; p < num_planes && p < PISPBE_MAX_PLANES; p++) {
 329		addr[p] = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, p);
 330		plane_factor += node->pisp_format->plane_factor[p];
 331	}
 332
 333	for (; p < PISPBE_MAX_PLANES && node->pisp_format->plane_factor[p]; p++) {
 334		/*
 335		 * Calculate the address offset of this plane as needed
 336		 * by the hardware. This is specifically for non-mplane
 337		 * buffer formats, where there are 3 image planes, e.g.
 338		 * for the V4L2_PIX_FMT_YUV420 format.
 339		 */
 340		addr[p] = addr[0] + ((size * plane_factor) >> 3);
 341		plane_factor += node->pisp_format->plane_factor[p];
 342	}
 343
 344	return num_planes;
 345}
 346
 347static dma_addr_t pispbe_get_addr(struct pispbe_buffer *buf)
 348{
 349	if (buf)
 350		return vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
 351
 352	return 0;
 353}
 354
 355static void pispbe_xlate_addrs(struct pispbe_dev *pispbe,
 356			       struct pispbe_job_descriptor *job,
 357			       struct pispbe_buffer *buf[PISPBE_NUM_NODES])
 358{
 359	struct pispbe_hw_enables *hw_en = &job->hw_enables;
 360	struct pisp_be_tiles_config *config = job->config;
 361	dma_addr_t *addrs = job->hw_dma_addrs;
 362	int ret;
 363
 364	/* Take a copy of the "enable" bitmaps so we can modify them. */
 365	hw_en->bayer_enables = config->config.global.bayer_enables;
 366	hw_en->rgb_enables = config->config.global.rgb_enables;
 367
 368	/*
 369	 * Main input first. There are 3 address pointers, corresponding to up
 370	 * to 3 planes.
 371	 */
 372	ret = pispbe_get_planes_addr(addrs, buf[MAIN_INPUT_NODE],
 373				     &pispbe->node[MAIN_INPUT_NODE]);
 374	if (ret <= 0) {
 375		/* Shouldn't happen, we have validated an input is available. */
 376		dev_warn(pispbe->dev, "ISP-BE missing input\n");
 377		hw_en->bayer_enables = 0;
 378		hw_en->rgb_enables = 0;
 379		return;
 380	}
 381
 382	/*
 383	 * Now TDN/Stitch inputs and outputs. These are single-plane and only
 384	 * used with Bayer input. Input enables must match the requirements
 385	 * of the processing stages, otherwise the hardware can lock up!
 386	 */
 387	if (hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) {
 388		addrs[3] = pispbe_get_addr(buf[TDN_INPUT_NODE]);
 389		if (addrs[3] == 0 ||
 390		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN_INPUT) ||
 391		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_TDN) ||
 392		    (config->config.tdn.reset & 1)) {
 393			hw_en->bayer_enables &=
 394				~(PISP_BE_BAYER_ENABLE_TDN_INPUT |
 395				  PISP_BE_BAYER_ENABLE_TDN_DECOMPRESS);
 396			if (!(config->config.tdn.reset & 1))
 397				hw_en->bayer_enables &=
 398					~PISP_BE_BAYER_ENABLE_TDN;
 399		}
 400
 401		addrs[4] = pispbe_get_addr(buf[STITCH_INPUT_NODE]);
 402		if (addrs[4] == 0 ||
 403		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_INPUT) ||
 404		    !(hw_en->bayer_enables & PISP_BE_BAYER_ENABLE_STITCH)) {
 405			hw_en->bayer_enables &=
 406				~(PISP_BE_BAYER_ENABLE_STITCH_INPUT |
 407				  PISP_BE_BAYER_ENABLE_STITCH_DECOMPRESS |
 408				  PISP_BE_BAYER_ENABLE_STITCH);
 409		}
 410
 411		addrs[5] = pispbe_get_addr(buf[TDN_OUTPUT_NODE]);
 412		if (addrs[5] == 0)
 413			hw_en->bayer_enables &=
 414				~(PISP_BE_BAYER_ENABLE_TDN_COMPRESS |
 415				  PISP_BE_BAYER_ENABLE_TDN_OUTPUT);
 416
 417		addrs[6] = pispbe_get_addr(buf[STITCH_OUTPUT_NODE]);
 418		if (addrs[6] == 0)
 419			hw_en->bayer_enables &=
 420				~(PISP_BE_BAYER_ENABLE_STITCH_COMPRESS |
 421				  PISP_BE_BAYER_ENABLE_STITCH_OUTPUT);
 422	} else {
 423		/* No Bayer input? Disable entire Bayer pipe (else lockup) */
 424		hw_en->bayer_enables = 0;
 425	}
 426
 427	/* Main image output channels. */
 428	for (unsigned int i = 0; i < PISP_BACK_END_NUM_OUTPUTS; i++) {
 429		ret = pispbe_get_planes_addr(addrs + 7 + 3 * i,
 430					     buf[OUTPUT0_NODE + i],
 431					     &pispbe->node[OUTPUT0_NODE + i]);
 432		if (ret <= 0)
 433			hw_en->rgb_enables &= ~(PISP_BE_RGB_ENABLE_OUTPUT0 << i);
 434	}
 435}
 436
 437/*
 438 * Prepare a job description to be submitted to the HW.
 439 *
 440 * To schedule a job, we need all streaming nodes (apart from Output0,
 441 * Output1, Tdn and Stitch) to have a buffer ready, which must
 442 * include at least a config buffer and a main input image.
 443 *
 444 * For Output0, Output1, Tdn and Stitch, a buffer only needs to be
 445 * available if the blocks are enabled in the config.
 446 *
 447 * If all the buffers required to form a job are available, append the
 448 * job descriptor to the job queue to be later queued to the HW.
 449 *
 450 * Returns 0 if a job has been successfully prepared, < 0 otherwise.
 451 */
 452static int pispbe_prepare_job(struct pispbe_dev *pispbe)
 453{
 454	struct pispbe_job_descriptor __free(kfree) *job = NULL;
 455	struct pispbe_buffer *buf[PISPBE_NUM_NODES] = {};
 456	unsigned int streaming_map;
 457	unsigned int config_index;
 458	struct pispbe_node *node;
 459
 460	lockdep_assert_irqs_enabled();
 461
 462	scoped_guard(spinlock_irq, &pispbe->hw_lock) {
 463		static const u32 mask = BIT(CONFIG_NODE) | BIT(MAIN_INPUT_NODE);
 464
 465		if ((pispbe->streaming_map & mask) != mask)
 466			return -ENODEV;
 467
 468		/*
 469		 * Take a copy of streaming_map: nodes activated after this
 470		 * point are ignored when preparing this job.
 471		 */
 472		streaming_map = pispbe->streaming_map;
 473	}
 474
 475	job = kzalloc(sizeof(*job), GFP_KERNEL);
 476	if (!job)
 477		return -ENOMEM;
 478
 479	node = &pispbe->node[CONFIG_NODE];
 480	buf[CONFIG_NODE] = list_first_entry_or_null(&node->ready_queue,
 481						    struct pispbe_buffer,
 482						    ready_list);
 483	if (!buf[CONFIG_NODE])
 484		return -ENODEV;
 485
 486	list_del(&buf[CONFIG_NODE]->ready_list);
 487	job->buffers[CONFIG_NODE] = buf[CONFIG_NODE];
 488
 489	config_index = buf[CONFIG_NODE]->vb.vb2_buf.index;
 490	job->config = &pispbe->config[config_index];
 491	job->tiles = pispbe->config_dma_addr +
 492		     config_index * sizeof(struct pisp_be_tiles_config) +
 493		     offsetof(struct pisp_be_tiles_config, tiles);
 494
 495	/* remember: srcimages, captures then metadata */
 496	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {
 497		unsigned int bayer_en =
 498			job->config->config.global.bayer_enables;
 499		unsigned int rgb_en =
 500			job->config->config.global.rgb_enables;
 501		bool ignore_buffers = false;
 502
 503		/* Config node is handled outside the loop above. */
 504		if (i == CONFIG_NODE)
 505			continue;
 506
 507		buf[i] = NULL;
 508		if (!(streaming_map & BIT(i)))
 509			continue;
 510
 511		if ((!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT0) &&
 512		     i == OUTPUT0_NODE) ||
 513		    (!(rgb_en & PISP_BE_RGB_ENABLE_OUTPUT1) &&
 514		     i == OUTPUT1_NODE) ||
 515		    (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_INPUT) &&
 516		     i == TDN_INPUT_NODE) ||
 517		    (!(bayer_en & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) &&
 518		     i == TDN_OUTPUT_NODE) ||
 519		    (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_INPUT) &&
 520		     i == STITCH_INPUT_NODE) ||
 521		    (!(bayer_en & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) &&
 522		     i == STITCH_OUTPUT_NODE)) {
 523			/*
 524			 * Ignore Output0/Output1/Tdn/Stitch buffer check if the
 525			 * global enables aren't set for these blocks. If a
 526			 * buffer has been provided, we dequeue it back to the
 527			 * user with the other in-use buffers.
 528			 */
 529			ignore_buffers = true;
 530		}
 531
 532		node = &pispbe->node[i];
 533
 534		/* Pull a buffer from each V4L2 queue to form the queued job */
 535		buf[i] = list_first_entry_or_null(&node->ready_queue,
 536						  struct pispbe_buffer,
 537						  ready_list);
 538		if (buf[i]) {
 539			list_del(&buf[i]->ready_list);
 540			job->buffers[i] = buf[i];
 541		}
 542
 543		if (!buf[i] && !ignore_buffers)
 544			goto err_return_buffers;
 545	}
 546
 547	/* Convert buffers to DMA addresses for the hardware */
 548	pispbe_xlate_addrs(pispbe, job, buf);
 549
 550	scoped_guard(spinlock_irq, &pispbe->hw_lock) {
 551		list_add_tail(&job->queue, &pispbe->job_queue);
 552	}
 553
 554	/* Set job to NULL to avoid automatic release due to __free(). */
 555	job = NULL;
 556
 557	return 0;
 558
 559err_return_buffers:
 560	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {
 561		struct pispbe_node *n =  &pispbe->node[i];
 562
 563		if (!buf[i])
 564			continue;
 565
 566		/* Return the buffer to the ready_list queue */
 567		list_add(&buf[i]->ready_list, &n->ready_queue);
 568	}
 569
 570	return -ENODEV;
 571}
 572
 573static void pispbe_schedule(struct pispbe_dev *pispbe, bool clear_hw_busy)
 574{
 575	struct pispbe_job_descriptor *job;
 576
 577	scoped_guard(spinlock_irqsave, &pispbe->hw_lock) {
 578		if (clear_hw_busy)
 579			pispbe->hw_busy = false;
 580
 581		if (pispbe->hw_busy)
 582			return;
 583
 584		job = list_first_entry_or_null(&pispbe->job_queue,
 585					       struct pispbe_job_descriptor,
 586					       queue);
 587		if (!job)
 588			return;
 589
 590		list_del(&job->queue);
 591
 592		for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++)
 593			pispbe->queued_job.buf[i] = job->buffers[i];
 594		pispbe->queued_job.valid = true;
 595
 596		pispbe->hw_busy = true;
 597	}
 598
 599	/*
 600	 * We can kick the job off without the hw_lock, as this can
 601	 * never run again until hw_busy is cleared, which will happen
 602	 * only when the following job has been queued and an interrupt
 603	 * is rised.
 604	 */
 605	pispbe_queue_job(pispbe, job);
 606	kfree(job);
 607}
 608
 609static void pispbe_isr_jobdone(struct pispbe_dev *pispbe,
 610			       struct pispbe_job *job)
 611{
 612	struct pispbe_buffer **buf = job->buf;
 613	u64 ts = ktime_get_ns();
 614
 615	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++) {
 616		if (buf[i]) {
 617			buf[i]->vb.vb2_buf.timestamp = ts;
 618			buf[i]->vb.sequence = pispbe->sequence;
 619			vb2_buffer_done(&buf[i]->vb.vb2_buf,
 620					VB2_BUF_STATE_DONE);
 621		}
 622	}
 623
 624	pispbe->sequence++;
 625}
 626
 627static irqreturn_t pispbe_isr(int irq, void *dev)
 628{
 629	struct pispbe_dev *pispbe = (struct pispbe_dev *)dev;
 630	bool can_queue_another = false;
 631	u8 started, done;
 632	u32 u;
 633
 634	u = pispbe_rd(pispbe, PISP_BE_INTERRUPT_STATUS_REG);
 635	if (u == 0)
 636		return IRQ_NONE;
 637
 638	pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, u);
 639	u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG);
 640	done = (uint8_t)u;
 641	started = (uint8_t)(u >> 8);
 642
 643	/*
 644	 * Be aware that done can go up by 2 and started by 1 when: a job that
 645	 * we previously saw "start" now finishes, and we then queued a new job
 646	 * which we see both start and finish "simultaneously".
 647	 */
 648	if (pispbe->running_job.valid && pispbe->done != done) {
 649		pispbe_isr_jobdone(pispbe, &pispbe->running_job);
 650		memset(&pispbe->running_job, 0, sizeof(pispbe->running_job));
 651		pispbe->done++;
 652	}
 653
 654	if (pispbe->started != started) {
 655		pispbe->started++;
 656		can_queue_another = 1;
 657
 658		if (pispbe->done != done && pispbe->queued_job.valid) {
 659			pispbe_isr_jobdone(pispbe, &pispbe->queued_job);
 660			pispbe->done++;
 661		} else {
 662			pispbe->running_job = pispbe->queued_job;
 663		}
 664
 665		memset(&pispbe->queued_job, 0, sizeof(pispbe->queued_job));
 666	}
 667
 668	if (pispbe->done != done || pispbe->started != started) {
 669		dev_dbg(pispbe->dev,
 670			"Job counters not matching: done = %u, expected %u - started = %u, expected %u\n",
 671			pispbe->done, done, pispbe->started, started);
 672		pispbe->started = started;
 673		pispbe->done = done;
 674	}
 675
 676	/* check if there's more to do before going to sleep */
 677	pispbe_schedule(pispbe, can_queue_another);
 678
 679	return IRQ_HANDLED;
 680}
 681
 682static int pisp_be_validate_config(struct pispbe_dev *pispbe,
 683				   struct pisp_be_tiles_config *config)
 684{
 685	u32 bayer_enables = config->config.global.bayer_enables;
 686	u32 rgb_enables = config->config.global.rgb_enables;
 687	struct device *dev = pispbe->dev;
 688	struct v4l2_format *fmt;
 689	unsigned int bpl, size;
 690
 691	if (!(bayer_enables & PISP_BE_BAYER_ENABLE_INPUT) ==
 692	    !(rgb_enables & PISP_BE_RGB_ENABLE_INPUT)) {
 693		dev_dbg(dev, "%s: Not one input enabled\n", __func__);
 694		return -EIO;
 695	}
 696
 697	if (config->num_tiles == 0 ||
 698	    config->num_tiles > PISP_BACK_END_NUM_TILES) {
 699		dev_dbg(dev, "%s: Invalid number of tiles: %d\n", __func__,
 700			config->num_tiles);
 701		return -EINVAL;
 702	}
 703
 704	/* Ensure output config strides and buffer sizes match the V4L2 formats. */
 705	fmt = &pispbe->node[TDN_OUTPUT_NODE].format;
 706	if (bayer_enables & PISP_BE_BAYER_ENABLE_TDN_OUTPUT) {
 707		bpl = config->config.tdn_output_format.stride;
 708		size = bpl * config->config.tdn_output_format.height;
 709
 710		if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) {
 711			dev_dbg(dev, "%s: bpl mismatch on tdn_output\n",
 712				__func__);
 713			return -EINVAL;
 714		}
 715
 716		if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) {
 717			dev_dbg(dev, "%s: size mismatch on tdn_output\n",
 718				__func__);
 719			return -EINVAL;
 720		}
 721	}
 722
 723	fmt = &pispbe->node[STITCH_OUTPUT_NODE].format;
 724	if (bayer_enables & PISP_BE_BAYER_ENABLE_STITCH_OUTPUT) {
 725		bpl = config->config.stitch_output_format.stride;
 726		size = bpl * config->config.stitch_output_format.height;
 727
 728		if (fmt->fmt.pix_mp.plane_fmt[0].bytesperline < bpl) {
 729			dev_dbg(dev, "%s: bpl mismatch on stitch_output\n",
 730				__func__);
 731			return -EINVAL;
 732		}
 733
 734		if (fmt->fmt.pix_mp.plane_fmt[0].sizeimage < size) {
 735			dev_dbg(dev, "%s: size mismatch on stitch_output\n",
 736				__func__);
 737			return -EINVAL;
 738		}
 739	}
 740
 741	for (unsigned int j = 0; j < PISP_BACK_END_NUM_OUTPUTS; j++) {
 742		if (!(rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT(j)))
 743			continue;
 744
 745		if (config->config.output_format[j].image.format &
 746		    PISP_IMAGE_FORMAT_WALLPAPER_ROLL)
 747			continue; /* TODO: Size checks for wallpaper formats */
 748
 749		fmt = &pispbe->node[OUTPUT0_NODE + j].format;
 750		for (unsigned int i = 0; i < fmt->fmt.pix_mp.num_planes; i++) {
 751			bpl = !i ? config->config.output_format[j].image.stride
 752			    : config->config.output_format[j].image.stride2;
 753			size = bpl * config->config.output_format[j].image.height;
 754
 755			if (config->config.output_format[j].image.format &
 756						PISP_IMAGE_FORMAT_SAMPLING_420)
 757				size >>= 1;
 758
 759			if (fmt->fmt.pix_mp.plane_fmt[i].bytesperline < bpl) {
 760				dev_dbg(dev, "%s: bpl mismatch on output %d\n",
 761					__func__, j);
 762				return -EINVAL;
 763			}
 764
 765			if (fmt->fmt.pix_mp.plane_fmt[i].sizeimage < size) {
 766				dev_dbg(dev, "%s: size mismatch on output\n",
 767					__func__);
 768				return -EINVAL;
 769			}
 770		}
 771	}
 772
 773	return 0;
 774}
 775
 776static int pispbe_node_queue_setup(struct vb2_queue *q, unsigned int *nbuffers,
 777				   unsigned int *nplanes, unsigned int sizes[],
 778				   struct device *alloc_devs[])
 779{
 780	struct pispbe_node *node = vb2_get_drv_priv(q);
 781	struct pispbe_dev *pispbe = node->pispbe;
 782	unsigned int num_planes = NODE_IS_MPLANE(node) ?
 783				  node->format.fmt.pix_mp.num_planes : 1;
 784
 785	if (*nplanes) {
 786		if (*nplanes != num_planes)
 787			return -EINVAL;
 788
 789		for (unsigned int i = 0; i < *nplanes; i++) {
 790			unsigned int size = NODE_IS_MPLANE(node) ?
 791				node->format.fmt.pix_mp.plane_fmt[i].sizeimage :
 792				node->format.fmt.meta.buffersize;
 793
 794			if (sizes[i] < size)
 795				return -EINVAL;
 796		}
 797
 798		return 0;
 799	}
 800
 801	*nplanes = num_planes;
 802	for (unsigned int i = 0; i < *nplanes; i++) {
 803		unsigned int size = NODE_IS_MPLANE(node) ?
 804				node->format.fmt.pix_mp.plane_fmt[i].sizeimage :
 805				node->format.fmt.meta.buffersize;
 806		sizes[i] = size;
 807	}
 808
 809	dev_dbg(pispbe->dev,
 810		"Image (or metadata) size %u, nbuffers %u for node %s\n",
 811		sizes[0], *nbuffers, NODE_NAME(node));
 812
 813	return 0;
 814}
 815
 816static int pispbe_node_buffer_prepare(struct vb2_buffer *vb)
 817{
 818	struct pispbe_node *node = vb2_get_drv_priv(vb->vb2_queue);
 819	struct pispbe_dev *pispbe = node->pispbe;
 820	unsigned int num_planes = NODE_IS_MPLANE(node) ?
 821				  node->format.fmt.pix_mp.num_planes : 1;
 822
 823	for (unsigned int i = 0; i < num_planes; i++) {
 824		unsigned long size = NODE_IS_MPLANE(node) ?
 825				node->format.fmt.pix_mp.plane_fmt[i].sizeimage :
 826				node->format.fmt.meta.buffersize;
 827
 828		if (vb2_plane_size(vb, i) < size) {
 829			dev_dbg(pispbe->dev,
 830				"data will not fit into plane %d (%lu < %lu)\n",
 831				i, vb2_plane_size(vb, i), size);
 832			return -EINVAL;
 833		}
 834
 835		vb2_set_plane_payload(vb, i, size);
 836	}
 837
 838	if (node->id == CONFIG_NODE) {
 839		void *dst = &node->pispbe->config[vb->index];
 840		void *src = vb2_plane_vaddr(vb, 0);
 841
 842		memcpy(dst, src, sizeof(struct pisp_be_tiles_config));
 843
 844		return pisp_be_validate_config(pispbe, dst);
 845	}
 846
 847	return 0;
 848}
 849
 850static void pispbe_node_buffer_queue(struct vb2_buffer *buf)
 851{
 852	struct vb2_v4l2_buffer *vbuf =
 853		container_of(buf, struct vb2_v4l2_buffer, vb2_buf);
 854	struct pispbe_buffer *buffer =
 855		container_of(vbuf, struct pispbe_buffer, vb);
 856	struct pispbe_node *node = vb2_get_drv_priv(buf->vb2_queue);
 857	struct pispbe_dev *pispbe = node->pispbe;
 858
 859	dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node));
 860	list_add_tail(&buffer->ready_list, &node->ready_queue);
 861
 862	/*
 863	 * Every time we add a buffer, check if there's now some work for the hw
 864	 * to do.
 865	 */
 866	if (!pispbe_prepare_job(pispbe))
 867		pispbe_schedule(pispbe, false);
 868}
 869
 870static int pispbe_node_start_streaming(struct vb2_queue *q, unsigned int count)
 871{
 872	struct pispbe_node *node = vb2_get_drv_priv(q);
 873	struct pispbe_dev *pispbe = node->pispbe;
 874	struct pispbe_buffer *buf, *tmp;
 875	int ret;
 876
 877	ret = pm_runtime_resume_and_get(pispbe->dev);
 878	if (ret < 0)
 879		goto err_return_buffers;
 880
 881	scoped_guard(spinlock_irq, &pispbe->hw_lock) {
 882		node->pispbe->streaming_map |=  BIT(node->id);
 883		node->pispbe->sequence = 0;
 884	}
 885
 886	dev_dbg(pispbe->dev, "%s: for node %s (count %u)\n",
 887		__func__, NODE_NAME(node), count);
 888	dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n",
 889		node->pispbe->streaming_map);
 890
 891	/* Maybe we're ready to run. */
 892	if (!pispbe_prepare_job(pispbe))
 893		pispbe_schedule(pispbe, false);
 894
 895	return 0;
 896
 897err_return_buffers:
 898	list_for_each_entry_safe(buf, tmp, &node->ready_queue, ready_list) {
 899		list_del(&buf->ready_list);
 900		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
 901	}
 902
 903	return ret;
 904}
 905
 906static void pispbe_node_stop_streaming(struct vb2_queue *q)
 907{
 908	struct pispbe_node *node = vb2_get_drv_priv(q);
 909	struct pispbe_dev *pispbe = node->pispbe;
 910	struct pispbe_job_descriptor *job, *temp;
 911	struct pispbe_buffer *buf;
 912	LIST_HEAD(tmp_list);
 913
 914	/*
 915	 * Now this is a bit awkward. In a simple M2M device we could just wait
 916	 * for all queued jobs to complete, but here there's a risk that a
 917	 * partial set of buffers was queued and cannot be run. For now, just
 918	 * cancel all buffers stuck in the "ready queue", then wait for any
 919	 * running job.
 920	 *
 921	 * This may return buffers out of order.
 922	 */
 923	dev_dbg(pispbe->dev, "%s: for node %s\n", __func__, NODE_NAME(node));
 924	do {
 925		buf = list_first_entry_or_null(&node->ready_queue,
 926					       struct pispbe_buffer,
 927					       ready_list);
 928		if (buf) {
 929			list_del(&buf->ready_list);
 930			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
 931		}
 932	} while (buf);
 933
 934	vb2_wait_for_all_buffers(&node->queue);
 935
 936	spin_lock_irq(&pispbe->hw_lock);
 937	pispbe->streaming_map &= ~BIT(node->id);
 938
 939	if (pispbe->streaming_map == 0) {
 940		/*
 941		 * If all nodes have stopped streaming release all jobs
 942		 * without holding the lock.
 943		 */
 944		list_splice_init(&pispbe->job_queue, &tmp_list);
 945	}
 946	spin_unlock_irq(&pispbe->hw_lock);
 947
 948	list_for_each_entry_safe(job, temp, &tmp_list, queue) {
 949		list_del(&job->queue);
 950		kfree(job);
 951	}
 952
 953	pm_runtime_put_autosuspend(pispbe->dev);
 954
 955	dev_dbg(pispbe->dev, "Nodes streaming now 0x%x\n",
 956		pispbe->streaming_map);
 957}
 958
 959static const struct vb2_ops pispbe_node_queue_ops = {
 960	.queue_setup = pispbe_node_queue_setup,
 961	.buf_prepare = pispbe_node_buffer_prepare,
 962	.buf_queue = pispbe_node_buffer_queue,
 963	.start_streaming = pispbe_node_start_streaming,
 964	.stop_streaming = pispbe_node_stop_streaming,
 965};
 966
 967static const struct v4l2_file_operations pispbe_fops = {
 968	.owner          = THIS_MODULE,
 969	.open           = v4l2_fh_open,
 970	.release        = vb2_fop_release,
 971	.poll           = vb2_fop_poll,
 972	.unlocked_ioctl = video_ioctl2,
 973	.mmap           = vb2_fop_mmap
 974};
 975
 976static int pispbe_node_querycap(struct file *file, void *priv,
 977				struct v4l2_capability *cap)
 978{
 979	struct pispbe_node *node = video_drvdata(file);
 980	struct pispbe_dev *pispbe = node->pispbe;
 981
 982	strscpy(cap->driver, PISPBE_NAME, sizeof(cap->driver));
 983	strscpy(cap->card, PISPBE_NAME, sizeof(cap->card));
 984
 985	dev_dbg(pispbe->dev, "Caps for node %s: %x and %x (dev %x)\n",
 986		NODE_NAME(node), cap->capabilities, cap->device_caps,
 987		node->vfd.device_caps);
 988
 989	return 0;
 990}
 991
 992static int pispbe_node_g_fmt_vid_cap(struct file *file, void *priv,
 993				     struct v4l2_format *f)
 994{
 995	struct pispbe_node *node = video_drvdata(file);
 996	struct pispbe_dev *pispbe = node->pispbe;
 997
 998	if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {
 999		dev_dbg(pispbe->dev,
1000			"Cannot get capture fmt for output node %s\n",
1001			NODE_NAME(node));
1002		return -EINVAL;
1003	}
1004
1005	*f = node->format;
1006	dev_dbg(pispbe->dev, "Get capture format for node %s\n",
1007		NODE_NAME(node));
1008
1009	return 0;
1010}
1011
1012static int pispbe_node_g_fmt_vid_out(struct file *file, void *priv,
1013				     struct v4l2_format *f)
1014{
1015	struct pispbe_node *node = video_drvdata(file);
1016	struct pispbe_dev *pispbe = node->pispbe;
1017
1018	if (NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {
1019		dev_dbg(pispbe->dev,
1020			"Cannot get capture fmt for output node %s\n",
1021			 NODE_NAME(node));
1022		return -EINVAL;
1023	}
1024
1025	*f = node->format;
1026	dev_dbg(pispbe->dev, "Get output format for node %s\n",
1027		NODE_NAME(node));
1028
1029	return 0;
1030}
1031
1032static int pispbe_node_g_fmt_meta_out(struct file *file, void *priv,
1033				      struct v4l2_format *f)
1034{
1035	struct pispbe_node *node = video_drvdata(file);
1036	struct pispbe_dev *pispbe = node->pispbe;
1037
1038	if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) {
1039		dev_dbg(pispbe->dev,
1040			"Cannot get capture fmt for meta output node %s\n",
1041			NODE_NAME(node));
1042		return -EINVAL;
1043	}
1044
1045	*f = node->format;
1046	dev_dbg(pispbe->dev, "Get output format for meta node %s\n",
1047		NODE_NAME(node));
1048
1049	return 0;
1050}
1051
1052static const struct pisp_be_format *pispbe_find_fmt(unsigned int fourcc)
1053{
1054	for (unsigned int i = 0; i < ARRAY_SIZE(supported_formats); i++) {
1055		if (supported_formats[i].fourcc == fourcc)
1056			return &supported_formats[i];
1057	}
1058
1059	return NULL;
1060}
1061
1062static void pispbe_set_plane_params(struct v4l2_format *f,
1063				    const struct pisp_be_format *fmt)
1064{
1065	unsigned int nplanes = f->fmt.pix_mp.num_planes;
1066	unsigned int total_plane_factor = 0;
1067
1068	for (unsigned int i = 0; i < PISPBE_MAX_PLANES; i++)
1069		total_plane_factor += fmt->plane_factor[i];
1070
1071	for (unsigned int i = 0; i < nplanes; i++) {
1072		struct v4l2_plane_pix_format *p = &f->fmt.pix_mp.plane_fmt[i];
1073		unsigned int bpl, plane_size;
1074
1075		bpl = (f->fmt.pix_mp.width * fmt->bit_depth) >> 3;
1076		bpl = ALIGN(max(p->bytesperline, bpl), fmt->align);
1077
1078		plane_size = bpl * f->fmt.pix_mp.height *
1079		      (nplanes > 1 ? fmt->plane_factor[i] : total_plane_factor);
1080		/*
1081		 * The shift is to divide out the plane_factor fixed point
1082		 * scaling of 8.
1083		 */
1084		plane_size = max(p->sizeimage, plane_size >> 3);
1085
1086		p->bytesperline = bpl;
1087		p->sizeimage = plane_size;
1088	}
1089}
1090
1091static void pispbe_try_format(struct v4l2_format *f, struct pispbe_node *node)
1092{
1093	struct pispbe_dev *pispbe = node->pispbe;
1094	u32 pixfmt = f->fmt.pix_mp.pixelformat;
1095	const struct pisp_be_format *fmt;
1096	bool is_rgb;
1097
1098	dev_dbg(pispbe->dev,
1099		"%s: [%s] req %ux%u %p4cc, planes %d\n",
1100		__func__, NODE_NAME(node), f->fmt.pix_mp.width,
1101		f->fmt.pix_mp.height, &pixfmt,
1102		f->fmt.pix_mp.num_planes);
1103
1104	fmt = pispbe_find_fmt(pixfmt);
1105	if (!fmt) {
1106		dev_dbg(pispbe->dev,
1107			"%s: [%s] Format not found, defaulting to YUV420\n",
1108			__func__, NODE_NAME(node));
1109		fmt = pispbe_find_fmt(V4L2_PIX_FMT_YUV420);
1110	}
1111
1112	f->fmt.pix_mp.pixelformat = fmt->fourcc;
1113	f->fmt.pix_mp.num_planes = fmt->num_planes;
1114	f->fmt.pix_mp.field = V4L2_FIELD_NONE;
1115	f->fmt.pix_mp.width = clamp(f->fmt.pix_mp.width,
1116				    PISP_BACK_END_MIN_TILE_WIDTH,
1117				    PISP_BACK_END_MAX_TILE_WIDTH);
1118	f->fmt.pix_mp.height = clamp(f->fmt.pix_mp.height,
1119				     PISP_BACK_END_MIN_TILE_HEIGHT,
1120				     PISP_BACK_END_MAX_TILE_HEIGHT);
1121
1122	/*
1123	 * Fill in the actual colour space when the requested one was
1124	 * not supported. This also catches the case when the "default"
1125	 * colour space was requested (as that's never in the mask).
1126	 */
1127	if (!(V4L2_COLORSPACE_MASK(f->fmt.pix_mp.colorspace) &
1128	    fmt->colorspace_mask))
1129		f->fmt.pix_mp.colorspace = fmt->colorspace_default;
1130
1131	/* In all cases, we only support the defaults for these: */
1132	f->fmt.pix_mp.ycbcr_enc =
1133		V4L2_MAP_YCBCR_ENC_DEFAULT(f->fmt.pix_mp.colorspace);
1134	f->fmt.pix_mp.xfer_func =
1135		V4L2_MAP_XFER_FUNC_DEFAULT(f->fmt.pix_mp.colorspace);
1136
1137	is_rgb = f->fmt.pix_mp.colorspace == V4L2_COLORSPACE_SRGB;
1138	f->fmt.pix_mp.quantization =
1139		V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb, f->fmt.pix_mp.colorspace,
1140					      f->fmt.pix_mp.ycbcr_enc);
1141
1142	/* Set plane size and bytes/line for each plane. */
1143	pispbe_set_plane_params(f, fmt);
1144
1145	for (unsigned int i = 0; i < f->fmt.pix_mp.num_planes; i++) {
1146		dev_dbg(pispbe->dev,
1147			"%s: [%s] calc plane %d, %ux%u, depth %u, bpl %u size %u\n",
1148			__func__, NODE_NAME(node), i, f->fmt.pix_mp.width,
1149			f->fmt.pix_mp.height, fmt->bit_depth,
1150			f->fmt.pix_mp.plane_fmt[i].bytesperline,
1151			f->fmt.pix_mp.plane_fmt[i].sizeimage);
1152	}
1153}
1154
1155static int pispbe_node_try_fmt_vid_cap(struct file *file, void *priv,
1156				       struct v4l2_format *f)
1157{
1158	struct pispbe_node *node = video_drvdata(file);
1159	struct pispbe_dev *pispbe = node->pispbe;
1160
1161	if (!NODE_IS_CAPTURE(node) || NODE_IS_META(node)) {
1162		dev_dbg(pispbe->dev,
1163			"Cannot set capture fmt for output node %s\n",
1164			NODE_NAME(node));
1165		return -EINVAL;
1166	}
1167
1168	pispbe_try_format(f, node);
1169
1170	return 0;
1171}
1172
1173static int pispbe_node_try_fmt_vid_out(struct file *file, void *priv,
1174				       struct v4l2_format *f)
1175{
1176	struct pispbe_node *node = video_drvdata(file);
1177	struct pispbe_dev *pispbe = node->pispbe;
1178
1179	if (!NODE_IS_OUTPUT(node) || NODE_IS_META(node)) {
1180		dev_dbg(pispbe->dev,
1181			"Cannot set capture fmt for output node %s\n",
1182			NODE_NAME(node));
1183		return -EINVAL;
1184	}
1185
1186	pispbe_try_format(f, node);
1187
1188	return 0;
1189}
1190
1191static int pispbe_node_try_fmt_meta_out(struct file *file, void *priv,
1192					struct v4l2_format *f)
1193{
1194	struct pispbe_node *node = video_drvdata(file);
1195	struct pispbe_dev *pispbe = node->pispbe;
1196
1197	if (!NODE_IS_META(node) || NODE_IS_CAPTURE(node)) {
1198		dev_dbg(pispbe->dev,
1199			"Cannot set capture fmt for meta output node %s\n",
1200			NODE_NAME(node));
1201		return -EINVAL;
1202	}
1203
1204	f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG;
1205	f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config);
1206
1207	return 0;
1208}
1209
1210static int pispbe_node_s_fmt_vid_cap(struct file *file, void *priv,
1211				     struct v4l2_format *f)
1212{
1213	struct pispbe_node *node = video_drvdata(file);
1214	struct pispbe_dev *pispbe = node->pispbe;
1215	int ret;
1216
1217	ret = pispbe_node_try_fmt_vid_cap(file, priv, f);
1218	if (ret < 0)
1219		return ret;
1220
1221	if (vb2_is_busy(&node->queue))
1222		return -EBUSY;
1223
1224	node->format = *f;
1225	node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat);
1226
1227	dev_dbg(pispbe->dev, "Set capture format for node %s to %p4cc\n",
1228		NODE_NAME(node), &f->fmt.pix_mp.pixelformat);
1229
1230	return 0;
1231}
1232
1233static int pispbe_node_s_fmt_vid_out(struct file *file, void *priv,
1234				     struct v4l2_format *f)
1235{
1236	struct pispbe_node *node = video_drvdata(file);
1237	struct pispbe_dev *pispbe = node->pispbe;
1238	int ret;
1239
1240	ret = pispbe_node_try_fmt_vid_out(file, priv, f);
1241	if (ret < 0)
1242		return ret;
1243
1244	if (vb2_is_busy(&node->queue))
1245		return -EBUSY;
1246
1247	node->format = *f;
1248	node->pisp_format = pispbe_find_fmt(f->fmt.pix_mp.pixelformat);
1249
1250	dev_dbg(pispbe->dev, "Set output format for node %s to %p4cc\n",
1251		NODE_NAME(node), &f->fmt.pix_mp.pixelformat);
1252
1253	return 0;
1254}
1255
1256static int pispbe_node_s_fmt_meta_out(struct file *file, void *priv,
1257				      struct v4l2_format *f)
1258{
1259	struct pispbe_node *node = video_drvdata(file);
1260	struct pispbe_dev *pispbe = node->pispbe;
1261	int ret;
1262
1263	ret = pispbe_node_try_fmt_meta_out(file, priv, f);
1264	if (ret < 0)
1265		return ret;
1266
1267	if (vb2_is_busy(&node->queue))
1268		return -EBUSY;
1269
1270	node->format = *f;
1271	node->pisp_format = &meta_out_supported_formats[0];
1272
1273	dev_dbg(pispbe->dev, "Set output format for meta node %s to %p4cc\n",
1274		NODE_NAME(node), &f->fmt.meta.dataformat);
1275
1276	return 0;
1277}
1278
1279static int pispbe_node_enum_fmt(struct file *file, void  *priv,
1280				struct v4l2_fmtdesc *f)
1281{
1282	struct pispbe_node *node = video_drvdata(file);
1283
1284	if (f->type != node->queue.type)
1285		return -EINVAL;
1286
1287	if (NODE_IS_META(node)) {
1288		if (f->index)
1289			return -EINVAL;
1290
1291		f->pixelformat = V4L2_META_FMT_RPI_BE_CFG;
1292		f->flags = 0;
1293		return 0;
1294	}
1295
1296	if (f->index >= ARRAY_SIZE(supported_formats))
1297		return -EINVAL;
1298
1299	f->pixelformat = supported_formats[f->index].fourcc;
1300	f->flags = 0;
1301
1302	return 0;
1303}
1304
1305static int pispbe_enum_framesizes(struct file *file, void *priv,
1306				  struct v4l2_frmsizeenum *fsize)
1307{
1308	struct pispbe_node *node = video_drvdata(file);
1309	struct pispbe_dev *pispbe = node->pispbe;
1310
1311	if (NODE_IS_META(node) || fsize->index)
1312		return -EINVAL;
1313
1314	if (!pispbe_find_fmt(fsize->pixel_format)) {
1315		dev_dbg(pispbe->dev, "Invalid pixel code: %x\n",
1316			fsize->pixel_format);
1317		return -EINVAL;
1318	}
1319
1320	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
1321	fsize->stepwise.min_width = 32;
1322	fsize->stepwise.max_width = 65535;
1323	fsize->stepwise.step_width = 2;
1324
1325	fsize->stepwise.min_height = 32;
1326	fsize->stepwise.max_height = 65535;
1327	fsize->stepwise.step_height = 2;
1328
1329	return 0;
1330}
1331
1332static const struct v4l2_ioctl_ops pispbe_node_ioctl_ops = {
1333	.vidioc_querycap = pispbe_node_querycap,
1334	.vidioc_g_fmt_vid_cap_mplane = pispbe_node_g_fmt_vid_cap,
1335	.vidioc_g_fmt_vid_out_mplane = pispbe_node_g_fmt_vid_out,
1336	.vidioc_g_fmt_meta_out = pispbe_node_g_fmt_meta_out,
1337	.vidioc_try_fmt_vid_cap_mplane = pispbe_node_try_fmt_vid_cap,
1338	.vidioc_try_fmt_vid_out_mplane = pispbe_node_try_fmt_vid_out,
1339	.vidioc_try_fmt_meta_out = pispbe_node_try_fmt_meta_out,
1340	.vidioc_s_fmt_vid_cap_mplane = pispbe_node_s_fmt_vid_cap,
1341	.vidioc_s_fmt_vid_out_mplane = pispbe_node_s_fmt_vid_out,
1342	.vidioc_s_fmt_meta_out = pispbe_node_s_fmt_meta_out,
1343	.vidioc_enum_fmt_vid_cap = pispbe_node_enum_fmt,
1344	.vidioc_enum_fmt_vid_out = pispbe_node_enum_fmt,
1345	.vidioc_enum_fmt_meta_out = pispbe_node_enum_fmt,
1346	.vidioc_enum_framesizes = pispbe_enum_framesizes,
1347	.vidioc_create_bufs = vb2_ioctl_create_bufs,
1348	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
1349	.vidioc_querybuf = vb2_ioctl_querybuf,
1350	.vidioc_qbuf = vb2_ioctl_qbuf,
1351	.vidioc_dqbuf = vb2_ioctl_dqbuf,
1352	.vidioc_expbuf = vb2_ioctl_expbuf,
1353	.vidioc_reqbufs = vb2_ioctl_reqbufs,
1354	.vidioc_streamon = vb2_ioctl_streamon,
1355	.vidioc_streamoff = vb2_ioctl_streamoff,
1356};
1357
1358static const struct video_device pispbe_videodev = {
1359	.name = PISPBE_NAME,
1360	.vfl_dir = VFL_DIR_M2M, /* gets overwritten */
1361	.fops = &pispbe_fops,
1362	.ioctl_ops = &pispbe_node_ioctl_ops,
1363	.minor = -1,
1364	.release = video_device_release_empty,
1365};
1366
1367static void pispbe_node_def_fmt(struct pispbe_node *node)
1368{
1369	if (NODE_IS_META(node) && NODE_IS_OUTPUT(node)) {
1370		/* Config node */
1371		struct v4l2_format *f = &node->format;
1372
1373		f->fmt.meta.dataformat = V4L2_META_FMT_RPI_BE_CFG;
1374		f->fmt.meta.buffersize = sizeof(struct pisp_be_tiles_config);
1375		f->type = node->buf_type;
1376	} else {
1377		struct v4l2_format f = {
1378			.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420,
1379			.fmt.pix_mp.width = 1920,
1380			.fmt.pix_mp.height = 1080,
1381			.type = node->buf_type,
1382		};
1383		pispbe_try_format(&f, node);
1384		node->format = f;
1385	}
1386
1387	node->pisp_format = pispbe_find_fmt(node->format.fmt.pix_mp.pixelformat);
1388}
1389
1390/*
1391 * Initialise a struct pispbe_node and register it as /dev/video<N>
1392 * to represent one of the PiSP Back End's input or output streams.
1393 */
1394static int pispbe_init_node(struct pispbe_dev *pispbe, unsigned int id)
1395{
1396	bool output = NODE_DESC_IS_OUTPUT(&node_desc[id]);
1397	struct pispbe_node *node = &pispbe->node[id];
1398	struct media_entity *entity = &node->vfd.entity;
1399	struct video_device *vdev = &node->vfd;
1400	struct vb2_queue *q = &node->queue;
1401	int ret;
1402
1403	node->id = id;
1404	node->pispbe = pispbe;
1405	node->buf_type = node_desc[id].buf_type;
1406
1407	mutex_init(&node->node_lock);
1408	mutex_init(&node->queue_lock);
1409	INIT_LIST_HEAD(&node->ready_queue);
1410
1411	node->format.type = node->buf_type;
1412	pispbe_node_def_fmt(node);
1413
1414	q->type = node->buf_type;
1415	q->io_modes = VB2_MMAP | VB2_DMABUF;
1416	q->mem_ops = &vb2_dma_contig_memops;
1417	q->drv_priv = node;
1418	q->ops = &pispbe_node_queue_ops;
1419	q->buf_struct_size = sizeof(struct pispbe_buffer);
1420	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
1421	q->dev = pispbe->dev;
1422	/* get V4L2 to handle node->queue locking */
1423	q->lock = &node->queue_lock;
1424
1425	ret = vb2_queue_init(q);
1426	if (ret < 0) {
1427		dev_err(pispbe->dev, "vb2_queue_init failed\n");
1428		goto err_mutex_destroy;
1429	}
1430
1431	*vdev = pispbe_videodev; /* default initialization */
1432	strscpy(vdev->name, node_desc[id].ent_name, sizeof(vdev->name));
1433	vdev->v4l2_dev = &pispbe->v4l2_dev;
1434	vdev->vfl_dir = output ? VFL_DIR_TX : VFL_DIR_RX;
1435	/* get V4L2 to serialise our ioctls */
1436	vdev->lock = &node->node_lock;
1437	vdev->queue = &node->queue;
1438	vdev->device_caps = V4L2_CAP_STREAMING | node_desc[id].caps;
1439
1440	node->pad.flags = output ? MEDIA_PAD_FL_SOURCE : MEDIA_PAD_FL_SINK;
1441	ret = media_entity_pads_init(entity, 1, &node->pad);
1442	if (ret) {
1443		dev_err(pispbe->dev,
1444			"Failed to register media pads for %s device node\n",
1445			NODE_NAME(node));
1446		goto err_unregister_queue;
1447	}
1448
1449	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
1450	if (ret) {
1451		dev_err(pispbe->dev,
1452			"Failed to register video %s device node\n",
1453			NODE_NAME(node));
1454		goto err_unregister_queue;
1455	}
1456	video_set_drvdata(vdev, node);
1457
1458	if (output)
1459		ret = media_create_pad_link(entity, 0, &pispbe->sd.entity,
1460					    id, MEDIA_LNK_FL_IMMUTABLE |
1461					    MEDIA_LNK_FL_ENABLED);
1462	else
1463		ret = media_create_pad_link(&pispbe->sd.entity, id, entity,
1464					    0, MEDIA_LNK_FL_IMMUTABLE |
1465					    MEDIA_LNK_FL_ENABLED);
1466	if (ret)
1467		goto err_unregister_video_dev;
1468
1469	dev_dbg(pispbe->dev, "%s device node registered as /dev/video%d\n",
1470		NODE_NAME(node), node->vfd.num);
1471
1472	return 0;
1473
1474err_unregister_video_dev:
1475	video_unregister_device(&node->vfd);
1476err_unregister_queue:
1477	vb2_queue_release(&node->queue);
1478err_mutex_destroy:
1479	mutex_destroy(&node->node_lock);
1480	mutex_destroy(&node->queue_lock);
1481	return ret;
1482}
1483
1484static const struct v4l2_subdev_pad_ops pispbe_pad_ops = {
1485	.link_validate = v4l2_subdev_link_validate_default,
1486};
1487
1488static const struct v4l2_subdev_ops pispbe_sd_ops = {
1489	.pad = &pispbe_pad_ops,
1490};
1491
1492static int pispbe_init_subdev(struct pispbe_dev *pispbe)
1493{
1494	struct v4l2_subdev *sd = &pispbe->sd;
1495	int ret;
1496
1497	v4l2_subdev_init(sd, &pispbe_sd_ops);
1498	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
1499	sd->owner = THIS_MODULE;
1500	sd->dev = pispbe->dev;
1501	strscpy(sd->name, PISPBE_NAME, sizeof(sd->name));
1502
1503	for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++)
1504		pispbe->pad[i].flags =
1505			NODE_DESC_IS_OUTPUT(&node_desc[i]) ?
1506			MEDIA_PAD_FL_SINK : MEDIA_PAD_FL_SOURCE;
1507
1508	ret = media_entity_pads_init(&sd->entity, PISPBE_NUM_NODES,
1509				     pispbe->pad);
1510	if (ret)
1511		goto error;
1512
1513	ret = v4l2_device_register_subdev(&pispbe->v4l2_dev, sd);
1514	if (ret)
1515		goto error;
1516
1517	return 0;
1518
1519error:
1520	media_entity_cleanup(&sd->entity);
1521	return ret;
1522}
1523
1524static int pispbe_init_devices(struct pispbe_dev *pispbe)
1525{
1526	struct v4l2_device *v4l2_dev;
1527	struct media_device *mdev;
1528	unsigned int num_regist;
1529	int ret;
1530
1531	/* Register v4l2_device and media_device */
1532	mdev = &pispbe->mdev;
1533	mdev->hw_revision = pispbe->hw_version;
1534	mdev->dev = pispbe->dev;
1535	strscpy(mdev->model, PISPBE_NAME, sizeof(mdev->model));
1536	media_device_init(mdev);
1537
1538	v4l2_dev = &pispbe->v4l2_dev;
1539	v4l2_dev->mdev = &pispbe->mdev;
1540	strscpy(v4l2_dev->name, PISPBE_NAME, sizeof(v4l2_dev->name));
1541
1542	ret = v4l2_device_register(pispbe->dev, v4l2_dev);
1543	if (ret)
1544		goto err_media_dev_cleanup;
1545
1546	/* Register the PISPBE subdevice. */
1547	ret = pispbe_init_subdev(pispbe);
1548	if (ret)
1549		goto err_unregister_v4l2;
1550
1551	/* Create device video nodes */
1552	for (num_regist = 0; num_regist < PISPBE_NUM_NODES; num_regist++) {
1553		ret = pispbe_init_node(pispbe, num_regist);
1554		if (ret)
1555			goto err_unregister_nodes;
1556	}
1557
1558	ret = media_device_register(mdev);
1559	if (ret)
1560		goto err_unregister_nodes;
1561
1562	pispbe->config =
1563		dma_alloc_coherent(pispbe->dev,
1564				   sizeof(struct pisp_be_tiles_config) *
1565					PISP_BE_NUM_CONFIG_BUFFERS,
1566				   &pispbe->config_dma_addr, GFP_KERNEL);
1567	if (!pispbe->config) {
1568		dev_err(pispbe->dev, "Unable to allocate cached config buffers.\n");
1569		ret = -ENOMEM;
1570		goto err_unregister_mdev;
1571	}
1572
1573	return 0;
1574
1575err_unregister_mdev:
1576	media_device_unregister(mdev);
1577err_unregister_nodes:
1578	while (num_regist-- > 0) {
1579		video_unregister_device(&pispbe->node[num_regist].vfd);
1580		vb2_queue_release(&pispbe->node[num_regist].queue);
1581	}
1582	v4l2_device_unregister_subdev(&pispbe->sd);
1583	media_entity_cleanup(&pispbe->sd.entity);
1584err_unregister_v4l2:
1585	v4l2_device_unregister(v4l2_dev);
1586err_media_dev_cleanup:
1587	media_device_cleanup(mdev);
1588	return ret;
1589}
1590
1591static void pispbe_destroy_devices(struct pispbe_dev *pispbe)
1592{
1593	if (pispbe->config) {
1594		dma_free_coherent(pispbe->dev,
1595				  sizeof(struct pisp_be_tiles_config) *
1596					PISP_BE_NUM_CONFIG_BUFFERS,
1597				  pispbe->config,
1598				  pispbe->config_dma_addr);
1599	}
1600
1601	dev_dbg(pispbe->dev, "Unregister from media controller\n");
1602
1603	v4l2_device_unregister_subdev(&pispbe->sd);
1604	media_entity_cleanup(&pispbe->sd.entity);
1605	media_device_unregister(&pispbe->mdev);
1606
1607	for (int i = PISPBE_NUM_NODES - 1; i >= 0; i--) {
1608		video_unregister_device(&pispbe->node[i].vfd);
1609		vb2_queue_release(&pispbe->node[i].queue);
1610		mutex_destroy(&pispbe->node[i].node_lock);
1611		mutex_destroy(&pispbe->node[i].queue_lock);
1612	}
1613
1614	media_device_cleanup(&pispbe->mdev);
1615	v4l2_device_unregister(&pispbe->v4l2_dev);
1616}
1617
1618static int pispbe_runtime_suspend(struct device *dev)
1619{
1620	struct pispbe_dev *pispbe = dev_get_drvdata(dev);
1621
1622	clk_disable_unprepare(pispbe->clk);
1623
1624	return 0;
1625}
1626
1627static int pispbe_runtime_resume(struct device *dev)
1628{
1629	struct pispbe_dev *pispbe = dev_get_drvdata(dev);
1630	int ret;
1631
1632	ret = clk_prepare_enable(pispbe->clk);
1633	if (ret) {
1634		dev_err(dev, "Unable to enable clock\n");
1635		return ret;
1636	}
1637
1638	dev_dbg(dev, "%s: Enabled clock, rate=%lu\n",
1639		__func__, clk_get_rate(pispbe->clk));
1640
1641	return 0;
1642}
1643
1644static int pispbe_hw_init(struct pispbe_dev *pispbe)
1645{
1646	u32 u;
1647
1648	/* Check the HW is present and has a known version */
1649	u = pispbe_rd(pispbe, PISP_BE_VERSION_REG);
1650	dev_dbg(pispbe->dev, "pispbe_probe: HW version:  0x%08x", u);
1651	pispbe->hw_version = u;
1652	if ((u & ~PISP_BE_VERSION_MINOR_BITS) != PISP_BE_VERSION_2712)
1653		return -ENODEV;
1654
1655	/* Clear leftover interrupts */
1656	pispbe_wr(pispbe, PISP_BE_INTERRUPT_STATUS_REG, 0xFFFFFFFFu);
1657	u = pispbe_rd(pispbe, PISP_BE_BATCH_STATUS_REG);
1658	dev_dbg(pispbe->dev, "pispbe_probe: BatchStatus: 0x%08x", u);
1659
1660	pispbe->done = (uint8_t)u;
1661	pispbe->started = (uint8_t)(u >> 8);
1662	u = pispbe_rd(pispbe, PISP_BE_STATUS_REG);
1663	dev_dbg(pispbe->dev, "pispbe_probe: Status:      0x%08x", u);
1664
1665	if (u != 0 || pispbe->done != pispbe->started) {
1666		dev_err(pispbe->dev, "pispbe_probe: HW is stuck or busy\n");
1667		return -EBUSY;
1668	}
1669
1670	/*
1671	 * AXI QOS=0, CACHE=4'b0010, PROT=3'b011
1672	 * Also set "chicken bits" 22:20 which enable sub-64-byte bursts
1673	 * and AXI AWID/BID variability (on versions which support this).
1674	 */
1675	pispbe_wr(pispbe, PISP_BE_AXI_REG, 0x32703200u);
1676
1677	/* Enable both interrupt flags */
1678	pispbe_wr(pispbe, PISP_BE_INTERRUPT_EN_REG, 0x00000003u);
1679
1680	return 0;
1681}
1682
1683/* Probe the ISP-BE hardware block, as a single platform device. */
1684static int pispbe_probe(struct platform_device *pdev)
1685{
1686	struct pispbe_dev *pispbe;
1687	int ret;
1688
1689	pispbe = devm_kzalloc(&pdev->dev, sizeof(*pispbe), GFP_KERNEL);
1690	if (!pispbe)
1691		return -ENOMEM;
1692
1693	INIT_LIST_HEAD(&pispbe->job_queue);
1694
1695	dev_set_drvdata(&pdev->dev, pispbe);
1696	pispbe->dev = &pdev->dev;
1697	platform_set_drvdata(pdev, pispbe);
1698
1699	pispbe->be_reg_base = devm_platform_ioremap_resource(pdev, 0);
1700	if (IS_ERR(pispbe->be_reg_base)) {
1701		dev_err(&pdev->dev, "Failed to get ISP-BE registers address\n");
1702		return PTR_ERR(pispbe->be_reg_base);
1703	}
1704
1705	pispbe->irq = platform_get_irq(pdev, 0);
1706	if (pispbe->irq <= 0)
1707		return -EINVAL;
1708
1709	ret = devm_request_irq(&pdev->dev, pispbe->irq, pispbe_isr, 0,
1710			       PISPBE_NAME, pispbe);
1711	if (ret) {
1712		dev_err(&pdev->dev, "Unable to request interrupt\n");
1713		return ret;
1714	}
1715
1716	ret = dma_set_mask_and_coherent(pispbe->dev, DMA_BIT_MASK(36));
1717	if (ret)
1718		return ret;
1719
1720	pispbe->clk = devm_clk_get(&pdev->dev, NULL);
1721	if (IS_ERR(pispbe->clk))
1722		return dev_err_probe(&pdev->dev, PTR_ERR(pispbe->clk),
1723				     "Failed to get clock");
1724
1725	/* Hardware initialisation */
1726	pm_runtime_set_autosuspend_delay(pispbe->dev, 200);
1727	pm_runtime_use_autosuspend(pispbe->dev);
1728	pm_runtime_enable(pispbe->dev);
1729
1730	ret = pm_runtime_resume_and_get(pispbe->dev);
1731	if (ret)
1732		goto pm_runtime_disable_err;
1733
1734	pispbe->hw_busy = false;
1735	spin_lock_init(&pispbe->hw_lock);
1736	ret = pispbe_hw_init(pispbe);
1737	if (ret)
1738		goto pm_runtime_suspend_err;
1739
1740	ret = pispbe_init_devices(pispbe);
1741	if (ret)
1742		goto disable_devs_err;
1743
1744	pm_runtime_put_autosuspend(pispbe->dev);
1745
1746	return 0;
1747
1748disable_devs_err:
1749	pispbe_destroy_devices(pispbe);
1750pm_runtime_suspend_err:
1751	pm_runtime_put(pispbe->dev);
1752pm_runtime_disable_err:
1753	pm_runtime_dont_use_autosuspend(pispbe->dev);
1754	pm_runtime_disable(pispbe->dev);
1755
1756	return ret;
1757}
1758
1759static void pispbe_remove(struct platform_device *pdev)
1760{
1761	struct pispbe_dev *pispbe = platform_get_drvdata(pdev);
1762
1763	pispbe_destroy_devices(pispbe);
1764
1765	pm_runtime_dont_use_autosuspend(pispbe->dev);
1766	pm_runtime_disable(pispbe->dev);
1767}
1768
1769static const struct dev_pm_ops pispbe_pm_ops = {
1770	SET_RUNTIME_PM_OPS(pispbe_runtime_suspend, pispbe_runtime_resume, NULL)
1771};
1772
1773static const struct of_device_id pispbe_of_match[] = {
1774	{
1775		.compatible = "raspberrypi,pispbe",
1776	},
1777	{ /* sentinel */ },
1778};
1779MODULE_DEVICE_TABLE(of, pispbe_of_match);
1780
1781static struct platform_driver pispbe_pdrv = {
1782	.probe		= pispbe_probe,
1783	.remove		= pispbe_remove,
1784	.driver		= {
1785		.name	= PISPBE_NAME,
1786		.of_match_table = pispbe_of_match,
1787		.pm = &pispbe_pm_ops,
1788	},
1789};
1790
1791module_platform_driver(pispbe_pdrv);
1792
1793MODULE_DESCRIPTION("PiSP Back End driver");
1794MODULE_AUTHOR("David Plowman <david.plowman@raspberrypi.com>");
1795MODULE_AUTHOR("Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>");
1796MODULE_LICENSE("GPL");