ipu3-css.c

   1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (C) 2018 Intel Corporation
   3
   4#include <linux/device.h>
   5#include <linux/iopoll.h>
   6#include <linux/slab.h>
   7#include <linux/string_choices.h>
   8
   9#include "ipu3.h"
  10#include "ipu3-css.h"
  11#include "ipu3-css-fw.h"
  12#include "ipu3-css-params.h"
  13#include "ipu3-dmamap.h"
  14#include "ipu3-tables.h"
  15
  16/* IRQ configuration */
  17#define IMGU_IRQCTRL_IRQ_MASK	(IMGU_IRQCTRL_IRQ_SP1 | \
  18				 IMGU_IRQCTRL_IRQ_SP2 | \
  19				 IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
  20				 IMGU_IRQCTRL_IRQ_SW_PIN(1))
  21
  22#define IPU3_CSS_FORMAT_BPP_DEN	50	/* Denominator */
  23
  24/* Some sane limits for resolutions */
  25#define IPU3_CSS_MIN_RES	32
  26#define IPU3_CSS_MAX_H		3136
  27#define IPU3_CSS_MAX_W		4224
  28
  29/* minimal envelope size(GDC in - out) should be 4 */
  30#define MIN_ENVELOPE            4
  31
  32/*
  33 * pre-allocated buffer size for CSS ABI, auxiliary frames
  34 * after BDS and before GDC. Those values should be tuned
  35 * to big enough to avoid buffer re-allocation when
  36 * streaming to lower streaming latency.
  37 */
  38#define CSS_ABI_SIZE    136
  39#define CSS_BDS_SIZE    (4480 * 3200 * 3)
  40#define CSS_GDC_SIZE    (4224 * 3200 * 12 / 8)
  41
  42#define IPU3_CSS_QUEUE_TO_FLAGS(q)	(1 << (q))
  43#define IPU3_CSS_FORMAT_FL_IN		\
  44			IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
  45#define IPU3_CSS_FORMAT_FL_OUT		\
  46			IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
  47#define IPU3_CSS_FORMAT_FL_VF		\
  48			IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
  49
  50/* Formats supported by IPU3 Camera Sub System */
  51static const struct imgu_css_format imgu_css_formats[] = {
  52	{
  53		.pixelformat = V4L2_PIX_FMT_NV12,
  54		.colorspace = V4L2_COLORSPACE_SRGB,
  55		.frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
  56		.osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
  57		.osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
  58		.chroma_decim = 4,
  59		.width_align = IPU3_UAPI_ISP_VEC_ELEMS,
  60		.flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
  61	}, {
  62		/* Each 32 bytes contains 25 10-bit pixels */
  63		.pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
  64		.colorspace = V4L2_COLORSPACE_RAW,
  65		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  66		.bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
  67		.bit_depth = 10,
  68		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  69		.flags = IPU3_CSS_FORMAT_FL_IN,
  70	}, {
  71		.pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
  72		.colorspace = V4L2_COLORSPACE_RAW,
  73		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  74		.bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
  75		.bit_depth = 10,
  76		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  77		.flags = IPU3_CSS_FORMAT_FL_IN,
  78	}, {
  79		.pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
  80		.colorspace = V4L2_COLORSPACE_RAW,
  81		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  82		.bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
  83		.bit_depth = 10,
  84		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  85		.flags = IPU3_CSS_FORMAT_FL_IN,
  86	}, {
  87		.pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
  88		.colorspace = V4L2_COLORSPACE_RAW,
  89		.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
  90		.bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
  91		.bit_depth = 10,
  92		.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
  93		.flags = IPU3_CSS_FORMAT_FL_IN,
  94	},
  95};
  96
  97static const struct {
  98	enum imgu_abi_queue_id qid;
  99	size_t ptr_ofs;
 100} imgu_css_queues[IPU3_CSS_QUEUES] = {
 101	[IPU3_CSS_QUEUE_IN] = {
 102		IMGU_ABI_QUEUE_C_ID,
 103		offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 104	},
 105	[IPU3_CSS_QUEUE_OUT] = {
 106		IMGU_ABI_QUEUE_D_ID,
 107		offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 108	},
 109	[IPU3_CSS_QUEUE_VF] = {
 110		IMGU_ABI_QUEUE_E_ID,
 111		offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
 112	},
 113	[IPU3_CSS_QUEUE_STAT_3A] = {
 114		IMGU_ABI_QUEUE_F_ID,
 115		offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
 116	},
 117};
 118
 119/* Initialize queue based on given format, adjust format as needed */
 120static int imgu_css_queue_init(struct imgu_css_queue *queue,
 121			       struct v4l2_pix_format_mplane *fmt, u32 flags)
 122{
 123	struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
 124	unsigned int i;
 125	u32 sizeimage;
 126
 127	INIT_LIST_HEAD(&queue->bufs);
 128
 129	queue->css_fmt = NULL;	/* Disable */
 130	if (!fmt)
 131		return 0;
 132
 133	for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
 134		if (!(imgu_css_formats[i].flags & flags))
 135			continue;
 136		queue->css_fmt = &imgu_css_formats[i];
 137		if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
 138			break;
 139	}
 140	if (!queue->css_fmt)
 141		return -EINVAL;	/* Could not find any suitable format */
 142
 143	queue->fmt.mpix = *fmt;
 144
 145	f->width = ALIGN(clamp_t(u32, f->width,
 146				 IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
 147	f->height = ALIGN(clamp_t(u32, f->height,
 148				  IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
 149	queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
 150	f->plane_fmt[0].bytesperline =
 151		imgu_bytesperline(f->width, queue->css_fmt->frame_format);
 152	sizeimage = f->height * f->plane_fmt[0].bytesperline;
 153	if (queue->css_fmt->chroma_decim)
 154		sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
 155
 156	f->plane_fmt[0].sizeimage = sizeimage;
 157	f->field = V4L2_FIELD_NONE;
 158	f->num_planes = 1;
 159	f->colorspace = queue->css_fmt->colorspace;
 160	f->flags = 0;
 161	f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
 162	f->quantization = V4L2_QUANTIZATION_DEFAULT;
 163	f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
 164	memset(f->reserved, 0, sizeof(f->reserved));
 165
 166	return 0;
 167}
 168
 169static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
 170{
 171	return q->css_fmt;
 172}
 173
 174/******************* css hw *******************/
 175
 176/* In the style of writesl() defined in include/asm-generic/io.h */
 177static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
 178{
 179	if (count >= 4) {
 180		const u32 *buf = mem;
 181
 182		count /= 4;
 183		do {
 184			writel(*buf++, addr);
 185			addr += 4;
 186		} while (--count);
 187	}
 188}
 189
 190/* Wait until register `reg', masked with `mask', becomes `cmp' */
 191static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
 192{
 193	u32 val;
 194
 195	return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
 196				  1000, 100 * 1000);
 197}
 198
 199/* Initialize the IPU3 CSS hardware and associated h/w blocks */
 200
 201int imgu_css_set_powerup(struct device *dev, void __iomem *base,
 202			 unsigned int freq)
 203{
 204	u32 pm_ctrl, state, val;
 205
 206	dev_dbg(dev, "%s with freq %u\n", __func__, freq);
 207	/* Clear the CSS busy signal */
 208	readl(base + IMGU_REG_GP_BUSY);
 209	writel(0, base + IMGU_REG_GP_BUSY);
 210
 211	/* Wait for idle signal */
 212	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 213			 IMGU_STATE_IDLE_STS)) {
 214		dev_err(dev, "failed to set CSS idle\n");
 215		goto fail;
 216	}
 217
 218	/* Reset the css */
 219	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
 220	       base + IMGU_REG_PM_CTRL);
 221
 222	usleep_range(200, 300);
 223
 224	/** Prepare CSS */
 225
 226	pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
 227	state = readl(base + IMGU_REG_STATE);
 228
 229	dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
 230		pm_ctrl, state, str_down_up(state & IMGU_STATE_POWER_DOWN));
 231
 232	/* Power up CSS using wrapper */
 233	if (state & IMGU_STATE_POWER_DOWN) {
 234		writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
 235		       base + IMGU_REG_PM_CTRL);
 236		if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
 237				 IMGU_PM_CTRL_START, 0)) {
 238			dev_err(dev, "failed to power up CSS\n");
 239			goto fail;
 240		}
 241		usleep_range(2000, 3000);
 242	} else {
 243		writel(IMGU_PM_CTRL_RACE_TO_HALT, base + IMGU_REG_PM_CTRL);
 244	}
 245
 246	/* Set the busy bit */
 247	writel(readl(base + IMGU_REG_GP_BUSY) | 1, base + IMGU_REG_GP_BUSY);
 248
 249	/* Set CSS clock frequency */
 250	pm_ctrl = readl(base + IMGU_REG_PM_CTRL);
 251	val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 252	writel(val, base + IMGU_REG_PM_CTRL);
 253	writel(0, base + IMGU_REG_GP_BUSY);
 254	if (imgu_hw_wait(base, IMGU_REG_STATE,
 255			 IMGU_STATE_PWRDNM_FSM_MASK, 0)) {
 256		dev_err(dev, "failed to pwrdn CSS\n");
 257		goto fail;
 258	}
 259	val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
 260	writel(val, base + IMGU_REG_SYSTEM_REQ);
 261	writel(1, base + IMGU_REG_GP_BUSY);
 262	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
 263	       base + IMGU_REG_PM_CTRL);
 264	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
 265			 IMGU_STATE_HALT_STS)) {
 266		dev_err(dev, "failed to halt CSS\n");
 267		goto fail;
 268	}
 269
 270	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
 271	       base + IMGU_REG_PM_CTRL);
 272	if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, 0)) {
 273		dev_err(dev, "failed to start CSS\n");
 274		goto fail;
 275	}
 276	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
 277	       base + IMGU_REG_PM_CTRL);
 278
 279	val = readl(base + IMGU_REG_PM_CTRL);	/* get pm_ctrl */
 280	val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 281	val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
 282	writel(val, base + IMGU_REG_PM_CTRL);
 283
 284	return 0;
 285
 286fail:
 287	imgu_css_set_powerdown(dev, base);
 288	return -EIO;
 289}
 290
 291void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
 292{
 293	dev_dbg(dev, "%s\n", __func__);
 294	/* wait for cio idle signal */
 295	if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
 296			 IMGU_CIO_GATE_BURST_MASK, 0))
 297		dev_warn(dev, "wait cio gate idle timeout");
 298
 299	/* wait for css idle signal */
 300	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 301			 IMGU_STATE_IDLE_STS))
 302		dev_warn(dev, "wait css idle timeout\n");
 303
 304	/* do halt-halted handshake with css */
 305	writel(1, base + IMGU_REG_GP_HALT);
 306	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
 307			 IMGU_STATE_HALT_STS))
 308		dev_warn(dev, "failed to halt css");
 309
 310	/* de-assert the busy bit */
 311	writel(0, base + IMGU_REG_GP_BUSY);
 312}
 313
 314static void imgu_css_hw_enable_irq(struct imgu_css *css)
 315{
 316	void __iomem *const base = css->base;
 317	u32 val, i;
 318
 319	/* Set up interrupts */
 320
 321	/*
 322	 * Enable IRQ on the SP which signals that SP goes to idle
 323	 * (aka ready state) and set trigger to pulse
 324	 */
 325	val = readl(base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
 326	writel(val, base + IMGU_REG_SP_CTRL(0));
 327	writel(val | IMGU_CTRL_IRQ_CLEAR, base + IMGU_REG_SP_CTRL(0));
 328
 329	/* Enable IRQs from the IMGU wrapper */
 330	writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_ENABLE);
 331	/* Clear */
 332	writel(IMGU_REG_INT_CSS_IRQ, base + IMGU_REG_INT_STATUS);
 333
 334	/* Enable IRQs from main IRQ controller */
 335	writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
 336	writel(0, base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
 337	writel(IMGU_IRQCTRL_IRQ_MASK,
 338	       base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
 339	writel(IMGU_IRQCTRL_IRQ_MASK,
 340	       base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
 341	writel(IMGU_IRQCTRL_IRQ_MASK,
 342	       base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
 343	writel(IMGU_IRQCTRL_IRQ_MASK,
 344	       base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
 345	/* Wait for write complete */
 346	readl(base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
 347
 348	/* Enable IRQs from SP0 and SP1 controllers */
 349	for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
 350		writel(~0, base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
 351		writel(0, base + IMGU_REG_IRQCTRL_MASK(i));
 352		writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_EDGE(i));
 353		writel(IMGU_IRQCTRL_IRQ_MASK,
 354		       base + IMGU_REG_IRQCTRL_ENABLE(i));
 355		writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_CLEAR(i));
 356		writel(IMGU_IRQCTRL_IRQ_MASK, base + IMGU_REG_IRQCTRL_MASK(i));
 357		/* Wait for write complete */
 358		readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
 359	}
 360}
 361
 362static int imgu_css_hw_init(struct imgu_css *css)
 363{
 364	/* For checking that streaming monitor statuses are valid */
 365	static const struct {
 366		u32 reg;
 367		u32 mask;
 368		const char *name;
 369	} stream_monitors[] = {
 370		{
 371			IMGU_REG_GP_SP1_STRMON_STAT,
 372			IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
 373			"ISP0 to SP0"
 374		}, {
 375			IMGU_REG_GP_ISP_STRMON_STAT,
 376			IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
 377			"SP0 to ISP0"
 378		}, {
 379			IMGU_REG_GP_MOD_STRMON_STAT,
 380			IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
 381			"ISP0 to DMA0"
 382		}, {
 383			IMGU_REG_GP_ISP_STRMON_STAT,
 384			IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
 385			"DMA0 to ISP0"
 386		}, {
 387			IMGU_REG_GP_MOD_STRMON_STAT,
 388			IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
 389			"ISP0 to GDC0"
 390		}, {
 391			IMGU_REG_GP_MOD_STRMON_STAT,
 392			IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
 393			"GDC0 to ISP0"
 394		}, {
 395			IMGU_REG_GP_MOD_STRMON_STAT,
 396			IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
 397			"SP0 to DMA0"
 398		}, {
 399			IMGU_REG_GP_SP1_STRMON_STAT,
 400			IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
 401			"DMA0 to SP0"
 402		}, {
 403			IMGU_REG_GP_MOD_STRMON_STAT,
 404			IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
 405			"SP0 to GDC0"
 406		}, {
 407			IMGU_REG_GP_MOD_STRMON_STAT,
 408			IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
 409			"GDC0 to SP0"
 410		},
 411	};
 412
 413	struct device *dev = css->dev;
 414	void __iomem *const base = css->base;
 415	u32 val, i;
 416
 417	/* Set instruction cache address and inv bit for ISP, SP, and SP1 */
 418	for (i = 0; i < IMGU_NUM_SP; i++) {
 419		struct imgu_fw_info *bi =
 420					&css->fwp->binary_header[css->fw_sp[i]];
 421
 422		writel(css->binary[css->fw_sp[i]].daddr,
 423		       base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
 424		writel(readl(base + IMGU_REG_SP_CTRL(bi->type)) |
 425		       IMGU_CTRL_ICACHE_INV,
 426		       base + IMGU_REG_SP_CTRL(bi->type));
 427	}
 428	writel(css->binary[css->fw_bl].daddr, base + IMGU_REG_ISP_ICACHE_ADDR);
 429	writel(readl(base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
 430	       base + IMGU_REG_ISP_CTRL);
 431
 432	/* Check that IMGU hardware is ready */
 433
 434	if (!(readl(base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
 435		dev_err(dev, "SP is not idle\n");
 436		return -EIO;
 437	}
 438	if (!(readl(base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
 439		dev_err(dev, "ISP is not idle\n");
 440		return -EIO;
 441	}
 442
 443	for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
 444		val = readl(base + stream_monitors[i].reg);
 445		if (val & stream_monitors[i].mask) {
 446			dev_err(dev, "error: Stream monitor %s is valid\n",
 447				stream_monitors[i].name);
 448			return -EIO;
 449		}
 450	}
 451
 452	/* Initialize GDC with default values */
 453
 454	for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
 455		u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
 456		u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
 457		u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
 458		u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
 459
 460		writel(val0 | (val1 << 16),
 461		       base + IMGU_REG_GDC_LUT_BASE + i * 8);
 462		writel(val2 | (val3 << 16),
 463		       base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
 464	}
 465
 466	return 0;
 467}
 468
 469/* Boot the given IPU3 CSS SP */
 470static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
 471{
 472	void __iomem *const base = css->base;
 473	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
 474	struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
 475		.ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
 476			+ bi->blob.data_source,
 477		.dmem_data_addr = bi->blob.data_target,
 478		.dmem_bss_addr = bi->blob.bss_target,
 479		.data_size = bi->blob.data_size,
 480		.bss_size = bi->blob.bss_size,
 481		.sp_id = sp,
 482	};
 483
 484	writes(&dmem_cfg, sizeof(dmem_cfg), base +
 485	       IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
 486
 487	writel(bi->info.sp.sp_entry, base + IMGU_REG_SP_START_ADDR(sp));
 488
 489	writel(readl(base + IMGU_REG_SP_CTRL(sp))
 490		| IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_SP_CTRL(sp));
 491
 492	if (imgu_hw_wait(css->base, IMGU_REG_SP_DMEM_BASE(sp)
 493			 + bi->info.sp.sw_state,
 494			 ~0, IMGU_ABI_SP_SWSTATE_INITIALIZED))
 495		return -EIO;
 496
 497	return 0;
 498}
 499
 500/* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
 501static int imgu_css_hw_start(struct imgu_css *css)
 502{
 503	static const u32 event_mask =
 504		((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
 505		(1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
 506		(1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
 507		(1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
 508		(1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
 509		(1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
 510		(1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
 511		(1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
 512		(1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
 513		(1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
 514		(1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
 515		<< IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
 516
 517	void __iomem *const base = css->base;
 518	struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
 519	unsigned int i;
 520
 521	writel(IMGU_TLB_INVALIDATE, base + IMGU_REG_TLB_INVALIDATE);
 522
 523	/* Start bootloader */
 524
 525	writel(IMGU_ABI_BL_SWSTATE_BUSY,
 526	       base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
 527	writel(IMGU_NUM_SP,
 528	       base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
 529
 530	for (i = 0; i < IMGU_NUM_SP; i++) {
 531		int j = IMGU_NUM_SP - i - 1;	/* load sp1 first, then sp0 */
 532		struct imgu_fw_info *sp =
 533					&css->fwp->binary_header[css->fw_sp[j]];
 534		struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
 535			.src_addr = css->binary[css->fw_sp[j]].daddr
 536				+ sp->blob.text_source,
 537			.size = sp->blob.text_size,
 538			.dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
 539			.dst_addr = IMGU_SP_PMEM_BASE(j),
 540		};
 541
 542		writes(&dma_cmd, sizeof(dma_cmd),
 543		       base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
 544		       bl->info.bl.dma_cmd_list);
 545	}
 546
 547	writel(bl->info.bl.bl_entry, base + IMGU_REG_ISP_START_ADDR);
 548
 549	writel(readl(base + IMGU_REG_ISP_CTRL)
 550		| IMGU_CTRL_START | IMGU_CTRL_RUN, base + IMGU_REG_ISP_CTRL);
 551	if (imgu_hw_wait(css->base, IMGU_REG_ISP_DMEM_BASE
 552			 + bl->info.bl.sw_state, ~0,
 553			 IMGU_ABI_BL_SWSTATE_OK)) {
 554		dev_err(css->dev, "failed to start bootloader\n");
 555		return -EIO;
 556	}
 557
 558	/* Start ISP */
 559
 560	memset(css->xmem_sp_group_ptrs.vaddr, 0,
 561	       sizeof(struct imgu_abi_sp_group));
 562
 563	bi = &css->fwp->binary_header[css->fw_sp[0]];
 564
 565	writel(css->xmem_sp_group_ptrs.daddr,
 566	       base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
 567
 568	writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
 569	       base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
 570	writel(1, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
 571
 572	if (imgu_css_hw_start_sp(css, 0))
 573		return -EIO;
 574
 575	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
 576	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) +
 577		bi->info.sp.host_sp_queues_initialized);
 578	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
 579	writel(0, base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
 580	writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(0)
 581		+ bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 582
 583	/* Enable all events for all queues */
 584
 585	for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
 586		writel(event_mask, base + IMGU_REG_SP_DMEM_BASE(0)
 587			+ bi->info.sp.host_sp_com
 588			+ IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
 589	writel(1, base + IMGU_REG_SP_DMEM_BASE(0) +
 590		bi->info.sp.host_sp_queues_initialized);
 591
 592	/* Start SP1 */
 593
 594	bi = &css->fwp->binary_header[css->fw_sp[1]];
 595
 596	writel(IMGU_ABI_SP_SWSTATE_TERMINATED,
 597	       base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
 598
 599	if (imgu_css_hw_start_sp(css, 1))
 600		return -EIO;
 601
 602	writel(IMGU_ABI_SP_COMM_COMMAND_READY, base + IMGU_REG_SP_DMEM_BASE(1)
 603		+ bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 604
 605	return 0;
 606}
 607
 608static void imgu_css_hw_stop(struct imgu_css *css)
 609{
 610	void __iomem *const base = css->base;
 611	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
 612
 613	/* Stop fw */
 614	writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
 615	       base + IMGU_REG_SP_DMEM_BASE(0) +
 616	       bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
 617	if (imgu_hw_wait(css->base, IMGU_REG_SP_CTRL(0),
 618			 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
 619		dev_err(css->dev, "wait sp0 idle timeout.\n");
 620	if (readl(base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
 621		  IMGU_ABI_SP_SWSTATE_TERMINATED)
 622		dev_err(css->dev, "sp0 is not terminated.\n");
 623	if (imgu_hw_wait(css->base, IMGU_REG_ISP_CTRL,
 624			 IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
 625		dev_err(css->dev, "wait isp idle timeout\n");
 626}
 627
 628static void imgu_css_hw_cleanup(struct imgu_css *css)
 629{
 630	void __iomem *const base = css->base;
 631
 632	/** Reset CSS **/
 633
 634	/* Clear the CSS busy signal */
 635	readl(base + IMGU_REG_GP_BUSY);
 636	writel(0, base + IMGU_REG_GP_BUSY);
 637
 638	/* Wait for idle signal */
 639	if (imgu_hw_wait(css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
 640			 IMGU_STATE_IDLE_STS))
 641		dev_err(css->dev, "failed to shut down hw cleanly\n");
 642
 643	/* Reset the css */
 644	writel(readl(base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
 645	       base + IMGU_REG_PM_CTRL);
 646
 647	usleep_range(200, 300);
 648}
 649
 650static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
 651{
 652	struct imgu_device *imgu = dev_get_drvdata(css->dev);
 653	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
 654	unsigned int i;
 655
 656	imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
 657	imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
 658	imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
 659	imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
 660
 661	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
 662		imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
 663}
 664
 665/*
 666 * This function initializes various stages of the
 667 * IPU3 CSS ISP pipeline
 668 */
 669static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
 670{
 671	static const int BYPC = 2;	/* Bytes per component */
 672	static const struct imgu_abi_buffer_sp buffer_sp_init = {
 673		.buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
 674		.buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
 675	};
 676
 677	struct imgu_abi_isp_iterator_config *cfg_iter;
 678	struct imgu_abi_isp_ref_config *cfg_ref;
 679	struct imgu_abi_isp_dvs_config *cfg_dvs;
 680	struct imgu_abi_isp_tnr3_config *cfg_tnr;
 681	struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
 682	struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
 683
 684	const int stage = 0;
 685	unsigned int i, j;
 686
 687	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
 688	struct imgu_css_queue *css_queue_in =
 689			&css_pipe->queue[IPU3_CSS_QUEUE_IN];
 690	struct imgu_css_queue *css_queue_out =
 691			&css_pipe->queue[IPU3_CSS_QUEUE_OUT];
 692	struct imgu_css_queue *css_queue_vf =
 693			&css_pipe->queue[IPU3_CSS_QUEUE_VF];
 694	const struct imgu_fw_info *bi =
 695			&css->fwp->binary_header[css_pipe->bindex];
 696	const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
 697
 698	struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
 699		bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
 700	struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
 701		bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
 702
 703	struct imgu_abi_isp_stage *isp_stage;
 704	struct imgu_abi_sp_stage *sp_stage;
 705	struct imgu_abi_sp_group *sp_group;
 706	struct imgu_abi_frames_sp *frames_sp;
 707	struct imgu_abi_frame_sp *frame_sp;
 708	struct imgu_abi_frame_sp_info *frame_sp_info;
 709
 710	const unsigned int bds_width_pad =
 711				ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
 712				      2 * IPU3_UAPI_ISP_VEC_ELEMS);
 713
 714	const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
 715	enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
 716	void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
 717
 718	struct imgu_device *imgu = dev_get_drvdata(css->dev);
 719
 720	dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
 721
 722	/* Configure iterator */
 723
 724	cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 725					       &cofs->dmem.iterator,
 726					       sizeof(*cfg_iter), vaddr);
 727	if (!cfg_iter)
 728		goto bad_firmware;
 729
 730	frame_sp_info = &cfg_iter->input_info;
 731	frame_sp_info->res.width	= css_queue_in->fmt.mpix.width;
 732	frame_sp_info->res.height	= css_queue_in->fmt.mpix.height;
 733	frame_sp_info->padded_width	= css_queue_in->width_pad;
 734	frame_sp_info->format		= css_queue_in->css_fmt->frame_format;
 735	frame_sp_info->raw_bit_depth	= css_queue_in->css_fmt->bit_depth;
 736	frame_sp_info->raw_bayer_order	= css_queue_in->css_fmt->bayer_order;
 737	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 738
 739	frame_sp_info = &cfg_iter->internal_info;
 740	frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
 741	frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
 742	frame_sp_info->padded_width	= bds_width_pad;
 743	frame_sp_info->format		= css_queue_out->css_fmt->frame_format;
 744	frame_sp_info->raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
 745	frame_sp_info->raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
 746	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 747
 748	frame_sp_info = &cfg_iter->output_info;
 749	frame_sp_info->res.width	= css_queue_out->fmt.mpix.width;
 750	frame_sp_info->res.height	= css_queue_out->fmt.mpix.height;
 751	frame_sp_info->padded_width	= css_queue_out->width_pad;
 752	frame_sp_info->format		= css_queue_out->css_fmt->frame_format;
 753	frame_sp_info->raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
 754	frame_sp_info->raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
 755	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 756
 757	frame_sp_info = &cfg_iter->vf_info;
 758	frame_sp_info->res.width	= css_queue_vf->fmt.mpix.width;
 759	frame_sp_info->res.height	= css_queue_vf->fmt.mpix.height;
 760	frame_sp_info->padded_width	= css_queue_vf->width_pad;
 761	frame_sp_info->format		= css_queue_vf->css_fmt->frame_format;
 762	frame_sp_info->raw_bit_depth	= css_queue_vf->css_fmt->bit_depth;
 763	frame_sp_info->raw_bayer_order	= css_queue_vf->css_fmt->bayer_order;
 764	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 765
 766	cfg_iter->dvs_envelope.width =
 767				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
 768	cfg_iter->dvs_envelope.height =
 769				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
 770
 771	/* Configure reference (delay) frames */
 772
 773	cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 774					      &cofs->dmem.ref,
 775					      sizeof(*cfg_ref), vaddr);
 776	if (!cfg_ref)
 777		goto bad_firmware;
 778
 779	cfg_ref->port_b.crop = 0;
 780	cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
 781	cfg_ref->port_b.width =
 782		css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
 783	cfg_ref->port_b.stride =
 784		css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
 785	cfg_ref->width_a_over_b =
 786				IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
 787	cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
 788	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
 789		cfg_ref->ref_frame_addr_y[i] =
 790			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
 791		cfg_ref->ref_frame_addr_c[i] =
 792			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
 793			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
 794			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
 795	}
 796	for (; i < IMGU_ABI_FRAMES_REF; i++) {
 797		cfg_ref->ref_frame_addr_y[i] = 0;
 798		cfg_ref->ref_frame_addr_c[i] = 0;
 799	}
 800
 801	/* Configure DVS (digital video stabilization) */
 802
 803	cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 804					      &cofs->dmem.dvs, sizeof(*cfg_dvs),
 805					      vaddr);
 806	if (!cfg_dvs)
 807		goto bad_firmware;
 808
 809	cfg_dvs->num_horizontal_blocks =
 810			ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
 811					   IMGU_DVS_BLOCK_W), 2);
 812	cfg_dvs->num_vertical_blocks =
 813			DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
 814				     IMGU_DVS_BLOCK_H);
 815
 816	/* Configure TNR (temporal noise reduction) */
 817
 818	if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
 819		cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 820						      &cofs->dmem.tnr3,
 821						      sizeof(*cfg_tnr),
 822						      vaddr);
 823		if (!cfg_tnr)
 824			goto bad_firmware;
 825
 826		cfg_tnr->port_b.crop = 0;
 827		cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
 828		cfg_tnr->port_b.width =
 829			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
 830		cfg_tnr->port_b.stride =
 831			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
 832		cfg_tnr->width_a_over_b =
 833			IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
 834		cfg_tnr->frame_height =
 835			css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
 836		cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
 837		for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
 838			cfg_tnr->frame_addr[i] =
 839				css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
 840					.mem[i].daddr;
 841		for (; i < IMGU_ABI_FRAMES_TNR; i++)
 842			cfg_tnr->frame_addr[i] = 0;
 843	}
 844
 845	/* Configure ref dmem state parameters */
 846
 847	cfg = IMGU_ABI_PARAM_CLASS_STATE;
 848	vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
 849
 850	cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 851						    &sofs->dmem.ref,
 852						    sizeof(*cfg_ref_state),
 853						    vaddr);
 854	if (!cfg_ref_state)
 855		goto bad_firmware;
 856
 857	cfg_ref_state->ref_in_buf_idx = 0;
 858	cfg_ref_state->ref_out_buf_idx = 1;
 859
 860	/* Configure tnr dmem state parameters */
 861	if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
 862		cfg_tnr_state =
 863			imgu_css_fw_pipeline_params(css, pipe, cfg, m0,
 864						    &sofs->dmem.tnr3,
 865						    sizeof(*cfg_tnr_state),
 866						    vaddr);
 867		if (!cfg_tnr_state)
 868			goto bad_firmware;
 869
 870		cfg_tnr_state->in_bufidx = 0;
 871		cfg_tnr_state->out_bufidx = 1;
 872		cfg_tnr_state->bypass_filter = 0;
 873		cfg_tnr_state->total_frame_counter = 0;
 874		for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
 875			cfg_tnr_state->buffer_frame_counter[i] = 0;
 876	}
 877
 878	/* Configure ISP stage */
 879
 880	isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
 881	memset(isp_stage, 0, sizeof(*isp_stage));
 882	isp_stage->blob_info = bi->blob;
 883	isp_stage->binary_info = bi->info.isp.sp;
 884	strscpy(isp_stage->binary_name,
 885		(char *)css->fwp + bi->blob.prog_name_offset,
 886		sizeof(isp_stage->binary_name));
 887	isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
 888	for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
 889		for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
 890			isp_stage->mem_initializers.params[i][j].address =
 891					css_pipe->binary_params_cs[i - 1][j].daddr;
 892
 893	/* Configure SP stage */
 894
 895	sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
 896	memset(sp_stage, 0, sizeof(*sp_stage));
 897
 898	frames_sp = &sp_stage->frames;
 899	frames_sp->in.buf_attr = buffer_sp_init;
 900	for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
 901		frames_sp->out[i].buf_attr = buffer_sp_init;
 902	frames_sp->out_vf.buf_attr = buffer_sp_init;
 903	frames_sp->s3a_buf = buffer_sp_init;
 904	frames_sp->dvs_buf = buffer_sp_init;
 905
 906	sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
 907	sp_stage->num = stage;
 908	sp_stage->isp_online = 0;
 909	sp_stage->isp_copy_vf = 0;
 910	sp_stage->isp_copy_output = 0;
 911
 912	sp_stage->enable.vf_output = css_pipe->vf_output_en;
 913
 914	frames_sp->effective_in_res.width =
 915				css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
 916	frames_sp->effective_in_res.height =
 917				css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
 918
 919	frame_sp = &frames_sp->in;
 920	frame_sp->info.res.width	= css_queue_in->fmt.mpix.width;
 921	frame_sp->info.res.height	= css_queue_in->fmt.mpix.height;
 922	frame_sp->info.padded_width	= css_queue_in->width_pad;
 923	frame_sp->info.format		= css_queue_in->css_fmt->frame_format;
 924	frame_sp->info.raw_bit_depth	= css_queue_in->css_fmt->bit_depth;
 925	frame_sp->info.raw_bayer_order	= css_queue_in->css_fmt->bayer_order;
 926	frame_sp->info.raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 927	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
 928	frame_sp->buf_attr.buf_type	= IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
 929
 930	frame_sp = &frames_sp->out[0];
 931	frame_sp->info.res.width	= css_queue_out->fmt.mpix.width;
 932	frame_sp->info.res.height	= css_queue_out->fmt.mpix.height;
 933	frame_sp->info.padded_width	= css_queue_out->width_pad;
 934	frame_sp->info.format		= css_queue_out->css_fmt->frame_format;
 935	frame_sp->info.raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
 936	frame_sp->info.raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
 937	frame_sp->info.raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 938	frame_sp->planes.nv.uv.offset	= css_queue_out->width_pad *
 939					  css_queue_out->fmt.mpix.height;
 940	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
 941	frame_sp->buf_attr.buf_type	= IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
 942
 943	frame_sp = &frames_sp->out[1];
 944	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
 945
 946	frame_sp_info = &frames_sp->internal_frame_info;
 947	frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
 948	frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
 949	frame_sp_info->padded_width	= bds_width_pad;
 950	frame_sp_info->format		= css_queue_out->css_fmt->frame_format;
 951	frame_sp_info->raw_bit_depth	= css_queue_out->css_fmt->bit_depth;
 952	frame_sp_info->raw_bayer_order	= css_queue_out->css_fmt->bayer_order;
 953	frame_sp_info->raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 954
 955	frame_sp = &frames_sp->out_vf;
 956	frame_sp->info.res.width	= css_queue_vf->fmt.mpix.width;
 957	frame_sp->info.res.height	= css_queue_vf->fmt.mpix.height;
 958	frame_sp->info.padded_width	= css_queue_vf->width_pad;
 959	frame_sp->info.format		= css_queue_vf->css_fmt->frame_format;
 960	frame_sp->info.raw_bit_depth	= css_queue_vf->css_fmt->bit_depth;
 961	frame_sp->info.raw_bayer_order	= css_queue_vf->css_fmt->bayer_order;
 962	frame_sp->info.raw_type		= IMGU_ABI_RAW_TYPE_BAYER;
 963	frame_sp->planes.yuv.u.offset	= css_queue_vf->width_pad *
 964					  css_queue_vf->fmt.mpix.height;
 965	frame_sp->planes.yuv.v.offset	= css_queue_vf->width_pad *
 966					  css_queue_vf->fmt.mpix.height * 5 / 4;
 967	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
 968	frame_sp->buf_attr.buf_type	= IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
 969
 970	frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
 971	frames_sp->s3a_buf.buf_type	= IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
 972
 973	frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
 974	frames_sp->dvs_buf.buf_type	= IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
 975
 976	sp_stage->dvs_envelope.width =
 977				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
 978	sp_stage->dvs_envelope.height =
 979				css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
 980
 981	sp_stage->isp_pipe_version =
 982				bi->info.isp.sp.pipeline.isp_pipe_version;
 983	sp_stage->isp_deci_log_factor =
 984			clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
 985				      IMGU_MAX_BQ_GRID_WIDTH),
 986				  fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
 987				      IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
 988	sp_stage->isp_vf_downscale_bits = 0;
 989	sp_stage->if_config_index = 255;
 990	sp_stage->sp_enable_xnr = 0;
 991	sp_stage->num_stripes = stripes;
 992	sp_stage->enable.s3a = 1;
 993	sp_stage->enable.dvs_stats = 0;
 994
 995	sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
 996	sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
 997	sp_stage->isp_stage_addr =
 998		css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
 999
1000	/* Configure SP group */
1001
1002	sp_group = css->xmem_sp_group_ptrs.vaddr;
1003	memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1004
1005	sp_group->pipe[pipe].num_stages = 1;
1006	sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1007	sp_group->pipe[pipe].thread_id = pipe;
1008	sp_group->pipe[pipe].pipe_num = pipe;
1009	sp_group->pipe[pipe].num_execs = -1;
1010	sp_group->pipe[pipe].pipe_qos_config = -1;
1011	sp_group->pipe[pipe].required_bds_factor = 0;
1012	sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1013	sp_group->pipe[pipe].inout_port_config =
1014					IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1015					IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1016	sp_group->pipe[pipe].scaler_pp_lut = 0;
1017	sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1018	sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1019	sp_group->pipe[pipe].sp_stage_addr[stage] =
1020			css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1021	sp_group->pipe[pipe].pipe_config =
1022			bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1023	sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1024
1025	/* Initialize parameter pools */
1026
1027	if (imgu_css_pool_init(imgu, &css_pipe->pool.parameter_set_info,
1028			       sizeof(struct imgu_abi_parameter_set_info)) ||
1029	    imgu_css_pool_init(imgu, &css_pipe->pool.acc,
1030			       sizeof(struct imgu_abi_acc_param)) ||
1031	    imgu_css_pool_init(imgu, &css_pipe->pool.gdc,
1032			       sizeof(struct imgu_abi_gdc_warp_param) *
1033			       3 * cfg_dvs->num_horizontal_blocks / 2 *
1034			       cfg_dvs->num_vertical_blocks) ||
1035	    imgu_css_pool_init(imgu, &css_pipe->pool.obgrid,
1036			       imgu_css_fw_obgrid_size(
1037			       &css->fwp->binary_header[css_pipe->bindex])))
1038		goto out_of_memory;
1039
1040	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1041		if (imgu_css_pool_init(imgu,
1042				       &css_pipe->pool.binary_params_p[i],
1043				       bi->info.isp.sp.mem_initializers.params
1044				       [IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1045			goto out_of_memory;
1046
1047	return 0;
1048
1049bad_firmware:
1050	imgu_css_pipeline_cleanup(css, pipe);
1051	return -EPROTO;
1052
1053out_of_memory:
1054	imgu_css_pipeline_cleanup(css, pipe);
1055	return -ENOMEM;
1056}
1057
1058static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1059{
1060	static const unsigned int sp;
1061	void __iomem *const base = css->base;
1062	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1063	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1064						bi->info.sp.host_sp_queue;
1065
1066	return queue >= 0 ? readb(&q->host2sp_bufq_info[thread][queue].end) :
1067			    readb(&q->host2sp_evtq_info.end);
1068}
1069
1070/* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
1071static int imgu_css_queue_data(struct imgu_css *css,
1072			       int queue, int thread, u32 data)
1073{
1074	static const unsigned int sp;
1075	void __iomem *const base = css->base;
1076	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1077	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1078						bi->info.sp.host_sp_queue;
1079	u8 size, start, end, end2;
1080
1081	if (queue >= 0) {
1082		size = readb(&q->host2sp_bufq_info[thread][queue].size);
1083		start = readb(&q->host2sp_bufq_info[thread][queue].start);
1084		end = readb(&q->host2sp_bufq_info[thread][queue].end);
1085	} else {
1086		size = readb(&q->host2sp_evtq_info.size);
1087		start = readb(&q->host2sp_evtq_info.start);
1088		end = readb(&q->host2sp_evtq_info.end);
1089	}
1090
1091	if (size == 0)
1092		return -EIO;
1093
1094	end2 = (end + 1) % size;
1095	if (end2 == start)
1096		return -EBUSY;	/* Queue full */
1097
1098	if (queue >= 0) {
1099		writel(data, &q->host2sp_bufq[thread][queue][end]);
1100		writeb(end2, &q->host2sp_bufq_info[thread][queue].end);
1101	} else {
1102		writel(data, &q->host2sp_evtq[end]);
1103		writeb(end2, &q->host2sp_evtq_info.end);
1104	}
1105
1106	return 0;
1107}
1108
1109/* Receive data using given buffer queue, or if queue < 0, event queue. */
1110static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1111{
1112	static const unsigned int sp;
1113	void __iomem *const base = css->base;
1114	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1115	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1116						bi->info.sp.host_sp_queue;
1117	u8 size, start, end, start2;
1118
1119	if (queue >= 0) {
1120		size = readb(&q->sp2host_bufq_info[queue].size);
1121		start = readb(&q->sp2host_bufq_info[queue].start);
1122		end = readb(&q->sp2host_bufq_info[queue].end);
1123	} else {
1124		size = readb(&q->sp2host_evtq_info.size);
1125		start = readb(&q->sp2host_evtq_info.start);
1126		end = readb(&q->sp2host_evtq_info.end);
1127	}
1128
1129	if (size == 0)
1130		return -EIO;
1131
1132	if (end == start)
1133		return -EBUSY;	/* Queue empty */
1134
1135	start2 = (start + 1) % size;
1136
1137	if (queue >= 0) {
1138		*data = readl(&q->sp2host_bufq[queue][start]);
1139		writeb(start2, &q->sp2host_bufq_info[queue].start);
1140	} else {
1141		int r;
1142
1143		*data = readl(&q->sp2host_evtq[start]);
1144		writeb(start2, &q->sp2host_evtq_info.start);
1145
1146		/* Acknowledge events dequeued from event queue */
1147		r = imgu_css_queue_data(css, queue, 0,
1148					IMGU_ABI_EVENT_EVENT_DEQUEUED);
1149		if (r < 0)
1150			return r;
1151	}
1152
1153	return 0;
1154}
1155
1156/* Free binary-specific resources */
1157static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1158{
1159	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1160	unsigned int i, j;
1161
1162	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1163
1164	for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1165		for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1166			imgu_dmamap_free(imgu,
1167					 &css_pipe->binary_params_cs[j][i]);
1168
1169	j = IPU3_CSS_AUX_FRAME_REF;
1170	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1171		imgu_dmamap_free(imgu,
1172				 &css_pipe->aux_frames[j].mem[i]);
1173
1174	j = IPU3_CSS_AUX_FRAME_TNR;
1175	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1176		imgu_dmamap_free(imgu,
1177				 &css_pipe->aux_frames[j].mem[i]);
1178}
1179
1180static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1181{
1182	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1183	unsigned int i, j;
1184
1185	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1186
1187	for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1188	     j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1189		for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1190			if (!imgu_dmamap_alloc(imgu,
1191					       &css_pipe->binary_params_cs[j - 1][i],
1192					       CSS_ABI_SIZE))
1193				goto out_of_memory;
1194
1195	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1196		if (!imgu_dmamap_alloc(imgu,
1197				       &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1198				       CSS_BDS_SIZE))
1199			goto out_of_memory;
1200
1201	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1202		if (!imgu_dmamap_alloc(imgu,
1203				       &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1204				       CSS_GDC_SIZE))
1205			goto out_of_memory;
1206
1207	return 0;
1208
1209out_of_memory:
1210	imgu_css_binary_cleanup(css, pipe);
1211	return -ENOMEM;
1212}
1213
1214/* allocate binary-specific resources */
1215static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1216{
1217	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1218	struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1219	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1220	int i, j, size;
1221	static const int BYPC = 2;	/* Bytes per component */
1222	unsigned int w, h;
1223
1224	/* Allocate parameter memory blocks for this binary */
1225
1226	for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1227		for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1228			if (imgu_css_dma_buffer_resize(
1229			    imgu,
1230			    &css_pipe->binary_params_cs[j - 1][i],
1231			    bi->info.isp.sp.mem_initializers.params[j][i].size))
1232				goto out_of_memory;
1233		}
1234
1235	/* Allocate internal frame buffers */
1236
1237	/* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1238	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1239	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1240					css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1241	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1242				ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1243				      IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1244	h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1245	w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1246		  2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1247	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1248		css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1249	size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1250	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1251		if (imgu_css_dma_buffer_resize(
1252			imgu,
1253			&css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1254			size))
1255			goto out_of_memory;
1256
1257	/* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1258	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1259	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1260			roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1261				bi->info.isp.sp.block.block_width *
1262				IPU3_UAPI_ISP_VEC_ELEMS);
1263	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1264			roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1265				bi->info.isp.sp.block.output_block_height);
1266
1267	w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1268	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1269	h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1270	size = w * ALIGN(h * 3 / 2 + 3, 2);	/* +3 for vf_pp prefetch */
1271	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1272		if (imgu_css_dma_buffer_resize(
1273			imgu,
1274			&css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1275			size))
1276			goto out_of_memory;
1277
1278	return 0;
1279
1280out_of_memory:
1281	imgu_css_binary_cleanup(css, pipe);
1282	return -ENOMEM;
1283}
1284
1285int imgu_css_start_streaming(struct imgu_css *css)
1286{
1287	u32 data;
1288	int r, pipe;
1289
1290	if (css->streaming)
1291		return -EPROTO;
1292
1293	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1294		r = imgu_css_binary_setup(css, pipe);
1295		if (r < 0)
1296			return r;
1297	}
1298
1299	r = imgu_css_hw_init(css);
1300	if (r < 0)
1301		return r;
1302
1303	r = imgu_css_hw_start(css);
1304	if (r < 0)
1305		goto fail;
1306
1307	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1308		r = imgu_css_pipeline_init(css, pipe);
1309		if (r < 0)
1310			goto fail;
1311	}
1312
1313	css->streaming = true;
1314
1315	imgu_css_hw_enable_irq(css);
1316
1317	/* Initialize parameters to default */
1318	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1319		r = imgu_css_set_parameters(css, pipe, NULL);
1320		if (r < 0)
1321			goto fail;
1322	}
1323
1324	while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_A_ID, &data)))
1325		;
1326	if (r != -EBUSY)
1327		goto fail;
1328
1329	while (!(r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_B_ID, &data)))
1330		;
1331	if (r != -EBUSY)
1332		goto fail;
1333
1334	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1335		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1336					IMGU_ABI_EVENT_START_STREAM |
1337					pipe << 16);
1338		if (r < 0)
1339			goto fail;
1340	}
1341
1342	return 0;
1343
1344fail:
1345	css->streaming = false;
1346	imgu_css_hw_cleanup(css);
1347	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1348		imgu_css_pipeline_cleanup(css, pipe);
1349		imgu_css_binary_cleanup(css, pipe);
1350	}
1351
1352	return r;
1353}
1354
1355void imgu_css_stop_streaming(struct imgu_css *css)
1356{
1357	struct imgu_css_buffer *b, *b0;
1358	int q, r, pipe;
1359
1360	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1361		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
1362					IMGU_ABI_EVENT_STOP_STREAM);
1363		if (r < 0)
1364			dev_warn(css->dev, "failed on stop stream event\n");
1365	}
1366
1367	if (!css->streaming)
1368		return;
1369
1370	imgu_css_hw_stop(css);
1371
1372	imgu_css_hw_cleanup(css);
1373
1374	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1375		struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1376
1377		imgu_css_pipeline_cleanup(css, pipe);
1378
1379		spin_lock(&css_pipe->qlock);
1380		for (q = 0; q < IPU3_CSS_QUEUES; q++)
1381			list_for_each_entry_safe(b, b0,
1382						 &css_pipe->queue[q].bufs,
1383						 list) {
1384				b->state = IPU3_CSS_BUFFER_FAILED;
1385				list_del(&b->list);
1386			}
1387		spin_unlock(&css_pipe->qlock);
1388	}
1389
1390	css->streaming = false;
1391}
1392
1393bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1394{
1395	int q;
1396	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1397
1398	spin_lock(&css_pipe->qlock);
1399	for (q = 0; q < IPU3_CSS_QUEUES; q++)
1400		if (!list_empty(&css_pipe->queue[q].bufs))
1401			break;
1402	spin_unlock(&css_pipe->qlock);
1403	return (q == IPU3_CSS_QUEUES);
1404}
1405
1406bool imgu_css_queue_empty(struct imgu_css *css)
1407{
1408	unsigned int pipe;
1409	bool ret = false;
1410
1411	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1412		ret &= imgu_css_pipe_queue_empty(css, pipe);
1413
1414	return ret;
1415}
1416
1417bool imgu_css_is_streaming(struct imgu_css *css)
1418{
1419	return css->streaming;
1420}
1421
1422static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1423{
1424	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1425	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1426	unsigned int p, q, i;
1427
1428	/* Allocate and map common structures with imgu hardware */
1429	for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1430		for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1431			if (!imgu_dmamap_alloc(imgu,
1432					       &css_pipe->xmem_sp_stage_ptrs[p][i],
1433					       sizeof(struct imgu_abi_sp_stage)))
1434				return -ENOMEM;
1435			if (!imgu_dmamap_alloc(imgu,
1436					       &css_pipe->xmem_isp_stage_ptrs[p][i],
1437					       sizeof(struct imgu_abi_isp_stage)))
1438				return -ENOMEM;
1439		}
1440
1441	if (!imgu_dmamap_alloc(imgu, &css_pipe->sp_ddr_ptrs,
1442			       ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1443				     IMGU_ABI_ISP_DDR_WORD_BYTES)))
1444		return -ENOMEM;
1445
1446	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1447		unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1448
1449		for (i = 0; i < abi_buf_num; i++)
1450			if (!imgu_dmamap_alloc(imgu,
1451					       &css_pipe->abi_buffers[q][i],
1452					       sizeof(struct imgu_abi_buffer)))
1453				return -ENOMEM;
1454	}
1455
1456	if (imgu_css_binary_preallocate(css, pipe)) {
1457		imgu_css_binary_cleanup(css, pipe);
1458		return -ENOMEM;
1459	}
1460
1461	return 0;
1462}
1463
1464static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1465{
1466	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1467	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1468	unsigned int p, q, i, abi_buf_num;
1469
1470	imgu_css_binary_cleanup(css, pipe);
1471
1472	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1473		abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1474		for (i = 0; i < abi_buf_num; i++)
1475			imgu_dmamap_free(imgu, &css_pipe->abi_buffers[q][i]);
1476	}
1477
1478	for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1479		for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1480			imgu_dmamap_free(imgu,
1481					 &css_pipe->xmem_sp_stage_ptrs[p][i]);
1482			imgu_dmamap_free(imgu,
1483					 &css_pipe->xmem_isp_stage_ptrs[p][i]);
1484		}
1485
1486	imgu_dmamap_free(imgu, &css_pipe->sp_ddr_ptrs);
1487}
1488
1489void imgu_css_cleanup(struct imgu_css *css)
1490{
1491	struct imgu_device *imgu = dev_get_drvdata(css->dev);
1492	unsigned int pipe;
1493
1494	imgu_css_stop_streaming(css);
1495	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1496		imgu_css_pipe_cleanup(css, pipe);
1497	imgu_dmamap_free(imgu, &css->xmem_sp_group_ptrs);
1498	imgu_css_fw_cleanup(css);
1499}
1500
1501int imgu_css_init(struct device *dev, struct imgu_css *css,
1502		  void __iomem *base, int length)
1503{
1504	struct imgu_device *imgu = dev_get_drvdata(dev);
1505	int r, q, pipe;
1506
1507	/* Initialize main data structure */
1508	css->dev = dev;
1509	css->base = base;
1510	css->iomem_length = length;
1511
1512	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1513		struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1514
1515		css_pipe->vf_output_en = false;
1516		spin_lock_init(&css_pipe->qlock);
1517		css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1518		css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1519		for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1520			r = imgu_css_queue_init(&css_pipe->queue[q], NULL, 0);
1521			if (r)
1522				return r;
1523		}
1524		r = imgu_css_map_init(css, pipe);
1525		if (r) {
1526			imgu_css_cleanup(css);
1527			return r;
1528		}
1529	}
1530	if (!imgu_dmamap_alloc(imgu, &css->xmem_sp_group_ptrs,
1531			       sizeof(struct imgu_abi_sp_group)))
1532		return -ENOMEM;
1533
1534	r = imgu_css_fw_init(css);
1535	if (r)
1536		return r;
1537
1538	return 0;
1539}
1540
1541static u32 imgu_css_adjust(u32 res, u32 align)
1542{
1543	u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1544
1545	return DIV_ROUND_CLOSEST(val, align) * align;
1546}
1547
1548/* Select a binary matching the required resolutions and formats */
1549static int imgu_css_find_binary(struct imgu_css *css,
1550				unsigned int pipe,
1551				struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1552				struct v4l2_rect rects[IPU3_CSS_RECTS])
1553{
1554	const int binary_nr = css->fwp->file_header.binary_nr;
1555	unsigned int binary_mode =
1556		(css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1557		IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1558	const struct v4l2_pix_format_mplane *in =
1559					&queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1560	const struct v4l2_pix_format_mplane *out =
1561					&queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1562	const struct v4l2_pix_format_mplane *vf =
1563					&queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1564	u32 stripe_w = 0, stripe_h = 0;
1565	const char *name;
1566	int i, j;
1567
1568	if (!imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_IN]))
1569		return -EINVAL;
1570
1571	/* Find out the strip size boundary */
1572	for (i = 0; i < binary_nr; i++) {
1573		struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1574
1575		u32 max_width = bi->info.isp.sp.output.max_width;
1576		u32 max_height = bi->info.isp.sp.output.max_height;
1577
1578		if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1579			stripe_w = stripe_w ?
1580				min(stripe_w, max_width) : max_width;
1581			stripe_h = stripe_h ?
1582				min(stripe_h, max_height) : max_height;
1583		}
1584	}
1585
1586	for (i = 0; i < binary_nr; i++) {
1587		struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1588		enum imgu_abi_frame_format q_fmt;
1589
1590		name = (void *)css->fwp + bi->blob.prog_name_offset;
1591
1592		/* Check that binary supports memory-to-memory processing */
1593		if (bi->info.isp.sp.input.source !=
1594		    IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1595			continue;
1596
1597		/* Check that binary supports raw10 input */
1598		if (!bi->info.isp.sp.enable.input_feeder &&
1599		    !bi->info.isp.sp.enable.input_raw)
1600			continue;
1601
1602		/* Check binary mode */
1603		if (bi->info.isp.sp.pipeline.mode != binary_mode)
1604			continue;
1605
1606		/* Since input is RGGB bayer, need to process colors */
1607		if (bi->info.isp.sp.enable.luma_only)
1608			continue;
1609
1610		if (in->width < bi->info.isp.sp.input.min_width ||
1611		    in->width > bi->info.isp.sp.input.max_width ||
1612		    in->height < bi->info.isp.sp.input.min_height ||
1613		    in->height > bi->info.isp.sp.input.max_height)
1614			continue;
1615
1616		if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_OUT])) {
1617			if (bi->info.isp.num_output_pins <= 0)
1618				continue;
1619
1620			q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1621			for (j = 0; j < bi->info.isp.num_output_formats; j++)
1622				if (bi->info.isp.output_formats[j] == q_fmt)
1623					break;
1624			if (j >= bi->info.isp.num_output_formats)
1625				continue;
1626
1627			if (out->width < bi->info.isp.sp.output.min_width ||
1628			    out->width > bi->info.isp.sp.output.max_width ||
1629			    out->height < bi->info.isp.sp.output.min_height ||
1630			    out->height > bi->info.isp.sp.output.max_height)
1631				continue;
1632
1633			if (out->width > bi->info.isp.sp.internal.max_width ||
1634			    out->height > bi->info.isp.sp.internal.max_height)
1635				continue;
1636		}
1637
1638		if (imgu_css_queue_enabled(&queue[IPU3_CSS_QUEUE_VF])) {
1639			if (bi->info.isp.num_output_pins <= 1)
1640				continue;
1641
1642			q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1643			for (j = 0; j < bi->info.isp.num_output_formats; j++)
1644				if (bi->info.isp.output_formats[j] == q_fmt)
1645					break;
1646			if (j >= bi->info.isp.num_output_formats)
1647				continue;
1648
1649			if (vf->width < bi->info.isp.sp.output.min_width ||
1650			    vf->width > bi->info.isp.sp.output.max_width ||
1651			    vf->height < bi->info.isp.sp.output.min_height ||
1652			    vf->height > bi->info.isp.sp.output.max_height)
1653				continue;
1654		}
1655
1656		/* All checks passed, select the binary */
1657		dev_dbg(css->dev, "using binary %s id = %u\n", name,
1658			bi->info.isp.sp.id);
1659		return i;
1660	}
1661
1662	/* Can not find suitable binary for these parameters */
1663	return -EINVAL;
1664}
1665
1666/*
1667 * Check that there is a binary matching requirements. Parameters may be
1668 * NULL indicating disabled input/output. Return negative if given
1669 * parameters can not be supported or on error, zero or positive indicating
1670 * found binary number. May modify the given parameters if not exact match
1671 * is found.
1672 */
1673int imgu_css_fmt_try(struct imgu_css *css,
1674		     struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1675		     struct v4l2_rect *rects[IPU3_CSS_RECTS],
1676		     unsigned int pipe)
1677{
1678	static const u32 EFF_ALIGN_W = 2;
1679	static const u32 BDS_ALIGN_W = 4;
1680	static const u32 OUT_ALIGN_W = 8;
1681	static const u32 OUT_ALIGN_H = 4;
1682	static const u32 VF_ALIGN_W  = 2;
1683	static const char *qnames[IPU3_CSS_QUEUES] = {
1684		[IPU3_CSS_QUEUE_IN] = "in",
1685		[IPU3_CSS_QUEUE_PARAMS]    = "params",
1686		[IPU3_CSS_QUEUE_OUT] = "out",
1687		[IPU3_CSS_QUEUE_VF] = "vf",
1688		[IPU3_CSS_QUEUE_STAT_3A]   = "3a",
1689	};
1690	static const char *rnames[IPU3_CSS_RECTS] = {
1691		[IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1692		[IPU3_CSS_RECT_BDS]       = "bayer-domain scaled resolution",
1693		[IPU3_CSS_RECT_ENVELOPE]  = "DVS envelope size",
1694		[IPU3_CSS_RECT_GDC]  = "GDC output res",
1695	};
1696	struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1697	struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1698	struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1699	struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1700	struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1701	struct imgu_css_queue *q;
1702	struct v4l2_pix_format_mplane *in, *out, *vf;
1703	int i, s, ret;
1704
1705	q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
1706	if (!q)
1707		return -ENOMEM;
1708
1709	in  = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1710	out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1711	vf  = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1712
1713	/* Adjust all formats, get statistics buffer sizes and formats */
1714	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1715		if (fmts[i])
1716			dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1717				qnames[i], fmts[i]->width, fmts[i]->height,
1718				fmts[i]->pixelformat);
1719		else
1720			dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1721				qnames[i]);
1722		if (imgu_css_queue_init(&q[i], fmts[i],
1723					IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1724			dev_notice(css->dev, "can not initialize queue %s\n",
1725				   qnames[i]);
1726			ret = -EINVAL;
1727			goto out;
1728		}
1729	}
1730	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1731		if (rects[i]) {
1732			dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1733				rnames[i], rects[i]->width, rects[i]->height);
1734			r[i].width  = rects[i]->width;
1735			r[i].height = rects[i]->height;
1736		} else {
1737			dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1738				rnames[i]);
1739		}
1740		/* For now, force known good resolutions */
1741		r[i].left = 0;
1742		r[i].top  = 0;
1743	}
1744
1745	/* Always require one input and vf only if out is also enabled */
1746	if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
1747	    !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1748		dev_warn(css->dev, "required queues are disabled\n");
1749		ret = -EINVAL;
1750		goto out;
1751	}
1752
1753	if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
1754		out->width = in->width;
1755		out->height = in->height;
1756	}
1757	if (eff->width <= 0 || eff->height <= 0) {
1758		eff->width = in->width;
1759		eff->height = in->height;
1760	}
1761	if (bds->width <= 0 || bds->height <= 0) {
1762		bds->width = out->width;
1763		bds->height = out->height;
1764	}
1765	if (gdc->width <= 0 || gdc->height <= 0) {
1766		gdc->width = out->width;
1767		gdc->height = out->height;
1768	}
1769
1770	in->width   = imgu_css_adjust(in->width, 1);
1771	in->height  = imgu_css_adjust(in->height, 1);
1772	eff->width  = imgu_css_adjust(eff->width, EFF_ALIGN_W);
1773	eff->height = imgu_css_adjust(eff->height, 1);
1774	bds->width  = imgu_css_adjust(bds->width, BDS_ALIGN_W);
1775	bds->height = imgu_css_adjust(bds->height, 1);
1776	gdc->width  = imgu_css_adjust(gdc->width, OUT_ALIGN_W);
1777	gdc->height = imgu_css_adjust(gdc->height, OUT_ALIGN_H);
1778	out->width  = imgu_css_adjust(out->width, OUT_ALIGN_W);
1779	out->height = imgu_css_adjust(out->height, OUT_ALIGN_H);
1780	vf->width   = imgu_css_adjust(vf->width, VF_ALIGN_W);
1781	vf->height  = imgu_css_adjust(vf->height, 1);
1782
1783	s = (bds->width - gdc->width) / 2;
1784	env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1785	s = (bds->height - gdc->height) / 2;
1786	env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1787
1788	ret = imgu_css_find_binary(css, pipe, q, r);
1789	if (ret < 0) {
1790		dev_err(css->dev, "failed to find suitable binary\n");
1791		ret = -EINVAL;
1792		goto out;
1793	}
1794	css->pipes[pipe].bindex = ret;
1795
1796	dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1797		css->pipes[pipe].bindex, pipe);
1798
1799	/* Final adjustment and set back the queried formats */
1800	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1801		if (fmts[i]) {
1802			if (imgu_css_queue_init(&q[i], &q[i].fmt.mpix,
1803						IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1804				dev_err(css->dev,
1805					"final resolution adjustment failed\n");
1806				ret = -EINVAL;
1807				goto out;
1808			}
1809			*fmts[i] = q[i].fmt.mpix;
1810		}
1811	}
1812
1813	for (i = 0; i < IPU3_CSS_RECTS; i++)
1814		if (rects[i])
1815			*rects[i] = r[i];
1816
1817	dev_dbg(css->dev,
1818		"in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1819		 in->width, in->height, eff->width, eff->height,
1820		 bds->width, bds->height, gdc->width, gdc->height,
1821		 out->width, out->height, vf->width, vf->height);
1822
1823	ret = 0;
1824out:
1825	kfree(q);
1826	return ret;
1827}
1828
1829int imgu_css_fmt_set(struct imgu_css *css,
1830		     struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1831		     struct v4l2_rect *rects[IPU3_CSS_RECTS],
1832		     unsigned int pipe)
1833{
1834	struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1835	struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1836	int i, r;
1837	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1838
1839	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1840		if (rects[i])
1841			rect_data[i] = *rects[i];
1842		else
1843			memset(&rect_data[i], 0, sizeof(rect_data[i]));
1844		all_rects[i] = &rect_data[i];
1845	}
1846	r = imgu_css_fmt_try(css, fmts, all_rects, pipe);
1847	if (r < 0)
1848		return r;
1849
1850	for (i = 0; i < IPU3_CSS_QUEUES; i++)
1851		if (imgu_css_queue_init(&css_pipe->queue[i], fmts[i],
1852					IPU3_CSS_QUEUE_TO_FLAGS(i)))
1853			return -EINVAL;
1854	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1855		css_pipe->rect[i] = rect_data[i];
1856		if (rects[i])
1857			*rects[i] = rect_data[i];
1858	}
1859
1860	return 0;
1861}
1862
1863int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1864{
1865	switch (fmt->dataformat) {
1866	case V4L2_META_FMT_IPU3_PARAMS:
1867		fmt->buffersize = sizeof(struct ipu3_uapi_params);
1868
1869		/*
1870		 * Sanity check for the parameter struct size. This must
1871		 * not change!
1872		 */
1873		BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1874
1875		break;
1876	case V4L2_META_FMT_IPU3_STAT_3A:
1877		fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1878		break;
1879	default:
1880		return -EINVAL;
1881	}
1882
1883	return 0;
1884}
1885
1886/*
1887 * Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1888 * called for the buffer. May be called from interrupt context.
1889 * Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1890 * code on error conditions.
1891 */
1892int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1893		       struct imgu_css_buffer *b)
1894{
1895	struct imgu_abi_buffer *abi_buf;
1896	struct imgu_addr_t *buf_addr;
1897	u32 data;
1898	int r;
1899	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1900
1901	if (!css->streaming)
1902		return -EPROTO;	/* CSS or buffer in wrong state */
1903
1904	if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1905		return -EINVAL;
1906
1907	b->queue_pos = imgu_css_queue_pos(css, imgu_css_queues[b->queue].qid,
1908					  pipe);
1909
1910	if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1911		return -EIO;
1912	abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1913
1914	/* Fill struct abi_buffer for firmware */
1915	memset(abi_buf, 0, sizeof(*abi_buf));
1916
1917	buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1918	*(imgu_addr_t *)buf_addr = b->daddr;
1919
1920	if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1921		abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1922
1923	if (b->queue == IPU3_CSS_QUEUE_OUT)
1924		abi_buf->payload.frame.padded_width =
1925				css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1926
1927	if (b->queue == IPU3_CSS_QUEUE_VF)
1928		abi_buf->payload.frame.padded_width =
1929					css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1930
1931	spin_lock(&css_pipe->qlock);
1932	list_add_tail(&b->list, &css_pipe->queue[b->queue].bufs);
1933	spin_unlock(&css_pipe->qlock);
1934	b->state = IPU3_CSS_BUFFER_QUEUED;
1935
1936	data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1937	r = imgu_css_queue_data(css, imgu_css_queues[b->queue].qid,
1938				pipe, data);
1939	if (r < 0)
1940		goto queueing_failed;
1941
1942	data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1943					      imgu_css_queues[b->queue].qid);
1944	r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe, data);
1945	if (r < 0)
1946		goto queueing_failed;
1947
1948	dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1949		b, b->queue, pipe);
1950
1951	return 0;
1952
1953queueing_failed:
1954	b->state = (r == -EBUSY || r == -EAGAIN) ?
1955		IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1956	list_del(&b->list);
1957
1958	return r;
1959}
1960
1961/*
1962 * Get next ready CSS buffer. Returns -EAGAIN in which case the function
1963 * should be called again, or -EBUSY which means that there are no more
1964 * buffers available. May be called from interrupt context.
1965 */
1966struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1967{
1968	static const unsigned char evtype_to_queue[] = {
1969		[IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1970		[IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1971		[IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1972		[IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1973	};
1974	struct imgu_css_buffer *b = ERR_PTR(-EAGAIN);
1975	u32 event, daddr;
1976	int evtype, pipe, pipeid, queue, qid, r;
1977	struct imgu_css_pipe *css_pipe;
1978
1979	if (!css->streaming)
1980		return ERR_PTR(-EPROTO);
1981
1982	r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
1983	if (r < 0)
1984		return ERR_PTR(r);
1985
1986	evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1987		  IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1988
1989	switch (evtype) {
1990	case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1991	case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1992	case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1993	case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1994		pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1995			IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1996		pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1997			IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1998		queue = evtype_to_queue[evtype];
1999		qid = imgu_css_queues[queue].qid;
2000
2001		if (pipe >= IMGU_MAX_PIPE_NUM) {
2002			dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2003			return ERR_PTR(-EIO);
2004		}
2005
2006		if (qid >= IMGU_ABI_QUEUE_NUM) {
2007			dev_err(css->dev, "Invalid qid: %i\n", qid);
2008			return ERR_PTR(-EIO);
2009		}
2010		css_pipe = &css->pipes[pipe];
2011		dev_dbg(css->dev,
2012			"event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2013			event, queue, pipe, pipeid);
2014
2015		r = imgu_css_dequeue_data(css, qid, &daddr);
2016		if (r < 0) {
2017			dev_err(css->dev, "failed to dequeue buffer\n");
2018			/* Force real error, not -EBUSY */
2019			return ERR_PTR(-EIO);
2020		}
2021
2022		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2023					IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2024		if (r < 0) {
2025			dev_err(css->dev, "failed to queue event\n");
2026			return ERR_PTR(-EIO);
2027		}
2028
2029		spin_lock(&css_pipe->qlock);
2030		if (list_empty(&css_pipe->queue[queue].bufs)) {
2031			spin_unlock(&css_pipe->qlock);
2032			dev_err(css->dev, "event on empty queue\n");
2033			return ERR_PTR(-EIO);
2034		}
2035		b = list_first_entry(&css_pipe->queue[queue].bufs,
2036				     struct imgu_css_buffer, list);
2037		if (queue != b->queue ||
2038		    daddr != css_pipe->abi_buffers
2039			[b->queue][b->queue_pos].daddr) {
2040			spin_unlock(&css_pipe->qlock);
2041			dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2042			return ERR_PTR(-EIO);
2043		}
2044
2045		dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2046		b->pipe = pipe;
2047		b->state = IPU3_CSS_BUFFER_DONE;
2048		list_del(&b->list);
2049		spin_unlock(&css_pipe->qlock);
2050		break;
2051	case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2052		pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2053			IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2054		if (pipe >= IMGU_MAX_PIPE_NUM) {
2055			dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2056			return ERR_PTR(-EIO);
2057		}
2058
2059		dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2060			event, pipe);
2061		break;
2062	case IMGU_ABI_EVTTYPE_TIMER:
2063		r = imgu_css_dequeue_data(css, IMGU_ABI_QUEUE_EVENT_ID, &event);
2064		if (r < 0)
2065			return ERR_PTR(r);
2066
2067		if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2068		    IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2069			dev_dbg(css->dev, "event: timer\n");
2070		else
2071			dev_warn(css->dev, "half of timer event missing\n");
2072		break;
2073	case IMGU_ABI_EVTTYPE_FW_WARNING:
2074		dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2075		break;
2076	case IMGU_ABI_EVTTYPE_FW_ASSERT:
2077		dev_err(css->dev,
2078			"event: firmware assert 0x%x module_id %i line_no %i\n",
2079			event,
2080			(event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2081			IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2082			swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2083			       IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2084		break;
2085	default:
2086		dev_warn(css->dev, "received unknown event 0x%x\n", event);
2087	}
2088
2089	return b;
2090}
2091
2092/*
2093 * Get a new set of parameters from pool and initialize them based on
2094 * the parameters params, gdc, and obgrid. Any of these may be NULL,
2095 * in which case the previously set parameters are used.
2096 * If parameters haven't been set previously, initialize from scratch.
2097 *
2098 * Return index to css->parameter_set_info which has the newly created
2099 * parameters or negative value on error.
2100 */
2101int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2102			    struct ipu3_uapi_params *set_params)
2103{
2104	static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2105	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2106	const int stage = 0;
2107	const struct imgu_fw_info *bi;
2108	int obgrid_size;
2109	unsigned int stripes, i;
2110	struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2111
2112	/* Destination buffers which are filled here */
2113	struct imgu_abi_parameter_set_info *param_set;
2114	struct imgu_abi_acc_param *acc = NULL;
2115	struct imgu_abi_gdc_warp_param *gdc = NULL;
2116	struct ipu3_uapi_obgrid_param *obgrid = NULL;
2117	const struct imgu_css_map *map;
2118	void *vmem0 = NULL;
2119	void *dmem0 = NULL;
2120
2121	enum imgu_abi_memories m;
2122	int r = -EBUSY;
2123
2124	if (!css->streaming)
2125		return -EPROTO;
2126
2127	dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2128
2129	bi = &css->fwp->binary_header[css_pipe->bindex];
2130	obgrid_size = imgu_css_fw_obgrid_size(bi);
2131	stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2132
2133	imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
2134	param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
2135				       0)->vaddr;
2136
2137	/* Get a new acc only if new parameters given, or none yet */
2138	map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2139	if (set_params || !map->vaddr) {
2140		imgu_css_pool_get(&css_pipe->pool.acc);
2141		map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2142		acc = map->vaddr;
2143	}
2144
2145	/* Get new VMEM0 only if needed, or none yet */
2146	m = IMGU_ABI_MEM_ISP_VMEM0;
2147	map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2148	if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2149					   set_params->use.tnr3_vmem_params ||
2150					   set_params->use.xnr3_vmem_params))) {
2151		imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2152		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2153		vmem0 = map->vaddr;
2154	}
2155
2156	/* Get new DMEM0 only if needed, or none yet */
2157	m = IMGU_ABI_MEM_ISP_DMEM0;
2158	map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2159	if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2160					   set_params->use.xnr3_dmem_params))) {
2161		imgu_css_pool_get(&css_pipe->pool.binary_params_p[m]);
2162		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2163		dmem0 = map->vaddr;
2164	}
2165
2166	/* Configure acc parameter cluster */
2167	if (acc) {
2168		/* get acc_old */
2169		map = imgu_css_pool_last(&css_pipe->pool.acc, 1);
2170		/* user acc */
2171		r = imgu_css_cfg_acc(css, pipe, use, acc, map->vaddr,
2172			set_params ? &set_params->acc_param : NULL);
2173		if (r < 0)
2174			goto fail;
2175	}
2176
2177	/* Configure late binding parameters */
2178	if (vmem0) {
2179		m = IMGU_ABI_MEM_ISP_VMEM0;
2180		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2181		r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2182				       map->vaddr, set_params);
2183		if (r < 0)
2184			goto fail;
2185	}
2186
2187	if (dmem0) {
2188		m = IMGU_ABI_MEM_ISP_DMEM0;
2189		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 1);
2190		r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2191				       map->vaddr, set_params);
2192		if (r < 0)
2193			goto fail;
2194	}
2195
2196	/* Get a new gdc only if a new gdc is given, or none yet */
2197	if (bi->info.isp.sp.enable.dvs_6axis) {
2198		unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2199		unsigned int g = IPU3_CSS_RECT_GDC;
2200		unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2201
2202		map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2203		if (!map->vaddr) {
2204			imgu_css_pool_get(&css_pipe->pool.gdc);
2205			map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2206			gdc = map->vaddr;
2207			imgu_css_cfg_gdc_table(map->vaddr,
2208				css_pipe->aux_frames[a].bytesperline /
2209				css_pipe->aux_frames[a].bytesperpixel,
2210				css_pipe->aux_frames[a].height,
2211				css_pipe->rect[g].width,
2212				css_pipe->rect[g].height,
2213				css_pipe->rect[e].width,
2214				css_pipe->rect[e].height);
2215		}
2216	}
2217
2218	/* Get a new obgrid only if a new obgrid is given, or none yet */
2219	map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2220	if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2221		imgu_css_pool_get(&css_pipe->pool.obgrid);
2222		map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2223		obgrid = map->vaddr;
2224
2225		/* Configure optical black level grid (obgrid) */
2226		if (set_params && set_params->use.obgrid_param)
2227			for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2228				obgrid[i] = set_params->obgrid_param;
2229		else
2230			memset(obgrid, 0, obgrid_size);
2231	}
2232
2233	/* Configure parameter set info, queued to `queue_id' */
2234
2235	memset(param_set, 0, sizeof(*param_set));
2236	map = imgu_css_pool_last(&css_pipe->pool.acc, 0);
2237	param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2238
2239	map = imgu_css_pool_last(&css_pipe->pool.gdc, 0);
2240	param_set->mem_map.dvs_6axis_params_y = map->daddr;
2241
2242	for (i = 0; i < stripes; i++) {
2243		map = imgu_css_pool_last(&css_pipe->pool.obgrid, 0);
2244		param_set->mem_map.obgrid_tbl[i] =
2245			map->daddr + (obgrid_size / stripes) * i;
2246	}
2247
2248	for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2249		map = imgu_css_pool_last(&css_pipe->pool.binary_params_p[m], 0);
2250		param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2251	}
2252
2253	/* Then queue the new parameter buffer */
2254	map = imgu_css_pool_last(&css_pipe->pool.parameter_set_info, 0);
2255	r = imgu_css_queue_data(css, queue_id, pipe, map->daddr);
2256	if (r < 0)
2257		goto fail;
2258
2259	r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2260				IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2261							       queue_id));
2262	if (r < 0)
2263		goto fail_no_put;
2264
2265	/* Finally dequeue all old parameter buffers */
2266
2267	do {
2268		u32 daddr;
2269
2270		r = imgu_css_dequeue_data(css, queue_id, &daddr);
2271		if (r == -EBUSY)
2272			break;
2273		if (r)
2274			goto fail_no_put;
2275		r = imgu_css_queue_data(css, IMGU_ABI_QUEUE_EVENT_ID, pipe,
2276					IMGU_ABI_EVENT_BUFFER_DEQUEUED
2277					(queue_id));
2278		if (r < 0) {
2279			dev_err(css->dev, "failed to queue parameter event\n");
2280			goto fail_no_put;
2281		}
2282	} while (1);
2283
2284	return 0;
2285
2286fail:
2287	/*
2288	 * A failure, most likely the parameter queue was full.
2289	 * Return error but continue streaming. User can try submitting new
2290	 * parameters again later.
2291	 */
2292
2293	imgu_css_pool_put(&css_pipe->pool.parameter_set_info);
2294	if (acc)
2295		imgu_css_pool_put(&css_pipe->pool.acc);
2296	if (gdc)
2297		imgu_css_pool_put(&css_pipe->pool.gdc);
2298	if (obgrid)
2299		imgu_css_pool_put(&css_pipe->pool.obgrid);
2300	if (vmem0)
2301		imgu_css_pool_put(
2302			&css_pipe->pool.binary_params_p
2303			[IMGU_ABI_MEM_ISP_VMEM0]);
2304	if (dmem0)
2305		imgu_css_pool_put(
2306			&css_pipe->pool.binary_params_p
2307			[IMGU_ABI_MEM_ISP_DMEM0]);
2308
2309fail_no_put:
2310	return r;
2311}
2312
2313int imgu_css_irq_ack(struct imgu_css *css)
2314{
2315	static const int NUM_SWIRQS = 3;
2316	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2317	void __iomem *const base = css->base;
2318	u32 irq_status[IMGU_IRQCTRL_NUM];
2319	int i;
2320
2321	u32 imgu_status = readl(base + IMGU_REG_INT_STATUS);
2322
2323	writel(imgu_status, base + IMGU_REG_INT_STATUS);
2324	for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2325		irq_status[i] = readl(base + IMGU_REG_IRQCTRL_STATUS(i));
2326
2327	for (i = 0; i < NUM_SWIRQS; i++) {
2328		if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2329			/* SP SW interrupt */
2330			u32 cnt = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2331					bi->info.sp.output);
2332			u32 val = readl(base + IMGU_REG_SP_DMEM_BASE(0) +
2333					bi->info.sp.output + 4 + 4 * i);
2334
2335			dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2336				__func__, i, cnt, val);
2337		}
2338	}
2339
2340	for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2341		if (irq_status[i]) {
2342			writel(irq_status[i], base + IMGU_REG_IRQCTRL_CLEAR(i));
2343			/* Wait for write to complete */
2344			readl(base + IMGU_REG_IRQCTRL_ENABLE(i));
2345		}
2346
2347	dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2348		__func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2349		irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2350
2351	if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2352		return -ENOMSG;
2353
2354	return 0;
2355}