tegra210.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
   4 */
   5
   6/*
   7 * This source file contains Tegra210 supported video formats,
   8 * VI and CSI SoC specific data, operations and registers accessors.
   9 */
  10#include <linux/bitfield.h>
  11#include <linux/clk.h>
  12#include <linux/clk/tegra.h>
  13#include <linux/delay.h>
  14#include <linux/host1x.h>
  15#include <linux/kthread.h>
  16
  17#include "csi.h"
  18#include "vi.h"
  19
  20#define TEGRA210_MIN_WIDTH	32U
  21#define TEGRA210_MAX_WIDTH	32768U
  22#define TEGRA210_MIN_HEIGHT	32U
  23#define TEGRA210_MAX_HEIGHT	32768U
  24
  25#define SURFACE_ALIGN_BYTES	64
  26
  27#define TEGRA_VI_SYNCPT_WAIT_TIMEOUT			msecs_to_jiffies(200)
  28
  29/* Tegra210 VI registers */
  30#define TEGRA_VI_CFG_VI_INCR_SYNCPT			0x000
  31#define   VI_CFG_VI_INCR_SYNCPT_COND(x)			(((x) & 0xff) << 8)
  32#define   VI_CSI_PP_FRAME_START(port)			(5 + (port) * 4)
  33#define   VI_CSI_MW_ACK_DONE(port)			(7 + (port) * 4)
  34#define TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL		0x004
  35#define   VI_INCR_SYNCPT_NO_STALL			BIT(8)
  36#define TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR		0x008
  37#define TEGRA_VI_CFG_CG_CTRL				0x0b8
  38#define   VI_CG_2ND_LEVEL_EN				0x1
  39
  40/* Tegra210 VI CSI registers */
  41#define TEGRA_VI_CSI_SW_RESET				0x000
  42#define TEGRA_VI_CSI_SINGLE_SHOT			0x004
  43#define   SINGLE_SHOT_CAPTURE				0x1
  44#define TEGRA_VI_CSI_IMAGE_DEF				0x00c
  45#define   BYPASS_PXL_TRANSFORM_OFFSET			24
  46#define   IMAGE_DEF_FORMAT_OFFSET			16
  47#define   IMAGE_DEF_DEST_MEM				0x1
  48#define TEGRA_VI_CSI_IMAGE_SIZE				0x018
  49#define   IMAGE_SIZE_HEIGHT_OFFSET			16
  50#define TEGRA_VI_CSI_IMAGE_SIZE_WC			0x01c
  51#define TEGRA_VI_CSI_IMAGE_DT				0x020
  52#define TEGRA_VI_CSI_SURFACE0_OFFSET_MSB		0x024
  53#define TEGRA_VI_CSI_SURFACE0_OFFSET_LSB		0x028
  54#define TEGRA_VI_CSI_SURFACE1_OFFSET_MSB		0x02c
  55#define TEGRA_VI_CSI_SURFACE1_OFFSET_LSB		0x030
  56#define TEGRA_VI_CSI_SURFACE2_OFFSET_MSB		0x034
  57#define TEGRA_VI_CSI_SURFACE2_OFFSET_LSB		0x038
  58#define TEGRA_VI_CSI_SURFACE0_STRIDE			0x054
  59#define TEGRA_VI_CSI_SURFACE1_STRIDE			0x058
  60#define TEGRA_VI_CSI_SURFACE2_STRIDE			0x05c
  61#define TEGRA_VI_CSI_SURFACE_HEIGHT0			0x060
  62#define TEGRA_VI_CSI_ERROR_STATUS			0x084
  63
  64/* Tegra210 CSI Pixel Parser registers: Starts from 0x838, offset 0x0 */
  65#define TEGRA_CSI_INPUT_STREAM_CONTROL                  0x000
  66#define   CSI_SKIP_PACKET_THRESHOLD_OFFSET		16
  67#define TEGRA_CSI_PIXEL_STREAM_CONTROL0			0x004
  68#define   CSI_PP_PACKET_HEADER_SENT			BIT(4)
  69#define   CSI_PP_DATA_IDENTIFIER_ENABLE			BIT(5)
  70#define   CSI_PP_WORD_COUNT_SELECT_HEADER		BIT(6)
  71#define   CSI_PP_CRC_CHECK_ENABLE			BIT(7)
  72#define   CSI_PP_WC_CHECK				BIT(8)
  73#define   CSI_PP_OUTPUT_FORMAT_STORE			(0x3 << 16)
  74#define   CSI_PPA_PAD_LINE_NOPAD			(0x2 << 24)
  75#define   CSI_PP_HEADER_EC_DISABLE			(0x1 << 27)
  76#define   CSI_PPA_PAD_FRAME_NOPAD			(0x2 << 28)
  77#define TEGRA_CSI_PIXEL_STREAM_CONTROL1                 0x008
  78#define   CSI_PP_TOP_FIELD_FRAME_OFFSET			0
  79#define   CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET		4
  80#define TEGRA_CSI_PIXEL_STREAM_GAP                      0x00c
  81#define   PP_FRAME_MIN_GAP_OFFSET			16
  82#define TEGRA_CSI_PIXEL_STREAM_PP_COMMAND               0x010
  83#define   CSI_PP_ENABLE					0x1
  84#define   CSI_PP_DISABLE				0x2
  85#define   CSI_PP_RST					0x3
  86#define   CSI_PP_SINGLE_SHOT_ENABLE			(0x1 << 2)
  87#define   CSI_PP_START_MARKER_FRAME_MAX_OFFSET		12
  88#define TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME           0x014
  89#define TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK           0x018
  90#define TEGRA_CSI_PIXEL_PARSER_STATUS                   0x01c
  91
  92/* Tegra210 CSI PHY registers */
  93/* CSI_PHY_CIL_COMMAND_0 offset 0x0d0 from TEGRA_CSI_PIXEL_PARSER_0_BASE */
  94#define TEGRA_CSI_PHY_CIL_COMMAND                       0x0d0
  95#define   CSI_A_PHY_CIL_NOP				0x0
  96#define   CSI_A_PHY_CIL_ENABLE				0x1
  97#define   CSI_A_PHY_CIL_DISABLE				0x2
  98#define   CSI_A_PHY_CIL_ENABLE_MASK			0x3
  99#define   CSI_B_PHY_CIL_NOP				(0x0 << 8)
 100#define   CSI_B_PHY_CIL_ENABLE				(0x1 << 8)
 101#define   CSI_B_PHY_CIL_DISABLE				(0x2 << 8)
 102#define   CSI_B_PHY_CIL_ENABLE_MASK			(0x3 << 8)
 103
 104#define TEGRA_CSI_CIL_PAD_CONFIG0                       0x000
 105#define   BRICK_CLOCK_A_4X				(0x1 << 16)
 106#define   BRICK_CLOCK_B_4X				(0x2 << 16)
 107#define TEGRA_CSI_CIL_PAD_CONFIG1                       0x004
 108#define TEGRA_CSI_CIL_PHY_CONTROL                       0x008
 109#define   CLK_SETTLE_MASK				GENMASK(13, 8)
 110#define   THS_SETTLE_MASK				GENMASK(5, 0)
 111#define TEGRA_CSI_CIL_INTERRUPT_MASK                    0x00c
 112#define TEGRA_CSI_CIL_STATUS                            0x010
 113#define TEGRA_CSI_CILX_STATUS                           0x014
 114#define TEGRA_CSI_CIL_SW_SENSOR_RESET                   0x020
 115
 116#define TEGRA_CSI_PATTERN_GENERATOR_CTRL		0x000
 117#define   PG_MODE_OFFSET				2
 118#define   PG_ENABLE					0x1
 119#define   PG_DISABLE					0x0
 120#define TEGRA_CSI_PG_BLANK				0x004
 121#define   PG_VBLANK_OFFSET				16
 122#define TEGRA_CSI_PG_PHASE				0x008
 123#define TEGRA_CSI_PG_RED_FREQ				0x00c
 124#define   PG_RED_VERT_INIT_FREQ_OFFSET			16
 125#define   PG_RED_HOR_INIT_FREQ_OFFSET			0
 126#define TEGRA_CSI_PG_RED_FREQ_RATE			0x010
 127#define TEGRA_CSI_PG_GREEN_FREQ				0x014
 128#define   PG_GREEN_VERT_INIT_FREQ_OFFSET		16
 129#define   PG_GREEN_HOR_INIT_FREQ_OFFSET			0
 130#define TEGRA_CSI_PG_GREEN_FREQ_RATE			0x018
 131#define TEGRA_CSI_PG_BLUE_FREQ				0x01c
 132#define   PG_BLUE_VERT_INIT_FREQ_OFFSET			16
 133#define   PG_BLUE_HOR_INIT_FREQ_OFFSET			0
 134#define TEGRA_CSI_PG_BLUE_FREQ_RATE			0x020
 135#define TEGRA_CSI_PG_AOHDR				0x024
 136#define TEGRA_CSI_CSI_SW_STATUS_RESET			0x214
 137#define TEGRA_CSI_CLKEN_OVERRIDE			0x218
 138
 139#define TEGRA210_CSI_PORT_OFFSET			0x34
 140#define TEGRA210_CSI_CIL_OFFSET				0x0f4
 141#define TEGRA210_CSI_TPG_OFFSET				0x18c
 142
 143#define CSI_PP_OFFSET(block)				((block) * 0x800)
 144#define TEGRA210_VI_CSI_BASE(x)				(0x100 + (x) * 0x100)
 145
 146/* Tegra210 VI registers accessors */
 147static void tegra_vi_write(struct tegra_vi_channel *chan, unsigned int addr,
 148			   u32 val)
 149{
 150	writel_relaxed(val, chan->vi->iomem + addr);
 151}
 152
 153static u32 tegra_vi_read(struct tegra_vi_channel *chan, unsigned int addr)
 154{
 155	return readl_relaxed(chan->vi->iomem + addr);
 156}
 157
 158/* Tegra210 VI_CSI registers accessors */
 159static void vi_csi_write(struct tegra_vi_channel *chan, u8 portno,
 160			 unsigned int addr, u32 val)
 161{
 162	void __iomem *vi_csi_base;
 163
 164	vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
 165
 166	writel_relaxed(val, vi_csi_base + addr);
 167}
 168
 169static u32 vi_csi_read(struct tegra_vi_channel *chan, u8 portno,
 170		       unsigned int addr)
 171{
 172	void __iomem *vi_csi_base;
 173
 174	vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(portno);
 175
 176	return readl_relaxed(vi_csi_base + addr);
 177}
 178
 179/*
 180 * Tegra210 VI channel capture operations
 181 */
 182
 183static int tegra210_channel_host1x_syncpt_init(struct tegra_vi_channel *chan)
 184{
 185	struct tegra_vi *vi = chan->vi;
 186	unsigned long flags = HOST1X_SYNCPT_CLIENT_MANAGED;
 187	struct host1x_syncpt *fs_sp;
 188	struct host1x_syncpt *mw_sp;
 189	int ret, i;
 190
 191	for (i = 0; i < chan->numgangports; i++) {
 192		fs_sp = host1x_syncpt_request(&vi->client, flags);
 193		if (!fs_sp) {
 194			dev_err(vi->dev, "failed to request frame start syncpoint\n");
 195			ret = -ENOMEM;
 196			goto free_syncpts;
 197		}
 198
 199		mw_sp = host1x_syncpt_request(&vi->client, flags);
 200		if (!mw_sp) {
 201			dev_err(vi->dev, "failed to request memory ack syncpoint\n");
 202			host1x_syncpt_put(fs_sp);
 203			ret = -ENOMEM;
 204			goto free_syncpts;
 205		}
 206
 207		chan->frame_start_sp[i] = fs_sp;
 208		chan->mw_ack_sp[i] = mw_sp;
 209		spin_lock_init(&chan->sp_incr_lock[i]);
 210	}
 211
 212	return 0;
 213
 214free_syncpts:
 215	for (i = 0; i < chan->numgangports; i++) {
 216		host1x_syncpt_put(chan->mw_ack_sp[i]);
 217		host1x_syncpt_put(chan->frame_start_sp[i]);
 218	}
 219	return ret;
 220}
 221
 222static void tegra210_channel_host1x_syncpt_free(struct tegra_vi_channel *chan)
 223{
 224	int i;
 225
 226	for (i = 0; i < chan->numgangports; i++) {
 227		host1x_syncpt_put(chan->mw_ack_sp[i]);
 228		host1x_syncpt_put(chan->frame_start_sp[i]);
 229	}
 230}
 231
 232static void tegra210_fmt_align(struct v4l2_pix_format *pix, unsigned int bpp)
 233{
 234	unsigned int min_bpl;
 235	unsigned int max_bpl;
 236	unsigned int bpl;
 237
 238	/*
 239	 * The transfer alignment requirements are expressed in bytes.
 240	 * Clamp the requested width and height to the limits.
 241	 */
 242	pix->width = clamp(pix->width, TEGRA210_MIN_WIDTH, TEGRA210_MAX_WIDTH);
 243	pix->height = clamp(pix->height, TEGRA210_MIN_HEIGHT, TEGRA210_MAX_HEIGHT);
 244
 245	/* Clamp the requested bytes per line value. If the maximum bytes per
 246	 * line value is zero, the module doesn't support user configurable
 247	 * line sizes. Override the requested value with the minimum in that
 248	 * case.
 249	 */
 250	min_bpl = pix->width * bpp;
 251	max_bpl = rounddown(TEGRA210_MAX_WIDTH, SURFACE_ALIGN_BYTES);
 252	bpl = roundup(pix->bytesperline, SURFACE_ALIGN_BYTES);
 253
 254	pix->bytesperline = clamp(bpl, min_bpl, max_bpl);
 255	pix->sizeimage = pix->bytesperline * pix->height;
 256	if (pix->pixelformat == V4L2_PIX_FMT_NV16)
 257		pix->sizeimage *= 2;
 258}
 259
 260static int tegra_channel_capture_setup(struct tegra_vi_channel *chan,
 261				       u8 portno)
 262{
 263	u32 height = chan->format.height;
 264	u32 width = chan->format.width;
 265	u32 format = chan->fmtinfo->img_fmt;
 266	u32 data_type = chan->fmtinfo->img_dt;
 267	u32 word_count = (width * chan->fmtinfo->bit_width) / 8;
 268	u32 bypass_pixel_transform = BIT(BYPASS_PXL_TRANSFORM_OFFSET);
 269
 270	/*
 271	 * VI Pixel transformation unit converts source pixels data format
 272	 * into selected destination pixel format and aligns properly while
 273	 * interfacing with memory packer.
 274	 * This pixel transformation should be enabled for YUV and RGB
 275	 * formats and should be bypassed for RAW formats as RAW formats
 276	 * only support direct to memory.
 277	 */
 278	if (chan->pg_mode || data_type == TEGRA_IMAGE_DT_YUV422_8 ||
 279	    data_type == TEGRA_IMAGE_DT_RGB888)
 280		bypass_pixel_transform = 0;
 281
 282	/*
 283	 * For x8 source streaming, the source image is split onto two x4 ports
 284	 * with left half to first x4 port and right half to second x4 port.
 285	 * So, use split width and corresponding word count for each x4 port.
 286	 */
 287	if (chan->numgangports > 1) {
 288		width = width >> 1;
 289		word_count = (width * chan->fmtinfo->bit_width) / 8;
 290	}
 291
 292	vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, 0xffffffff);
 293	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DEF,
 294		     bypass_pixel_transform |
 295		     (format << IMAGE_DEF_FORMAT_OFFSET) |
 296		     IMAGE_DEF_DEST_MEM);
 297	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_DT, data_type);
 298	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE_WC, word_count);
 299	vi_csi_write(chan, portno, TEGRA_VI_CSI_IMAGE_SIZE,
 300		     (height << IMAGE_SIZE_HEIGHT_OFFSET) | width);
 301	return 0;
 302}
 303
 304static void tegra_channel_vi_soft_reset(struct tegra_vi_channel *chan,
 305					u8 portno)
 306{
 307	/* disable clock gating to enable continuous clock */
 308	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, 0);
 309	/*
 310	 * Soft reset memory client interface, pixel format logic, sensor
 311	 * control logic, and a shadow copy logic to bring VI to clean state.
 312	 */
 313	vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0xf);
 314	usleep_range(100, 200);
 315	vi_csi_write(chan, portno, TEGRA_VI_CSI_SW_RESET, 0x0);
 316
 317	/* enable back VI clock gating */
 318	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
 319}
 320
 321static void tegra_channel_capture_error_recover(struct tegra_vi_channel *chan,
 322						u8 portno)
 323{
 324	struct v4l2_subdev *subdev;
 325	u32 val;
 326
 327	/*
 328	 * Recover VI and CSI hardware blocks in case of missing frame start
 329	 * events due to source not streaming or noisy csi inputs from the
 330	 * external source or many outstanding frame start or MW_ACK_DONE
 331	 * events which can cause CSI and VI hardware hang.
 332	 * This helps to have a clean capture for next frame.
 333	 */
 334	val = vi_csi_read(chan, portno, TEGRA_VI_CSI_ERROR_STATUS);
 335	dev_dbg(&chan->video.dev, "TEGRA_VI_CSI_ERROR_STATUS 0x%08x\n", val);
 336	vi_csi_write(chan, portno, TEGRA_VI_CSI_ERROR_STATUS, val);
 337
 338	val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
 339	dev_dbg(&chan->video.dev,
 340		"TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR 0x%08x\n", val);
 341	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
 342
 343	/* recover VI by issuing software reset and re-setup for capture */
 344	tegra_channel_vi_soft_reset(chan, portno);
 345	tegra_channel_capture_setup(chan, portno);
 346
 347	/* recover CSI block */
 348	subdev = tegra_channel_get_remote_csi_subdev(chan);
 349	tegra_csi_error_recover(subdev);
 350}
 351
 352static struct tegra_channel_buffer *
 353dequeue_buf_done(struct tegra_vi_channel *chan)
 354{
 355	struct tegra_channel_buffer *buf = NULL;
 356
 357	spin_lock(&chan->done_lock);
 358	if (list_empty(&chan->done)) {
 359		spin_unlock(&chan->done_lock);
 360		return NULL;
 361	}
 362
 363	buf = list_first_entry(&chan->done,
 364			       struct tegra_channel_buffer, queue);
 365	if (buf)
 366		list_del_init(&buf->queue);
 367	spin_unlock(&chan->done_lock);
 368
 369	return buf;
 370}
 371
 372static void release_buffer(struct tegra_vi_channel *chan,
 373			   struct tegra_channel_buffer *buf,
 374			   enum vb2_buffer_state state)
 375{
 376	struct vb2_v4l2_buffer *vb = &buf->buf;
 377
 378	vb->sequence = chan->sequence++;
 379	vb->field = V4L2_FIELD_NONE;
 380	vb->vb2_buf.timestamp = ktime_get_ns();
 381	vb2_buffer_done(&vb->vb2_buf, state);
 382}
 383
 384static void tegra_channel_vi_buffer_setup(struct tegra_vi_channel *chan,
 385					  u8 portno, u32 buf_offset,
 386					  struct tegra_channel_buffer *buf)
 387{
 388	int bytesperline = chan->format.bytesperline;
 389	u32 sizeimage = chan->format.sizeimage;
 390
 391	/* program buffer address by using surface 0 */
 392	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_MSB,
 393		     ((u64)buf->addr + buf_offset) >> 32);
 394	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_OFFSET_LSB,
 395		     buf->addr + buf_offset);
 396	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE0_STRIDE, bytesperline);
 397
 398	if (chan->fmtinfo->fourcc != V4L2_PIX_FMT_NV16)
 399		return;
 400	/*
 401	 * Program surface 1 for UV plane with offset sizeimage from Y plane.
 402	 */
 403	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_MSB,
 404		     (((u64)buf->addr + sizeimage / 2) + buf_offset) >> 32);
 405	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_OFFSET_LSB,
 406		     buf->addr + sizeimage / 2 + buf_offset);
 407	vi_csi_write(chan, portno, TEGRA_VI_CSI_SURFACE1_STRIDE, bytesperline);
 408}
 409
 410static int tegra_channel_capture_frame(struct tegra_vi_channel *chan,
 411				       struct tegra_channel_buffer *buf)
 412{
 413	u32 thresh, value, frame_start, mw_ack_done;
 414	u32 fs_thresh[GANG_PORTS_MAX];
 415	u8 *portnos = chan->portnos;
 416	int gang_bpl = (chan->format.width >> 1) * chan->fmtinfo->bpp;
 417	u32 buf_offset;
 418	bool capture_timedout = false;
 419	int err, i;
 420
 421	for (i = 0; i < chan->numgangports; i++) {
 422		/*
 423		 * Align buffers side-by-side for all consecutive x4 ports
 424		 * in gang ports using bytes per line based on source split
 425		 * width.
 426		 */
 427		buf_offset = i * roundup(gang_bpl, SURFACE_ALIGN_BYTES);
 428		tegra_channel_vi_buffer_setup(chan, portnos[i], buf_offset,
 429					      buf);
 430
 431		/*
 432		 * Tegra VI block interacts with host1x syncpt to synchronize
 433		 * programmed condition and hardware operation for capture.
 434		 * Frame start and Memory write acknowledge syncpts has their
 435		 * own FIFO of depth 2.
 436		 *
 437		 * Syncpoint trigger conditions set through VI_INCR_SYNCPT
 438		 * register are added to HW syncpt FIFO and when HW triggers,
 439		 * syncpt condition is removed from the FIFO and counter at
 440		 * syncpoint index will be incremented by the hardware and
 441		 * software can wait for counter to reach threshold to
 442		 * synchronize capturing frame with hardware capture events.
 443		 */
 444
 445		/* increase channel syncpoint threshold for FRAME_START */
 446		thresh = host1x_syncpt_incr_max(chan->frame_start_sp[i], 1);
 447		fs_thresh[i] = thresh;
 448
 449		/* Program FRAME_START trigger condition syncpt request */
 450		frame_start = VI_CSI_PP_FRAME_START(portnos[i]);
 451		value = VI_CFG_VI_INCR_SYNCPT_COND(frame_start) |
 452			host1x_syncpt_id(chan->frame_start_sp[i]);
 453		tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
 454
 455		/* increase channel syncpoint threshold for MW_ACK_DONE */
 456		thresh = host1x_syncpt_incr_max(chan->mw_ack_sp[i], 1);
 457		buf->mw_ack_sp_thresh[i] = thresh;
 458
 459		/* Program MW_ACK_DONE trigger condition syncpt request */
 460		mw_ack_done = VI_CSI_MW_ACK_DONE(portnos[i]);
 461		value = VI_CFG_VI_INCR_SYNCPT_COND(mw_ack_done) |
 462			host1x_syncpt_id(chan->mw_ack_sp[i]);
 463		tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
 464	}
 465
 466	/* enable single shot capture after all ganged ports are ready */
 467	for (i = 0; i < chan->numgangports; i++)
 468		vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_SINGLE_SHOT,
 469			     SINGLE_SHOT_CAPTURE);
 470
 471	for (i = 0; i < chan->numgangports; i++) {
 472		/*
 473		 * Wait for syncpt counter to reach frame start event threshold
 474		 */
 475		err = host1x_syncpt_wait(chan->frame_start_sp[i], fs_thresh[i],
 476					 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
 477		if (err) {
 478			capture_timedout = true;
 479			/* increment syncpoint counter for timedout events */
 480			host1x_syncpt_incr(chan->frame_start_sp[i]);
 481			spin_lock(&chan->sp_incr_lock[i]);
 482			host1x_syncpt_incr(chan->mw_ack_sp[i]);
 483			spin_unlock(&chan->sp_incr_lock[i]);
 484			/* clear errors and recover */
 485			tegra_channel_capture_error_recover(chan, portnos[i]);
 486		}
 487	}
 488
 489	if (capture_timedout) {
 490		dev_err_ratelimited(&chan->video.dev,
 491				    "frame start syncpt timeout: %d\n", err);
 492		release_buffer(chan, buf, VB2_BUF_STATE_ERROR);
 493		return err;
 494	}
 495
 496	/* move buffer to capture done queue */
 497	spin_lock(&chan->done_lock);
 498	list_add_tail(&buf->queue, &chan->done);
 499	spin_unlock(&chan->done_lock);
 500
 501	/* wait up kthread for capture done */
 502	wake_up_interruptible(&chan->done_wait);
 503
 504	return 0;
 505}
 506
 507static void tegra_channel_capture_done(struct tegra_vi_channel *chan,
 508				       struct tegra_channel_buffer *buf)
 509{
 510	enum vb2_buffer_state state = VB2_BUF_STATE_DONE;
 511	u32 value;
 512	bool capture_timedout = false;
 513	int ret, i;
 514
 515	for (i = 0; i < chan->numgangports; i++) {
 516		/*
 517		 * Wait for syncpt counter to reach MW_ACK_DONE event threshold
 518		 */
 519		ret = host1x_syncpt_wait(chan->mw_ack_sp[i],
 520					 buf->mw_ack_sp_thresh[i],
 521					 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
 522		if (ret) {
 523			capture_timedout = true;
 524			state = VB2_BUF_STATE_ERROR;
 525			/* increment syncpoint counter for timedout event */
 526			spin_lock(&chan->sp_incr_lock[i]);
 527			host1x_syncpt_incr(chan->mw_ack_sp[i]);
 528			spin_unlock(&chan->sp_incr_lock[i]);
 529		}
 530	}
 531
 532	if (capture_timedout)
 533		dev_err_ratelimited(&chan->video.dev,
 534				    "MW_ACK_DONE syncpt timeout: %d\n", ret);
 535	release_buffer(chan, buf, state);
 536}
 537
 538static int chan_capture_kthread_start(void *data)
 539{
 540	struct tegra_vi_channel *chan = data;
 541	struct tegra_channel_buffer *buf;
 542	unsigned int retries = 0;
 543	int err = 0;
 544
 545	while (1) {
 546		/*
 547		 * Source is not streaming if error is non-zero.
 548		 * So, do not dequeue buffers on error and let the thread sleep
 549		 * till kthread stop signal is received.
 550		 */
 551		wait_event_interruptible(chan->start_wait,
 552					 kthread_should_stop() ||
 553					 (!list_empty(&chan->capture) &&
 554					 !err));
 555
 556		if (kthread_should_stop())
 557			break;
 558
 559		/* dequeue the buffer and start capture */
 560		spin_lock(&chan->start_lock);
 561		if (list_empty(&chan->capture)) {
 562			spin_unlock(&chan->start_lock);
 563			continue;
 564		}
 565
 566		buf = list_first_entry(&chan->capture,
 567				       struct tegra_channel_buffer, queue);
 568		list_del_init(&buf->queue);
 569		spin_unlock(&chan->start_lock);
 570
 571		err = tegra_channel_capture_frame(chan, buf);
 572		if (!err) {
 573			retries = 0;
 574			continue;
 575		}
 576
 577		if (retries++ > chan->syncpt_timeout_retry)
 578			vb2_queue_error(&chan->queue);
 579		else
 580			err = 0;
 581	}
 582
 583	return 0;
 584}
 585
 586static int chan_capture_kthread_finish(void *data)
 587{
 588	struct tegra_vi_channel *chan = data;
 589	struct tegra_channel_buffer *buf;
 590
 591	while (1) {
 592		wait_event_interruptible(chan->done_wait,
 593					 !list_empty(&chan->done) ||
 594					 kthread_should_stop());
 595
 596		/* dequeue buffers and finish capture */
 597		buf = dequeue_buf_done(chan);
 598		while (buf) {
 599			tegra_channel_capture_done(chan, buf);
 600			buf = dequeue_buf_done(chan);
 601		}
 602
 603		if (kthread_should_stop())
 604			break;
 605	}
 606
 607	return 0;
 608}
 609
 610static int tegra210_vi_start_streaming(struct vb2_queue *vq, u32 count)
 611{
 612	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
 613	struct media_pipeline *pipe = &chan->video.pipe;
 614	u32 val;
 615	u8 *portnos = chan->portnos;
 616	int ret, i;
 617
 618	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
 619
 620	/* clear syncpt errors */
 621	val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
 622	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
 623
 624	/*
 625	 * Sync point FIFO full stalls the host interface.
 626	 * Setting NO_STALL will drop INCR_SYNCPT methods when fifos are
 627	 * full and the corresponding condition bits in INCR_SYNCPT_ERROR
 628	 * register will be set.
 629	 * This allows SW to process error recovery.
 630	 */
 631	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL,
 632		       VI_INCR_SYNCPT_NO_STALL);
 633
 634	/* start the pipeline */
 635	ret = video_device_pipeline_start(&chan->video, pipe);
 636	if (ret < 0)
 637		goto error_pipeline_start;
 638
 639	/* clear csi errors and do capture setup for all ports in gang mode */
 640	for (i = 0; i < chan->numgangports; i++) {
 641		val = vi_csi_read(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS);
 642		vi_csi_write(chan, portnos[i], TEGRA_VI_CSI_ERROR_STATUS, val);
 643
 644		tegra_channel_capture_setup(chan, portnos[i]);
 645	}
 646
 647	ret = tegra_channel_set_stream(chan, true);
 648	if (ret < 0)
 649		goto error_set_stream;
 650
 651	chan->sequence = 0;
 652
 653	/* start kthreads to capture data to buffer and return them */
 654	chan->kthread_start_capture = kthread_run(chan_capture_kthread_start,
 655						  chan, "%s:0",
 656						  chan->video.name);
 657	if (IS_ERR(chan->kthread_start_capture)) {
 658		ret = PTR_ERR(chan->kthread_start_capture);
 659		chan->kthread_start_capture = NULL;
 660		dev_err(&chan->video.dev,
 661			"failed to run capture start kthread: %d\n", ret);
 662		goto error_kthread_start;
 663	}
 664
 665	chan->kthread_finish_capture = kthread_run(chan_capture_kthread_finish,
 666						   chan, "%s:1",
 667						   chan->video.name);
 668	if (IS_ERR(chan->kthread_finish_capture)) {
 669		ret = PTR_ERR(chan->kthread_finish_capture);
 670		chan->kthread_finish_capture = NULL;
 671		dev_err(&chan->video.dev,
 672			"failed to run capture finish kthread: %d\n", ret);
 673		goto error_kthread_done;
 674	}
 675
 676	return 0;
 677
 678error_kthread_done:
 679	kthread_stop(chan->kthread_start_capture);
 680error_kthread_start:
 681	tegra_channel_set_stream(chan, false);
 682error_set_stream:
 683	video_device_pipeline_stop(&chan->video);
 684error_pipeline_start:
 685	tegra_channel_release_buffers(chan, VB2_BUF_STATE_QUEUED);
 686	return ret;
 687}
 688
 689static void tegra210_vi_stop_streaming(struct vb2_queue *vq)
 690{
 691	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
 692
 693	if (chan->kthread_start_capture) {
 694		kthread_stop(chan->kthread_start_capture);
 695		chan->kthread_start_capture = NULL;
 696	}
 697
 698	if (chan->kthread_finish_capture) {
 699		kthread_stop(chan->kthread_finish_capture);
 700		chan->kthread_finish_capture = NULL;
 701	}
 702
 703	tegra_channel_release_buffers(chan, VB2_BUF_STATE_ERROR);
 704	tegra_channel_set_stream(chan, false);
 705	video_device_pipeline_stop(&chan->video);
 706}
 707
 708/*
 709 * Tegra210 VI Pixel memory format enum.
 710 * These format enum value gets programmed into corresponding Tegra VI
 711 * channel register bits.
 712 */
 713enum tegra210_image_format {
 714	TEGRA210_IMAGE_FORMAT_T_L8 = 16,
 715
 716	TEGRA210_IMAGE_FORMAT_T_R16_I = 32,
 717	TEGRA210_IMAGE_FORMAT_T_B5G6R5,
 718	TEGRA210_IMAGE_FORMAT_T_R5G6B5,
 719	TEGRA210_IMAGE_FORMAT_T_A1B5G5R5,
 720	TEGRA210_IMAGE_FORMAT_T_A1R5G5B5,
 721	TEGRA210_IMAGE_FORMAT_T_B5G5R5A1,
 722	TEGRA210_IMAGE_FORMAT_T_R5G5B5A1,
 723	TEGRA210_IMAGE_FORMAT_T_A4B4G4R4,
 724	TEGRA210_IMAGE_FORMAT_T_A4R4G4B4,
 725	TEGRA210_IMAGE_FORMAT_T_B4G4R4A4,
 726	TEGRA210_IMAGE_FORMAT_T_R4G4B4A4,
 727
 728	TEGRA210_IMAGE_FORMAT_T_A8B8G8R8 = 64,
 729	TEGRA210_IMAGE_FORMAT_T_A8R8G8B8,
 730	TEGRA210_IMAGE_FORMAT_T_B8G8R8A8,
 731	TEGRA210_IMAGE_FORMAT_T_R8G8B8A8,
 732	TEGRA210_IMAGE_FORMAT_T_A2B10G10R10,
 733	TEGRA210_IMAGE_FORMAT_T_A2R10G10B10,
 734	TEGRA210_IMAGE_FORMAT_T_B10G10R10A2,
 735	TEGRA210_IMAGE_FORMAT_T_R10G10B10A2,
 736
 737	TEGRA210_IMAGE_FORMAT_T_A8Y8U8V8 = 193,
 738	TEGRA210_IMAGE_FORMAT_T_V8U8Y8A8,
 739
 740	TEGRA210_IMAGE_FORMAT_T_A2Y10U10V10 = 197,
 741	TEGRA210_IMAGE_FORMAT_T_V10U10Y10A2,
 742	TEGRA210_IMAGE_FORMAT_T_Y8_U8__Y8_V8,
 743	TEGRA210_IMAGE_FORMAT_T_Y8_V8__Y8_U8,
 744	TEGRA210_IMAGE_FORMAT_T_U8_Y8__V8_Y8,
 745	TEGRA210_IMAGE_FORMAT_T_V8_Y8__U8_Y8,
 746
 747	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N444 = 224,
 748	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N444,
 749	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N444,
 750	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N422,
 751	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N422,
 752	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N422,
 753	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N420,
 754	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N420,
 755	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N420,
 756	TEGRA210_IMAGE_FORMAT_T_X2LC10LB10LA10,
 757	TEGRA210_IMAGE_FORMAT_T_A2R6R6R6R6R6,
 758};
 759
 760#define TEGRA210_VIDEO_FMT(DATA_TYPE, BIT_WIDTH, MBUS_CODE, BPP,	\
 761			   FORMAT, FOURCC)				\
 762{									\
 763	TEGRA_IMAGE_DT_##DATA_TYPE,					\
 764	BIT_WIDTH,							\
 765	MEDIA_BUS_FMT_##MBUS_CODE,					\
 766	BPP,								\
 767	TEGRA210_IMAGE_FORMAT_##FORMAT,					\
 768	V4L2_PIX_FMT_##FOURCC,						\
 769}
 770
 771/* Tegra210 supported video formats */
 772static const struct tegra_video_format tegra210_video_formats[] = {
 773	/* RAW 8 */
 774	TEGRA210_VIDEO_FMT(RAW8, 8, SRGGB8_1X8, 1, T_L8, SRGGB8),
 775	TEGRA210_VIDEO_FMT(RAW8, 8, SGRBG8_1X8, 1, T_L8, SGRBG8),
 776	TEGRA210_VIDEO_FMT(RAW8, 8, SGBRG8_1X8, 1, T_L8, SGBRG8),
 777	TEGRA210_VIDEO_FMT(RAW8, 8, SBGGR8_1X8, 1, T_L8, SBGGR8),
 778	/* RAW 10 */
 779	TEGRA210_VIDEO_FMT(RAW10, 10, SRGGB10_1X10, 2, T_R16_I, SRGGB10),
 780	TEGRA210_VIDEO_FMT(RAW10, 10, SGRBG10_1X10, 2, T_R16_I, SGRBG10),
 781	TEGRA210_VIDEO_FMT(RAW10, 10, SGBRG10_1X10, 2, T_R16_I, SGBRG10),
 782	TEGRA210_VIDEO_FMT(RAW10, 10, SBGGR10_1X10, 2, T_R16_I, SBGGR10),
 783	/* RAW 12 */
 784	TEGRA210_VIDEO_FMT(RAW12, 12, SRGGB12_1X12, 2, T_R16_I, SRGGB12),
 785	TEGRA210_VIDEO_FMT(RAW12, 12, SGRBG12_1X12, 2, T_R16_I, SGRBG12),
 786	TEGRA210_VIDEO_FMT(RAW12, 12, SGBRG12_1X12, 2, T_R16_I, SGBRG12),
 787	TEGRA210_VIDEO_FMT(RAW12, 12, SBGGR12_1X12, 2, T_R16_I, SBGGR12),
 788	/* RGB888 */
 789	TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X24, 4, T_A8R8G8B8, XBGR32),
 790	TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X32_PADHI, 4, T_A8B8G8R8,
 791			   RGBX32),
 792	/* YUV422 */
 793	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 2, T_U8_Y8__V8_Y8, YVYU),
 794	TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_1X16, 2, T_V8_Y8__U8_Y8, YUYV),
 795	TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_1X16, 2, T_Y8_U8__Y8_V8, VYUY),
 796	TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_1X16, 2, T_Y8_V8__Y8_U8, UYVY),
 797	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, T_Y8__V8U8_N422, NV16),
 798	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 2, T_U8_Y8__V8_Y8, YVYU),
 799	TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_2X8, 2, T_V8_Y8__U8_Y8, YUYV),
 800	TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_2X8, 2, T_Y8_U8__Y8_V8, VYUY),
 801	TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_2X8, 2, T_Y8_V8__Y8_U8, UYVY),
 802};
 803
 804/* Tegra210 VI operations */
 805static const struct tegra_vi_ops tegra210_vi_ops = {
 806	.channel_host1x_syncpt_init = tegra210_channel_host1x_syncpt_init,
 807	.channel_host1x_syncpt_free = tegra210_channel_host1x_syncpt_free,
 808	.vi_fmt_align = tegra210_fmt_align,
 809	.vi_start_streaming = tegra210_vi_start_streaming,
 810	.vi_stop_streaming = tegra210_vi_stop_streaming,
 811};
 812
 813/* Tegra210 VI SoC data */
 814const struct tegra_vi_soc tegra210_vi_soc = {
 815	.video_formats = tegra210_video_formats,
 816	.nformats = ARRAY_SIZE(tegra210_video_formats),
 817	.ops = &tegra210_vi_ops,
 818	.hw_revision = 3,
 819	.vi_max_channels = 6,
 820#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
 821	.default_video_format = &tegra210_video_formats[0],
 822	.vi_max_clk_hz = 499200000,
 823#else
 824	.default_video_format = &tegra210_video_formats[4],
 825	.vi_max_clk_hz = 998400000,
 826#endif
 827};
 828
 829/* Tegra210 CSI PHY registers accessors */
 830static void csi_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
 831		      u32 val)
 832{
 833	void __iomem *csi_pp_base;
 834
 835	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 836
 837	writel_relaxed(val, csi_pp_base + addr);
 838}
 839
 840/* Tegra210 CSI Pixel parser registers accessors */
 841static void pp_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
 842{
 843	void __iomem *csi_pp_base;
 844	unsigned int offset;
 845
 846	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 847	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 848
 849	writel_relaxed(val, csi_pp_base + offset + addr);
 850}
 851
 852static u32 pp_read(struct tegra_csi *csi, u8 portno, u32 addr)
 853{
 854	void __iomem *csi_pp_base;
 855	unsigned int offset;
 856
 857	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 858	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 859
 860	return readl_relaxed(csi_pp_base + offset + addr);
 861}
 862
 863/* Tegra210 CSI CIL A/B port registers accessors */
 864static void cil_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
 865{
 866	void __iomem *csi_cil_base;
 867	unsigned int offset;
 868
 869	csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
 870		       TEGRA210_CSI_CIL_OFFSET;
 871	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 872
 873	writel_relaxed(val, csi_cil_base + offset + addr);
 874}
 875
 876static u32 cil_read(struct tegra_csi *csi, u8 portno, u32 addr)
 877{
 878	void __iomem *csi_cil_base;
 879	unsigned int offset;
 880
 881	csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
 882		       TEGRA210_CSI_CIL_OFFSET;
 883	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
 884
 885	return readl_relaxed(csi_cil_base + offset + addr);
 886}
 887
 888/* Tegra210 CSI Test pattern generator registers accessor */
 889static void tpg_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
 890		      u32 val)
 891{
 892	void __iomem *csi_pp_base;
 893	unsigned int offset;
 894
 895	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
 896	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET +
 897		 TEGRA210_CSI_TPG_OFFSET;
 898
 899	writel_relaxed(val, csi_pp_base + offset + addr);
 900}
 901
 902/*
 903 * Tegra210 CSI operations
 904 */
 905static void tegra210_csi_port_recover(struct tegra_csi_channel *csi_chan,
 906				      u8 portno)
 907{
 908	struct tegra_csi *csi = csi_chan->csi;
 909	u32 val;
 910
 911	/*
 912	 * Recover CSI hardware in case of capture errors by issuing
 913	 * software reset to CSICIL sensor, pixel parser, and clear errors
 914	 * to have clean capture on  next streaming.
 915	 */
 916	val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
 917	dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
 918
 919	val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
 920	dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
 921
 922	val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
 923	dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
 924
 925	if (csi_chan->numlanes == 4) {
 926		/* reset CSI CIL sensor */
 927		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 928		cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 929		/*
 930		 * SW_STATUS_RESET resets all status bits of PPA, PPB, CILA,
 931		 * CILB status registers and debug counters.
 932		 * So, SW_STATUS_RESET can be used only when CSI brick is in
 933		 * x4 mode.
 934		 */
 935		csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x1);
 936
 937		/* sleep for 20 clock cycles to drain the FIFO */
 938		usleep_range(10, 20);
 939
 940		cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 941		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 942		csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x0);
 943	} else {
 944		/* reset CSICIL sensor */
 945		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
 946		usleep_range(10, 20);
 947		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
 948
 949		/* clear the errors */
 950		pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS,
 951			 0xffffffff);
 952		cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 953		cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 954	}
 955}
 956
 957static void tegra210_csi_error_recover(struct tegra_csi_channel *csi_chan)
 958{
 959	u8 *portnos = csi_chan->csi_port_nums;
 960	int i;
 961
 962	for (i = 0; i < csi_chan->numgangports; i++)
 963		tegra210_csi_port_recover(csi_chan, portnos[i]);
 964}
 965
 966static int
 967tegra210_csi_port_start_streaming(struct tegra_csi_channel *csi_chan,
 968				  u8 portno)
 969{
 970	struct tegra_csi *csi = csi_chan->csi;
 971	u8 clk_settle_time = 0;
 972	u8 ths_settle_time = 10;
 973	u32 val;
 974
 975	if (!csi_chan->pg_mode)
 976		tegra_csi_calc_settle_time(csi_chan, portno, &clk_settle_time,
 977					   &ths_settle_time);
 978
 979	csi_write(csi, portno, TEGRA_CSI_CLKEN_OVERRIDE, 0);
 980
 981	/* clean up status */
 982	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, 0xffffffff);
 983	cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
 984	cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
 985	cil_write(csi, portno, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
 986
 987	/* CIL PHY registers setup */
 988	cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
 989	cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL,
 990		  FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
 991		  FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
 992
 993	/*
 994	 * The CSI unit provides for connection of up to six cameras in
 995	 * the system and is organized as three identical instances of
 996	 * two MIPI support blocks, each with a separate 4-lane
 997	 * interface that can be configured as a single camera with 4
 998	 * lanes or as a dual camera with 2 lanes available for each
 999	 * camera.
1000	 */
1001	if (csi_chan->numlanes == 4) {
1002		cil_write(csi, portno + 1, TEGRA_CSI_CIL_STATUS, 0xffffffff);
1003		cil_write(csi, portno + 1, TEGRA_CSI_CILX_STATUS, 0xffffffff);
1004		cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
1005
1006		cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0,
1007			  BRICK_CLOCK_A_4X);
1008		cil_write(csi, portno + 1, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
1009		cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
1010		cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL,
1011			  FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
1012			  FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
1013		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
1014			  CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_ENABLE);
1015	} else {
1016		val = ((portno & 1) == PORT_A) ?
1017		      CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_NOP :
1018		      CSI_B_PHY_CIL_ENABLE | CSI_A_PHY_CIL_NOP;
1019		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
1020	}
1021
1022	/* CSI pixel parser registers setup */
1023	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1024		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1025		 CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_RST);
1026	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK, 0x0);
1027	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL0,
1028		 CSI_PP_PACKET_HEADER_SENT |
1029		 CSI_PP_DATA_IDENTIFIER_ENABLE |
1030		 CSI_PP_WORD_COUNT_SELECT_HEADER |
1031		 CSI_PP_CRC_CHECK_ENABLE |  CSI_PP_WC_CHECK |
1032		 CSI_PP_OUTPUT_FORMAT_STORE | CSI_PPA_PAD_LINE_NOPAD |
1033		 CSI_PP_HEADER_EC_DISABLE | CSI_PPA_PAD_FRAME_NOPAD |
1034		 (portno & 1));
1035	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL1,
1036		 (0x1 << CSI_PP_TOP_FIELD_FRAME_OFFSET) |
1037		 (0x1 << CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET));
1038	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_GAP,
1039		 0x14 << PP_FRAME_MIN_GAP_OFFSET);
1040	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME, 0x0);
1041	pp_write(csi, portno, TEGRA_CSI_INPUT_STREAM_CONTROL,
1042		 (0x3f << CSI_SKIP_PACKET_THRESHOLD_OFFSET) |
1043		 (csi_chan->numlanes - 1));
1044
1045	/* TPG setup */
1046	if (csi_chan->pg_mode) {
1047		tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
1048			  ((csi_chan->pg_mode - 1) << PG_MODE_OFFSET) |
1049			  PG_ENABLE);
1050		tpg_write(csi, portno, TEGRA_CSI_PG_BLANK,
1051			  csi_chan->v_blank << PG_VBLANK_OFFSET |
1052			  csi_chan->h_blank);
1053		tpg_write(csi, portno, TEGRA_CSI_PG_PHASE, 0x0);
1054		tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ,
1055			  (0x10 << PG_RED_VERT_INIT_FREQ_OFFSET) |
1056			  (0x10 << PG_RED_HOR_INIT_FREQ_OFFSET));
1057		tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ_RATE, 0x0);
1058		tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ,
1059			  (0x10 << PG_GREEN_VERT_INIT_FREQ_OFFSET) |
1060			  (0x10 << PG_GREEN_HOR_INIT_FREQ_OFFSET));
1061		tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ_RATE, 0x0);
1062		tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ,
1063			  (0x10 << PG_BLUE_VERT_INIT_FREQ_OFFSET) |
1064			  (0x10 << PG_BLUE_HOR_INIT_FREQ_OFFSET));
1065		tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ_RATE, 0x0);
1066	}
1067
1068	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1069		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1070		 CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_ENABLE);
1071
1072	return 0;
1073}
1074
1075static void
1076tegra210_csi_port_stop_streaming(struct tegra_csi_channel *csi_chan, u8 portno)
1077{
1078	struct tegra_csi *csi = csi_chan->csi;
1079	u32 val;
1080
1081	val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
1082
1083	dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
1084	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, val);
1085
1086	val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
1087	dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
1088	cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, val);
1089
1090	val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
1091	dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
1092	cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, val);
1093
1094	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
1095		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
1096		 CSI_PP_DISABLE);
1097
1098	if (csi_chan->pg_mode) {
1099		tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
1100			  PG_DISABLE);
1101		return;
1102	}
1103
1104	if (csi_chan->numlanes == 4) {
1105		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
1106			  CSI_A_PHY_CIL_DISABLE |
1107			  CSI_B_PHY_CIL_DISABLE);
1108	} else {
1109		val = ((portno & 1) == PORT_A) ?
1110		      CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_NOP :
1111		      CSI_B_PHY_CIL_DISABLE | CSI_A_PHY_CIL_NOP;
1112		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
1113	}
1114}
1115
1116static int tegra210_csi_start_streaming(struct tegra_csi_channel *csi_chan)
1117{
1118	u8 *portnos = csi_chan->csi_port_nums;
1119	int ret, i;
1120
1121	for (i = 0; i < csi_chan->numgangports; i++) {
1122		ret = tegra210_csi_port_start_streaming(csi_chan, portnos[i]);
1123		if (ret)
1124			goto stream_start_fail;
1125	}
1126
1127	return 0;
1128
1129stream_start_fail:
1130	for (i = i - 1; i >= 0; i--)
1131		tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
1132
1133	return ret;
1134}
1135
1136static void tegra210_csi_stop_streaming(struct tegra_csi_channel *csi_chan)
1137{
1138	u8 *portnos = csi_chan->csi_port_nums;
1139	int i;
1140
1141	for (i = 0; i < csi_chan->numgangports; i++)
1142		tegra210_csi_port_stop_streaming(csi_chan, portnos[i]);
1143}
1144
1145/*
1146 * Tegra210 CSI TPG frame rate table with horizontal and vertical
1147 * blanking intervals for corresponding format and resolution.
1148 * Blanking intervals are tuned values from design team for max TPG
1149 * clock rate.
1150 */
1151static const struct tpg_framerate tegra210_tpg_frmrate_table[] = {
1152	{
1153		.frmsize = { 1280, 720 },
1154		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
1155		.framerate = 120,
1156		.h_blank = 512,
1157		.v_blank = 8,
1158	},
1159	{
1160		.frmsize = { 1920, 1080 },
1161		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
1162		.framerate = 60,
1163		.h_blank = 512,
1164		.v_blank = 8,
1165	},
1166	{
1167		.frmsize = { 3840, 2160 },
1168		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
1169		.framerate = 20,
1170		.h_blank = 8,
1171		.v_blank = 8,
1172	},
1173	{
1174		.frmsize = { 1280, 720 },
1175		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1176		.framerate = 60,
1177		.h_blank = 512,
1178		.v_blank = 8,
1179	},
1180	{
1181		.frmsize = { 1920, 1080 },
1182		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1183		.framerate = 30,
1184		.h_blank = 512,
1185		.v_blank = 8,
1186	},
1187	{
1188		.frmsize = { 3840, 2160 },
1189		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
1190		.framerate = 8,
1191		.h_blank = 8,
1192		.v_blank = 8,
1193	},
1194};
1195
1196static const char * const tegra210_csi_cil_clks[] = {
1197	"csi",
1198	"cilab",
1199	"cilcd",
1200	"cile",
1201#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
1202	"csi_tpg",
1203#endif
1204};
1205
1206/* Tegra210 CSI operations */
1207static const struct tegra_csi_ops tegra210_csi_ops = {
1208	.csi_start_streaming = tegra210_csi_start_streaming,
1209	.csi_stop_streaming = tegra210_csi_stop_streaming,
1210	.csi_err_recover = tegra210_csi_error_recover,
1211};
1212
1213/* Tegra210 CSI SoC data */
1214const struct tegra_csi_soc tegra210_csi_soc = {
1215	.ops = &tegra210_csi_ops,
1216	.csi_max_channels = 6,
1217	.clk_names = tegra210_csi_cil_clks,
1218	.num_clks = ARRAY_SIZE(tegra210_csi_cil_clks),
1219	.tpg_frmrate_table = tegra210_tpg_frmrate_table,
1220	.tpg_frmrate_table_size = ARRAY_SIZE(tegra210_tpg_frmrate_table),
1221};