ipu7-isys.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2013 - 2025 Intel Corporation
   4 */
   5
   6#include <linux/auxiliary_bus.h>
   7#include <linux/bitfield.h>
   8#include <linux/bits.h>
   9#include <linux/bug.h>
  10#include <linux/completion.h>
  11#include <linux/container_of.h>
  12#include <linux/delay.h>
  13#include <linux/device.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/err.h>
  16#include <linux/firmware.h>
  17#include <linux/list.h>
  18#include <linux/module.h>
  19#include <linux/mutex.h>
  20#include <linux/pci.h>
  21#include <linux/pm_runtime.h>
  22#include <linux/pm_qos.h>
  23#include <linux/slab.h>
  24#include <linux/spinlock.h>
  25#include <linux/string.h>
  26#include <linux/types.h>
  27
  28#include <media/ipu-bridge.h>
  29#include <media/media-entity.h>
  30#include <media/v4l2-async.h>
  31#include <media/v4l2-device.h>
  32#include <media/v4l2-fwnode.h>
  33#include <media/v4l2-subdev.h>
  34
  35#include "abi/ipu7_fw_isys_abi.h"
  36
  37#include "ipu7-bus.h"
  38#include "ipu7-buttress-regs.h"
  39#include "ipu7-cpd.h"
  40#include "ipu7-dma.h"
  41#include "ipu7-fw-isys.h"
  42#include "ipu7-mmu.h"
  43#include "ipu7-isys.h"
  44#include "ipu7-isys-csi2.h"
  45#include "ipu7-isys-csi-phy.h"
  46#include "ipu7-isys-csi2-regs.h"
  47#include "ipu7-isys-video.h"
  48#include "ipu7-platform-regs.h"
  49
  50#define ISYS_PM_QOS_VALUE	300
  51
  52static int
  53isys_complete_ext_device_registration(struct ipu7_isys *isys,
  54				      struct v4l2_subdev *sd,
  55				      struct ipu7_isys_csi2_config *csi2)
  56{
  57	struct device *dev = &isys->adev->auxdev.dev;
  58	unsigned int i;
  59	int ret;
  60
  61	v4l2_set_subdev_hostdata(sd, csi2);
  62
  63	for (i = 0; i < sd->entity.num_pads; i++) {
  64		if (sd->entity.pads[i].flags & MEDIA_PAD_FL_SOURCE)
  65			break;
  66	}
  67
  68	if (i == sd->entity.num_pads) {
  69		dev_warn(dev, "no source pad in external entity\n");
  70		ret = -ENOENT;
  71		goto skip_unregister_subdev;
  72	}
  73
  74	ret = media_create_pad_link(&sd->entity, i,
  75				    &isys->csi2[csi2->port].asd.sd.entity,
  76				    0, MEDIA_LNK_FL_ENABLED |
  77				    MEDIA_LNK_FL_IMMUTABLE);
  78	if (ret) {
  79		dev_warn(dev, "can't create link\n");
  80		goto skip_unregister_subdev;
  81	}
  82
  83	isys->csi2[csi2->port].nlanes = csi2->nlanes;
  84	if (csi2->bus_type == V4L2_MBUS_CSI2_DPHY)
  85		isys->csi2[csi2->port].phy_mode = PHY_MODE_DPHY;
  86	else
  87		isys->csi2[csi2->port].phy_mode = PHY_MODE_CPHY;
  88
  89	return 0;
  90
  91skip_unregister_subdev:
  92	v4l2_device_unregister_subdev(sd);
  93	return ret;
  94}
  95
  96static void isys_stream_init(struct ipu7_isys *isys)
  97{
  98	unsigned int i;
  99
 100	for (i = 0; i < IPU_ISYS_MAX_STREAMS; i++) {
 101		mutex_init(&isys->streams[i].mutex);
 102		init_completion(&isys->streams[i].stream_open_completion);
 103		init_completion(&isys->streams[i].stream_close_completion);
 104		init_completion(&isys->streams[i].stream_start_completion);
 105		init_completion(&isys->streams[i].stream_stop_completion);
 106		INIT_LIST_HEAD(&isys->streams[i].queues);
 107		isys->streams[i].isys = isys;
 108		isys->streams[i].stream_handle = i;
 109		isys->streams[i].vc = INVALID_VC_ID;
 110	}
 111}
 112
 113static int isys_fw_log_init(struct ipu7_isys *isys)
 114{
 115	struct device *dev = &isys->adev->auxdev.dev;
 116	struct isys_fw_log *fw_log;
 117	void *log_buf;
 118
 119	if (isys->fw_log)
 120		return 0;
 121
 122	fw_log = devm_kzalloc(dev, sizeof(*fw_log), GFP_KERNEL);
 123	if (!fw_log)
 124		return -ENOMEM;
 125
 126	mutex_init(&fw_log->mutex);
 127
 128	log_buf = devm_kzalloc(dev, FW_LOG_BUF_SIZE, GFP_KERNEL);
 129	if (!log_buf)
 130		return -ENOMEM;
 131
 132	fw_log->head = log_buf;
 133	fw_log->addr = log_buf;
 134	fw_log->count = 0;
 135	fw_log->size = 0;
 136
 137	isys->fw_log = fw_log;
 138
 139	return 0;
 140}
 141
 142/* The .bound() notifier callback when a match is found */
 143static int isys_notifier_bound(struct v4l2_async_notifier *notifier,
 144			       struct v4l2_subdev *sd,
 145			       struct v4l2_async_connection *asc)
 146{
 147	struct ipu7_isys *isys = container_of(notifier,
 148					      struct ipu7_isys, notifier);
 149	struct sensor_async_sd *s_asd =
 150		container_of(asc, struct sensor_async_sd, asc);
 151	struct device *dev = &isys->adev->auxdev.dev;
 152	int ret;
 153
 154	ret = ipu_bridge_instantiate_vcm(sd->dev);
 155	if (ret) {
 156		dev_err(dev, "instantiate vcm failed\n");
 157		return ret;
 158	}
 159
 160	dev_info(dev, "bind %s nlanes is %d port is %d\n",
 161		 sd->name, s_asd->csi2.nlanes, s_asd->csi2.port);
 162	isys_complete_ext_device_registration(isys, sd, &s_asd->csi2);
 163
 164	return v4l2_device_register_subdev_nodes(&isys->v4l2_dev);
 165}
 166
 167static int isys_notifier_complete(struct v4l2_async_notifier *notifier)
 168{
 169	struct ipu7_isys *isys = container_of(notifier,
 170					      struct ipu7_isys, notifier);
 171
 172	dev_info(&isys->adev->auxdev.dev,
 173		 "All sensor registration completed.\n");
 174
 175	return v4l2_device_register_subdev_nodes(&isys->v4l2_dev);
 176}
 177
 178static const struct v4l2_async_notifier_operations isys_async_ops = {
 179	.bound = isys_notifier_bound,
 180	.complete = isys_notifier_complete,
 181};
 182
 183static int isys_notifier_init(struct ipu7_isys *isys)
 184{
 185	const struct ipu7_isys_internal_csi2_pdata *csi2 =
 186		&isys->pdata->ipdata->csi2;
 187	struct ipu7_device *isp = isys->adev->isp;
 188	struct device *dev = &isp->pdev->dev;
 189	unsigned int i;
 190	int ret;
 191
 192	v4l2_async_nf_init(&isys->notifier, &isys->v4l2_dev);
 193
 194	for (i = 0; i < csi2->nports; i++) {
 195		struct v4l2_fwnode_endpoint vep = {
 196			.bus_type = V4L2_MBUS_UNKNOWN
 197		};
 198		struct sensor_async_sd *s_asd;
 199		struct fwnode_handle *ep;
 200
 201		ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), i, 0,
 202						     FWNODE_GRAPH_ENDPOINT_NEXT);
 203		if (!ep)
 204			continue;
 205
 206		ret = v4l2_fwnode_endpoint_parse(ep, &vep);
 207		if (ret)
 208			goto err_parse;
 209
 210		if (vep.bus_type != V4L2_MBUS_CSI2_DPHY &&
 211		    vep.bus_type != V4L2_MBUS_CSI2_CPHY) {
 212			ret = -EINVAL;
 213			dev_err(dev, "unsupported bus type %d!\n",
 214				vep.bus_type);
 215			goto err_parse;
 216		}
 217
 218		s_asd = v4l2_async_nf_add_fwnode_remote(&isys->notifier, ep,
 219							struct
 220							sensor_async_sd);
 221		if (IS_ERR(s_asd)) {
 222			ret = PTR_ERR(s_asd);
 223			goto err_parse;
 224		}
 225
 226		s_asd->csi2.port = vep.base.port;
 227		s_asd->csi2.nlanes = vep.bus.mipi_csi2.num_data_lanes;
 228		s_asd->csi2.bus_type = vep.bus_type;
 229
 230		fwnode_handle_put(ep);
 231
 232		continue;
 233
 234err_parse:
 235		fwnode_handle_put(ep);
 236		return ret;
 237	}
 238
 239	if (list_empty(&isys->notifier.waiting_list)) {
 240		/* isys probe could continue with async subdevs missing */
 241		dev_warn(dev, "no subdev found in graph\n");
 242		return 0;
 243	}
 244
 245	isys->notifier.ops = &isys_async_ops;
 246	ret = v4l2_async_nf_register(&isys->notifier);
 247	if (ret) {
 248		dev_err(dev, "failed to register async notifier(%d)\n", ret);
 249		v4l2_async_nf_cleanup(&isys->notifier);
 250	}
 251
 252	return ret;
 253}
 254
 255static void isys_notifier_cleanup(struct ipu7_isys *isys)
 256{
 257	v4l2_async_nf_unregister(&isys->notifier);
 258	v4l2_async_nf_cleanup(&isys->notifier);
 259}
 260
 261static void isys_unregister_video_devices(struct ipu7_isys *isys)
 262{
 263	const struct ipu7_isys_internal_csi2_pdata *csi2_pdata =
 264		&isys->pdata->ipdata->csi2;
 265	unsigned int i, j;
 266
 267	for (i = 0; i < csi2_pdata->nports; i++)
 268		for (j = 0; j < IPU7_NR_OF_CSI2_SRC_PADS; j++)
 269			ipu7_isys_video_cleanup(&isys->csi2[i].av[j]);
 270}
 271
 272static int isys_register_video_devices(struct ipu7_isys *isys)
 273{
 274	const struct ipu7_isys_internal_csi2_pdata *csi2_pdata =
 275		&isys->pdata->ipdata->csi2;
 276	unsigned int i, j;
 277	int ret;
 278
 279	for (i = 0; i < csi2_pdata->nports; i++) {
 280		for (j = 0; j < IPU7_NR_OF_CSI2_SRC_PADS; j++) {
 281			struct ipu7_isys_video *av = &isys->csi2[i].av[j];
 282
 283			snprintf(av->vdev.name, sizeof(av->vdev.name),
 284				 IPU_ISYS_ENTITY_PREFIX " ISYS Capture %u",
 285				 i * IPU7_NR_OF_CSI2_SRC_PADS + j);
 286			av->isys = isys;
 287			av->aq.vbq.buf_struct_size =
 288				sizeof(struct ipu7_isys_video_buffer);
 289
 290			ret = ipu7_isys_video_init(av);
 291			if (ret)
 292				goto fail;
 293		}
 294	}
 295
 296	return 0;
 297
 298fail:
 299	i = i + 1U;
 300	while (i--) {
 301		while (j--)
 302			ipu7_isys_video_cleanup(&isys->csi2[i].av[j]);
 303		j = IPU7_NR_OF_CSI2_SRC_PADS;
 304	}
 305
 306	return ret;
 307}
 308
 309static void isys_csi2_unregister_subdevices(struct ipu7_isys *isys)
 310{
 311	const struct ipu7_isys_internal_csi2_pdata *csi2 =
 312		&isys->pdata->ipdata->csi2;
 313	unsigned int i;
 314
 315	for (i = 0; i < csi2->nports; i++)
 316		ipu7_isys_csi2_cleanup(&isys->csi2[i]);
 317}
 318
 319static int isys_csi2_register_subdevices(struct ipu7_isys *isys)
 320{
 321	const struct ipu7_isys_internal_csi2_pdata *csi2_pdata =
 322		&isys->pdata->ipdata->csi2;
 323	unsigned int i;
 324	int ret;
 325
 326	for (i = 0; i < csi2_pdata->nports; i++) {
 327		ret = ipu7_isys_csi2_init(&isys->csi2[i], isys,
 328					  isys->pdata->base +
 329					  csi2_pdata->offsets[i], i);
 330		if (ret)
 331			goto fail;
 332	}
 333
 334	isys->isr_csi2_mask = IPU7_CSI_RX_LEGACY_IRQ_MASK;
 335
 336	return 0;
 337
 338fail:
 339	while (i--)
 340		ipu7_isys_csi2_cleanup(&isys->csi2[i]);
 341
 342	return ret;
 343}
 344
 345static int isys_csi2_create_media_links(struct ipu7_isys *isys)
 346{
 347	const struct ipu7_isys_internal_csi2_pdata *csi2_pdata =
 348		&isys->pdata->ipdata->csi2;
 349	struct device *dev = &isys->adev->auxdev.dev;
 350	struct media_entity *sd;
 351	unsigned int i, j;
 352	int ret;
 353
 354	for (i = 0; i < csi2_pdata->nports; i++) {
 355		sd = &isys->csi2[i].asd.sd.entity;
 356
 357		for (j = 0; j < IPU7_NR_OF_CSI2_SRC_PADS; j++) {
 358			struct ipu7_isys_video *av = &isys->csi2[i].av[j];
 359
 360			ret = media_create_pad_link(sd, IPU7_CSI2_PAD_SRC + j,
 361						    &av->vdev.entity, 0, 0);
 362			if (ret) {
 363				dev_err(dev, "CSI2 can't create link\n");
 364				return ret;
 365			}
 366
 367			av->csi2 = &isys->csi2[i];
 368		}
 369	}
 370
 371	return 0;
 372}
 373
 374static int isys_register_devices(struct ipu7_isys *isys)
 375{
 376	struct device *dev = &isys->adev->auxdev.dev;
 377	struct pci_dev *pdev = isys->adev->isp->pdev;
 378	int ret;
 379
 380	media_device_pci_init(&isys->media_dev,
 381			      pdev, IPU_MEDIA_DEV_MODEL_NAME);
 382
 383	strscpy(isys->v4l2_dev.name, isys->media_dev.model,
 384		sizeof(isys->v4l2_dev.name));
 385
 386	ret = media_device_register(&isys->media_dev);
 387	if (ret < 0)
 388		goto out_media_device_unregister;
 389
 390	isys->v4l2_dev.mdev = &isys->media_dev;
 391	isys->v4l2_dev.ctrl_handler = NULL;
 392
 393	ret = v4l2_device_register(dev, &isys->v4l2_dev);
 394	if (ret < 0)
 395		goto out_media_device_unregister;
 396
 397	ret = isys_register_video_devices(isys);
 398	if (ret)
 399		goto out_v4l2_device_unregister;
 400
 401	ret = isys_csi2_register_subdevices(isys);
 402	if (ret)
 403		goto out_video_unregister_device;
 404
 405	ret = isys_csi2_create_media_links(isys);
 406	if (ret)
 407		goto out_csi2_unregister_subdevices;
 408
 409	ret = isys_notifier_init(isys);
 410	if (ret)
 411		goto out_csi2_unregister_subdevices;
 412
 413	return 0;
 414
 415out_csi2_unregister_subdevices:
 416	isys_csi2_unregister_subdevices(isys);
 417
 418out_video_unregister_device:
 419	isys_unregister_video_devices(isys);
 420
 421out_v4l2_device_unregister:
 422	v4l2_device_unregister(&isys->v4l2_dev);
 423
 424out_media_device_unregister:
 425	media_device_unregister(&isys->media_dev);
 426	media_device_cleanup(&isys->media_dev);
 427
 428	dev_err(dev, "failed to register isys devices\n");
 429
 430	return ret;
 431}
 432
 433static void isys_unregister_devices(struct ipu7_isys *isys)
 434{
 435	isys_unregister_video_devices(isys);
 436	isys_csi2_unregister_subdevices(isys);
 437	v4l2_device_unregister(&isys->v4l2_dev);
 438	media_device_unregister(&isys->media_dev);
 439	media_device_cleanup(&isys->media_dev);
 440}
 441
 442static void enable_csi2_legacy_irq(struct ipu7_isys *isys, bool enable)
 443{
 444	u32 offset = IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE;
 445	void __iomem *base = isys->pdata->base;
 446	u32 mask = isys->isr_csi2_mask;
 447
 448	if (!enable) {
 449		writel(mask, base + offset + IRQ_CTL_CLEAR);
 450		writel(0, base + offset + IRQ_CTL_ENABLE);
 451		return;
 452	}
 453
 454	writel(mask, base + offset + IRQ_CTL_EDGE);
 455	writel(mask, base + offset + IRQ_CTL_CLEAR);
 456	writel(mask, base + offset + IRQ_CTL_MASK);
 457	writel(mask, base + offset + IRQ_CTL_ENABLE);
 458}
 459
 460static void enable_to_sw_irq(struct ipu7_isys *isys, bool enable)
 461{
 462	void __iomem *base = isys->pdata->base;
 463	u32 mask = IS_UC_TO_SW_IRQ_MASK;
 464	u32 offset = IS_UC_CTRL_BASE;
 465
 466	if (!enable) {
 467		writel(0, base + offset + TO_SW_IRQ_CNTL_ENABLE);
 468		return;
 469	}
 470
 471	writel(mask, base + offset + TO_SW_IRQ_CNTL_CLEAR);
 472	writel(mask, base + offset + TO_SW_IRQ_CNTL_MASK_N);
 473	writel(mask, base + offset + TO_SW_IRQ_CNTL_ENABLE);
 474}
 475
 476void ipu7_isys_setup_hw(struct ipu7_isys *isys)
 477{
 478	u32 offset;
 479	void __iomem *base = isys->pdata->base;
 480
 481	/* soft reset */
 482	offset = IS_IO_GPREGS_BASE;
 483
 484	writel(0x0, base + offset + CLK_EN_TXCLKESC);
 485	/* Update if ISYS freq updated (0: 400/1, 1:400/2, 63:400/64) */
 486	writel(0x0, base + offset + CLK_DIV_FACTOR_IS_CLK);
 487	/* correct the initial printf configuration */
 488	writel(0x200, base + IS_UC_CTRL_BASE + PRINTF_AXI_CNTL);
 489
 490	enable_to_sw_irq(isys, 1);
 491	enable_csi2_legacy_irq(isys, 1);
 492}
 493
 494static void isys_cleanup_hw(struct ipu7_isys *isys)
 495{
 496	enable_csi2_legacy_irq(isys, 0);
 497	enable_to_sw_irq(isys, 0);
 498}
 499
 500static int isys_runtime_pm_resume(struct device *dev)
 501{
 502	struct ipu7_bus_device *adev = to_ipu7_bus_device(dev);
 503	struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev);
 504	struct ipu7_device *isp = adev->isp;
 505	unsigned long flags;
 506	int ret;
 507
 508	if (!isys)
 509		return 0;
 510
 511	ret = ipu7_mmu_hw_init(adev->mmu);
 512	if (ret)
 513		return ret;
 514
 515	cpu_latency_qos_update_request(&isys->pm_qos, ISYS_PM_QOS_VALUE);
 516
 517	ret = ipu_buttress_start_tsc_sync(isp);
 518	if (ret)
 519		return ret;
 520
 521	spin_lock_irqsave(&isys->power_lock, flags);
 522	isys->power = 1;
 523	spin_unlock_irqrestore(&isys->power_lock, flags);
 524
 525	return 0;
 526}
 527
 528static int isys_runtime_pm_suspend(struct device *dev)
 529{
 530	struct ipu7_bus_device *adev = to_ipu7_bus_device(dev);
 531	struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev);
 532	unsigned long flags;
 533
 534	if (!isys)
 535		return 0;
 536
 537	isys_cleanup_hw(isys);
 538
 539	spin_lock_irqsave(&isys->power_lock, flags);
 540	isys->power = 0;
 541	spin_unlock_irqrestore(&isys->power_lock, flags);
 542
 543	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 544
 545	ipu7_mmu_hw_cleanup(adev->mmu);
 546
 547	return 0;
 548}
 549
 550static int isys_suspend(struct device *dev)
 551{
 552	struct ipu7_isys *isys = dev_get_drvdata(dev);
 553
 554	/* If stream is open, refuse to suspend */
 555	if (isys->stream_opened)
 556		return -EBUSY;
 557
 558	return 0;
 559}
 560
 561static int isys_resume(struct device *dev)
 562{
 563	return 0;
 564}
 565
 566static const struct dev_pm_ops isys_pm_ops = {
 567	.runtime_suspend = isys_runtime_pm_suspend,
 568	.runtime_resume = isys_runtime_pm_resume,
 569	.suspend = isys_suspend,
 570	.resume = isys_resume,
 571};
 572
 573static void isys_remove(struct auxiliary_device *auxdev)
 574{
 575	struct ipu7_isys *isys = dev_get_drvdata(&auxdev->dev);
 576	struct isys_fw_msgs *fwmsg, *safe;
 577	struct ipu7_bus_device *adev = auxdev_to_adev(auxdev);
 578
 579	for (int i = 0; i < IPU_ISYS_MAX_STREAMS; i++)
 580		mutex_destroy(&isys->streams[i].mutex);
 581
 582	list_for_each_entry_safe(fwmsg, safe, &isys->framebuflist, head)
 583		ipu7_dma_free(adev, sizeof(struct isys_fw_msgs),
 584			      fwmsg, fwmsg->dma_addr, 0);
 585
 586	list_for_each_entry_safe(fwmsg, safe, &isys->framebuflist_fw, head)
 587		ipu7_dma_free(adev, sizeof(struct isys_fw_msgs),
 588			      fwmsg, fwmsg->dma_addr, 0);
 589
 590	isys_notifier_cleanup(isys);
 591	isys_unregister_devices(isys);
 592
 593	cpu_latency_qos_remove_request(&isys->pm_qos);
 594
 595	mutex_destroy(&isys->stream_mutex);
 596	mutex_destroy(&isys->mutex);
 597}
 598
 599static int alloc_fw_msg_bufs(struct ipu7_isys *isys, int amount)
 600{
 601	struct ipu7_bus_device *adev = isys->adev;
 602	struct isys_fw_msgs *addr;
 603	dma_addr_t dma_addr;
 604	unsigned long flags;
 605	unsigned int i;
 606
 607	for (i = 0; i < amount; i++) {
 608		addr = ipu7_dma_alloc(adev, sizeof(struct isys_fw_msgs),
 609				      &dma_addr, GFP_KERNEL, 0);
 610		if (!addr)
 611			break;
 612		addr->dma_addr = dma_addr;
 613
 614		spin_lock_irqsave(&isys->listlock, flags);
 615		list_add(&addr->head, &isys->framebuflist);
 616		spin_unlock_irqrestore(&isys->listlock, flags);
 617	}
 618
 619	if (i == amount)
 620		return 0;
 621
 622	spin_lock_irqsave(&isys->listlock, flags);
 623	while (!list_empty(&isys->framebuflist)) {
 624		addr = list_first_entry(&isys->framebuflist,
 625					struct isys_fw_msgs, head);
 626		list_del(&addr->head);
 627		spin_unlock_irqrestore(&isys->listlock, flags);
 628		ipu7_dma_free(adev, sizeof(struct isys_fw_msgs),
 629			      addr, addr->dma_addr, 0);
 630		spin_lock_irqsave(&isys->listlock, flags);
 631	}
 632	spin_unlock_irqrestore(&isys->listlock, flags);
 633
 634	return -ENOMEM;
 635}
 636
 637struct isys_fw_msgs *ipu7_get_fw_msg_buf(struct ipu7_isys_stream *stream)
 638{
 639	struct device *dev = &stream->isys->adev->auxdev.dev;
 640	struct ipu7_isys *isys = stream->isys;
 641	struct isys_fw_msgs *msg;
 642	unsigned long flags;
 643	int ret;
 644
 645	spin_lock_irqsave(&isys->listlock, flags);
 646	if (list_empty(&isys->framebuflist)) {
 647		spin_unlock_irqrestore(&isys->listlock, flags);
 648		dev_dbg(dev, "Frame buffer list empty\n");
 649
 650		ret = alloc_fw_msg_bufs(isys, 5);
 651		if (ret < 0)
 652			return NULL;
 653
 654		spin_lock_irqsave(&isys->listlock, flags);
 655		if (list_empty(&isys->framebuflist)) {
 656			spin_unlock_irqrestore(&isys->listlock, flags);
 657			dev_err(dev, "Frame list empty\n");
 658			return NULL;
 659		}
 660	}
 661	msg = list_last_entry(&isys->framebuflist, struct isys_fw_msgs, head);
 662	list_move(&msg->head, &isys->framebuflist_fw);
 663	spin_unlock_irqrestore(&isys->listlock, flags);
 664	memset(&msg->fw_msg, 0, sizeof(msg->fw_msg));
 665
 666	return msg;
 667}
 668
 669void ipu7_cleanup_fw_msg_bufs(struct ipu7_isys *isys)
 670{
 671	struct isys_fw_msgs *fwmsg, *fwmsg0;
 672	unsigned long flags;
 673
 674	spin_lock_irqsave(&isys->listlock, flags);
 675	list_for_each_entry_safe(fwmsg, fwmsg0, &isys->framebuflist_fw, head)
 676		list_move(&fwmsg->head, &isys->framebuflist);
 677	spin_unlock_irqrestore(&isys->listlock, flags);
 678}
 679
 680void ipu7_put_fw_msg_buf(struct ipu7_isys *isys, uintptr_t data)
 681{
 682	struct isys_fw_msgs *msg;
 683	void *ptr = (void *)data;
 684	unsigned long flags;
 685
 686	if (WARN_ON_ONCE(!ptr))
 687		return;
 688
 689	spin_lock_irqsave(&isys->listlock, flags);
 690	msg = container_of(ptr, struct isys_fw_msgs, fw_msg.dummy);
 691	list_move(&msg->head, &isys->framebuflist);
 692	spin_unlock_irqrestore(&isys->listlock, flags);
 693}
 694
 695static int isys_probe(struct auxiliary_device *auxdev,
 696		      const struct auxiliary_device_id *auxdev_id)
 697{
 698	const struct ipu7_isys_internal_csi2_pdata *csi2_pdata;
 699	struct ipu7_bus_device *adev = auxdev_to_adev(auxdev);
 700	struct ipu7_device *isp = adev->isp;
 701	struct ipu7_isys *isys;
 702	int ret = 0;
 703
 704	if (!isp->ipu7_bus_ready_to_probe)
 705		return -EPROBE_DEFER;
 706
 707	isys = devm_kzalloc(&auxdev->dev, sizeof(*isys), GFP_KERNEL);
 708	if (!isys)
 709		return -ENOMEM;
 710
 711	ret = pm_runtime_resume_and_get(&auxdev->dev);
 712	if (ret < 0)
 713		return ret;
 714
 715	adev->auxdrv_data =
 716		(const struct ipu7_auxdrv_data *)auxdev_id->driver_data;
 717	adev->auxdrv = to_auxiliary_drv(auxdev->dev.driver);
 718	isys->adev = adev;
 719	isys->pdata = adev->pdata;
 720
 721	INIT_LIST_HEAD(&isys->requests);
 722	csi2_pdata = &isys->pdata->ipdata->csi2;
 723
 724	isys->csi2 = devm_kcalloc(&auxdev->dev, csi2_pdata->nports,
 725				  sizeof(*isys->csi2), GFP_KERNEL);
 726	if (!isys->csi2) {
 727		ret = -ENOMEM;
 728		goto out_runtime_put;
 729	}
 730
 731	ret = ipu7_mmu_hw_init(adev->mmu);
 732	if (ret)
 733		goto out_runtime_put;
 734
 735	spin_lock_init(&isys->streams_lock);
 736	spin_lock_init(&isys->power_lock);
 737	isys->power = 0;
 738
 739	mutex_init(&isys->mutex);
 740	mutex_init(&isys->stream_mutex);
 741
 742	spin_lock_init(&isys->listlock);
 743	INIT_LIST_HEAD(&isys->framebuflist);
 744	INIT_LIST_HEAD(&isys->framebuflist_fw);
 745
 746	dev_set_drvdata(&auxdev->dev, isys);
 747
 748	isys->icache_prefetch = 0;
 749	isys->phy_rext_cal = 0;
 750
 751	isys_stream_init(isys);
 752
 753	cpu_latency_qos_add_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 754	ret = alloc_fw_msg_bufs(isys, 20);
 755	if (ret < 0)
 756		goto out_cleanup_isys;
 757
 758	ret = ipu7_fw_isys_init(isys);
 759	if (ret)
 760		goto out_cleanup_isys;
 761
 762	ret = isys_register_devices(isys);
 763	if (ret)
 764		goto out_cleanup_fw;
 765
 766	ret = isys_fw_log_init(isys);
 767	if (ret)
 768		goto out_cleanup;
 769
 770	ipu7_mmu_hw_cleanup(adev->mmu);
 771	pm_runtime_put(&auxdev->dev);
 772
 773	return 0;
 774
 775out_cleanup:
 776	isys_unregister_devices(isys);
 777out_cleanup_fw:
 778	ipu7_fw_isys_release(isys);
 779out_cleanup_isys:
 780	cpu_latency_qos_remove_request(&isys->pm_qos);
 781
 782	for (unsigned int i = 0; i < IPU_ISYS_MAX_STREAMS; i++)
 783		mutex_destroy(&isys->streams[i].mutex);
 784
 785	mutex_destroy(&isys->mutex);
 786	mutex_destroy(&isys->stream_mutex);
 787
 788	ipu7_mmu_hw_cleanup(adev->mmu);
 789
 790out_runtime_put:
 791	pm_runtime_put(&auxdev->dev);
 792
 793	return ret;
 794}
 795
 796struct ipu7_csi2_error {
 797	const char *error_string;
 798	bool is_info_only;
 799};
 800
 801/*
 802 * Strings corresponding to CSI-2 receiver errors are here.
 803 * Corresponding macros are defined in the header file.
 804 */
 805static const struct ipu7_csi2_error dphy_rx_errors[] = {
 806	{ "Error handler FIFO full", false },
 807	{ "Reserved Short Packet encoding detected", true },
 808	{ "Reserved Long Packet encoding detected", true },
 809	{ "Received packet is too short", false},
 810	{ "Received packet is too long", false},
 811	{ "Short packet discarded due to errors", false },
 812	{ "Long packet discarded due to errors", false },
 813	{ "CSI Combo Rx interrupt", false },
 814	{ "IDI CDC FIFO overflow(remaining bits are reserved as 0)", false },
 815	{ "Received NULL packet", true },
 816	{ "Received blanking packet", true },
 817	{ "Tie to 0", true },
 818	{ }
 819};
 820
 821static void ipu7_isys_register_errors(struct ipu7_isys_csi2 *csi2)
 822{
 823	u32 offset = IS_IO_CSI2_ERR_LEGACY_IRQ_CTL_BASE(csi2->port);
 824	u32 status = readl(csi2->base + offset + IRQ_CTL_STATUS);
 825	u32 mask = IPU7_CSI_RX_ERROR_IRQ_MASK;
 826
 827	if (!status)
 828		return;
 829
 830	dev_dbg(&csi2->isys->adev->auxdev.dev, "csi2-%u error status 0x%08x\n",
 831		csi2->port, status);
 832
 833	writel(status & mask, csi2->base + offset + IRQ_CTL_CLEAR);
 834	csi2->receiver_errors |= status & mask;
 835}
 836
 837static void ipu7_isys_csi2_error(struct ipu7_isys_csi2 *csi2)
 838{
 839	struct ipu7_csi2_error const *errors;
 840	unsigned int i;
 841	u32 status;
 842
 843	/* Register errors once more in case of error interrupts are disabled */
 844	ipu7_isys_register_errors(csi2);
 845	status = csi2->receiver_errors;
 846	csi2->receiver_errors = 0;
 847	errors = dphy_rx_errors;
 848
 849	for (i = 0; i < CSI_RX_NUM_ERRORS_IN_IRQ; i++) {
 850		if (status & BIT(i))
 851			dev_err_ratelimited(&csi2->isys->adev->auxdev.dev,
 852					    "csi2-%i error: %s\n",
 853					    csi2->port,
 854					    errors[i].error_string);
 855	}
 856}
 857
 858struct resp_to_msg {
 859	enum ipu7_insys_resp_type type;
 860	const char *msg;
 861};
 862
 863static const struct resp_to_msg is_fw_msg[] = {
 864	{IPU_INSYS_RESP_TYPE_STREAM_OPEN_DONE,
 865	 "IPU_INSYS_RESP_TYPE_STREAM_OPEN_DONE"},
 866	{IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK,
 867	 "IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK"},
 868	{IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK,
 869	 "IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK"},
 870	{IPU_INSYS_RESP_TYPE_STREAM_ABORT_ACK,
 871	 "IPU_INSYS_RESP_TYPE_STREAM_ABORT_ACK"},
 872	{IPU_INSYS_RESP_TYPE_STREAM_FLUSH_ACK,
 873	 "IPU_INSYS_RESP_TYPE_STREAM_FLUSH_ACK"},
 874	{IPU_INSYS_RESP_TYPE_STREAM_CLOSE_ACK,
 875	 "IPU_INSYS_RESP_TYPE_STREAM_CLOSE_ACK"},
 876	{IPU_INSYS_RESP_TYPE_PIN_DATA_READY,
 877	 "IPU_INSYS_RESP_TYPE_PIN_DATA_READY"},
 878	{IPU_INSYS_RESP_TYPE_FRAME_SOF, "IPU_INSYS_RESP_TYPE_FRAME_SOF"},
 879	{IPU_INSYS_RESP_TYPE_FRAME_EOF, "IPU_INSYS_RESP_TYPE_FRAME_EOF"},
 880	{IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE,
 881	 "IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE"},
 882	{IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE,
 883	 "IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE"},
 884	{N_IPU_INSYS_RESP_TYPE, "N_IPU_INSYS_RESP_TYPE"},
 885};
 886
 887int isys_isr_one(struct ipu7_bus_device *adev)
 888{
 889	struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev);
 890	struct ipu7_isys_stream *stream = NULL;
 891	struct device *dev = &adev->auxdev.dev;
 892	struct ipu7_isys_csi2 *csi2 = NULL;
 893	struct ia_gofo_msg_err err_info;
 894	struct ipu7_insys_resp *resp;
 895	u64 ts;
 896
 897	if (!isys->adev->syscom)
 898		return 1;
 899
 900	resp = ipu7_fw_isys_get_resp(isys);
 901	if (!resp)
 902		return 1;
 903	if (resp->type >= N_IPU_INSYS_RESP_TYPE) {
 904		dev_err(dev, "Unknown response type %u stream %u\n",
 905			resp->type, resp->stream_id);
 906		ipu7_fw_isys_put_resp(isys);
 907		return 1;
 908	}
 909
 910	err_info = resp->error_info;
 911	ts = ((u64)resp->timestamp[1] << 32) | resp->timestamp[0];
 912	if (err_info.err_group == INSYS_MSG_ERR_GROUP_CAPTURE &&
 913	    err_info.err_code == INSYS_MSG_ERR_CAPTURE_SYNC_FRAME_DROP) {
 914		/* receive a sp w/o command, firmware drop it */
 915		dev_dbg(dev, "FRAME DROP: %02u %s stream %u\n",
 916			resp->type, is_fw_msg[resp->type].msg,
 917			resp->stream_id);
 918		dev_dbg(dev, "\tpin %u buf_id %llx frame %u\n",
 919			resp->pin_id, resp->buf_id, resp->frame_id);
 920		dev_dbg(dev, "\terror group %u code %u details [%u %u]\n",
 921			err_info.err_group, err_info.err_code,
 922			err_info.err_detail[0], err_info.err_detail[1]);
 923	} else if (!IA_GOFO_MSG_ERR_IS_OK(err_info)) {
 924		dev_err(dev, "%02u %s stream %u pin %u buf_id %llx frame %u\n",
 925			resp->type, is_fw_msg[resp->type].msg, resp->stream_id,
 926			resp->pin_id, resp->buf_id, resp->frame_id);
 927		dev_err(dev, "\terror group %u code %u details [%u %u]\n",
 928			err_info.err_group, err_info.err_code,
 929			err_info.err_detail[0], err_info.err_detail[1]);
 930	} else {
 931		dev_dbg(dev, "%02u %s stream %u pin %u buf_id %llx frame %u\n",
 932			resp->type, is_fw_msg[resp->type].msg, resp->stream_id,
 933			resp->pin_id, resp->buf_id, resp->frame_id);
 934		dev_dbg(dev, "\tts %llu\n", ts);
 935	}
 936
 937	if (resp->stream_id >= IPU_ISYS_MAX_STREAMS) {
 938		dev_err(dev, "bad stream handle %u\n",
 939			resp->stream_id);
 940		goto leave;
 941	}
 942
 943	stream = ipu7_isys_query_stream_by_handle(isys, resp->stream_id);
 944	if (!stream) {
 945		dev_err(dev, "stream of stream_handle %u is unused\n",
 946			resp->stream_id);
 947		goto leave;
 948	}
 949
 950	stream->error = err_info.err_code;
 951
 952	if (stream->asd)
 953		csi2 = ipu7_isys_subdev_to_csi2(stream->asd);
 954
 955	switch (resp->type) {
 956	case IPU_INSYS_RESP_TYPE_STREAM_OPEN_DONE:
 957		complete(&stream->stream_open_completion);
 958		break;
 959	case IPU_INSYS_RESP_TYPE_STREAM_CLOSE_ACK:
 960		complete(&stream->stream_close_completion);
 961		break;
 962	case IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK:
 963		complete(&stream->stream_start_completion);
 964		break;
 965	case IPU_INSYS_RESP_TYPE_STREAM_ABORT_ACK:
 966		complete(&stream->stream_stop_completion);
 967		break;
 968	case IPU_INSYS_RESP_TYPE_STREAM_FLUSH_ACK:
 969		complete(&stream->stream_stop_completion);
 970		break;
 971	case IPU_INSYS_RESP_TYPE_PIN_DATA_READY:
 972		/*
 973		 * firmware only release the capture msg until software
 974		 * get pin_data_ready event
 975		 */
 976		ipu7_put_fw_msg_buf(ipu7_bus_get_drvdata(adev), resp->buf_id);
 977		if (resp->pin_id < IPU_INSYS_OUTPUT_PINS &&
 978		    stream->output_pins[resp->pin_id].pin_ready)
 979			stream->output_pins[resp->pin_id].pin_ready(stream,
 980								    resp);
 981		else
 982			dev_err(dev, "No handler for pin %u ready\n",
 983				resp->pin_id);
 984		if (csi2)
 985			ipu7_isys_csi2_error(csi2);
 986
 987		break;
 988	case IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK:
 989		break;
 990	case IPU_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE:
 991	case IPU_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE:
 992		break;
 993	case IPU_INSYS_RESP_TYPE_FRAME_SOF:
 994		if (csi2)
 995			ipu7_isys_csi2_sof_event_by_stream(stream);
 996
 997		stream->seq[stream->seq_index].sequence =
 998			atomic_read(&stream->sequence) - 1U;
 999		stream->seq[stream->seq_index].timestamp = ts;
1000		dev_dbg(dev,
1001			"SOF: stream %u frame %u (index %u), ts 0x%16.16llx\n",
1002			resp->stream_id, resp->frame_id,
1003			stream->seq[stream->seq_index].sequence, ts);
1004		stream->seq_index = (stream->seq_index + 1U)
1005			% IPU_ISYS_MAX_PARALLEL_SOF;
1006		break;
1007	case IPU_INSYS_RESP_TYPE_FRAME_EOF:
1008		if (csi2)
1009			ipu7_isys_csi2_eof_event_by_stream(stream);
1010
1011		dev_dbg(dev, "eof: stream %d(index %u) ts 0x%16.16llx\n",
1012			resp->stream_id,
1013			stream->seq[stream->seq_index].sequence, ts);
1014		break;
1015	default:
1016		dev_err(dev, "Unknown response type %u stream %u\n",
1017			resp->type, resp->stream_id);
1018		break;
1019	}
1020
1021	ipu7_isys_put_stream(stream);
1022leave:
1023	ipu7_fw_isys_put_resp(isys);
1024
1025	return 0;
1026}
1027
1028static void ipu7_isys_csi2_isr(struct ipu7_isys_csi2 *csi2)
1029{
1030	struct device *dev = &csi2->isys->adev->auxdev.dev;
1031	struct ipu7_device *isp = csi2->isys->adev->isp;
1032	struct ipu7_isys_stream *s;
1033	u32 sync, offset;
1034	u32 fe = 0;
1035	u8 vc;
1036
1037	ipu7_isys_register_errors(csi2);
1038
1039	offset = IS_IO_CSI2_SYNC_LEGACY_IRQ_CTL_BASE(csi2->port);
1040	sync = readl(csi2->base + offset + IRQ_CTL_STATUS);
1041	writel(sync, csi2->base + offset + IRQ_CTL_CLEAR);
1042	dev_dbg(dev, "csi2-%u sync status 0x%08x\n", csi2->port, sync);
1043
1044	if (!is_ipu7(isp->hw_ver)) {
1045		fe = readl(csi2->base + offset + IRQ1_CTL_STATUS);
1046		writel(fe, csi2->base + offset + IRQ1_CTL_CLEAR);
1047		dev_dbg(dev, "csi2-%u FE status 0x%08x\n", csi2->port, fe);
1048	}
1049
1050	for (vc = 0; vc < IPU7_NR_OF_CSI2_VC && (sync || fe); vc++) {
1051		s = ipu7_isys_query_stream_by_source(csi2->isys,
1052						     csi2->asd.source, vc);
1053		if (!s)
1054			continue;
1055
1056		if (!is_ipu7(isp->hw_ver)) {
1057			if (sync & IPU7P5_CSI_RX_SYNC_FS_VC & (1U << vc))
1058				ipu7_isys_csi2_sof_event_by_stream(s);
1059
1060			if (fe & IPU7P5_CSI_RX_SYNC_FE_VC & (1U << vc))
1061				ipu7_isys_csi2_eof_event_by_stream(s);
1062		} else {
1063			if (sync & IPU7_CSI_RX_SYNC_FS_VC & (1U << (vc * 2)))
1064				ipu7_isys_csi2_sof_event_by_stream(s);
1065
1066			if (sync & IPU7_CSI_RX_SYNC_FE_VC & (2U << (vc * 2)))
1067				ipu7_isys_csi2_eof_event_by_stream(s);
1068		}
1069	}
1070}
1071
1072static irqreturn_t isys_isr(struct ipu7_bus_device *adev)
1073{
1074	struct ipu7_isys *isys = ipu7_bus_get_drvdata(adev);
1075	u32 status_csi, status_sw, csi_offset, sw_offset;
1076	struct device *dev = &isys->adev->auxdev.dev;
1077	void __iomem *base = isys->pdata->base;
1078
1079	spin_lock(&isys->power_lock);
1080	if (!isys->power) {
1081		spin_unlock(&isys->power_lock);
1082		return IRQ_NONE;
1083	}
1084
1085	csi_offset = IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE;
1086	sw_offset = IS_BASE;
1087
1088	status_csi = readl(base + csi_offset + IRQ_CTL_STATUS);
1089	status_sw = readl(base + sw_offset + TO_SW_IRQ_CNTL_STATUS);
1090	if (!status_csi && !status_sw) {
1091		spin_unlock(&isys->power_lock);
1092		return IRQ_NONE;
1093	}
1094
1095	if (status_csi)
1096		dev_dbg(dev, "status csi 0x%08x\n", status_csi);
1097	if (status_sw)
1098		dev_dbg(dev, "status to_sw 0x%08x\n", status_sw);
1099
1100	do {
1101		writel(status_sw, base + sw_offset + TO_SW_IRQ_CNTL_CLEAR);
1102		writel(status_csi, base + csi_offset + IRQ_CTL_CLEAR);
1103
1104		if (isys->isr_csi2_mask & status_csi) {
1105			unsigned int i;
1106
1107			for (i = 0; i < isys->pdata->ipdata->csi2.nports; i++) {
1108				/* irq from not enabled port */
1109				if (!isys->csi2[i].base)
1110					continue;
1111				if (status_csi & isys->csi2[i].legacy_irq_mask)
1112					ipu7_isys_csi2_isr(&isys->csi2[i]);
1113			}
1114		}
1115
1116		if (!isys_isr_one(adev))
1117			status_sw = TO_SW_IRQ_FW;
1118		else
1119			status_sw = 0;
1120
1121		status_csi = readl(base + csi_offset + IRQ_CTL_STATUS);
1122		status_sw |= readl(base + sw_offset + TO_SW_IRQ_CNTL_STATUS);
1123	} while ((status_csi & isys->isr_csi2_mask) ||
1124		 (status_sw & TO_SW_IRQ_FW));
1125
1126	writel(TO_SW_IRQ_MASK, base + sw_offset + TO_SW_IRQ_CNTL_MASK_N);
1127
1128	spin_unlock(&isys->power_lock);
1129
1130	return IRQ_HANDLED;
1131}
1132
1133static const struct ipu7_auxdrv_data ipu7_isys_auxdrv_data = {
1134	.isr = isys_isr,
1135	.isr_threaded = NULL,
1136	.wake_isr_thread = false,
1137};
1138
1139static const struct auxiliary_device_id ipu7_isys_id_table[] = {
1140	{
1141		.name = "intel_ipu7.isys",
1142		.driver_data = (kernel_ulong_t)&ipu7_isys_auxdrv_data,
1143	},
1144	{ }
1145};
1146MODULE_DEVICE_TABLE(auxiliary, ipu7_isys_id_table);
1147
1148static struct auxiliary_driver isys_driver = {
1149	.name = IPU_ISYS_NAME,
1150	.probe = isys_probe,
1151	.remove = isys_remove,
1152	.id_table = ipu7_isys_id_table,
1153	.driver = {
1154		.pm = &isys_pm_ops,
1155	},
1156};
1157
1158module_auxiliary_driver(isys_driver);
1159
1160MODULE_AUTHOR("Bingbu Cao <bingbu.cao@intel.com>");
1161MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>");
1162MODULE_AUTHOR("Qingwu Zhang <qingwu.zhang@intel.com>");
1163MODULE_LICENSE("GPL");
1164MODULE_DESCRIPTION("Intel ipu7 input system driver");
1165MODULE_IMPORT_NS("INTEL_IPU7");
1166MODULE_IMPORT_NS("INTEL_IPU_BRIDGE");