mtk_vpu.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3* Copyright (c) 2016 MediaTek Inc.
   4* Author: Andrew-CT Chen <andrew-ct.chen@mediatek.com>
   5*/
   6#include <linux/clk.h>
   7#include <linux/debugfs.h>
   8#include <linux/firmware.h>
   9#include <linux/interrupt.h>
  10#include <linux/iommu.h>
  11#include <linux/module.h>
  12#include <linux/of.h>
  13#include <linux/of_platform.h>
  14#include <linux/of_reserved_mem.h>
  15#include <linux/platform_device.h>
  16#include <linux/sched.h>
  17#include <linux/sizes.h>
  18#include <linux/dma-mapping.h>
  19
  20#include "mtk_vpu.h"
  21
  22/*
  23 * VPU (video processor unit) is a tiny processor controlling video hardware
  24 * related to video codec, scaling and color format converting.
  25 * VPU interfaces with other blocks by share memory and interrupt.
  26 */
  27
  28#define INIT_TIMEOUT_MS		2000U
  29#define IPI_TIMEOUT_MS		2000U
  30#define VPU_IDLE_TIMEOUT_MS	1000U
  31#define VPU_FW_VER_LEN		16
  32
  33/* maximum program/data TCM (Tightly-Coupled Memory) size */
  34#define VPU_PTCM_SIZE		(96 * SZ_1K)
  35#define VPU_DTCM_SIZE		(32 * SZ_1K)
  36/* the offset to get data tcm address */
  37#define VPU_DTCM_OFFSET		0x18000UL
  38/* daynamic allocated maximum extended memory size */
  39#define VPU_EXT_P_SIZE		SZ_1M
  40#define VPU_EXT_D_SIZE		SZ_4M
  41/* maximum binary firmware size */
  42#define VPU_P_FW_SIZE		(VPU_PTCM_SIZE + VPU_EXT_P_SIZE)
  43#define VPU_D_FW_SIZE		(VPU_DTCM_SIZE + VPU_EXT_D_SIZE)
  44/* the size of share buffer between Host and  VPU */
  45#define SHARE_BUF_SIZE		48
  46
  47/* binary firmware name */
  48#define VPU_P_FW		"vpu_p.bin"
  49#define VPU_D_FW		"vpu_d.bin"
  50#define VPU_P_FW_NEW		"mediatek/mt8173/vpu_p.bin"
  51#define VPU_D_FW_NEW		"mediatek/mt8173/vpu_d.bin"
  52
  53#define VPU_RESET		0x0
  54#define VPU_TCM_CFG		0x0008
  55#define VPU_PMEM_EXT0_ADDR	0x000C
  56#define VPU_PMEM_EXT1_ADDR	0x0010
  57#define VPU_TO_HOST		0x001C
  58#define VPU_DMEM_EXT0_ADDR	0x0014
  59#define VPU_DMEM_EXT1_ADDR	0x0018
  60#define HOST_TO_VPU		0x0024
  61#define VPU_IDLE_REG		0x002C
  62#define VPU_INT_STATUS		0x0034
  63#define VPU_PC_REG		0x0060
  64#define VPU_SP_REG		0x0064
  65#define VPU_RA_REG		0x0068
  66#define VPU_WDT_REG		0x0084
  67
  68/* vpu inter-processor communication interrupt */
  69#define VPU_IPC_INT		BIT(8)
  70/* vpu idle state */
  71#define VPU_IDLE_STATE		BIT(23)
  72
  73/**
  74 * enum vpu_fw_type - VPU firmware type
  75 *
  76 * @P_FW: program firmware
  77 * @D_FW: data firmware
  78 *
  79 */
  80enum vpu_fw_type {
  81	P_FW,
  82	D_FW,
  83};
  84
  85/**
  86 * struct vpu_mem - VPU extended program/data memory information
  87 *
  88 * @va:		the kernel virtual memory address of VPU extended memory
  89 * @pa:		the physical memory address of VPU extended memory
  90 *
  91 */
  92struct vpu_mem {
  93	void *va;
  94	dma_addr_t pa;
  95};
  96
  97/**
  98 * struct vpu_regs - VPU TCM and configuration registers
  99 *
 100 * @tcm:	the register for VPU Tightly-Coupled Memory
 101 * @cfg:	the register for VPU configuration
 102 * @irq:	the irq number for VPU interrupt
 103 */
 104struct vpu_regs {
 105	void __iomem *tcm;
 106	void __iomem *cfg;
 107	int irq;
 108};
 109
 110/**
 111 * struct vpu_wdt_handler - VPU watchdog reset handler
 112 *
 113 * @reset_func:	reset handler
 114 * @priv:	private data
 115 */
 116struct vpu_wdt_handler {
 117	void (*reset_func)(void *);
 118	void *priv;
 119};
 120
 121/**
 122 * struct vpu_wdt - VPU watchdog workqueue
 123 *
 124 * @handler:	VPU watchdog reset handler
 125 * @ws:		workstruct for VPU watchdog
 126 * @wq:		workqueue for VPU watchdog
 127 */
 128struct vpu_wdt {
 129	struct vpu_wdt_handler handler[VPU_RST_MAX];
 130	struct work_struct ws;
 131	struct workqueue_struct *wq;
 132};
 133
 134/**
 135 * struct vpu_run - VPU initialization status
 136 *
 137 * @signaled:		the signal of vpu initialization completed
 138 * @fw_ver:		VPU firmware version
 139 * @dec_capability:	decoder capability which is not used for now and
 140 *			the value is reserved for future use
 141 * @enc_capability:	encoder capability which is not used for now and
 142 *			the value is reserved for future use
 143 * @wq:			wait queue for VPU initialization status
 144 */
 145struct vpu_run {
 146	u32 signaled;
 147	char fw_ver[VPU_FW_VER_LEN];
 148	unsigned int	dec_capability;
 149	unsigned int	enc_capability;
 150	wait_queue_head_t wq;
 151};
 152
 153/**
 154 * struct vpu_ipi_desc - VPU IPI descriptor
 155 *
 156 * @handler:	IPI handler
 157 * @name:	the name of IPI handler
 158 * @priv:	the private data of IPI handler
 159 */
 160struct vpu_ipi_desc {
 161	ipi_handler_t handler;
 162	const char *name;
 163	void *priv;
 164};
 165
 166/**
 167 * struct share_obj - DTCM (Data Tightly-Coupled Memory) buffer shared with
 168 *		      AP and VPU
 169 *
 170 * @id:		IPI id
 171 * @len:	share buffer length
 172 * @share_buf:	share buffer data
 173 */
 174struct share_obj {
 175	s32 id;
 176	u32 len;
 177	unsigned char share_buf[SHARE_BUF_SIZE];
 178};
 179
 180/**
 181 * struct mtk_vpu - vpu driver data
 182 * @extmem:		VPU extended memory information
 183 * @reg:		VPU TCM and configuration registers
 184 * @run:		VPU initialization status
 185 * @wdt:		VPU watchdog workqueue
 186 * @ipi_desc:		VPU IPI descriptor
 187 * @recv_buf:		VPU DTCM share buffer for receiving. The
 188 *			receive buffer is only accessed in interrupt context.
 189 * @send_buf:		VPU DTCM share buffer for sending
 190 * @dev:		VPU struct device
 191 * @clk:		VPU clock on/off
 192 * @fw_loaded:		indicate VPU firmware loaded
 193 * @enable_4GB:		VPU 4GB mode on/off
 194 * @vpu_mutex:		protect mtk_vpu (except recv_buf) and ensure only
 195 *			one client to use VPU service at a time. For example,
 196 *			suppose a client is using VPU to decode VP8.
 197 *			If the other client wants to encode VP8,
 198 *			it has to wait until VP8 decode completes.
 199 * @wdt_refcnt:		WDT reference count to make sure the watchdog can be
 200 *			disabled if no other client is using VPU service
 201 * @ack_wq:		The wait queue for each codec and mdp. When sleeping
 202 *			processes wake up, they will check the condition
 203 *			"ipi_id_ack" to run the corresponding action or
 204 *			go back to sleep.
 205 * @ipi_id_ack:		The ACKs for registered IPI function sending
 206 *			interrupt to VPU
 207 *
 208 */
 209struct mtk_vpu {
 210	struct vpu_mem extmem[2];
 211	struct vpu_regs reg;
 212	struct vpu_run run;
 213	struct vpu_wdt wdt;
 214	struct vpu_ipi_desc ipi_desc[IPI_MAX];
 215	struct share_obj __iomem *recv_buf;
 216	struct share_obj __iomem *send_buf;
 217	struct device *dev;
 218	struct clk *clk;
 219	bool fw_loaded;
 220	bool enable_4GB;
 221	struct mutex vpu_mutex; /* for protecting vpu data data structure */
 222	u32 wdt_refcnt;
 223	wait_queue_head_t ack_wq;
 224	bool ipi_id_ack[IPI_MAX];
 225};
 226
 227static inline void vpu_cfg_writel(struct mtk_vpu *vpu, u32 val, u32 offset)
 228{
 229	writel(val, vpu->reg.cfg + offset);
 230}
 231
 232static inline u32 vpu_cfg_readl(struct mtk_vpu *vpu, u32 offset)
 233{
 234	return readl(vpu->reg.cfg + offset);
 235}
 236
 237static inline bool vpu_running(struct mtk_vpu *vpu)
 238{
 239	return vpu_cfg_readl(vpu, VPU_RESET) & BIT(0);
 240}
 241
 242static void vpu_clock_disable(struct mtk_vpu *vpu)
 243{
 244	/* Disable VPU watchdog */
 245	mutex_lock(&vpu->vpu_mutex);
 246	if (!--vpu->wdt_refcnt)
 247		vpu_cfg_writel(vpu,
 248			       vpu_cfg_readl(vpu, VPU_WDT_REG) & ~(1L << 31),
 249			       VPU_WDT_REG);
 250	mutex_unlock(&vpu->vpu_mutex);
 251
 252	clk_disable(vpu->clk);
 253}
 254
 255static int vpu_clock_enable(struct mtk_vpu *vpu)
 256{
 257	int ret;
 258
 259	ret = clk_enable(vpu->clk);
 260	if (ret)
 261		return ret;
 262	/* Enable VPU watchdog */
 263	mutex_lock(&vpu->vpu_mutex);
 264	if (!vpu->wdt_refcnt++)
 265		vpu_cfg_writel(vpu,
 266			       vpu_cfg_readl(vpu, VPU_WDT_REG) | (1L << 31),
 267			       VPU_WDT_REG);
 268	mutex_unlock(&vpu->vpu_mutex);
 269
 270	return ret;
 271}
 272
 273static void vpu_dump_status(struct mtk_vpu *vpu)
 274{
 275	dev_info(vpu->dev,
 276		 "vpu: run %x, pc = 0x%x, ra = 0x%x, sp = 0x%x, idle = 0x%x\n"
 277		 "vpu: int %x, hv = 0x%x, vh = 0x%x, wdt = 0x%x\n",
 278		 vpu_running(vpu), vpu_cfg_readl(vpu, VPU_PC_REG),
 279		 vpu_cfg_readl(vpu, VPU_RA_REG), vpu_cfg_readl(vpu, VPU_SP_REG),
 280		 vpu_cfg_readl(vpu, VPU_IDLE_REG),
 281		 vpu_cfg_readl(vpu, VPU_INT_STATUS),
 282		 vpu_cfg_readl(vpu, HOST_TO_VPU),
 283		 vpu_cfg_readl(vpu, VPU_TO_HOST),
 284		 vpu_cfg_readl(vpu, VPU_WDT_REG));
 285}
 286
 287int vpu_ipi_register(struct platform_device *pdev,
 288		     enum ipi_id id, ipi_handler_t handler,
 289		     const char *name, void *priv)
 290{
 291	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 292	struct vpu_ipi_desc *ipi_desc;
 293
 294	if (!vpu) {
 295		dev_err(&pdev->dev, "vpu device in not ready\n");
 296		return -EPROBE_DEFER;
 297	}
 298
 299	if (id < IPI_MAX && handler) {
 300		ipi_desc = vpu->ipi_desc;
 301		ipi_desc[id].name = name;
 302		ipi_desc[id].handler = handler;
 303		ipi_desc[id].priv = priv;
 304		return 0;
 305	}
 306
 307	dev_err(&pdev->dev, "register vpu ipi id %d with invalid arguments\n",
 308		id);
 309	return -EINVAL;
 310}
 311EXPORT_SYMBOL_GPL(vpu_ipi_register);
 312
 313int vpu_ipi_send(struct platform_device *pdev,
 314		 enum ipi_id id, void *buf,
 315		 unsigned int len)
 316{
 317	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 318	struct share_obj __iomem *send_obj = vpu->send_buf;
 319	unsigned long timeout;
 320	int ret = 0;
 321
 322	if (id <= IPI_VPU_INIT || id >= IPI_MAX ||
 323	    len > sizeof(send_obj->share_buf) || !buf) {
 324		dev_err(vpu->dev, "failed to send ipi message\n");
 325		return -EINVAL;
 326	}
 327
 328	ret = vpu_clock_enable(vpu);
 329	if (ret) {
 330		dev_err(vpu->dev, "failed to enable vpu clock\n");
 331		return ret;
 332	}
 333	if (!vpu_running(vpu)) {
 334		dev_err(vpu->dev, "vpu_ipi_send: VPU is not running\n");
 335		ret = -EINVAL;
 336		goto clock_disable;
 337	}
 338
 339	mutex_lock(&vpu->vpu_mutex);
 340
 341	 /* Wait until VPU receives the last command */
 342	timeout = jiffies + msecs_to_jiffies(IPI_TIMEOUT_MS);
 343	do {
 344		if (time_after(jiffies, timeout)) {
 345			dev_err(vpu->dev, "vpu_ipi_send: IPI timeout!\n");
 346			ret = -EIO;
 347			vpu_dump_status(vpu);
 348			goto mut_unlock;
 349		}
 350	} while (vpu_cfg_readl(vpu, HOST_TO_VPU));
 351
 352	memcpy_toio(send_obj->share_buf, buf, len);
 353	writel(len, &send_obj->len);
 354	writel(id, &send_obj->id);
 355
 356	vpu->ipi_id_ack[id] = false;
 357	/* send the command to VPU */
 358	vpu_cfg_writel(vpu, 0x1, HOST_TO_VPU);
 359
 360	mutex_unlock(&vpu->vpu_mutex);
 361
 362	/* wait for VPU's ACK */
 363	timeout = msecs_to_jiffies(IPI_TIMEOUT_MS);
 364	ret = wait_event_timeout(vpu->ack_wq, vpu->ipi_id_ack[id], timeout);
 365	vpu->ipi_id_ack[id] = false;
 366	if (ret == 0) {
 367		dev_err(vpu->dev, "vpu ipi %d ack time out !\n", id);
 368		ret = -EIO;
 369		vpu_dump_status(vpu);
 370		goto clock_disable;
 371	}
 372	vpu_clock_disable(vpu);
 373
 374	return 0;
 375
 376mut_unlock:
 377	mutex_unlock(&vpu->vpu_mutex);
 378clock_disable:
 379	vpu_clock_disable(vpu);
 380
 381	return ret;
 382}
 383EXPORT_SYMBOL_GPL(vpu_ipi_send);
 384
 385static void vpu_wdt_reset_func(struct work_struct *ws)
 386{
 387	struct vpu_wdt *wdt = container_of(ws, struct vpu_wdt, ws);
 388	struct mtk_vpu *vpu = container_of(wdt, struct mtk_vpu, wdt);
 389	struct vpu_wdt_handler *handler = wdt->handler;
 390	int index, ret;
 391
 392	dev_info(vpu->dev, "vpu reset\n");
 393	ret = vpu_clock_enable(vpu);
 394	if (ret) {
 395		dev_err(vpu->dev, "[VPU] wdt enables clock failed %d\n", ret);
 396		return;
 397	}
 398	mutex_lock(&vpu->vpu_mutex);
 399	vpu_cfg_writel(vpu, 0x0, VPU_RESET);
 400	vpu->fw_loaded = false;
 401	mutex_unlock(&vpu->vpu_mutex);
 402	vpu_clock_disable(vpu);
 403
 404	for (index = 0; index < VPU_RST_MAX; index++) {
 405		if (handler[index].reset_func) {
 406			handler[index].reset_func(handler[index].priv);
 407			dev_dbg(vpu->dev, "wdt handler func %d\n", index);
 408		}
 409	}
 410}
 411
 412int vpu_wdt_reg_handler(struct platform_device *pdev,
 413			void wdt_reset(void *),
 414			void *priv, enum rst_id id)
 415{
 416	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 417	struct vpu_wdt_handler *handler;
 418
 419	if (!vpu) {
 420		dev_err(&pdev->dev, "vpu device in not ready\n");
 421		return -EPROBE_DEFER;
 422	}
 423
 424	handler = vpu->wdt.handler;
 425
 426	if (id < VPU_RST_MAX && wdt_reset) {
 427		dev_dbg(vpu->dev, "wdt register id %d\n", id);
 428		mutex_lock(&vpu->vpu_mutex);
 429		handler[id].reset_func = wdt_reset;
 430		handler[id].priv = priv;
 431		mutex_unlock(&vpu->vpu_mutex);
 432		return 0;
 433	}
 434
 435	dev_err(vpu->dev, "register vpu wdt handler failed\n");
 436	return -EINVAL;
 437}
 438EXPORT_SYMBOL_GPL(vpu_wdt_reg_handler);
 439
 440unsigned int vpu_get_vdec_hw_capa(struct platform_device *pdev)
 441{
 442	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 443
 444	return vpu->run.dec_capability;
 445}
 446EXPORT_SYMBOL_GPL(vpu_get_vdec_hw_capa);
 447
 448unsigned int vpu_get_venc_hw_capa(struct platform_device *pdev)
 449{
 450	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 451
 452	return vpu->run.enc_capability;
 453}
 454EXPORT_SYMBOL_GPL(vpu_get_venc_hw_capa);
 455
 456void *vpu_mapping_dm_addr(struct platform_device *pdev,
 457			  u32 dtcm_dmem_addr)
 458{
 459	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 460
 461	if (!dtcm_dmem_addr ||
 462	    (dtcm_dmem_addr > (VPU_DTCM_SIZE + VPU_EXT_D_SIZE))) {
 463		dev_err(vpu->dev, "invalid virtual data memory address\n");
 464		return ERR_PTR(-EINVAL);
 465	}
 466
 467	if (dtcm_dmem_addr < VPU_DTCM_SIZE)
 468		return (__force void *)(dtcm_dmem_addr + vpu->reg.tcm +
 469					VPU_DTCM_OFFSET);
 470
 471	return vpu->extmem[D_FW].va + (dtcm_dmem_addr - VPU_DTCM_SIZE);
 472}
 473EXPORT_SYMBOL_GPL(vpu_mapping_dm_addr);
 474
 475struct platform_device *vpu_get_plat_device(struct platform_device *pdev)
 476{
 477	struct device *dev = &pdev->dev;
 478	struct device_node *vpu_node;
 479	struct platform_device *vpu_pdev;
 480
 481	vpu_node = of_parse_phandle(dev->of_node, "mediatek,vpu", 0);
 482	if (!vpu_node) {
 483		dev_err(dev, "can't get vpu node\n");
 484		return NULL;
 485	}
 486
 487	vpu_pdev = of_find_device_by_node(vpu_node);
 488	of_node_put(vpu_node);
 489	if (WARN_ON(!vpu_pdev)) {
 490		dev_err(dev, "vpu pdev failed\n");
 491		return NULL;
 492	}
 493
 494	return vpu_pdev;
 495}
 496EXPORT_SYMBOL_GPL(vpu_get_plat_device);
 497
 498/* load vpu program/data memory */
 499static int load_requested_vpu(struct mtk_vpu *vpu,
 500			      u8 fw_type)
 501{
 502	size_t tcm_size = fw_type ? VPU_DTCM_SIZE : VPU_PTCM_SIZE;
 503	size_t fw_size = fw_type ? VPU_D_FW_SIZE : VPU_P_FW_SIZE;
 504	char *fw_name = fw_type ? VPU_D_FW : VPU_P_FW;
 505	char *fw_new_name = fw_type ? VPU_D_FW_NEW : VPU_P_FW_NEW;
 506	const struct firmware *vpu_fw;
 507	size_t dl_size = 0;
 508	size_t extra_fw_size = 0;
 509	void *dest;
 510	int ret;
 511
 512	ret = request_firmware(&vpu_fw, fw_new_name, vpu->dev);
 513	if (ret < 0) {
 514		dev_info(vpu->dev, "Failed to load %s, %d, retry\n",
 515			 fw_new_name, ret);
 516
 517		ret = request_firmware(&vpu_fw, fw_name, vpu->dev);
 518		if (ret < 0) {
 519			dev_err(vpu->dev, "Failed to load %s, %d\n", fw_name,
 520				ret);
 521			return ret;
 522		}
 523	}
 524	dl_size = vpu_fw->size;
 525	if (dl_size > fw_size) {
 526		dev_err(vpu->dev, "fw %s size %zu is abnormal\n", fw_name,
 527			dl_size);
 528		release_firmware(vpu_fw);
 529		return  -EFBIG;
 530	}
 531	dev_dbg(vpu->dev, "Downloaded fw %s size: %zu.\n",
 532		fw_name,
 533		dl_size);
 534	/* reset VPU */
 535	vpu_cfg_writel(vpu, 0x0, VPU_RESET);
 536
 537	/* handle extended firmware size */
 538	if (dl_size > tcm_size) {
 539		dev_dbg(vpu->dev, "fw size %zu > limited fw size %zu\n",
 540			dl_size, tcm_size);
 541		extra_fw_size = dl_size - tcm_size;
 542		dev_dbg(vpu->dev, "extra_fw_size %zu\n", extra_fw_size);
 543		dl_size = tcm_size;
 544	}
 545	dest = (__force void *)vpu->reg.tcm;
 546	if (fw_type == D_FW)
 547		dest += VPU_DTCM_OFFSET;
 548	memcpy(dest, vpu_fw->data, dl_size);
 549	/* download to extended memory if need */
 550	if (extra_fw_size > 0) {
 551		dest = vpu->extmem[fw_type].va;
 552		dev_dbg(vpu->dev, "download extended memory type %x\n",
 553			fw_type);
 554		memcpy(dest, vpu_fw->data + tcm_size, extra_fw_size);
 555	}
 556
 557	release_firmware(vpu_fw);
 558
 559	return 0;
 560}
 561
 562int vpu_load_firmware(struct platform_device *pdev)
 563{
 564	struct mtk_vpu *vpu;
 565	struct device *dev;
 566	struct vpu_run *run;
 567	int ret;
 568
 569	if (!pdev) {
 570		pr_err("VPU platform device is invalid\n");
 571		return -EINVAL;
 572	}
 573
 574	dev = &pdev->dev;
 575
 576	vpu = platform_get_drvdata(pdev);
 577	run = &vpu->run;
 578
 579	mutex_lock(&vpu->vpu_mutex);
 580	if (vpu->fw_loaded) {
 581		mutex_unlock(&vpu->vpu_mutex);
 582		return 0;
 583	}
 584	mutex_unlock(&vpu->vpu_mutex);
 585
 586	ret = vpu_clock_enable(vpu);
 587	if (ret) {
 588		dev_err(dev, "enable clock failed %d\n", ret);
 589		return ret;
 590	}
 591
 592	mutex_lock(&vpu->vpu_mutex);
 593
 594	run->signaled = false;
 595	dev_dbg(vpu->dev, "firmware request\n");
 596	/* Downloading program firmware to device*/
 597	ret = load_requested_vpu(vpu, P_FW);
 598	if (ret < 0) {
 599		dev_err(dev, "Failed to request %s, %d\n", VPU_P_FW, ret);
 600		goto OUT_LOAD_FW;
 601	}
 602
 603	/* Downloading data firmware to device */
 604	ret = load_requested_vpu(vpu, D_FW);
 605	if (ret < 0) {
 606		dev_err(dev, "Failed to request %s, %d\n", VPU_D_FW, ret);
 607		goto OUT_LOAD_FW;
 608	}
 609
 610	vpu->fw_loaded = true;
 611	/* boot up vpu */
 612	vpu_cfg_writel(vpu, 0x1, VPU_RESET);
 613
 614	ret = wait_event_interruptible_timeout(run->wq,
 615					       run->signaled,
 616					       msecs_to_jiffies(INIT_TIMEOUT_MS)
 617					       );
 618	if (ret == 0) {
 619		ret = -ETIME;
 620		dev_err(dev, "wait vpu initialization timeout!\n");
 621		goto OUT_LOAD_FW;
 622	} else if (-ERESTARTSYS == ret) {
 623		dev_err(dev, "wait vpu interrupted by a signal!\n");
 624		goto OUT_LOAD_FW;
 625	}
 626
 627	ret = 0;
 628	dev_info(dev, "vpu is ready. Fw version %s\n", run->fw_ver);
 629
 630OUT_LOAD_FW:
 631	mutex_unlock(&vpu->vpu_mutex);
 632	vpu_clock_disable(vpu);
 633
 634	return ret;
 635}
 636EXPORT_SYMBOL_GPL(vpu_load_firmware);
 637
 638static void vpu_init_ipi_handler(void *data, unsigned int len, void *priv)
 639{
 640	struct mtk_vpu *vpu = priv;
 641	const struct vpu_run *run = data;
 642
 643	vpu->run.signaled = run->signaled;
 644	strscpy(vpu->run.fw_ver, run->fw_ver, sizeof(vpu->run.fw_ver));
 645	vpu->run.dec_capability = run->dec_capability;
 646	vpu->run.enc_capability = run->enc_capability;
 647	wake_up_interruptible(&vpu->run.wq);
 648}
 649
 650#ifdef CONFIG_DEBUG_FS
 651static ssize_t vpu_debug_read(struct file *file, char __user *user_buf,
 652			      size_t count, loff_t *ppos)
 653{
 654	char buf[256];
 655	unsigned int len;
 656	unsigned int running, pc, vpu_to_host, host_to_vpu, wdt, idle, ra, sp;
 657	int ret;
 658	struct device *dev = file->private_data;
 659	struct mtk_vpu *vpu = dev_get_drvdata(dev);
 660
 661	ret = vpu_clock_enable(vpu);
 662	if (ret) {
 663		dev_err(vpu->dev, "[VPU] enable clock failed %d\n", ret);
 664		return 0;
 665	}
 666
 667	/* vpu register status */
 668	running = vpu_running(vpu);
 669	pc = vpu_cfg_readl(vpu, VPU_PC_REG);
 670	wdt = vpu_cfg_readl(vpu, VPU_WDT_REG);
 671	host_to_vpu = vpu_cfg_readl(vpu, HOST_TO_VPU);
 672	vpu_to_host = vpu_cfg_readl(vpu, VPU_TO_HOST);
 673	ra = vpu_cfg_readl(vpu, VPU_RA_REG);
 674	sp = vpu_cfg_readl(vpu, VPU_SP_REG);
 675	idle = vpu_cfg_readl(vpu, VPU_IDLE_REG);
 676
 677	vpu_clock_disable(vpu);
 678
 679	if (running) {
 680		len = snprintf(buf, sizeof(buf), "VPU is running\n\n"
 681		"FW Version: %s\n"
 682		"PC: 0x%x\n"
 683		"WDT: 0x%x\n"
 684		"Host to VPU: 0x%x\n"
 685		"VPU to Host: 0x%x\n"
 686		"SP: 0x%x\n"
 687		"RA: 0x%x\n"
 688		"idle: 0x%x\n",
 689		vpu->run.fw_ver, pc, wdt,
 690		host_to_vpu, vpu_to_host, sp, ra, idle);
 691	} else {
 692		len = snprintf(buf, sizeof(buf), "VPU not running\n");
 693	}
 694
 695	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
 696}
 697
 698static const struct file_operations vpu_debug_fops = {
 699	.open = simple_open,
 700	.read = vpu_debug_read,
 701};
 702#endif /* CONFIG_DEBUG_FS */
 703
 704static void vpu_free_ext_mem(struct mtk_vpu *vpu, u8 fw_type)
 705{
 706	struct device *dev = vpu->dev;
 707	size_t fw_ext_size = fw_type ? VPU_EXT_D_SIZE : VPU_EXT_P_SIZE;
 708
 709	dma_free_coherent(dev, fw_ext_size, vpu->extmem[fw_type].va,
 710			  vpu->extmem[fw_type].pa);
 711}
 712
 713static int vpu_alloc_ext_mem(struct mtk_vpu *vpu, u32 fw_type)
 714{
 715	struct device *dev = vpu->dev;
 716	size_t fw_ext_size = fw_type ? VPU_EXT_D_SIZE : VPU_EXT_P_SIZE;
 717	u32 vpu_ext_mem0 = fw_type ? VPU_DMEM_EXT0_ADDR : VPU_PMEM_EXT0_ADDR;
 718	u32 vpu_ext_mem1 = fw_type ? VPU_DMEM_EXT1_ADDR : VPU_PMEM_EXT1_ADDR;
 719	u32 offset_4gb = vpu->enable_4GB ? 0x40000000 : 0;
 720
 721	vpu->extmem[fw_type].va = dma_alloc_coherent(dev,
 722					       fw_ext_size,
 723					       &vpu->extmem[fw_type].pa,
 724					       GFP_KERNEL);
 725	if (!vpu->extmem[fw_type].va) {
 726		dev_err(dev, "Failed to allocate the extended program memory\n");
 727		return -ENOMEM;
 728	}
 729
 730	/* Disable extend0. Enable extend1 */
 731	vpu_cfg_writel(vpu, 0x1, vpu_ext_mem0);
 732	vpu_cfg_writel(vpu, (vpu->extmem[fw_type].pa & 0xFFFFF000) + offset_4gb,
 733		       vpu_ext_mem1);
 734
 735	dev_info(dev, "%s extend memory phy=0x%llx virt=0x%p\n",
 736		 fw_type ? "Data" : "Program",
 737		 (unsigned long long)vpu->extmem[fw_type].pa,
 738		 vpu->extmem[fw_type].va);
 739
 740	return 0;
 741}
 742
 743static void vpu_ipi_handler(struct mtk_vpu *vpu)
 744{
 745	struct share_obj __iomem *rcv_obj = vpu->recv_buf;
 746	struct vpu_ipi_desc *ipi_desc = vpu->ipi_desc;
 747	unsigned char data[SHARE_BUF_SIZE];
 748	s32 id = readl(&rcv_obj->id);
 749
 750	memcpy_fromio(data, rcv_obj->share_buf, sizeof(data));
 751	if (id < IPI_MAX && ipi_desc[id].handler) {
 752		ipi_desc[id].handler(data, readl(&rcv_obj->len),
 753				     ipi_desc[id].priv);
 754		if (id > IPI_VPU_INIT) {
 755			vpu->ipi_id_ack[id] = true;
 756			wake_up(&vpu->ack_wq);
 757		}
 758	} else {
 759		dev_err(vpu->dev, "No such ipi id = %d\n", id);
 760	}
 761}
 762
 763static int vpu_ipi_init(struct mtk_vpu *vpu)
 764{
 765	/* Disable VPU to host interrupt */
 766	vpu_cfg_writel(vpu, 0x0, VPU_TO_HOST);
 767
 768	/* shared buffer initialization */
 769	vpu->recv_buf = vpu->reg.tcm + VPU_DTCM_OFFSET;
 770	vpu->send_buf = vpu->recv_buf + 1;
 771	memset_io(vpu->recv_buf, 0, sizeof(struct share_obj));
 772	memset_io(vpu->send_buf, 0, sizeof(struct share_obj));
 773
 774	return 0;
 775}
 776
 777static irqreturn_t vpu_irq_handler(int irq, void *priv)
 778{
 779	struct mtk_vpu *vpu = priv;
 780	u32 vpu_to_host;
 781	int ret;
 782
 783	/*
 784	 * Clock should have been enabled already.
 785	 * Enable again in case vpu_ipi_send times out
 786	 * and has disabled the clock.
 787	 */
 788	ret = clk_enable(vpu->clk);
 789	if (ret) {
 790		dev_err(vpu->dev, "[VPU] enable clock failed %d\n", ret);
 791		return IRQ_NONE;
 792	}
 793	vpu_to_host = vpu_cfg_readl(vpu, VPU_TO_HOST);
 794	if (vpu_to_host & VPU_IPC_INT) {
 795		vpu_ipi_handler(vpu);
 796	} else {
 797		dev_err(vpu->dev, "vpu watchdog timeout! 0x%x", vpu_to_host);
 798		queue_work(vpu->wdt.wq, &vpu->wdt.ws);
 799	}
 800
 801	/* VPU won't send another interrupt until we set VPU_TO_HOST to 0. */
 802	vpu_cfg_writel(vpu, 0x0, VPU_TO_HOST);
 803	clk_disable(vpu->clk);
 804
 805	return IRQ_HANDLED;
 806}
 807
 808#ifdef CONFIG_DEBUG_FS
 809static struct dentry *vpu_debugfs;
 810#endif
 811static int mtk_vpu_probe(struct platform_device *pdev)
 812{
 813	struct mtk_vpu *vpu;
 814	struct device *dev;
 815	int ret = 0;
 816
 817	dev_dbg(&pdev->dev, "initialization\n");
 818
 819	dev = &pdev->dev;
 820	vpu = devm_kzalloc(dev, sizeof(*vpu), GFP_KERNEL);
 821	if (!vpu)
 822		return -ENOMEM;
 823
 824	vpu->dev = &pdev->dev;
 825	vpu->reg.tcm = devm_platform_ioremap_resource_byname(pdev, "tcm");
 826	if (IS_ERR((__force void *)vpu->reg.tcm))
 827		return PTR_ERR((__force void *)vpu->reg.tcm);
 828
 829	vpu->reg.cfg = devm_platform_ioremap_resource_byname(pdev, "cfg_reg");
 830	if (IS_ERR((__force void *)vpu->reg.cfg))
 831		return PTR_ERR((__force void *)vpu->reg.cfg);
 832
 833	/* Get VPU clock */
 834	vpu->clk = devm_clk_get(dev, "main");
 835	if (IS_ERR(vpu->clk)) {
 836		dev_err(dev, "get vpu clock failed\n");
 837		return PTR_ERR(vpu->clk);
 838	}
 839
 840	platform_set_drvdata(pdev, vpu);
 841
 842	ret = clk_prepare(vpu->clk);
 843	if (ret) {
 844		dev_err(dev, "prepare vpu clock failed\n");
 845		return ret;
 846	}
 847
 848	/* VPU watchdog */
 849	vpu->wdt.wq = create_singlethread_workqueue("vpu_wdt");
 850	if (!vpu->wdt.wq) {
 851		dev_err(dev, "initialize wdt workqueue failed\n");
 852		ret = -ENOMEM;
 853		goto clk_unprepare;
 854	}
 855	INIT_WORK(&vpu->wdt.ws, vpu_wdt_reset_func);
 856	mutex_init(&vpu->vpu_mutex);
 857
 858	ret = vpu_clock_enable(vpu);
 859	if (ret) {
 860		dev_err(dev, "enable vpu clock failed\n");
 861		goto workqueue_destroy;
 862	}
 863
 864	dev_dbg(dev, "vpu ipi init\n");
 865	ret = vpu_ipi_init(vpu);
 866	if (ret) {
 867		dev_err(dev, "Failed to init ipi\n");
 868		goto disable_vpu_clk;
 869	}
 870
 871	/* register vpu initialization IPI */
 872	ret = vpu_ipi_register(pdev, IPI_VPU_INIT, vpu_init_ipi_handler,
 873			       "vpu_init", vpu);
 874	if (ret) {
 875		dev_err(dev, "Failed to register IPI_VPU_INIT\n");
 876		goto vpu_mutex_destroy;
 877	}
 878
 879#ifdef CONFIG_DEBUG_FS
 880	vpu_debugfs = debugfs_create_file("mtk_vpu", S_IRUGO, NULL, (void *)dev,
 881					  &vpu_debug_fops);
 882#endif
 883
 884	/* Set PTCM to 96K and DTCM to 32K */
 885	vpu_cfg_writel(vpu, 0x2, VPU_TCM_CFG);
 886
 887	vpu->enable_4GB = !!(totalram_pages() > (SZ_2G >> PAGE_SHIFT));
 888	dev_info(dev, "4GB mode %u\n", vpu->enable_4GB);
 889
 890	if (vpu->enable_4GB) {
 891		ret = of_reserved_mem_device_init(dev);
 892		if (ret)
 893			dev_info(dev, "init reserved memory failed\n");
 894			/* continue to use dynamic allocation if failed */
 895	}
 896
 897	ret = vpu_alloc_ext_mem(vpu, D_FW);
 898	if (ret) {
 899		dev_err(dev, "Allocate DM failed\n");
 900		goto remove_debugfs;
 901	}
 902
 903	ret = vpu_alloc_ext_mem(vpu, P_FW);
 904	if (ret) {
 905		dev_err(dev, "Allocate PM failed\n");
 906		goto free_d_mem;
 907	}
 908
 909	init_waitqueue_head(&vpu->run.wq);
 910	init_waitqueue_head(&vpu->ack_wq);
 911
 912	ret = platform_get_irq(pdev, 0);
 913	if (ret < 0)
 914		goto free_p_mem;
 915	vpu->reg.irq = ret;
 916	ret = devm_request_irq(dev, vpu->reg.irq, vpu_irq_handler, 0,
 917			       pdev->name, vpu);
 918	if (ret) {
 919		dev_err(dev, "failed to request irq\n");
 920		goto free_p_mem;
 921	}
 922
 923	vpu_clock_disable(vpu);
 924	dev_dbg(dev, "initialization completed\n");
 925
 926	return 0;
 927
 928free_p_mem:
 929	vpu_free_ext_mem(vpu, P_FW);
 930free_d_mem:
 931	vpu_free_ext_mem(vpu, D_FW);
 932remove_debugfs:
 933	of_reserved_mem_device_release(dev);
 934#ifdef CONFIG_DEBUG_FS
 935	debugfs_remove(vpu_debugfs);
 936#endif
 937	memset(vpu->ipi_desc, 0, sizeof(struct vpu_ipi_desc) * IPI_MAX);
 938vpu_mutex_destroy:
 939	mutex_destroy(&vpu->vpu_mutex);
 940disable_vpu_clk:
 941	vpu_clock_disable(vpu);
 942workqueue_destroy:
 943	destroy_workqueue(vpu->wdt.wq);
 944clk_unprepare:
 945	clk_unprepare(vpu->clk);
 946
 947	return ret;
 948}
 949
 950static const struct of_device_id mtk_vpu_match[] = {
 951	{
 952		.compatible = "mediatek,mt8173-vpu",
 953	},
 954	{},
 955};
 956MODULE_DEVICE_TABLE(of, mtk_vpu_match);
 957
 958static void mtk_vpu_remove(struct platform_device *pdev)
 959{
 960	struct mtk_vpu *vpu = platform_get_drvdata(pdev);
 961
 962#ifdef CONFIG_DEBUG_FS
 963	debugfs_remove(vpu_debugfs);
 964#endif
 965	if (vpu->wdt.wq)
 966		destroy_workqueue(vpu->wdt.wq);
 967	vpu_free_ext_mem(vpu, P_FW);
 968	vpu_free_ext_mem(vpu, D_FW);
 969	mutex_destroy(&vpu->vpu_mutex);
 970	clk_unprepare(vpu->clk);
 971}
 972
 973static int mtk_vpu_suspend(struct device *dev)
 974{
 975	struct mtk_vpu *vpu = dev_get_drvdata(dev);
 976	unsigned long timeout;
 977	int ret;
 978
 979	ret = vpu_clock_enable(vpu);
 980	if (ret) {
 981		dev_err(dev, "failed to enable vpu clock\n");
 982		return ret;
 983	}
 984
 985	if (!vpu_running(vpu)) {
 986		vpu_clock_disable(vpu);
 987		clk_unprepare(vpu->clk);
 988		return 0;
 989	}
 990
 991	mutex_lock(&vpu->vpu_mutex);
 992	/* disable vpu timer interrupt */
 993	vpu_cfg_writel(vpu, vpu_cfg_readl(vpu, VPU_INT_STATUS) | VPU_IDLE_STATE,
 994		       VPU_INT_STATUS);
 995	/* check if vpu is idle for system suspend */
 996	timeout = jiffies + msecs_to_jiffies(VPU_IDLE_TIMEOUT_MS);
 997	do {
 998		if (time_after(jiffies, timeout)) {
 999			dev_err(dev, "vpu idle timeout\n");
1000			mutex_unlock(&vpu->vpu_mutex);
1001			vpu_clock_disable(vpu);
1002			return -EIO;
1003		}
1004	} while (!vpu_cfg_readl(vpu, VPU_IDLE_REG));
1005
1006	mutex_unlock(&vpu->vpu_mutex);
1007	vpu_clock_disable(vpu);
1008	clk_unprepare(vpu->clk);
1009
1010	return 0;
1011}
1012
1013static int mtk_vpu_resume(struct device *dev)
1014{
1015	struct mtk_vpu *vpu = dev_get_drvdata(dev);
1016	int ret;
1017
1018	clk_prepare(vpu->clk);
1019	ret = vpu_clock_enable(vpu);
1020	if (ret) {
1021		clk_unprepare(vpu->clk);
1022		dev_err(dev, "failed to enable vpu clock\n");
1023		return ret;
1024	}
1025
1026	mutex_lock(&vpu->vpu_mutex);
1027	/* enable vpu timer interrupt */
1028	vpu_cfg_writel(vpu,
1029		       vpu_cfg_readl(vpu, VPU_INT_STATUS) & ~(VPU_IDLE_STATE),
1030		       VPU_INT_STATUS);
1031	mutex_unlock(&vpu->vpu_mutex);
1032	vpu_clock_disable(vpu);
1033
1034	return 0;
1035}
1036
1037static const struct dev_pm_ops mtk_vpu_pm = {
1038	.suspend = mtk_vpu_suspend,
1039	.resume = mtk_vpu_resume,
1040};
1041
1042static struct platform_driver mtk_vpu_driver = {
1043	.probe	= mtk_vpu_probe,
1044	.remove = mtk_vpu_remove,
1045	.driver	= {
1046		.name	= "mtk_vpu",
1047		.pm = &mtk_vpu_pm,
1048		.of_match_table = mtk_vpu_match,
1049	},
1050};
1051
1052module_platform_driver(mtk_vpu_driver);
1053
1054MODULE_LICENSE("GPL v2");
1055MODULE_DESCRIPTION("Mediatek Video Processor Unit driver");