ivpu_hw_ip.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2020-2024 Intel Corporation
   4 */
   5
   6#include "ivpu_drv.h"
   7#include "ivpu_fw.h"
   8#include "ivpu_hw.h"
   9#include "ivpu_hw_37xx_reg.h"
  10#include "ivpu_hw_40xx_reg.h"
  11#include "ivpu_hw_btrs.h"
  12#include "ivpu_hw_ip.h"
  13#include "ivpu_hw_reg_io.h"
  14#include "ivpu_mmu.h"
  15#include "ivpu_pm.h"
  16
  17#define PWR_ISLAND_STATUS_TIMEOUT_US        (5 * USEC_PER_MSEC)
  18
  19#define TIM_SAFE_ENABLE		            0xf1d0dead
  20#define TIM_WATCHDOG_RESET_VALUE            0xffffffff
  21
  22#define ICB_0_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
  23			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
  24			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
  25			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
  26			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
  27			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
  28			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
  29
  30#define ICB_1_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
  31			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
  32			     (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
  33
  34#define ICB_0_1_IRQ_MASK_37XX ((((u64)ICB_1_IRQ_MASK_37XX) << 32) | ICB_0_IRQ_MASK_37XX)
  35
  36#define ICB_0_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
  37			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
  38			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
  39			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
  40			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
  41			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
  42			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
  43
  44#define ICB_1_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
  45			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
  46			     (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
  47
  48#define ICB_0_1_IRQ_MASK_40XX ((((u64)ICB_1_IRQ_MASK_40XX) << 32) | ICB_0_IRQ_MASK_40XX)
  49
  50#define ITF_FIREWALL_VIOLATION_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
  51					  (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
  52					  (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
  53					  (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
  54					  (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
  55					  (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
  56					  (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
  57
  58#define ITF_FIREWALL_VIOLATION_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
  59					  (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
  60					  (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
  61					  (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
  62					  (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
  63					  (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
  64					  (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
  65
  66static int wait_for_ip_bar(struct ivpu_device *vdev)
  67{
  68	return REGV_POLL_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, AON, 0, 100);
  69}
  70
  71static void host_ss_rst_clr(struct ivpu_device *vdev)
  72{
  73	u32 val = 0;
  74
  75	val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, TOP_NOC, val);
  76	val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, DSS_MAS, val);
  77	val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, MSS_MAS, val);
  78
  79	REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_CLR, val);
  80}
  81
  82static int host_ss_noc_qreqn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
  83{
  84	u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN);
  85
  86	if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
  87		return -EIO;
  88
  89	return 0;
  90}
  91
  92static int host_ss_noc_qreqn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
  93{
  94	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
  95
  96	if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
  97		return -EIO;
  98
  99	return 0;
 100}
 101
 102static int host_ss_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
 103{
 104	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 105		return host_ss_noc_qreqn_check_37xx(vdev, exp_val);
 106	else
 107		return host_ss_noc_qreqn_check_40xx(vdev, exp_val);
 108}
 109
 110static int host_ss_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 111{
 112	u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QACCEPTN);
 113
 114	if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
 115		return -EIO;
 116
 117	return 0;
 118}
 119
 120static int host_ss_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 121{
 122	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN);
 123
 124	if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
 125		return -EIO;
 126
 127	return 0;
 128}
 129
 130static int host_ss_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 131{
 132	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 133		return host_ss_noc_qacceptn_check_37xx(vdev, exp_val);
 134	else
 135		return host_ss_noc_qacceptn_check_40xx(vdev, exp_val);
 136}
 137
 138static int host_ss_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 139{
 140	u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QDENY);
 141
 142	if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
 143		return -EIO;
 144
 145	return 0;
 146}
 147
 148static int host_ss_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 149{
 150	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY);
 151
 152	if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
 153		return -EIO;
 154
 155	return 0;
 156}
 157
 158static int host_ss_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 159{
 160	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 161		return host_ss_noc_qdeny_check_37xx(vdev, exp_val);
 162	else
 163		return host_ss_noc_qdeny_check_40xx(vdev, exp_val);
 164}
 165
 166static int top_noc_qrenqn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 167{
 168	u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN);
 169
 170	if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
 171	    !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
 172		return -EIO;
 173
 174	return 0;
 175}
 176
 177static int top_noc_qrenqn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 178{
 179	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
 180
 181	if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
 182	    !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
 183		return -EIO;
 184
 185	return 0;
 186}
 187
 188static int top_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
 189{
 190	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 191		return top_noc_qrenqn_check_37xx(vdev, exp_val);
 192	else
 193		return top_noc_qrenqn_check_40xx(vdev, exp_val);
 194}
 195
 196int ivpu_hw_ip_host_ss_configure(struct ivpu_device *vdev)
 197{
 198	int ret;
 199
 200	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 201		ret = wait_for_ip_bar(vdev);
 202		if (ret) {
 203			ivpu_err(vdev, "Timed out waiting for NPU IP bar\n");
 204			return ret;
 205		}
 206		host_ss_rst_clr(vdev);
 207	}
 208
 209	ret = host_ss_noc_qreqn_check(vdev, 0x0);
 210	if (ret) {
 211		ivpu_err(vdev, "Failed qreqn check: %d\n", ret);
 212		return ret;
 213	}
 214
 215	ret = host_ss_noc_qacceptn_check(vdev, 0x0);
 216	if (ret) {
 217		ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 218		return ret;
 219	}
 220
 221	ret = host_ss_noc_qdeny_check(vdev, 0x0);
 222	if (ret)
 223		ivpu_err(vdev, "Failed qdeny check %d\n", ret);
 224
 225	return ret;
 226}
 227
 228static void idle_gen_drive_37xx(struct ivpu_device *vdev, bool enable)
 229{
 230	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN);
 231
 232	if (enable)
 233		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val);
 234	else
 235		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val);
 236
 237	REGV_WR32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, val);
 238}
 239
 240static void idle_gen_drive_40xx(struct ivpu_device *vdev, bool enable)
 241{
 242	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN);
 243
 244	if (enable)
 245		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
 246	else
 247		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
 248
 249	REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val);
 250}
 251
 252void ivpu_hw_ip_idle_gen_enable(struct ivpu_device *vdev)
 253{
 254	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 255		idle_gen_drive_37xx(vdev, true);
 256	else
 257		idle_gen_drive_40xx(vdev, true);
 258}
 259
 260void ivpu_hw_ip_idle_gen_disable(struct ivpu_device *vdev)
 261{
 262	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 263		idle_gen_drive_37xx(vdev, false);
 264	else
 265		idle_gen_drive_40xx(vdev, false);
 266}
 267
 268static void
 269pwr_island_delay_set_50xx(struct ivpu_device *vdev, u32 post, u32 post1, u32 post2, u32 status)
 270{
 271	u32 val;
 272
 273	val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY);
 274	val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST_DLY, post, val);
 275	val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST1_DLY, post1, val);
 276	val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST2_DLY, post2, val);
 277	REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, val);
 278
 279	val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY);
 280	val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, STATUS_DLY, status, val);
 281	REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, val);
 282}
 283
 284static void pwr_island_trickle_drive_37xx(struct ivpu_device *vdev, bool enable)
 285{
 286	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
 287
 288	if (enable)
 289		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val);
 290	else
 291		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val);
 292
 293	REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
 294}
 295
 296static void pwr_island_trickle_drive_40xx(struct ivpu_device *vdev, bool enable)
 297{
 298	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
 299
 300	if (enable)
 301		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
 302	else
 303		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
 304
 305	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
 306}
 307
 308static void pwr_island_drive_37xx(struct ivpu_device *vdev, bool enable)
 309{
 310	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0);
 311
 312	if (enable)
 313		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 314	else
 315		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 316
 317	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 318}
 319
 320static void pwr_island_drive_40xx(struct ivpu_device *vdev, bool enable)
 321{
 322	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0);
 323
 324	if (enable)
 325		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 326	else
 327		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 328
 329	REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 330}
 331
 332static void pwr_island_enable(struct ivpu_device *vdev)
 333{
 334	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 335		pwr_island_trickle_drive_37xx(vdev, true);
 336		ndelay(500);
 337		pwr_island_drive_37xx(vdev, true);
 338	} else {
 339		pwr_island_trickle_drive_40xx(vdev, true);
 340		ndelay(500);
 341		pwr_island_drive_40xx(vdev, true);
 342	}
 343}
 344
 345static int wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val)
 346{
 347	if (IVPU_WA(punit_disabled))
 348		return 0;
 349
 350	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 351		return REGV_POLL_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0, MSS_CPU, exp_val,
 352				     PWR_ISLAND_STATUS_TIMEOUT_US);
 353	else
 354		return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, exp_val,
 355				     PWR_ISLAND_STATUS_TIMEOUT_US);
 356}
 357
 358static void pwr_island_isolation_drive_37xx(struct ivpu_device *vdev, bool enable)
 359{
 360	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0);
 361
 362	if (enable)
 363		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val);
 364	else
 365		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val);
 366
 367	REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, val);
 368}
 369
 370static void pwr_island_isolation_drive_40xx(struct ivpu_device *vdev, bool enable)
 371{
 372	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0);
 373
 374	if (enable)
 375		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
 376	else
 377		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
 378
 379	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val);
 380}
 381
 382static void pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable)
 383{
 384	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 385		pwr_island_isolation_drive_37xx(vdev, enable);
 386	else
 387		pwr_island_isolation_drive_40xx(vdev, enable);
 388}
 389
 390static void pwr_island_isolation_disable(struct ivpu_device *vdev)
 391{
 392	pwr_island_isolation_drive(vdev, false);
 393}
 394
 395static void host_ss_clk_drive_37xx(struct ivpu_device *vdev, bool enable)
 396{
 397	u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_CLK_SET);
 398
 399	if (enable) {
 400		val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val);
 401		val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val);
 402		val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val);
 403	} else {
 404		val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val);
 405		val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val);
 406		val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val);
 407	}
 408
 409	REGV_WR32(VPU_37XX_HOST_SS_CPR_CLK_SET, val);
 410}
 411
 412static void host_ss_clk_drive_40xx(struct ivpu_device *vdev, bool enable)
 413{
 414	u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN);
 415
 416	if (enable) {
 417		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
 418		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
 419		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
 420	} else {
 421		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
 422		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
 423		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
 424	}
 425
 426	REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val);
 427}
 428
 429static void host_ss_clk_drive(struct ivpu_device *vdev, bool enable)
 430{
 431	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 432		host_ss_clk_drive_37xx(vdev, enable);
 433	else
 434		host_ss_clk_drive_40xx(vdev, enable);
 435}
 436
 437static void host_ss_clk_enable(struct ivpu_device *vdev)
 438{
 439	host_ss_clk_drive(vdev, true);
 440}
 441
 442static void host_ss_rst_drive_37xx(struct ivpu_device *vdev, bool enable)
 443{
 444	u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_RST_SET);
 445
 446	if (enable) {
 447		val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val);
 448		val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val);
 449		val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val);
 450	} else {
 451		val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val);
 452		val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val);
 453		val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val);
 454	}
 455
 456	REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_SET, val);
 457}
 458
 459static void host_ss_rst_drive_40xx(struct ivpu_device *vdev, bool enable)
 460{
 461	u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN);
 462
 463	if (enable) {
 464		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
 465		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
 466		val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
 467	} else {
 468		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
 469		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
 470		val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
 471	}
 472
 473	REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val);
 474}
 475
 476static void host_ss_rst_drive(struct ivpu_device *vdev, bool enable)
 477{
 478	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 479		host_ss_rst_drive_37xx(vdev, enable);
 480	else
 481		host_ss_rst_drive_40xx(vdev, enable);
 482}
 483
 484static void host_ss_rst_enable(struct ivpu_device *vdev)
 485{
 486	host_ss_rst_drive(vdev, true);
 487}
 488
 489static void host_ss_noc_qreqn_top_socmmio_drive_37xx(struct ivpu_device *vdev, bool enable)
 490{
 491	u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN);
 492
 493	if (enable)
 494		val = REG_SET_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 495	else
 496		val = REG_CLR_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 497	REGV_WR32(VPU_37XX_HOST_SS_NOC_QREQN, val);
 498}
 499
 500static void host_ss_noc_qreqn_top_socmmio_drive_40xx(struct ivpu_device *vdev, bool enable)
 501{
 502	u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
 503
 504	if (enable)
 505		val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 506	else
 507		val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 508	REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val);
 509}
 510
 511static void host_ss_noc_qreqn_top_socmmio_drive(struct ivpu_device *vdev, bool enable)
 512{
 513	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 514		host_ss_noc_qreqn_top_socmmio_drive_37xx(vdev, enable);
 515	else
 516		host_ss_noc_qreqn_top_socmmio_drive_40xx(vdev, enable);
 517}
 518
 519static int host_ss_axi_drive(struct ivpu_device *vdev, bool enable)
 520{
 521	int ret;
 522
 523	host_ss_noc_qreqn_top_socmmio_drive(vdev, enable);
 524
 525	ret = host_ss_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 526	if (ret) {
 527		ivpu_err(vdev, "Failed HOST SS NOC QACCEPTN check: %d\n", ret);
 528		return ret;
 529	}
 530
 531	ret = host_ss_noc_qdeny_check(vdev, 0x0);
 532	if (ret)
 533		ivpu_err(vdev, "Failed HOST SS NOC QDENY check: %d\n", ret);
 534
 535	return ret;
 536}
 537
 538static void top_noc_qreqn_drive_40xx(struct ivpu_device *vdev, bool enable)
 539{
 540	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
 541
 542	if (enable) {
 543		val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
 544		val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 545	} else {
 546		val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
 547		val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 548	}
 549
 550	REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val);
 551}
 552
 553static void top_noc_qreqn_drive_37xx(struct ivpu_device *vdev, bool enable)
 554{
 555	u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN);
 556
 557	if (enable) {
 558		val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val);
 559		val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 560	} else {
 561		val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val);
 562		val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 563	}
 564
 565	REGV_WR32(VPU_37XX_TOP_NOC_QREQN, val);
 566}
 567
 568static void top_noc_qreqn_drive(struct ivpu_device *vdev, bool enable)
 569{
 570	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 571		top_noc_qreqn_drive_37xx(vdev, enable);
 572	else
 573		top_noc_qreqn_drive_40xx(vdev, enable);
 574}
 575
 576int ivpu_hw_ip_host_ss_axi_enable(struct ivpu_device *vdev)
 577{
 578	return host_ss_axi_drive(vdev, true);
 579}
 580
 581static int top_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 582{
 583	u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QACCEPTN);
 584
 585	if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
 586	    !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
 587		return -EIO;
 588
 589	return 0;
 590}
 591
 592static int top_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 593{
 594	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN);
 595
 596	if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
 597	    !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
 598		return -EIO;
 599
 600	return 0;
 601}
 602
 603static int top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 604{
 605	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 606		return top_noc_qacceptn_check_37xx(vdev, exp_val);
 607	else
 608		return top_noc_qacceptn_check_40xx(vdev, exp_val);
 609}
 610
 611static int top_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 612{
 613	u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QDENY);
 614
 615	if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
 616	    !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
 617		return -EIO;
 618
 619	return 0;
 620}
 621
 622static int top_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 623{
 624	u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY);
 625
 626	if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
 627	    !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
 628		return -EIO;
 629
 630	return 0;
 631}
 632
 633static int top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 634{
 635	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 636		return top_noc_qdeny_check_37xx(vdev, exp_val);
 637	else
 638		return top_noc_qdeny_check_40xx(vdev, exp_val);
 639}
 640
 641static int top_noc_drive(struct ivpu_device *vdev, bool enable)
 642{
 643	int ret;
 644
 645	top_noc_qreqn_drive(vdev, enable);
 646
 647	ret = top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 648	if (ret) {
 649		ivpu_err(vdev, "Failed TOP NOC QACCEPTN check: %d\n", ret);
 650		return ret;
 651	}
 652
 653	ret = top_noc_qdeny_check(vdev, 0x0);
 654	if (ret)
 655		ivpu_err(vdev, "Failed TOP NOC QDENY check: %d\n", ret);
 656
 657	return ret;
 658}
 659
 660int ivpu_hw_ip_top_noc_enable(struct ivpu_device *vdev)
 661{
 662	return top_noc_drive(vdev, true);
 663}
 664
 665static void dpu_active_drive_37xx(struct ivpu_device *vdev, bool enable)
 666{
 667	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE);
 668
 669	if (enable)
 670		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val);
 671	else
 672		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val);
 673
 674	REGV_WR32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, val);
 675}
 676
 677static void pwr_island_delay_set(struct ivpu_device *vdev)
 678{
 679	bool high = vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_HIGH;
 680	u32 post, post1, post2, status;
 681
 682	if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX)
 683		return;
 684
 685	switch (ivpu_device_id(vdev)) {
 686	case PCI_DEVICE_ID_WCL:
 687	case PCI_DEVICE_ID_PTL_P:
 688		post = high ? 18 : 0;
 689		post1 = 0;
 690		post2 = 0;
 691		status = high ? 46 : 3;
 692		break;
 693
 694	case PCI_DEVICE_ID_NVL:
 695		post = high ? 198 : 17;
 696		post1 = 0;
 697		post2 = high ? 198 : 17;
 698		status = 0;
 699		break;
 700
 701	default:
 702		dump_stack();
 703		ivpu_err(vdev, "Unknown device ID\n");
 704		return;
 705	}
 706
 707	pwr_island_delay_set_50xx(vdev, post, post1, post2, status);
 708}
 709
 710int ivpu_hw_ip_pwr_domain_enable(struct ivpu_device *vdev)
 711{
 712	int ret;
 713
 714	pwr_island_delay_set(vdev);
 715	pwr_island_enable(vdev);
 716
 717	ret = wait_for_pwr_island_status(vdev, 0x1);
 718	if (ret) {
 719		ivpu_err(vdev, "Timed out waiting for power island status\n");
 720		return ret;
 721	}
 722
 723	ret = top_noc_qreqn_check(vdev, 0x0);
 724	if (ret) {
 725		ivpu_err(vdev, "Failed TOP NOC QREQN check %d\n", ret);
 726		return ret;
 727	}
 728
 729	host_ss_clk_enable(vdev);
 730	pwr_island_isolation_disable(vdev);
 731	host_ss_rst_enable(vdev);
 732
 733	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 734		dpu_active_drive_37xx(vdev, true);
 735
 736	return ret;
 737}
 738
 739u64 ivpu_hw_ip_read_perf_timer_counter(struct ivpu_device *vdev)
 740{
 741	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 742		return REGV_RD64(VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT);
 743	else
 744		return REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT);
 745}
 746
 747static void ivpu_hw_ip_snoop_disable_37xx(struct ivpu_device *vdev)
 748{
 749	u32 val = REGV_RD32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES);
 750
 751	val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, NOSNOOP_OVERRIDE_EN, val);
 752	val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AW_NOSNOOP_OVERRIDE, val);
 753
 754	if (ivpu_is_force_snoop_enabled(vdev))
 755		val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val);
 756	else
 757		val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val);
 758
 759	REGV_WR32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, val);
 760}
 761
 762static void ivpu_hw_ip_snoop_disable_40xx(struct ivpu_device *vdev)
 763{
 764	u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES);
 765
 766	val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val);
 767	val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val);
 768
 769	if (ivpu_is_force_snoop_enabled(vdev))
 770		val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val);
 771	else
 772		val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val);
 773
 774	REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val);
 775}
 776
 777void ivpu_hw_ip_snoop_disable(struct ivpu_device *vdev)
 778{
 779	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 780		return ivpu_hw_ip_snoop_disable_37xx(vdev);
 781	else
 782		return ivpu_hw_ip_snoop_disable_40xx(vdev);
 783}
 784
 785static void ivpu_hw_ip_tbu_mmu_enable_37xx(struct ivpu_device *vdev)
 786{
 787	u32 val = REGV_RD32(VPU_37XX_HOST_IF_TBU_MMUSSIDV);
 788
 789	val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
 790	val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
 791	val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
 792	val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
 793
 794	REGV_WR32(VPU_37XX_HOST_IF_TBU_MMUSSIDV, val);
 795}
 796
 797static void ivpu_hw_ip_tbu_mmu_enable_40xx(struct ivpu_device *vdev)
 798{
 799	u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV);
 800
 801	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
 802	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
 803	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val);
 804	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val);
 805	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
 806	val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
 807
 808	REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val);
 809}
 810
 811void ivpu_hw_ip_tbu_mmu_enable(struct ivpu_device *vdev)
 812{
 813	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 814		return ivpu_hw_ip_tbu_mmu_enable_37xx(vdev);
 815	else
 816		return ivpu_hw_ip_tbu_mmu_enable_40xx(vdev);
 817}
 818
 819static int soc_cpu_boot_37xx(struct ivpu_device *vdev)
 820{
 821	u32 val;
 822
 823	val = REGV_RD32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC);
 824	val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTRUN0, val);
 825
 826	val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTVEC, val);
 827	REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 828
 829	val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val);
 830	REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 831
 832	val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val);
 833	REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 834
 835	val = vdev->fw->entry_point >> 9;
 836	REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val);
 837
 838	val = REG_SET_FLD(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, DONE, val);
 839	REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val);
 840
 841	ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
 842		 vdev->fw->entry_point == vdev->fw->cold_boot_entry_point ? "cold boot" : "resume");
 843
 844	return 0;
 845}
 846
 847static int cpu_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 848{
 849	u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN);
 850
 851	if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val))
 852		return -EIO;
 853
 854	return 0;
 855}
 856
 857static int cpu_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 858{
 859	u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY);
 860
 861	if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val))
 862		return -EIO;
 863
 864	return 0;
 865}
 866
 867static void cpu_noc_top_mmio_drive_40xx(struct ivpu_device *vdev, bool enable)
 868{
 869	u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN);
 870
 871	if (enable)
 872		val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
 873	else
 874		val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
 875	REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val);
 876}
 877
 878static int soc_cpu_drive_40xx(struct ivpu_device *vdev, bool enable)
 879{
 880	int ret;
 881
 882	cpu_noc_top_mmio_drive_40xx(vdev, enable);
 883
 884	ret = cpu_noc_qacceptn_check_40xx(vdev, enable ? 0x1 : 0x0);
 885	if (ret) {
 886		ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 887		return ret;
 888	}
 889
 890	ret = cpu_noc_qdeny_check_40xx(vdev, 0x0);
 891	if (ret)
 892		ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 893
 894	return ret;
 895}
 896
 897static int soc_cpu_enable(struct ivpu_device *vdev)
 898{
 899	if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_60XX)
 900		return 0;
 901
 902	return soc_cpu_drive_40xx(vdev, true);
 903}
 904
 905static int soc_cpu_boot_40xx(struct ivpu_device *vdev)
 906{
 907	int ret;
 908	u32 val;
 909	u64 val64;
 910
 911	ret = soc_cpu_enable(vdev);
 912	if (ret) {
 913		ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret);
 914		return ret;
 915	}
 916
 917	val64 = vdev->fw->entry_point;
 918	val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1;
 919	REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64);
 920
 921	val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO);
 922	val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val);
 923	REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val);
 924
 925	ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
 926		 ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume");
 927
 928	return 0;
 929}
 930
 931int ivpu_hw_ip_soc_cpu_boot(struct ivpu_device *vdev)
 932{
 933	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 934		return soc_cpu_boot_37xx(vdev);
 935	else
 936		return soc_cpu_boot_40xx(vdev);
 937}
 938
 939static void wdt_disable_37xx(struct ivpu_device *vdev)
 940{
 941	u32 val;
 942
 943	/* Enable writing and set non-zero WDT value */
 944	REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 945	REGV_WR32(VPU_37XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
 946
 947	/* Enable writing and disable watchdog timer */
 948	REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 949	REGV_WR32(VPU_37XX_CPU_SS_TIM_WDOG_EN, 0);
 950
 951	/* Now clear the timeout interrupt */
 952	val = REGV_RD32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG);
 953	val = REG_CLR_FLD(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
 954	REGV_WR32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, val);
 955}
 956
 957static void wdt_disable_40xx(struct ivpu_device *vdev)
 958{
 959	u32 val;
 960
 961	REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 962	REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
 963
 964	REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 965	REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0);
 966
 967	val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG);
 968	val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
 969	REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val);
 970}
 971
 972void ivpu_hw_ip_wdt_disable(struct ivpu_device *vdev)
 973{
 974	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 975		return wdt_disable_37xx(vdev);
 976	else
 977		return wdt_disable_40xx(vdev);
 978}
 979
 980static u32 ipc_rx_count_get_37xx(struct ivpu_device *vdev)
 981{
 982	u32 count = readl(vdev->regv + VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT);
 983
 984	return REG_GET_FLD(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
 985}
 986
 987static u32 ipc_rx_count_get_40xx(struct ivpu_device *vdev)
 988{
 989	u32 count = readl(vdev->regv + VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT);
 990
 991	return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
 992}
 993
 994u32 ivpu_hw_ip_ipc_rx_count_get(struct ivpu_device *vdev)
 995{
 996	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 997		return ipc_rx_count_get_37xx(vdev);
 998	else
 999		return ipc_rx_count_get_40xx(vdev);
1000}
1001
1002void ivpu_hw_ip_irq_enable(struct ivpu_device *vdev)
1003{
1004	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
1005		REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_37XX);
1006		REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_37XX);
1007	} else {
1008		REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_40XX);
1009		REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_40XX);
1010	}
1011}
1012
1013void ivpu_hw_ip_irq_disable(struct ivpu_device *vdev)
1014{
1015	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
1016		REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
1017		REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, 0x0);
1018	} else {
1019		REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
1020		REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul);
1021	}
1022}
1023
1024static void diagnose_failure_37xx(struct ivpu_device *vdev)
1025{
1026	u32 reg = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX;
1027
1028	if (ipc_rx_count_get_37xx(vdev))
1029		ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1030
1031	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg))
1032		ivpu_err(vdev, "WDT MSS timeout detected\n");
1033
1034	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg))
1035		ivpu_err(vdev, "WDT NCE timeout detected\n");
1036
1037	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg))
1038		ivpu_err(vdev, "NOC Firewall irq detected\n");
1039}
1040
1041static void diagnose_failure_40xx(struct ivpu_device *vdev)
1042{
1043	u32 reg = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX;
1044
1045	if (ipc_rx_count_get_40xx(vdev))
1046		ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1047
1048	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg))
1049		ivpu_err(vdev, "WDT MSS timeout detected\n");
1050
1051	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg))
1052		ivpu_err(vdev, "WDT NCE timeout detected\n");
1053
1054	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg))
1055		ivpu_err(vdev, "NOC Firewall irq detected\n");
1056}
1057
1058void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev)
1059{
1060	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1061		diagnose_failure_37xx(vdev);
1062	else
1063		diagnose_failure_40xx(vdev);
1064}
1065
1066void ivpu_hw_ip_irq_clear(struct ivpu_device *vdev)
1067{
1068	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1069		REGV_WR64(VPU_37XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_37XX);
1070	else
1071		REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_40XX);
1072}
1073
1074static void irq_wdt_nce_handler(struct ivpu_device *vdev)
1075{
1076	ivpu_pm_trigger_recovery(vdev, "WDT NCE IRQ");
1077}
1078
1079static void irq_wdt_mss_handler(struct ivpu_device *vdev)
1080{
1081	ivpu_hw_ip_wdt_disable(vdev);
1082	ivpu_pm_trigger_recovery(vdev, "WDT MSS IRQ");
1083}
1084
1085static void irq_noc_firewall_handler(struct ivpu_device *vdev)
1086{
1087	atomic_inc(&vdev->hw->firewall_irq_counter);
1088
1089	ivpu_dbg(vdev, IRQ, "NOC Firewall interrupt detected, counter %d\n",
1090		 atomic_read(&vdev->hw->firewall_irq_counter));
1091}
1092
1093/* Handler for IRQs from NPU core */
1094bool ivpu_hw_ip_irq_handler_37xx(struct ivpu_device *vdev, int irq)
1095{
1096	u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX;
1097
1098	if (!status)
1099		return false;
1100
1101	REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status);
1102
1103	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
1104		ivpu_mmu_irq_evtq_handler(vdev);
1105
1106	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
1107		ivpu_ipc_irq_handler(vdev);
1108
1109	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
1110		ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
1111
1112	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
1113		ivpu_mmu_irq_gerr_handler(vdev);
1114
1115	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
1116		irq_wdt_mss_handler(vdev);
1117
1118	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
1119		irq_wdt_nce_handler(vdev);
1120
1121	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
1122		irq_noc_firewall_handler(vdev);
1123
1124	return true;
1125}
1126
1127/* Handler for IRQs from NPU core */
1128bool ivpu_hw_ip_irq_handler_40xx(struct ivpu_device *vdev, int irq)
1129{
1130	u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX;
1131
1132	if (!status)
1133		return false;
1134
1135	REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status);
1136
1137	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
1138		ivpu_mmu_irq_evtq_handler(vdev);
1139
1140	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
1141		ivpu_ipc_irq_handler(vdev);
1142
1143	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
1144		ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
1145
1146	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
1147		ivpu_mmu_irq_gerr_handler(vdev);
1148
1149	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
1150		irq_wdt_mss_handler(vdev);
1151
1152	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
1153		irq_wdt_nce_handler(vdev);
1154
1155	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
1156		irq_noc_firewall_handler(vdev);
1157
1158	return true;
1159}
1160
1161static void db_set_37xx(struct ivpu_device *vdev, u32 db_id)
1162{
1163	u32 reg_stride = VPU_37XX_CPU_SS_DOORBELL_1 - VPU_37XX_CPU_SS_DOORBELL_0;
1164	u32 val = REG_FLD(VPU_37XX_CPU_SS_DOORBELL_0, SET);
1165
1166	REGV_WR32I(VPU_37XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
1167}
1168
1169static void db_set_40xx(struct ivpu_device *vdev, u32 db_id)
1170{
1171	u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0;
1172	u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET);
1173
1174	REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
1175}
1176
1177void ivpu_hw_ip_db_set(struct ivpu_device *vdev, u32 db_id)
1178{
1179	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1180		db_set_37xx(vdev, db_id);
1181	else
1182		db_set_40xx(vdev, db_id);
1183}
1184
1185u32 ivpu_hw_ip_ipc_rx_addr_get(struct ivpu_device *vdev)
1186{
1187	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1188		return REGV_RD32(VPU_37XX_HOST_SS_TIM_IPC_FIFO_ATM);
1189	else
1190		return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM);
1191}
1192
1193void ivpu_hw_ip_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
1194{
1195	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1196		REGV_WR32(VPU_37XX_CPU_SS_TIM_IPC_FIFO, vpu_addr);
1197	else
1198		REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr);
1199}