vcn_v4_0_5.c

   1/*
   2 * Copyright 2023 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include "amdgpu.h"
  26#include "amdgpu_vcn.h"
  27#include "amdgpu_pm.h"
  28#include "amdgpu_cs.h"
  29#include "soc15.h"
  30#include "soc15d.h"
  31#include "soc15_hw_ip.h"
  32#include "vcn_v2_0.h"
  33#include "mmsch_v4_0.h"
  34#include "vcn_v4_0_5.h"
  35
  36#include "vcn/vcn_4_0_5_offset.h"
  37#include "vcn/vcn_4_0_5_sh_mask.h"
  38#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  39
  40#include <drm/drm_drv.h>
  41
  42#define mmUVD_DPG_LMA_CTL							regUVD_DPG_LMA_CTL
  43#define mmUVD_DPG_LMA_CTL_BASE_IDX					regUVD_DPG_LMA_CTL_BASE_IDX
  44#define mmUVD_DPG_LMA_DATA							regUVD_DPG_LMA_DATA
  45#define mmUVD_DPG_LMA_DATA_BASE_IDX					regUVD_DPG_LMA_DATA_BASE_IDX
  46
  47#define VCN_VID_SOC_ADDRESS_2_0						0x1fb00
  48#define VCN1_VID_SOC_ADDRESS_3_0					(0x48300 + 0x38000)
  49#define VCN1_AON_SOC_ADDRESS_3_0					(0x48000 + 0x38000)
  50
  51#define VCN_HARVEST_MMSCH							0
  52
  53#define RDECODE_MSG_CREATE							0x00000000
  54#define RDECODE_MESSAGE_CREATE						0x00000001
  55
  56static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_5[] = {
  57	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS),
  58	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS),
  59	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID),
  60	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2),
  61	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0),
  62	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1),
  63	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD),
  64	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI),
  65	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO),
  66	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2),
  67	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2),
  68	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3),
  69	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3),
  70	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4),
  71	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4),
  72	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR),
  73	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR),
  74	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2),
  75	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2),
  76	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3),
  77	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3),
  78	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4),
  79	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4),
  80	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE),
  81	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2),
  82	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3),
  83	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4),
  84	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG),
  85	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS),
  86	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL),
  87	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA),
  88	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK),
  89	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE)
  90};
  91
  92static int amdgpu_ih_clientid_vcns[] = {
  93	SOC15_IH_CLIENTID_VCN,
  94	SOC15_IH_CLIENTID_VCN1
  95};
  96
  97static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev);
  98static void vcn_v4_0_5_set_irq_funcs(struct amdgpu_device *adev);
  99static int vcn_v4_0_5_set_pg_state(struct amdgpu_vcn_inst *vinst,
 100				   enum amd_powergating_state state);
 101static int vcn_v4_0_5_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
 102				     struct dpg_pause_state *new_state);
 103static void vcn_v4_0_5_unified_ring_set_wptr(struct amdgpu_ring *ring);
 104
 105/**
 106 * vcn_v4_0_5_early_init - set function pointers and load microcode
 107 *
 108 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 109 *
 110 * Set ring and irq function pointers
 111 * Load microcode from filesystem
 112 */
 113static int vcn_v4_0_5_early_init(struct amdgpu_ip_block *ip_block)
 114{
 115	struct amdgpu_device *adev = ip_block->adev;
 116	int i, r;
 117
 118	if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(4, 0, 6))
 119		adev->vcn.per_inst_fw = true;
 120
 121	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
 122		/* re-use enc ring as unified ring */
 123		adev->vcn.inst[i].num_enc_rings = 1;
 124	vcn_v4_0_5_set_unified_ring_funcs(adev);
 125	vcn_v4_0_5_set_irq_funcs(adev);
 126
 127	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 128		adev->vcn.inst[i].set_pg_state = vcn_v4_0_5_set_pg_state;
 129
 130		r = amdgpu_vcn_early_init(adev, i);
 131		if (r)
 132			return r;
 133	}
 134
 135	return 0;
 136}
 137
 138/**
 139 * vcn_v4_0_5_sw_init - sw init for VCN block
 140 *
 141 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 142 *
 143 * Load firmware and sw initialization
 144 */
 145static int vcn_v4_0_5_sw_init(struct amdgpu_ip_block *ip_block)
 146{
 147	struct amdgpu_ring *ring;
 148	struct amdgpu_device *adev = ip_block->adev;
 149	int i, r;
 150
 151	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 152		struct amdgpu_vcn4_fw_shared *fw_shared;
 153
 154		if (adev->vcn.harvest_config & (1 << i))
 155			continue;
 156
 157		r = amdgpu_vcn_sw_init(adev, i);
 158		if (r)
 159			return r;
 160
 161		amdgpu_vcn_setup_ucode(adev, i);
 162
 163		r = amdgpu_vcn_resume(adev, i);
 164		if (r)
 165			return r;
 166
 167		atomic_set(&adev->vcn.inst[i].sched_score, 0);
 168
 169		/* VCN UNIFIED TRAP */
 170		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 171				VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
 172		if (r)
 173			return r;
 174
 175		/* VCN POISON TRAP */
 176		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 177				VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].irq);
 178		if (r)
 179			return r;
 180
 181		ring = &adev->vcn.inst[i].ring_enc[0];
 182		ring->use_doorbell = true;
 183		if (amdgpu_sriov_vf(adev))
 184			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 185						i * (adev->vcn.inst[i].num_enc_rings + 1) + 1;
 186		else
 187			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 188						2 + 8 * i;
 189		ring->vm_hub = AMDGPU_MMHUB0(0);
 190		sprintf(ring->name, "vcn_unified_%d", i);
 191
 192		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 193				AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
 194		if (r)
 195			return r;
 196
 197		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 198		fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
 199		fw_shared->sq.is_enabled = 1;
 200
 201		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);
 202		fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?
 203			AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;
 204
 205		if (amdgpu_sriov_vf(adev))
 206			fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
 207
 208		fw_shared->present_flag_0 |= AMDGPU_FW_SHARED_FLAG_0_DRM_KEY_INJECT;
 209		fw_shared->drm_key_wa.method =
 210			AMDGPU_DRM_KEY_INJECT_WORKAROUND_VCNFW_ASD_HANDSHAKING;
 211
 212		if (amdgpu_vcnfw_log)
 213			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
 214
 215		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 216			adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_5_pause_dpg_mode;
 217	}
 218
 219	adev->vcn.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
 220	if (!amdgpu_sriov_vf(adev))
 221		adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
 222
 223	r = amdgpu_vcn_sysfs_reset_mask_init(adev);
 224	if (r)
 225		return r;
 226
 227	if (amdgpu_sriov_vf(adev)) {
 228		r = amdgpu_virt_alloc_mm_table(adev);
 229		if (r)
 230			return r;
 231	}
 232
 233	r = amdgpu_vcn_reg_dump_init(adev, vcn_reg_list_4_0_5, ARRAY_SIZE(vcn_reg_list_4_0_5));
 234
 235	return r;
 236}
 237
 238/**
 239 * vcn_v4_0_5_sw_fini - sw fini for VCN block
 240 *
 241 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 242 *
 243 * VCN suspend and free up sw allocation
 244 */
 245static int vcn_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block)
 246{
 247	struct amdgpu_device *adev = ip_block->adev;
 248	int i, r, idx;
 249
 250	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 251		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 252			struct amdgpu_vcn4_fw_shared *fw_shared;
 253
 254			if (adev->vcn.harvest_config & (1 << i))
 255				continue;
 256
 257			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 258			fw_shared->present_flag_0 = 0;
 259			fw_shared->sq.is_enabled = 0;
 260		}
 261
 262		drm_dev_exit(idx);
 263	}
 264
 265	if (amdgpu_sriov_vf(adev))
 266		amdgpu_virt_free_mm_table(adev);
 267
 268	amdgpu_vcn_sysfs_reset_mask_fini(adev);
 269
 270	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 271		r = amdgpu_vcn_suspend(adev, i);
 272		if (r)
 273			return r;
 274
 275		amdgpu_vcn_sw_fini(adev, i);
 276	}
 277
 278	return 0;
 279}
 280
 281/**
 282 * vcn_v4_0_5_hw_init - start and test VCN block
 283 *
 284 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 285 *
 286 * Initialize the hardware, boot up the VCPU and do some testing
 287 */
 288static int vcn_v4_0_5_hw_init(struct amdgpu_ip_block *ip_block)
 289{
 290	struct amdgpu_device *adev = ip_block->adev;
 291	struct amdgpu_ring *ring;
 292	int i, r;
 293
 294	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 295		if (adev->vcn.harvest_config & (1 << i))
 296			continue;
 297
 298		ring = &adev->vcn.inst[i].ring_enc[0];
 299
 300		adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 301				((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
 302
 303		r = amdgpu_ring_test_helper(ring);
 304		if (r)
 305			return r;
 306	}
 307
 308	return 0;
 309}
 310
 311/**
 312 * vcn_v4_0_5_hw_fini - stop the hardware block
 313 *
 314 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 315 *
 316 * Stop the VCN block, mark ring as not ready any more
 317 */
 318static int vcn_v4_0_5_hw_fini(struct amdgpu_ip_block *ip_block)
 319{
 320	struct amdgpu_device *adev = ip_block->adev;
 321	int i;
 322
 323	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 324		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
 325
 326		if (adev->vcn.harvest_config & (1 << i))
 327			continue;
 328
 329		cancel_delayed_work_sync(&vinst->idle_work);
 330
 331		if (!amdgpu_sriov_vf(adev)) {
 332			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 333			    (vinst->cur_state != AMD_PG_STATE_GATE &&
 334			     RREG32_SOC15(VCN, i, regUVD_STATUS))) {
 335				vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
 336			}
 337		}
 338	}
 339
 340	return 0;
 341}
 342
 343/**
 344 * vcn_v4_0_5_suspend - suspend VCN block
 345 *
 346 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 347 *
 348 * HW fini and suspend VCN block
 349 */
 350static int vcn_v4_0_5_suspend(struct amdgpu_ip_block *ip_block)
 351{
 352	struct amdgpu_device *adev = ip_block->adev;
 353	int r, i;
 354
 355	r = vcn_v4_0_5_hw_fini(ip_block);
 356	if (r)
 357		return r;
 358
 359	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 360		r = amdgpu_vcn_suspend(ip_block->adev, i);
 361		if (r)
 362			return r;
 363	}
 364
 365	return r;
 366}
 367
 368/**
 369 * vcn_v4_0_5_resume - resume VCN block
 370 *
 371 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 372 *
 373 * Resume firmware and hw init VCN block
 374 */
 375static int vcn_v4_0_5_resume(struct amdgpu_ip_block *ip_block)
 376{
 377	struct amdgpu_device *adev = ip_block->adev;
 378	int r, i;
 379
 380	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 381		r = amdgpu_vcn_resume(ip_block->adev, i);
 382		if (r)
 383			return r;
 384	}
 385
 386	r = vcn_v4_0_5_hw_init(ip_block);
 387
 388	return r;
 389}
 390
 391/**
 392 * vcn_v4_0_5_mc_resume - memory controller programming
 393 *
 394 * @vinst: VCN instance
 395 *
 396 * Let the VCN memory controller know it's offsets
 397 */
 398static void vcn_v4_0_5_mc_resume(struct amdgpu_vcn_inst *vinst)
 399{
 400	struct amdgpu_device *adev = vinst->adev;
 401	int inst = vinst->inst;
 402	uint32_t offset, size;
 403	const struct common_firmware_header *hdr;
 404
 405	hdr = (const struct common_firmware_header *)adev->vcn.inst[inst].fw->data;
 406	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 407
 408	/* cache window 0: fw */
 409	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 410		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 411			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
 412		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 413			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
 414		WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
 415		offset = 0;
 416	} else {
 417		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 418			lower_32_bits(adev->vcn.inst[inst].gpu_addr));
 419		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 420			upper_32_bits(adev->vcn.inst[inst].gpu_addr));
 421		offset = size;
 422		WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 423	}
 424	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
 425
 426	/* cache window 1: stack */
 427	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 428		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 429	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 430		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 431	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
 432	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 433
 434	/* cache window 2: context */
 435	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 436		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 437	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 438		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 439	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
 440	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 441
 442	/* non-cache window */
 443	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
 444		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 445	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
 446		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 447	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
 448	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
 449		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
 450}
 451
 452/**
 453 * vcn_v4_0_5_mc_resume_dpg_mode - memory controller programming for dpg mode
 454 *
 455 * @vinst: VCN instance
 456 * @indirect: indirectly write sram
 457 *
 458 * Let the VCN memory controller know it's offsets with dpg mode
 459 */
 460static void vcn_v4_0_5_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
 461					  bool indirect)
 462{
 463	struct amdgpu_device *adev = vinst->adev;
 464	int inst_idx = vinst->inst;
 465	uint32_t offset, size;
 466	const struct common_firmware_header *hdr;
 467
 468	hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data;
 469	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 470
 471	/* cache window 0: fw */
 472	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 473		if (!indirect) {
 474			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 475			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 476			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo),
 477			0, indirect);
 478			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 479			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 480			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi),
 481			0, indirect);
 482			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 483				VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 484		} else {
 485			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 486			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 487			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 488			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 489			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 490				VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 491		}
 492		offset = 0;
 493	} else {
 494		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 495			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 496			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 497		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 498			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 499			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 500		offset = size;
 501		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 502			VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
 503			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 504	}
 505
 506	if (!indirect)
 507		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 508			VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 509	else
 510		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 511			VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 512
 513	/* cache window 1: stack */
 514	if (!indirect) {
 515		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 516			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 517			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 518		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 519			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 520			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 521		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 522			VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 523	} else {
 524		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 525			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 526		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 527			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 528		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 529			VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 530	}
 531
 532	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 533		VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 534
 535	/* cache window 2: context */
 536	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 537		VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 538		lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
 539		0, indirect);
 540	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 541		VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 542		upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
 543		0, indirect);
 544	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 545		VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 546	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 547		VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 548
 549	/* non-cache window */
 550	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 551		VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
 552		lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 553	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 554		VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
 555		upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 556	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 557		VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 558	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 559		VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
 560		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
 561
 562	/* VCN global tiling registers */
 563	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 564		VCN, inst_idx, regUVD_GFX10_ADDR_CONFIG),
 565		adev->gfx.config.gb_addr_config, 0, indirect);
 566}
 567
 568/**
 569 * vcn_v4_0_5_disable_static_power_gating - disable VCN static power gating
 570 *
 571 * @vinst: VCN instance
 572 *
 573 * Disable static power gating for VCN block
 574 */
 575static void vcn_v4_0_5_disable_static_power_gating(struct amdgpu_vcn_inst *vinst)
 576{
 577	struct amdgpu_device *adev = vinst->adev;
 578	int inst = vinst->inst;
 579	uint32_t data = 0;
 580
 581	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 582		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 583					1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT);
 584		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 585					UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 586		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 587					2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT);
 588		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 589					1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
 590					UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 591		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 592					2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT);
 593		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 594					1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
 595					UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 596		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 597					2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT);
 598		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 599					1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
 600					UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 601	} else {
 602		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 603			1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT);
 604		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 605			0, UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 606		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 607			1 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT);
 608		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 609			0, UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 610		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 611			1 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT);
 612		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 613			0, UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 614		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 615			1 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT);
 616		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 617			0, UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 618	}
 619
 620	data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 621	data &= ~0x103;
 622	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 623		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 624			UVD_POWER_STATUS__UVD_PG_EN_MASK;
 625	WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 626}
 627
 628/**
 629 * vcn_v4_0_5_enable_static_power_gating - enable VCN static power gating
 630 *
 631 * @vinst: VCN instance
 632 *
 633 * Enable static power gating for VCN block
 634 */
 635static void vcn_v4_0_5_enable_static_power_gating(struct amdgpu_vcn_inst *vinst)
 636{
 637	struct amdgpu_device *adev = vinst->adev;
 638	int inst = vinst->inst;
 639	uint32_t data;
 640
 641	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 642		/* Before power off, this indicator has to be turned on */
 643		data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 644		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 645		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 646		WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 647
 648		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 649			2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT);
 650		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 651			1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
 652			UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 653		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 654			2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT);
 655		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 656			1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
 657			UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 658		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 659			2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT);
 660		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 661			1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
 662			UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 663		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
 664			2 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT);
 665		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 666			1 << UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS__SHIFT,
 667			UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 668	}
 669}
 670
 671/**
 672 * vcn_v4_0_5_disable_clock_gating - disable VCN clock gating
 673 *
 674 * @vinst: VCN instance
 675 *
 676 * Disable clock gating for VCN block
 677 */
 678static void vcn_v4_0_5_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
 679{
 680	struct amdgpu_device *adev = vinst->adev;
 681	int inst = vinst->inst;
 682	uint32_t data;
 683
 684	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 685		return;
 686
 687	/* VCN disable CGC */
 688	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 689	data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 690	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 691	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 692	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 693
 694	data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
 695	data &= ~(UVD_CGC_GATE__SYS_MASK
 696		| UVD_CGC_GATE__UDEC_MASK
 697		| UVD_CGC_GATE__MPEG2_MASK
 698		| UVD_CGC_GATE__REGS_MASK
 699		| UVD_CGC_GATE__RBC_MASK
 700		| UVD_CGC_GATE__LMI_MC_MASK
 701		| UVD_CGC_GATE__LMI_UMC_MASK
 702		| UVD_CGC_GATE__IDCT_MASK
 703		| UVD_CGC_GATE__MPRD_MASK
 704		| UVD_CGC_GATE__MPC_MASK
 705		| UVD_CGC_GATE__LBSI_MASK
 706		| UVD_CGC_GATE__LRBBM_MASK
 707		| UVD_CGC_GATE__UDEC_RE_MASK
 708		| UVD_CGC_GATE__UDEC_CM_MASK
 709		| UVD_CGC_GATE__UDEC_IT_MASK
 710		| UVD_CGC_GATE__UDEC_DB_MASK
 711		| UVD_CGC_GATE__UDEC_MP_MASK
 712		| UVD_CGC_GATE__WCB_MASK
 713		| UVD_CGC_GATE__VCPU_MASK
 714		| UVD_CGC_GATE__MMSCH_MASK);
 715
 716	WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
 717	SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0,  0xFFFFFFFF);
 718
 719	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 720	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 721		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 722		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 723		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 724		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 725		| UVD_CGC_CTRL__SYS_MODE_MASK
 726		| UVD_CGC_CTRL__UDEC_MODE_MASK
 727		| UVD_CGC_CTRL__MPEG2_MODE_MASK
 728		| UVD_CGC_CTRL__REGS_MODE_MASK
 729		| UVD_CGC_CTRL__RBC_MODE_MASK
 730		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
 731		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 732		| UVD_CGC_CTRL__IDCT_MODE_MASK
 733		| UVD_CGC_CTRL__MPRD_MODE_MASK
 734		| UVD_CGC_CTRL__MPC_MODE_MASK
 735		| UVD_CGC_CTRL__LBSI_MODE_MASK
 736		| UVD_CGC_CTRL__LRBBM_MODE_MASK
 737		| UVD_CGC_CTRL__WCB_MODE_MASK
 738		| UVD_CGC_CTRL__VCPU_MODE_MASK
 739		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
 740	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 741
 742	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
 743	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 744		| UVD_SUVD_CGC_GATE__SIT_MASK
 745		| UVD_SUVD_CGC_GATE__SMP_MASK
 746		| UVD_SUVD_CGC_GATE__SCM_MASK
 747		| UVD_SUVD_CGC_GATE__SDB_MASK
 748		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
 749		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 750		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
 751		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 752		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
 753		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 754		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
 755		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 756		| UVD_SUVD_CGC_GATE__SCLR_MASK
 757		| UVD_SUVD_CGC_GATE__UVD_SC_MASK
 758		| UVD_SUVD_CGC_GATE__ENT_MASK
 759		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 760		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 761		| UVD_SUVD_CGC_GATE__SITE_MASK
 762		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 763		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 764		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 765		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 766		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 767	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
 768
 769	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
 770	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 771		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 772		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 773		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 774		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 775		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 776		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 777		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 778		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 779		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 780	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
 781}
 782
 783/**
 784 * vcn_v4_0_5_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
 785 *
 786 * @vinst: VCN instance
 787 * @sram_sel: sram select
 788 * @indirect: indirectly write sram
 789 *
 790 * Disable clock gating for VCN block with dpg mode
 791 */
 792static void vcn_v4_0_5_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst,
 793						     uint8_t sram_sel,
 794						     uint8_t indirect)
 795{
 796	struct amdgpu_device *adev = vinst->adev;
 797	int inst_idx = vinst->inst;
 798	uint32_t reg_data = 0;
 799
 800	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 801		return;
 802
 803	/* enable sw clock gating control */
 804	reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 805	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 806	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 807	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 808		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 809		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 810		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 811		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 812		 UVD_CGC_CTRL__SYS_MODE_MASK |
 813		 UVD_CGC_CTRL__UDEC_MODE_MASK |
 814		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
 815		 UVD_CGC_CTRL__REGS_MODE_MASK |
 816		 UVD_CGC_CTRL__RBC_MODE_MASK |
 817		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 818		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 819		 UVD_CGC_CTRL__IDCT_MODE_MASK |
 820		 UVD_CGC_CTRL__MPRD_MODE_MASK |
 821		 UVD_CGC_CTRL__MPC_MODE_MASK |
 822		 UVD_CGC_CTRL__LBSI_MODE_MASK |
 823		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
 824		 UVD_CGC_CTRL__WCB_MODE_MASK |
 825		 UVD_CGC_CTRL__VCPU_MODE_MASK);
 826	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 827		VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
 828
 829	/* turn off clock gating */
 830	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 831		VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
 832
 833	/* turn on SUVD clock gating */
 834	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 835		VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
 836
 837	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
 838	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 839		VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
 840}
 841
 842/**
 843 * vcn_v4_0_5_enable_clock_gating - enable VCN clock gating
 844 *
 845 * @vinst: VCN instance
 846 *
 847 * Enable clock gating for VCN block
 848 */
 849static void vcn_v4_0_5_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
 850{
 851	struct amdgpu_device *adev = vinst->adev;
 852	int inst = vinst->inst;
 853	uint32_t data;
 854
 855	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 856		return;
 857
 858	/* enable VCN CGC */
 859	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 860	data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 861	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 862	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 863	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 864
 865	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
 866	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 867		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 868		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 869		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 870		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 871		| UVD_CGC_CTRL__SYS_MODE_MASK
 872		| UVD_CGC_CTRL__UDEC_MODE_MASK
 873		| UVD_CGC_CTRL__MPEG2_MODE_MASK
 874		| UVD_CGC_CTRL__REGS_MODE_MASK
 875		| UVD_CGC_CTRL__RBC_MODE_MASK
 876		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
 877		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 878		| UVD_CGC_CTRL__IDCT_MODE_MASK
 879		| UVD_CGC_CTRL__MPRD_MODE_MASK
 880		| UVD_CGC_CTRL__MPC_MODE_MASK
 881		| UVD_CGC_CTRL__LBSI_MODE_MASK
 882		| UVD_CGC_CTRL__LRBBM_MODE_MASK
 883		| UVD_CGC_CTRL__WCB_MODE_MASK
 884		| UVD_CGC_CTRL__VCPU_MODE_MASK
 885		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
 886	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
 887
 888	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
 889	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 890		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 891		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 892		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 893		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 894		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 895		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 896		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 897		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 898		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 899	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
 900}
 901
 902/**
 903 * vcn_v4_0_5_start_dpg_mode - VCN start with dpg mode
 904 *
 905 * @vinst: VCN instance
 906 * @indirect: indirectly write sram
 907 *
 908 * Start VCN block with dpg mode
 909 */
 910static int vcn_v4_0_5_start_dpg_mode(struct amdgpu_vcn_inst *vinst,
 911				     bool indirect)
 912{
 913	struct amdgpu_device *adev = vinst->adev;
 914	int inst_idx = vinst->inst;
 915	struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
 916	struct amdgpu_ring *ring;
 917	uint32_t tmp;
 918	int ret;
 919
 920	/* disable register anti-hang mechanism */
 921	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
 922		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 923	/* enable dynamic power gating mode */
 924	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
 925	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 926	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 927	WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
 928
 929	if (indirect)
 930		adev->vcn.inst[inst_idx].dpg_sram_curr_addr =
 931					(uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
 932
 933	/* enable clock gating */
 934	vcn_v4_0_5_disable_clock_gating_dpg_mode(vinst, 0, indirect);
 935
 936	/* enable VCPU clock */
 937	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 938	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
 939	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 940		VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 941
 942	/* disable master interrupt */
 943	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 944		VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
 945
 946	/* setup regUVD_LMI_CTRL */
 947	tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 948		UVD_LMI_CTRL__REQ_MODE_MASK |
 949		UVD_LMI_CTRL__CRC_RESET_MASK |
 950		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 951		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 952		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 953		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 954		0x00100000L);
 955	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 956		VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
 957
 958	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 959		VCN, inst_idx, regUVD_MPC_CNTL),
 960		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
 961
 962	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 963		VCN, inst_idx, regUVD_MPC_SET_MUXA0),
 964		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 965		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 966		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 967		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
 968
 969	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 970		VCN, inst_idx, regUVD_MPC_SET_MUXB0),
 971		 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 972		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 973		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 974		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
 975
 976	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 977		VCN, inst_idx, regUVD_MPC_SET_MUX),
 978		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 979		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 980		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
 981
 982	vcn_v4_0_5_mc_resume_dpg_mode(vinst, indirect);
 983
 984	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 985	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 986	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 987		VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 988
 989	/* enable LMI MC and UMC channels */
 990	tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
 991	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 992		VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
 993
 994	/* enable master interrupt */
 995	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
 996		VCN, inst_idx, regUVD_MASTINT_EN),
 997		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
 998
 999	if (indirect) {
1000		ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
1001		if (ret) {
1002			dev_err(adev->dev, "vcn sram load failed %d\n", ret);
1003			return ret;
1004		}
1005	}
1006
1007	ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1008
1009	WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
1010	WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1011	WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
1012
1013	tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1014	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1015	WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1016	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1017	WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
1018	WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
1019
1020	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
1021	WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
1022	ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1023
1024	tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1025	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1026	WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1027	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1028
1029	WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
1030			ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1031			VCN_RB1_DB_CTRL__EN_MASK);
1032
1033	/* Keeping one read-back to ensure all register writes are done, otherwise
1034	 * it may introduce race conditions */
1035	RREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL);
1036
1037	return 0;
1038}
1039
1040
1041/**
1042 * vcn_v4_0_5_start - VCN start
1043 *
1044 * @vinst: VCN instance
1045 *
1046 * Start VCN block
1047 */
1048static int vcn_v4_0_5_start(struct amdgpu_vcn_inst *vinst)
1049{
1050	struct amdgpu_device *adev = vinst->adev;
1051	int i = vinst->inst;
1052	struct amdgpu_vcn4_fw_shared *fw_shared;
1053	struct amdgpu_ring *ring;
1054	uint32_t tmp;
1055	int j, k, r;
1056
1057	if (adev->vcn.harvest_config & (1 << i))
1058		return 0;
1059
1060	if (adev->pm.dpm_enabled)
1061		amdgpu_dpm_enable_vcn(adev, true, i);
1062
1063	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1064
1065	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1066		return vcn_v4_0_5_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram);
1067
1068	/* disable VCN power gating */
1069	vcn_v4_0_5_disable_static_power_gating(vinst);
1070
1071	/* set VCN status busy */
1072	tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1073	WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1074
1075	/* SW clock gating */
1076	vcn_v4_0_5_disable_clock_gating(vinst);
1077
1078	/* enable VCPU clock */
1079	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1080		 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1081
1082	/* disable master interrupt */
1083	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
1084		 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1085
1086	/* enable LMI MC and UMC channels */
1087	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
1088		 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1089
1090	tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1091	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1092	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1093	WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1094
1095	/* setup regUVD_LMI_CTRL */
1096	tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1097	WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1098		     UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1099		     UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1100		     UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1101		     UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1102
1103	/* setup regUVD_MPC_CNTL */
1104	tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1105	tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1106	tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1107	WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1108
1109	/* setup UVD_MPC_SET_MUXA0 */
1110	WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1111		     ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1112		      (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1113		      (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1114		      (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1115
1116	/* setup UVD_MPC_SET_MUXB0 */
1117	WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1118		     ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1119		      (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1120		      (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1121		      (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1122
1123	/* setup UVD_MPC_SET_MUX */
1124	WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1125		     ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1126		      (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1127		      (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1128
1129	vcn_v4_0_5_mc_resume(vinst);
1130
1131	/* VCN global tiling registers */
1132	WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1133		     adev->gfx.config.gb_addr_config);
1134
1135	/* unblock VCPU register access */
1136	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1137		 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1138
1139	/* release VCPU reset to boot */
1140	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1141		 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1142
1143	for (j = 0; j < 10; ++j) {
1144		uint32_t status;
1145
1146		for (k = 0; k < 100; ++k) {
1147			status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1148			if (status & 2)
1149				break;
1150			mdelay(10);
1151			if (amdgpu_emu_mode == 1)
1152				msleep(1);
1153		}
1154
1155		if (amdgpu_emu_mode == 1) {
1156			r = -1;
1157			if (status & 2) {
1158				r = 0;
1159				break;
1160			}
1161		} else {
1162			r = 0;
1163			if (status & 2)
1164				break;
1165
1166			dev_err(adev->dev,
1167				"VCN[%d] is not responding, trying to reset VCPU!!!\n", i);
1168			WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1169				 UVD_VCPU_CNTL__BLK_RST_MASK,
1170				 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1171			mdelay(10);
1172			WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1173				 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1174
1175			mdelay(10);
1176			r = -1;
1177		}
1178	}
1179
1180	if (r) {
1181		dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1182		return r;
1183	}
1184
1185	/* enable master interrupt */
1186	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1187		 UVD_MASTINT_EN__VCPU_EN_MASK,
1188		 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1189
1190	/* clear the busy bit of VCN_STATUS */
1191	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1192		 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1193
1194	ring = &adev->vcn.inst[i].ring_enc[0];
1195	WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1196		     ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1197		     VCN_RB1_DB_CTRL__EN_MASK);
1198
1199	WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1200	WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1201	WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1202
1203	tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1204	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1205	WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1206	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1207	WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1208	WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
1209
1210	tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1211	WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1212	ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1213
1214	tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1215	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1216	WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1217	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1218
1219	/* Keeping one read-back to ensure all register writes are done, otherwise
1220	 * it may introduce race conditions */
1221	RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1222
1223	return 0;
1224}
1225
1226/**
1227 * vcn_v4_0_5_stop_dpg_mode - VCN stop with dpg mode
1228 *
1229 * @vinst: VCN instance
1230 *
1231 * Stop VCN block with dpg mode
1232 */
1233static void vcn_v4_0_5_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
1234{
1235	struct amdgpu_device *adev = vinst->adev;
1236	int inst_idx = vinst->inst;
1237	uint32_t tmp;
1238
1239	/* Wait for power status to be 1 */
1240	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1241		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1242
1243	/* wait for read ptr to be equal to write ptr */
1244	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1245	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1246
1247	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1248		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1249
1250	/* disable dynamic power gating mode */
1251	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1252		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1253
1254	/* Keeping one read-back to ensure all register writes are done,
1255	 * otherwise it may introduce race conditions.
1256	 */
1257	RREG32_SOC15(VCN, inst_idx, regUVD_STATUS);
1258}
1259
1260/**
1261 * vcn_v4_0_5_stop - VCN stop
1262 *
1263 * @vinst: VCN instance
1264 *
1265 * Stop VCN block
1266 */
1267static int vcn_v4_0_5_stop(struct amdgpu_vcn_inst *vinst)
1268{
1269	struct amdgpu_device *adev = vinst->adev;
1270	int i = vinst->inst;
1271	struct amdgpu_vcn4_fw_shared *fw_shared;
1272	uint32_t tmp;
1273	int r = 0;
1274
1275	if (adev->vcn.harvest_config & (1 << i))
1276		return 0;
1277
1278	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1279	fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1280
1281	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1282		vcn_v4_0_5_stop_dpg_mode(vinst);
1283		r = 0;
1284		goto done;
1285	}
1286
1287	/* wait for vcn idle */
1288	r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1289	if (r)
1290		goto done;
1291
1292	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1293		UVD_LMI_STATUS__READ_CLEAN_MASK |
1294		UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1295		UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1296	r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1297	if (r)
1298		goto done;
1299
1300	/* disable LMI UMC channel */
1301	tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1302	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1303	WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1304	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1305		UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1306	r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1307	if (r)
1308		goto done;
1309
1310	/* block VCPU register access */
1311	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1312		 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1313		 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1314
1315	/* reset VCPU */
1316	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1317		 UVD_VCPU_CNTL__BLK_RST_MASK,
1318		 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1319
1320	/* disable VCPU clock */
1321	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1322		 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1323
1324	/* apply soft reset */
1325	tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1326	tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1327	WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1328	tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1329	tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1330	WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1331
1332	/* clear status */
1333	WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1334
1335	/* apply HW clock gating */
1336	vcn_v4_0_5_enable_clock_gating(vinst);
1337
1338	/* enable VCN power gating */
1339	vcn_v4_0_5_enable_static_power_gating(vinst);
1340
1341	/* Keeping one read-back to ensure all register writes are done,
1342	 * otherwise it may introduce race conditions.
1343	 */
1344	RREG32_SOC15(VCN, i, regUVD_STATUS);
1345
1346done:
1347	if (adev->pm.dpm_enabled)
1348		amdgpu_dpm_enable_vcn(adev, false, i);
1349
1350	return r;
1351}
1352
1353/**
1354 * vcn_v4_0_5_pause_dpg_mode - VCN pause with dpg mode
1355 *
1356 * @vinst: VCN instance
1357 * @new_state: pause state
1358 *
1359 * Pause dpg mode for VCN block
1360 */
1361static int vcn_v4_0_5_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
1362				     struct dpg_pause_state *new_state)
1363{
1364	struct amdgpu_device *adev = vinst->adev;
1365	int inst_idx = vinst->inst;
1366	uint32_t reg_data = 0;
1367	int ret_code;
1368
1369	/* pause/unpause if state is changed */
1370	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1371		DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1372			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1373		reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1374			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1375
1376		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1377			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1378				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1379
1380			if (!ret_code) {
1381				/* pause DPG */
1382				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1383				WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1384
1385				/* wait for ACK */
1386				SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1387					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1388					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1389
1390				SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1391					UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1392					UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1393			}
1394		} else {
1395			/* unpause dpg, no need to wait */
1396			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1397			WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1398		}
1399		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1400	}
1401
1402	return 0;
1403}
1404
1405/**
1406 * vcn_v4_0_5_unified_ring_get_rptr - get unified read pointer
1407 *
1408 * @ring: amdgpu_ring pointer
1409 *
1410 * Returns the current hardware unified read pointer
1411 */
1412static uint64_t vcn_v4_0_5_unified_ring_get_rptr(struct amdgpu_ring *ring)
1413{
1414	struct amdgpu_device *adev = ring->adev;
1415
1416	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1417		DRM_ERROR("wrong ring id is identified in %s", __func__);
1418
1419	return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1420}
1421
1422/**
1423 * vcn_v4_0_5_unified_ring_get_wptr - get unified write pointer
1424 *
1425 * @ring: amdgpu_ring pointer
1426 *
1427 * Returns the current hardware unified write pointer
1428 */
1429static uint64_t vcn_v4_0_5_unified_ring_get_wptr(struct amdgpu_ring *ring)
1430{
1431	struct amdgpu_device *adev = ring->adev;
1432
1433	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1434		DRM_ERROR("wrong ring id is identified in %s", __func__);
1435
1436	if (ring->use_doorbell)
1437		return *ring->wptr_cpu_addr;
1438	else
1439		return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1440}
1441
1442/**
1443 * vcn_v4_0_5_unified_ring_set_wptr - set enc write pointer
1444 *
1445 * @ring: amdgpu_ring pointer
1446 *
1447 * Commits the enc write pointer to the hardware
1448 */
1449static void vcn_v4_0_5_unified_ring_set_wptr(struct amdgpu_ring *ring)
1450{
1451	struct amdgpu_device *adev = ring->adev;
1452
1453	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1454		DRM_ERROR("wrong ring id is identified in %s", __func__);
1455
1456	if (ring->use_doorbell) {
1457		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1458		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1459	} else {
1460		WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1461	}
1462}
1463
1464static int vcn_v4_0_5_ring_reset(struct amdgpu_ring *ring,
1465				 unsigned int vmid,
1466				 struct amdgpu_fence *timedout_fence)
1467{
1468	struct amdgpu_device *adev = ring->adev;
1469	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
1470	int r;
1471
1472	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
1473	r = vcn_v4_0_5_stop(vinst);
1474	if (r)
1475		return r;
1476	r = vcn_v4_0_5_start(vinst);
1477	if (r)
1478		return r;
1479	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
1480}
1481
1482static struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = {
1483	.type = AMDGPU_RING_TYPE_VCN_ENC,
1484	.align_mask = 0x3f,
1485	.nop = VCN_ENC_CMD_NO_OP,
1486	.get_rptr = vcn_v4_0_5_unified_ring_get_rptr,
1487	.get_wptr = vcn_v4_0_5_unified_ring_get_wptr,
1488	.set_wptr = vcn_v4_0_5_unified_ring_set_wptr,
1489	.emit_frame_size =
1490		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1491		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1492		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1493		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1494		1, /* vcn_v2_0_enc_ring_insert_end */
1495	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1496	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
1497	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
1498	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1499	.test_ring = amdgpu_vcn_enc_ring_test_ring,
1500	.test_ib = amdgpu_vcn_unified_ring_test_ib,
1501	.insert_nop = amdgpu_ring_insert_nop,
1502	.insert_end = vcn_v2_0_enc_ring_insert_end,
1503	.pad_ib = amdgpu_ring_generic_pad_ib,
1504	.begin_use = amdgpu_vcn_ring_begin_use,
1505	.end_use = amdgpu_vcn_ring_end_use,
1506	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1507	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1508	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1509	.reset = vcn_v4_0_5_ring_reset,
1510};
1511
1512/**
1513 * vcn_v4_0_5_set_unified_ring_funcs - set unified ring functions
1514 *
1515 * @adev: amdgpu_device pointer
1516 *
1517 * Set unified ring functions
1518 */
1519static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev)
1520{
1521	int i;
1522
1523	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1524		if (adev->vcn.harvest_config & (1 << i))
1525			continue;
1526
1527		if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 5))
1528			vcn_v4_0_5_unified_ring_vm_funcs.secure_submission_supported = true;
1529
1530		adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_5_unified_ring_vm_funcs;
1531		adev->vcn.inst[i].ring_enc[0].me = i;
1532	}
1533}
1534
1535/**
1536 * vcn_v4_0_5_is_idle - check VCN block is idle
1537 *
1538 * @ip_block: Pointer to the amdgpu_ip_block structure
1539 *
1540 * Check whether VCN block is idle
1541 */
1542static bool vcn_v4_0_5_is_idle(struct amdgpu_ip_block *ip_block)
1543{
1544	struct amdgpu_device *adev = ip_block->adev;
1545	int i, ret = 1;
1546
1547	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1548		if (adev->vcn.harvest_config & (1 << i))
1549			continue;
1550
1551		ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1552	}
1553
1554	return ret;
1555}
1556
1557/**
1558 * vcn_v4_0_5_wait_for_idle - wait for VCN block idle
1559 *
1560 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
1561 *
1562 * Wait for VCN block idle
1563 */
1564static int vcn_v4_0_5_wait_for_idle(struct amdgpu_ip_block *ip_block)
1565{
1566	struct amdgpu_device *adev = ip_block->adev;
1567	int i, ret = 0;
1568
1569	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1570		if (adev->vcn.harvest_config & (1 << i))
1571			continue;
1572
1573		ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1574			UVD_STATUS__IDLE);
1575		if (ret)
1576			return ret;
1577	}
1578
1579	return ret;
1580}
1581
1582/**
1583 * vcn_v4_0_5_set_clockgating_state - set VCN block clockgating state
1584 *
1585 * @ip_block: amdgpu_ip_block pointer
1586 * @state: clock gating state
1587 *
1588 * Set VCN block clockgating state
1589 */
1590static int vcn_v4_0_5_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1591					  enum amd_clockgating_state state)
1592{
1593	struct amdgpu_device *adev = ip_block->adev;
1594	bool enable = state == AMD_CG_STATE_GATE;
1595	int i;
1596
1597	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1598		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
1599
1600		if (adev->vcn.harvest_config & (1 << i))
1601			continue;
1602
1603		if (enable) {
1604			if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1605				return -EBUSY;
1606			vcn_v4_0_5_enable_clock_gating(vinst);
1607		} else {
1608			vcn_v4_0_5_disable_clock_gating(vinst);
1609		}
1610	}
1611
1612	return 0;
1613}
1614
1615static int vcn_v4_0_5_set_pg_state(struct amdgpu_vcn_inst *vinst,
1616				   enum amd_powergating_state state)
1617{
1618	int ret = 0;
1619
1620	if (state == vinst->cur_state)
1621		return 0;
1622
1623	if (state == AMD_PG_STATE_GATE)
1624		ret = vcn_v4_0_5_stop(vinst);
1625	else
1626		ret = vcn_v4_0_5_start(vinst);
1627
1628	if (!ret)
1629		vinst->cur_state = state;
1630
1631	return ret;
1632}
1633
1634/**
1635 * vcn_v4_0_5_process_interrupt - process VCN block interrupt
1636 *
1637 * @adev: amdgpu_device pointer
1638 * @source: interrupt sources
1639 * @entry: interrupt entry from clients and sources
1640 *
1641 * Process VCN block interrupt
1642 */
1643static int vcn_v4_0_5_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1644		struct amdgpu_iv_entry *entry)
1645{
1646	uint32_t ip_instance;
1647
1648	switch (entry->client_id) {
1649	case SOC15_IH_CLIENTID_VCN:
1650		ip_instance = 0;
1651		break;
1652	case SOC15_IH_CLIENTID_VCN1:
1653		ip_instance = 1;
1654		break;
1655	default:
1656		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1657		return 0;
1658	}
1659
1660	DRM_DEBUG("IH: VCN TRAP\n");
1661
1662	switch (entry->src_id) {
1663	case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1664		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1665		break;
1666	case VCN_4_0__SRCID_UVD_POISON:
1667		amdgpu_vcn_process_poison_irq(adev, source, entry);
1668		break;
1669	default:
1670		DRM_ERROR("Unhandled interrupt: %d %d\n",
1671			  entry->src_id, entry->src_data[0]);
1672		break;
1673	}
1674
1675	return 0;
1676}
1677
1678static const struct amdgpu_irq_src_funcs vcn_v4_0_5_irq_funcs = {
1679	.process = vcn_v4_0_5_process_interrupt,
1680};
1681
1682/**
1683 * vcn_v4_0_5_set_irq_funcs - set VCN block interrupt irq functions
1684 *
1685 * @adev: amdgpu_device pointer
1686 *
1687 * Set VCN block interrupt irq functions
1688 */
1689static void vcn_v4_0_5_set_irq_funcs(struct amdgpu_device *adev)
1690{
1691	int i;
1692
1693	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1694		if (adev->vcn.harvest_config & (1 << i))
1695			continue;
1696
1697		adev->vcn.inst[i].irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
1698		adev->vcn.inst[i].irq.funcs = &vcn_v4_0_5_irq_funcs;
1699	}
1700}
1701
1702static const struct amd_ip_funcs vcn_v4_0_5_ip_funcs = {
1703	.name = "vcn_v4_0_5",
1704	.early_init = vcn_v4_0_5_early_init,
1705	.sw_init = vcn_v4_0_5_sw_init,
1706	.sw_fini = vcn_v4_0_5_sw_fini,
1707	.hw_init = vcn_v4_0_5_hw_init,
1708	.hw_fini = vcn_v4_0_5_hw_fini,
1709	.suspend = vcn_v4_0_5_suspend,
1710	.resume = vcn_v4_0_5_resume,
1711	.is_idle = vcn_v4_0_5_is_idle,
1712	.wait_for_idle = vcn_v4_0_5_wait_for_idle,
1713	.set_clockgating_state = vcn_v4_0_5_set_clockgating_state,
1714	.set_powergating_state = vcn_set_powergating_state,
1715	.dump_ip_state = amdgpu_vcn_dump_ip_state,
1716	.print_ip_state = amdgpu_vcn_print_ip_state,
1717};
1718
1719const struct amdgpu_ip_block_version vcn_v4_0_5_ip_block = {
1720	.type = AMD_IP_BLOCK_TYPE_VCN,
1721	.major = 4,
1722	.minor = 0,
1723	.rev = 5,
1724	.funcs = &vcn_v4_0_5_ip_funcs,
1725};