jpeg_v4_0_3.c

   1/*
   2 * Copyright 2022 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include "amdgpu.h"
  25#include "amdgpu_jpeg.h"
  26#include "soc15.h"
  27#include "soc15d.h"
  28#include "jpeg_v2_0.h"
  29#include "jpeg_v4_0_3.h"
  30#include "mmsch_v4_0_3.h"
  31
  32#include "vcn/vcn_4_0_3_offset.h"
  33#include "vcn/vcn_4_0_3_sh_mask.h"
  34#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  35
  36#define NORMALIZE_JPEG_REG_OFFSET(offset) \
  37		(offset & 0x1FFFF)
  38
  39enum jpeg_engin_status {
  40	UVD_PGFSM_STATUS__UVDJ_PWR_ON  = 0,
  41	UVD_PGFSM_STATUS__UVDJ_PWR_OFF = 2,
  42};
  43
  44static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev);
  45static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev);
  46static int jpeg_v4_0_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
  47				enum amd_powergating_state state);
  48static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev);
  49static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring);
  50
  51static int amdgpu_ih_srcid_jpeg[] = {
  52	VCN_4_0__SRCID__JPEG_DECODE,
  53	VCN_4_0__SRCID__JPEG1_DECODE,
  54	VCN_4_0__SRCID__JPEG2_DECODE,
  55	VCN_4_0__SRCID__JPEG3_DECODE,
  56	VCN_4_0__SRCID__JPEG4_DECODE,
  57	VCN_4_0__SRCID__JPEG5_DECODE,
  58	VCN_4_0__SRCID__JPEG6_DECODE,
  59	VCN_4_0__SRCID__JPEG7_DECODE
  60};
  61
  62static const struct amdgpu_hwip_reg_entry jpeg_reg_list_4_0_3[] = {
  63	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_POWER_STATUS),
  64	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_INT_STAT),
  65	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_SYS_INT_STATUS),
  66	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_RPTR),
  67	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_WPTR),
  68	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC0_UVD_JRBC_STATUS),
  69	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_ADDR_MODE),
  70	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_GFX10_ADDR_CONFIG),
  71	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_Y_GFX10_TILING_SURFACE),
  72	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_UV_GFX10_TILING_SURFACE),
  73	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_PITCH),
  74	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_UV_PITCH),
  75	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC1_UVD_JRBC_RB_RPTR),
  76	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC1_UVD_JRBC_RB_WPTR),
  77	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC1_UVD_JRBC_STATUS),
  78	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC2_UVD_JRBC_RB_RPTR),
  79	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC2_UVD_JRBC_RB_WPTR),
  80	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC2_UVD_JRBC_STATUS),
  81	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC3_UVD_JRBC_RB_RPTR),
  82	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC3_UVD_JRBC_RB_WPTR),
  83	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC3_UVD_JRBC_STATUS),
  84	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC4_UVD_JRBC_RB_RPTR),
  85	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC4_UVD_JRBC_RB_WPTR),
  86	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC4_UVD_JRBC_STATUS),
  87	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC5_UVD_JRBC_RB_RPTR),
  88	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC5_UVD_JRBC_RB_WPTR),
  89	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC5_UVD_JRBC_STATUS),
  90	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC6_UVD_JRBC_RB_RPTR),
  91	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC6_UVD_JRBC_RB_WPTR),
  92	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC6_UVD_JRBC_STATUS),
  93	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC7_UVD_JRBC_RB_RPTR),
  94	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC7_UVD_JRBC_RB_WPTR),
  95	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC7_UVD_JRBC_STATUS),
  96};
  97
  98static inline bool jpeg_v4_0_3_normalizn_reqd(struct amdgpu_device *adev)
  99{
 100	return (adev->jpeg.caps & AMDGPU_JPEG_CAPS(RRMT_ENABLED)) == 0;
 101}
 102
 103static inline int jpeg_v4_0_3_core_reg_offset(u32 pipe)
 104{
 105	if (pipe)
 106		return ((0x40 * pipe) - 0xc80);
 107	else
 108		return 0;
 109}
 110
 111/**
 112 * jpeg_v4_0_3_early_init - set function pointers
 113 *
 114 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 115 *
 116 * Set ring and irq function pointers
 117 */
 118static int jpeg_v4_0_3_early_init(struct amdgpu_ip_block *ip_block)
 119{
 120	struct amdgpu_device *adev = ip_block->adev;
 121
 122	adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS_4_0_3;
 123
 124	jpeg_v4_0_3_set_dec_ring_funcs(adev);
 125	jpeg_v4_0_3_set_irq_funcs(adev);
 126	jpeg_v4_0_3_set_ras_funcs(adev);
 127
 128	return 0;
 129}
 130
 131/**
 132 * jpeg_v4_0_3_sw_init - sw init for JPEG block
 133 *
 134 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 135 *
 136 * Load firmware and sw initialization
 137 */
 138static int jpeg_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block)
 139{
 140	struct amdgpu_device *adev = ip_block->adev;
 141	struct amdgpu_ring *ring;
 142	int i, j, r, jpeg_inst;
 143
 144	for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 145		/* JPEG TRAP */
 146		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 147				amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
 148		if (r)
 149			return r;
 150	}
 151
 152	/* JPEG DJPEG POISON EVENT */
 153	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 154			VCN_4_0__SRCID_DJPEG0_POISON, &adev->jpeg.inst->ras_poison_irq);
 155	if (r)
 156		return r;
 157
 158	/* JPEG EJPEG POISON EVENT */
 159	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 160			VCN_4_0__SRCID_EJPEG0_POISON, &adev->jpeg.inst->ras_poison_irq);
 161	if (r)
 162		return r;
 163
 164	r = amdgpu_jpeg_sw_init(adev);
 165	if (r)
 166		return r;
 167
 168	r = amdgpu_jpeg_resume(adev);
 169	if (r)
 170		return r;
 171
 172	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 173		jpeg_inst = GET_INST(JPEG, i);
 174
 175		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 176			ring = &adev->jpeg.inst[i].ring_dec[j];
 177			ring->use_doorbell = true;
 178			ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
 179			if (!amdgpu_sriov_vf(adev)) {
 180				ring->doorbell_index =
 181					(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 182					1 + j + 9 * jpeg_inst;
 183			} else {
 184				if (j < 4)
 185					ring->doorbell_index =
 186						(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 187						4 + j + 32 * jpeg_inst;
 188				else
 189					ring->doorbell_index =
 190						(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 191						8 + j + 32 * jpeg_inst;
 192			}
 193			sprintf(ring->name, "jpeg_dec_%d.%d", adev->jpeg.inst[i].aid_id, j);
 194			r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 195						AMDGPU_RING_PRIO_DEFAULT, NULL);
 196			if (r)
 197				return r;
 198
 199			adev->jpeg.internal.jpeg_pitch[j] =
 200				regUVD_JRBC0_UVD_JRBC_SCRATCH0_INTERNAL_OFFSET;
 201			adev->jpeg.inst[i].external.jpeg_pitch[j] =
 202				SOC15_REG_OFFSET1(JPEG, jpeg_inst, regUVD_JRBC0_UVD_JRBC_SCRATCH0,
 203						  jpeg_v4_0_3_core_reg_offset(j));
 204		}
 205	}
 206
 207	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
 208		r = amdgpu_jpeg_ras_sw_init(adev);
 209		if (r) {
 210			dev_err(adev->dev, "Failed to initialize jpeg ras block!\n");
 211			return r;
 212		}
 213	}
 214
 215	r = amdgpu_jpeg_reg_dump_init(adev, jpeg_reg_list_4_0_3, ARRAY_SIZE(jpeg_reg_list_4_0_3));
 216	if (r)
 217		return r;
 218
 219	adev->jpeg.supported_reset =
 220		amdgpu_get_soft_full_reset_mask(adev->jpeg.inst[0].ring_dec);
 221	if (!amdgpu_sriov_vf(adev))
 222		adev->jpeg.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
 223	r = amdgpu_jpeg_sysfs_reset_mask_init(adev);
 224
 225	return 0;
 226}
 227
 228/**
 229 * jpeg_v4_0_3_sw_fini - sw fini for JPEG block
 230 *
 231 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 232 *
 233 * JPEG suspend and free up sw allocation
 234 */
 235static int jpeg_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block)
 236{
 237	struct amdgpu_device *adev = ip_block->adev;
 238	int r;
 239
 240	r = amdgpu_jpeg_suspend(adev);
 241	if (r)
 242		return r;
 243
 244	amdgpu_jpeg_sysfs_reset_mask_fini(adev);
 245
 246	r = amdgpu_jpeg_sw_fini(adev);
 247
 248	return r;
 249}
 250
 251static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
 252{
 253	struct amdgpu_ring *ring;
 254	uint64_t ctx_addr;
 255	uint32_t param, resp, expected;
 256	uint32_t tmp, timeout;
 257
 258	struct amdgpu_mm_table *table = &adev->virt.mm_table;
 259	uint32_t *table_loc;
 260	uint32_t table_size;
 261	uint32_t size, size_dw, item_offset;
 262	uint32_t init_status;
 263	int i, j, jpeg_inst;
 264
 265	struct mmsch_v4_0_cmd_direct_write
 266		direct_wt = { {0} };
 267	struct mmsch_v4_0_cmd_end end = { {0} };
 268	struct mmsch_v4_0_3_init_header header;
 269
 270	direct_wt.cmd_header.command_type =
 271		MMSCH_COMMAND__DIRECT_REG_WRITE;
 272	end.cmd_header.command_type =
 273		MMSCH_COMMAND__END;
 274
 275	for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
 276		jpeg_inst = GET_INST(JPEG, i);
 277
 278		memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header));
 279		header.version = MMSCH_VERSION;
 280		header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2;
 281
 282		table_loc = (uint32_t *)table->cpu_addr;
 283		table_loc += header.total_size;
 284
 285		item_offset = header.total_size;
 286
 287		for (j = 0; j < adev->jpeg.num_jpeg_rings; j++) {
 288			ring = &adev->jpeg.inst[i].ring_dec[j];
 289			table_size = 0;
 290
 291			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW);
 292			MMSCH_V4_0_INSERT_DIRECT_WT(tmp, lower_32_bits(ring->gpu_addr));
 293			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH);
 294			MMSCH_V4_0_INSERT_DIRECT_WT(tmp, upper_32_bits(ring->gpu_addr));
 295			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_SIZE);
 296			MMSCH_V4_0_INSERT_DIRECT_WT(tmp, ring->ring_size / 4);
 297
 298			if (j <= 3) {
 299				header.mjpegdec0[j].table_offset = item_offset;
 300				header.mjpegdec0[j].init_status = 0;
 301				header.mjpegdec0[j].table_size = table_size;
 302			} else {
 303				header.mjpegdec1[j - 4].table_offset = item_offset;
 304				header.mjpegdec1[j - 4].init_status = 0;
 305				header.mjpegdec1[j - 4].table_size = table_size;
 306			}
 307			header.total_size += table_size;
 308			item_offset += table_size;
 309		}
 310
 311		MMSCH_V4_0_INSERT_END();
 312
 313		/* send init table to MMSCH */
 314		size = sizeof(struct mmsch_v4_0_3_init_header);
 315		table_loc = (uint32_t *)table->cpu_addr;
 316		memcpy((void *)table_loc, &header, size);
 317
 318		ctx_addr = table->gpu_addr;
 319		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
 320		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
 321
 322		tmp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID);
 323		tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
 324		tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
 325		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID, tmp);
 326
 327		size = header.total_size;
 328		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_SIZE, size);
 329
 330		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP, 0);
 331
 332		param = 0x00000001;
 333		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_HOST, param);
 334		tmp = 0;
 335		timeout = 1000;
 336		resp = 0;
 337		expected = MMSCH_VF_MAILBOX_RESP__OK;
 338		init_status =
 339			((struct mmsch_v4_0_3_init_header *)(table_loc))->mjpegdec0[i].init_status;
 340		while (resp != expected) {
 341			resp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP);
 342
 343			if (resp != 0)
 344				break;
 345			udelay(10);
 346			tmp = tmp + 10;
 347			if (tmp >= timeout) {
 348				DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
 349					" waiting for regMMSCH_VF_MAILBOX_RESP "\
 350					"(expected=0x%08x, readback=0x%08x)\n",
 351					tmp, expected, resp);
 352				return -EBUSY;
 353			}
 354		}
 355		if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE &&
 356				init_status != MMSCH_VF_ENGINE_STATUS__PASS)
 357			DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n",
 358					resp, init_status);
 359
 360	}
 361	return 0;
 362}
 363
 364/**
 365 * jpeg_v4_0_3_hw_init - start and test JPEG block
 366 *
 367 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 368 *
 369 */
 370static int jpeg_v4_0_3_hw_init(struct amdgpu_ip_block *ip_block)
 371{
 372	struct amdgpu_device *adev = ip_block->adev;
 373	struct amdgpu_ring *ring;
 374	int i, j, r, jpeg_inst;
 375
 376	if (amdgpu_sriov_vf(adev)) {
 377		r = jpeg_v4_0_3_start_sriov(adev);
 378		if (r)
 379			return r;
 380
 381		for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 382			for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 383				ring = &adev->jpeg.inst[i].ring_dec[j];
 384				ring->wptr = 0;
 385				ring->wptr_old = 0;
 386				jpeg_v4_0_3_dec_ring_set_wptr(ring);
 387				ring->sched.ready = true;
 388			}
 389		}
 390	} else {
 391		/* This flag is not set for VF, assumed to be disabled always */
 392		if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) &
 393		    0x100)
 394			adev->jpeg.caps |= AMDGPU_JPEG_CAPS(RRMT_ENABLED);
 395
 396		for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 397			jpeg_inst = GET_INST(JPEG, i);
 398
 399			ring = adev->jpeg.inst[i].ring_dec;
 400
 401			if (ring->use_doorbell)
 402				adev->nbio.funcs->vcn_doorbell_range(
 403					adev, ring->use_doorbell,
 404					(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 405						9 * jpeg_inst,
 406					adev->jpeg.inst[i].aid_id);
 407
 408			for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 409				ring = &adev->jpeg.inst[i].ring_dec[j];
 410				if (ring->use_doorbell)
 411					WREG32_SOC15_OFFSET(
 412						VCN, GET_INST(VCN, i),
 413						regVCN_JPEG_DB_CTRL,
 414						(ring->pipe ? (ring->pipe - 0x15) : 0),
 415						ring->doorbell_index
 416							<< VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
 417							VCN_JPEG_DB_CTRL__EN_MASK);
 418				r = amdgpu_ring_test_helper(ring);
 419				if (r)
 420					return r;
 421			}
 422		}
 423	}
 424
 425	return 0;
 426}
 427
 428/**
 429 * jpeg_v4_0_3_hw_fini - stop the hardware block
 430 *
 431 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 432 *
 433 * Stop the JPEG block, mark ring as not ready any more
 434 */
 435static int jpeg_v4_0_3_hw_fini(struct amdgpu_ip_block *ip_block)
 436{
 437	struct amdgpu_device *adev = ip_block->adev;
 438	int ret = 0;
 439
 440	cancel_delayed_work_sync(&adev->jpeg.idle_work);
 441
 442	if (!amdgpu_sriov_vf(adev)) {
 443		if (adev->jpeg.cur_state != AMD_PG_STATE_GATE)
 444			ret = jpeg_v4_0_3_set_powergating_state(ip_block, AMD_PG_STATE_GATE);
 445	}
 446
 447	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG) && !amdgpu_sriov_vf(adev))
 448		amdgpu_irq_put(adev, &adev->jpeg.inst->ras_poison_irq, 0);
 449
 450	return ret;
 451}
 452
 453/**
 454 * jpeg_v4_0_3_suspend - suspend JPEG block
 455 *
 456 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 457 *
 458 * HW fini and suspend JPEG block
 459 */
 460static int jpeg_v4_0_3_suspend(struct amdgpu_ip_block *ip_block)
 461{
 462	int r;
 463
 464	r = jpeg_v4_0_3_hw_fini(ip_block);
 465	if (r)
 466		return r;
 467
 468	r = amdgpu_jpeg_suspend(ip_block->adev);
 469
 470	return r;
 471}
 472
 473/**
 474 * jpeg_v4_0_3_resume - resume JPEG block
 475 *
 476 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
 477 *
 478 * Resume firmware and hw init JPEG block
 479 */
 480static int jpeg_v4_0_3_resume(struct amdgpu_ip_block *ip_block)
 481{
 482	int r;
 483
 484	r = amdgpu_jpeg_resume(ip_block->adev);
 485	if (r)
 486		return r;
 487
 488	r = jpeg_v4_0_3_hw_init(ip_block);
 489
 490	return r;
 491}
 492
 493static void jpeg_v4_0_3_disable_clock_gating(struct amdgpu_device *adev, int inst_idx)
 494{
 495	int i, jpeg_inst;
 496	uint32_t data;
 497
 498	jpeg_inst = GET_INST(JPEG, inst_idx);
 499	data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
 500	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 501		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 502		data &= (~(JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1));
 503	} else {
 504		data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 505	}
 506
 507	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 508	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 509	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
 510
 511	data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
 512	data &= ~(JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
 513	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
 514		data &= ~(JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
 515	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
 516}
 517
 518static void jpeg_v4_0_3_enable_clock_gating(struct amdgpu_device *adev, int inst_idx)
 519{
 520	int i, jpeg_inst;
 521	uint32_t data;
 522
 523	jpeg_inst = GET_INST(JPEG, inst_idx);
 524	data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL);
 525	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
 526		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 527		data |= (JPEG_CGC_CTRL__JPEG0_DEC_MODE_MASK << 1);
 528	} else {
 529		data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 530	}
 531
 532	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 533	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 534	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_CTRL, data);
 535
 536	data = RREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE);
 537	data |= (JPEG_CGC_GATE__JMCIF_MASK | JPEG_CGC_GATE__JRBBM_MASK);
 538	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i)
 539		data |= (JPEG_CGC_GATE__JPEG0_DEC_MASK << i);
 540	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CGC_GATE, data);
 541}
 542
 543static void jpeg_v4_0_3_start_inst(struct amdgpu_device *adev, int inst)
 544{
 545	int jpeg_inst = GET_INST(JPEG, inst);
 546
 547	WREG32_SOC15(JPEG, jpeg_inst, regUVD_PGFSM_CONFIG,
 548		     1 << UVD_PGFSM_CONFIG__UVDJ_PWR_CONFIG__SHIFT);
 549	SOC15_WAIT_ON_RREG(JPEG, jpeg_inst, regUVD_PGFSM_STATUS,
 550			   UVD_PGFSM_STATUS__UVDJ_PWR_ON <<
 551			   UVD_PGFSM_STATUS__UVDJ_PWR_STATUS__SHIFT,
 552			   UVD_PGFSM_STATUS__UVDJ_PWR_STATUS_MASK);
 553
 554	/* disable anti hang mechanism */
 555	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS),
 556		 0, ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 557
 558	/* JPEG disable CGC */
 559	jpeg_v4_0_3_disable_clock_gating(adev, inst);
 560
 561	/* MJPEG global tiling registers */
 562	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX8_ADDR_CONFIG,
 563		     adev->gfx.config.gb_addr_config);
 564	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_DEC_GFX10_ADDR_CONFIG,
 565		     adev->gfx.config.gb_addr_config);
 566
 567	/* enable JMI channel */
 568	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL), 0,
 569		 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 570}
 571
 572static void jpeg_v4_0_3_start_jrbc(struct amdgpu_ring *ring)
 573{
 574	struct amdgpu_device *adev = ring->adev;
 575	int jpeg_inst = GET_INST(JPEG, ring->me);
 576	int reg_offset = jpeg_v4_0_3_core_reg_offset(ring->pipe);
 577
 578	/* enable System Interrupt for JRBC */
 579	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regJPEG_SYS_INT_EN),
 580		 JPEG_SYS_INT_EN__DJRBC0_MASK << ring->pipe,
 581		 ~(JPEG_SYS_INT_EN__DJRBC0_MASK << ring->pipe));
 582
 583	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 584			    regUVD_JMI0_UVD_LMI_JRBC_RB_VMID,
 585			    reg_offset, 0);
 586	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 587			    regUVD_JRBC0_UVD_JRBC_RB_CNTL,
 588			    reg_offset,
 589			    (0x00000001L | 0x00000002L));
 590	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 591			    regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_LOW,
 592			    reg_offset, lower_32_bits(ring->gpu_addr));
 593	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 594			    regUVD_JMI0_UVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
 595			    reg_offset, upper_32_bits(ring->gpu_addr));
 596	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 597			    regUVD_JRBC0_UVD_JRBC_RB_RPTR,
 598			    reg_offset, 0);
 599	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 600			    regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 601			    reg_offset, 0);
 602	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 603			    regUVD_JRBC0_UVD_JRBC_RB_CNTL,
 604			    reg_offset, 0x00000002L);
 605	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
 606			    regUVD_JRBC0_UVD_JRBC_RB_SIZE,
 607			    reg_offset, ring->ring_size / 4);
 608	ring->wptr = RREG32_SOC15_OFFSET(JPEG, jpeg_inst, regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 609					 reg_offset);
 610}
 611
 612/**
 613 * jpeg_v4_0_3_start - start JPEG block
 614 *
 615 * @adev: amdgpu_device pointer
 616 *
 617 * Setup and start the JPEG block
 618 */
 619static int jpeg_v4_0_3_start(struct amdgpu_device *adev)
 620{
 621	struct amdgpu_ring *ring;
 622	int i, j;
 623
 624	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 625		jpeg_v4_0_3_start_inst(adev, i);
 626		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 627			ring = &adev->jpeg.inst[i].ring_dec[j];
 628			jpeg_v4_0_3_start_jrbc(ring);
 629		}
 630	}
 631
 632	return 0;
 633}
 634
 635static void jpeg_v4_0_3_stop_inst(struct amdgpu_device *adev, int inst)
 636{
 637	int jpeg_inst = GET_INST(JPEG, inst);
 638	/* reset JMI */
 639	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JMI_CNTL),
 640		 UVD_JMI_CNTL__SOFT_RESET_MASK,
 641		 ~UVD_JMI_CNTL__SOFT_RESET_MASK);
 642
 643	jpeg_v4_0_3_enable_clock_gating(adev, inst);
 644
 645	/* enable anti hang mechanism */
 646	WREG32_P(SOC15_REG_OFFSET(JPEG, jpeg_inst, regUVD_JPEG_POWER_STATUS),
 647		 UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
 648		 ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
 649
 650}
 651
 652/**
 653 * jpeg_v4_0_3_stop - stop JPEG block
 654 *
 655 * @adev: amdgpu_device pointer
 656 *
 657 * stop the JPEG block
 658 */
 659static int jpeg_v4_0_3_stop(struct amdgpu_device *adev)
 660{
 661	int i;
 662
 663	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i)
 664		jpeg_v4_0_3_stop_inst(adev, i);
 665
 666	return 0;
 667}
 668
 669/**
 670 * jpeg_v4_0_3_dec_ring_get_rptr - get read pointer
 671 *
 672 * @ring: amdgpu_ring pointer
 673 *
 674 * Returns the current hardware read pointer
 675 */
 676static uint64_t jpeg_v4_0_3_dec_ring_get_rptr(struct amdgpu_ring *ring)
 677{
 678	struct amdgpu_device *adev = ring->adev;
 679
 680	return RREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC0_UVD_JRBC_RB_RPTR,
 681				   jpeg_v4_0_3_core_reg_offset(ring->pipe));
 682}
 683
 684/**
 685 * jpeg_v4_0_3_dec_ring_get_wptr - get write pointer
 686 *
 687 * @ring: amdgpu_ring pointer
 688 *
 689 * Returns the current hardware write pointer
 690 */
 691static uint64_t jpeg_v4_0_3_dec_ring_get_wptr(struct amdgpu_ring *ring)
 692{
 693	struct amdgpu_device *adev = ring->adev;
 694
 695	if (ring->use_doorbell)
 696		return adev->wb.wb[ring->wptr_offs];
 697
 698	return RREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 699				   jpeg_v4_0_3_core_reg_offset(ring->pipe));
 700}
 701
 702void jpeg_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 703{
 704	/* JPEG engine access for HDP flush doesn't work when RRMT is enabled.
 705	 * This is a workaround to avoid any HDP flush through JPEG ring.
 706	 */
 707}
 708
 709/**
 710 * jpeg_v4_0_3_dec_ring_set_wptr - set write pointer
 711 *
 712 * @ring: amdgpu_ring pointer
 713 *
 714 * Commits the write pointer to the hardware
 715 */
 716static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring)
 717{
 718	struct amdgpu_device *adev = ring->adev;
 719
 720	if (ring->use_doorbell) {
 721		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
 722		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
 723	} else {
 724		WREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, ring->me), regUVD_JRBC0_UVD_JRBC_RB_WPTR,
 725				    jpeg_v4_0_3_core_reg_offset(ring->pipe),
 726				    lower_32_bits(ring->wptr));
 727	}
 728}
 729
 730/**
 731 * jpeg_v4_0_3_dec_ring_insert_start - insert a start command
 732 *
 733 * @ring: amdgpu_ring pointer
 734 *
 735 * Write a start command to the ring.
 736 */
 737void jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring *ring)
 738{
 739	if (!amdgpu_sriov_vf(ring->adev)) {
 740		amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 741			0, 0, PACKETJ_TYPE0));
 742		amdgpu_ring_write(ring, 0x62a04); /* PCTL0_MMHUB_DEEPSLEEP_IB */
 743
 744		amdgpu_ring_write(ring,
 745				  PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR, 0,
 746					  0, PACKETJ_TYPE0));
 747		amdgpu_ring_write(ring, 0x80004000);
 748	}
 749}
 750
 751/**
 752 * jpeg_v4_0_3_dec_ring_insert_end - insert a end command
 753 *
 754 * @ring: amdgpu_ring pointer
 755 *
 756 * Write a end command to the ring.
 757 */
 758void jpeg_v4_0_3_dec_ring_insert_end(struct amdgpu_ring *ring)
 759{
 760	if (!amdgpu_sriov_vf(ring->adev)) {
 761		amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 762			0, 0, PACKETJ_TYPE0));
 763		amdgpu_ring_write(ring, 0x62a04);
 764
 765		amdgpu_ring_write(ring,
 766				  PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR, 0,
 767					  0, PACKETJ_TYPE0));
 768		amdgpu_ring_write(ring, 0x00004000);
 769	}
 770}
 771
 772/**
 773 * jpeg_v4_0_3_dec_ring_emit_fence - emit an fence & trap command
 774 *
 775 * @ring: amdgpu_ring pointer
 776 * @addr: address
 777 * @seq: sequence number
 778 * @flags: fence related flags
 779 *
 780 * Write a fence and a trap command to the ring.
 781 */
 782void jpeg_v4_0_3_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 783				unsigned int flags)
 784{
 785	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 786
 787	amdgpu_ring_write(ring, PACKETJ(regUVD_JPEG_GPCOM_DATA0_INTERNAL_OFFSET,
 788		0, 0, PACKETJ_TYPE0));
 789	amdgpu_ring_write(ring, seq);
 790
 791	amdgpu_ring_write(ring,	PACKETJ(regUVD_JPEG_GPCOM_DATA1_INTERNAL_OFFSET,
 792		0, 0, PACKETJ_TYPE0));
 793	amdgpu_ring_write(ring, seq);
 794
 795	amdgpu_ring_write(ring,	PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW_INTERNAL_OFFSET,
 796		0, 0, PACKETJ_TYPE0));
 797	amdgpu_ring_write(ring, lower_32_bits(addr));
 798
 799	amdgpu_ring_write(ring,	PACKETJ(regUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 800		0, 0, PACKETJ_TYPE0));
 801	amdgpu_ring_write(ring, upper_32_bits(addr));
 802
 803	amdgpu_ring_write(ring,	PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
 804		0, 0, PACKETJ_TYPE0));
 805	amdgpu_ring_write(ring, 0x8);
 806
 807	amdgpu_ring_write(ring,	PACKETJ(regUVD_JPEG_GPCOM_CMD_INTERNAL_OFFSET,
 808		0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
 809	amdgpu_ring_write(ring, 0);
 810
 811	amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 812	amdgpu_ring_write(ring, 0);
 813
 814	amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 815	amdgpu_ring_write(ring, 0);
 816
 817	amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
 818	amdgpu_ring_write(ring, 0);
 819}
 820
 821/**
 822 * jpeg_v4_0_3_dec_ring_emit_ib - execute indirect buffer
 823 *
 824 * @ring: amdgpu_ring pointer
 825 * @job: job to retrieve vmid from
 826 * @ib: indirect buffer to execute
 827 * @flags: unused
 828 *
 829 * Write ring commands to execute the indirect buffer.
 830 */
 831void jpeg_v4_0_3_dec_ring_emit_ib(struct amdgpu_ring *ring,
 832				struct amdgpu_job *job,
 833				struct amdgpu_ib *ib,
 834				uint32_t flags)
 835{
 836	unsigned int vmid = AMDGPU_JOB_GET_VMID(job);
 837
 838	amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JRBC_IB_VMID_INTERNAL_OFFSET,
 839		0, 0, PACKETJ_TYPE0));
 840
 841	if (ring->funcs->parse_cs)
 842		amdgpu_ring_write(ring, 0);
 843	else
 844		amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
 845
 846	amdgpu_ring_write(ring, PACKETJ(regUVD_LMI_JPEG_VMID_INTERNAL_OFFSET,
 847		0, 0, PACKETJ_TYPE0));
 848	amdgpu_ring_write(ring, (vmid | (vmid << 4) | (vmid << 8)));
 849
 850	amdgpu_ring_write(ring,	PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET,
 851		0, 0, PACKETJ_TYPE0));
 852	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 853
 854	amdgpu_ring_write(ring,	PACKETJ(regUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 855		0, 0, PACKETJ_TYPE0));
 856	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 857
 858	amdgpu_ring_write(ring,	PACKETJ(regUVD_JRBC_IB_SIZE_INTERNAL_OFFSET,
 859		0, 0, PACKETJ_TYPE0));
 860	amdgpu_ring_write(ring, ib->length_dw);
 861
 862	amdgpu_ring_write(ring,	PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW_INTERNAL_OFFSET,
 863		0, 0, PACKETJ_TYPE0));
 864	amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
 865
 866	amdgpu_ring_write(ring,	PACKETJ(regUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH_INTERNAL_OFFSET,
 867		0, 0, PACKETJ_TYPE0));
 868	amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
 869
 870	amdgpu_ring_write(ring,	PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
 871	amdgpu_ring_write(ring, 0);
 872
 873	amdgpu_ring_write(ring,	PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
 874		0, 0, PACKETJ_TYPE0));
 875	amdgpu_ring_write(ring, 0x01400200);
 876
 877	amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
 878		0, 0, PACKETJ_TYPE0));
 879	amdgpu_ring_write(ring, 0x2);
 880
 881	amdgpu_ring_write(ring,	PACKETJ(regUVD_JRBC_STATUS_INTERNAL_OFFSET,
 882		0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
 883	amdgpu_ring_write(ring, 0x2);
 884}
 885
 886void jpeg_v4_0_3_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
 887				uint32_t val, uint32_t mask)
 888{
 889	uint32_t reg_offset;
 890
 891	/* Use normalized offsets if required */
 892	if (jpeg_v4_0_3_normalizn_reqd(ring->adev))
 893		reg = NORMALIZE_JPEG_REG_OFFSET(reg);
 894
 895	reg_offset = (reg << 2);
 896
 897	amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_RB_COND_RD_TIMER_INTERNAL_OFFSET,
 898		0, 0, PACKETJ_TYPE0));
 899	amdgpu_ring_write(ring, 0x01400200);
 900
 901	amdgpu_ring_write(ring,	PACKETJ(regUVD_JRBC_RB_REF_DATA_INTERNAL_OFFSET,
 902		0, 0, PACKETJ_TYPE0));
 903	amdgpu_ring_write(ring, val);
 904
 905	amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 906		0, 0, PACKETJ_TYPE0));
 907	if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
 908		amdgpu_ring_write(ring, 0);
 909		amdgpu_ring_write(ring,
 910			PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
 911	} else {
 912		amdgpu_ring_write(ring, reg_offset);
 913		amdgpu_ring_write(ring,	PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 914			0, 0, PACKETJ_TYPE3));
 915	}
 916	amdgpu_ring_write(ring, mask);
 917}
 918
 919void jpeg_v4_0_3_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 920				unsigned int vmid, uint64_t pd_addr)
 921{
 922	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
 923	uint32_t data0, data1, mask;
 924
 925	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 926
 927	/* wait for register write */
 928	data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
 929	data1 = lower_32_bits(pd_addr);
 930	mask = 0xffffffff;
 931	jpeg_v4_0_3_dec_ring_emit_reg_wait(ring, data0, data1, mask);
 932}
 933
 934void jpeg_v4_0_3_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
 935{
 936	uint32_t reg_offset;
 937
 938	/* Use normalized offsets if required */
 939	if (jpeg_v4_0_3_normalizn_reqd(ring->adev))
 940		reg = NORMALIZE_JPEG_REG_OFFSET(reg);
 941
 942	reg_offset = (reg << 2);
 943
 944	amdgpu_ring_write(ring,	PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET,
 945		0, 0, PACKETJ_TYPE0));
 946	if (reg_offset >= 0x10000 && reg_offset <= 0x105ff) {
 947		amdgpu_ring_write(ring, 0);
 948		amdgpu_ring_write(ring,
 949			PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
 950	} else {
 951		amdgpu_ring_write(ring, reg_offset);
 952		amdgpu_ring_write(ring,	PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR,
 953			0, 0, PACKETJ_TYPE0));
 954	}
 955	amdgpu_ring_write(ring, val);
 956}
 957
 958void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count)
 959{
 960	int i;
 961
 962	WARN_ON(ring->wptr % 2 || count % 2);
 963
 964	for (i = 0; i < count / 2; i++) {
 965		amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
 966		amdgpu_ring_write(ring, 0);
 967	}
 968}
 969
 970static bool jpeg_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block)
 971{
 972	struct amdgpu_device *adev = ip_block->adev;
 973	bool ret = false;
 974	int i, j;
 975
 976	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 977		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 978			ret &= ((RREG32_SOC15_OFFSET(JPEG, GET_INST(JPEG, i),
 979				regUVD_JRBC0_UVD_JRBC_STATUS, jpeg_v4_0_3_core_reg_offset(j)) &
 980				UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
 981				UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
 982		}
 983	}
 984
 985	return ret;
 986}
 987
 988static int jpeg_v4_0_3_wait_for_idle(struct amdgpu_ip_block *ip_block)
 989{
 990	struct amdgpu_device *adev = ip_block->adev;
 991	int ret = 0;
 992	int i, j;
 993
 994	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
 995		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 996			ret &= (SOC15_WAIT_ON_RREG_OFFSET(JPEG, GET_INST(JPEG, i),
 997				regUVD_JRBC0_UVD_JRBC_STATUS, jpeg_v4_0_3_core_reg_offset(j),
 998				UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
 999				UVD_JRBC0_UVD_JRBC_STATUS__RB_JOB_DONE_MASK));
1000		}
1001	}
1002	return ret;
1003}
1004
1005static int jpeg_v4_0_3_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1006					  enum amd_clockgating_state state)
1007{
1008	struct amdgpu_device *adev = ip_block->adev;
1009	bool enable = state == AMD_CG_STATE_GATE;
1010	int i;
1011
1012	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1013		if (enable) {
1014			if (!jpeg_v4_0_3_is_idle(ip_block))
1015				return -EBUSY;
1016			jpeg_v4_0_3_enable_clock_gating(adev, i);
1017		} else {
1018			jpeg_v4_0_3_disable_clock_gating(adev, i);
1019		}
1020	}
1021	return 0;
1022}
1023
1024static int jpeg_v4_0_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
1025					  enum amd_powergating_state state)
1026{
1027	struct amdgpu_device *adev = ip_block->adev;
1028	int ret;
1029
1030	if (amdgpu_sriov_vf(adev)) {
1031		adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
1032		return 0;
1033	}
1034
1035	if (state == adev->jpeg.cur_state)
1036		return 0;
1037
1038	if (state == AMD_PG_STATE_GATE)
1039		ret = jpeg_v4_0_3_stop(adev);
1040	else
1041		ret = jpeg_v4_0_3_start(adev);
1042
1043	if (!ret)
1044		adev->jpeg.cur_state = state;
1045
1046	return ret;
1047}
1048
1049static int jpeg_v4_0_3_set_interrupt_state(struct amdgpu_device *adev,
1050					struct amdgpu_irq_src *source,
1051					unsigned int type,
1052					enum amdgpu_interrupt_state state)
1053{
1054	return 0;
1055}
1056
1057static int jpeg_v4_0_3_set_ras_interrupt_state(struct amdgpu_device *adev,
1058					struct amdgpu_irq_src *source,
1059					unsigned int type,
1060					enum amdgpu_interrupt_state state)
1061{
1062	return 0;
1063}
1064
1065static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device *adev,
1066				      struct amdgpu_irq_src *source,
1067				      struct amdgpu_iv_entry *entry)
1068{
1069	uint32_t i, inst;
1070
1071	i = node_id_to_phys_map[entry->node_id];
1072	DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
1073
1074	for (inst = 0; inst < adev->jpeg.num_jpeg_inst; ++inst)
1075		if (adev->jpeg.inst[inst].aid_id == i)
1076			break;
1077
1078	if (inst >= adev->jpeg.num_jpeg_inst) {
1079		dev_WARN_ONCE(adev->dev, 1,
1080			      "Interrupt received for unknown JPEG instance %d",
1081			      entry->node_id);
1082		return 0;
1083	}
1084
1085	switch (entry->src_id) {
1086	case VCN_4_0__SRCID__JPEG_DECODE:
1087		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
1088		break;
1089	case VCN_4_0__SRCID__JPEG1_DECODE:
1090		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
1091		break;
1092	case VCN_4_0__SRCID__JPEG2_DECODE:
1093		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
1094		break;
1095	case VCN_4_0__SRCID__JPEG3_DECODE:
1096		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
1097		break;
1098	case VCN_4_0__SRCID__JPEG4_DECODE:
1099		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
1100		break;
1101	case VCN_4_0__SRCID__JPEG5_DECODE:
1102		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
1103		break;
1104	case VCN_4_0__SRCID__JPEG6_DECODE:
1105		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
1106		break;
1107	case VCN_4_0__SRCID__JPEG7_DECODE:
1108		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
1109		break;
1110	default:
1111		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
1112			  entry->src_id, entry->src_data[0]);
1113		break;
1114	}
1115
1116	return 0;
1117}
1118
1119static void jpeg_v4_0_3_core_stall_reset(struct amdgpu_ring *ring)
1120{
1121	struct amdgpu_device *adev = ring->adev;
1122	int jpeg_inst = GET_INST(JPEG, ring->me);
1123	int reg_offset = jpeg_v4_0_3_core_reg_offset(ring->pipe);
1124
1125	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
1126			    regUVD_JMI0_UVD_JMI_CLIENT_STALL,
1127			    reg_offset, 0x1F);
1128	SOC15_WAIT_ON_RREG_OFFSET(JPEG, jpeg_inst,
1129				  regUVD_JMI0_UVD_JMI_CLIENT_CLEAN_STATUS,
1130				  reg_offset, 0x1F, 0x1F);
1131	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
1132			    regUVD_JMI0_JPEG_LMI_DROP,
1133			    reg_offset, 0x1F);
1134	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CORE_RST_CTRL, 1 << ring->pipe);
1135	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
1136			    regUVD_JMI0_UVD_JMI_CLIENT_STALL,
1137			    reg_offset, 0x00);
1138	WREG32_SOC15_OFFSET(JPEG, jpeg_inst,
1139			    regUVD_JMI0_JPEG_LMI_DROP,
1140			    reg_offset, 0x00);
1141	WREG32_SOC15(JPEG, jpeg_inst, regJPEG_CORE_RST_CTRL, 0x00);
1142}
1143
1144static int jpeg_v4_0_3_ring_reset(struct amdgpu_ring *ring,
1145				  unsigned int vmid,
1146				  struct amdgpu_fence *timedout_fence)
1147{
1148	if (amdgpu_sriov_vf(ring->adev))
1149		return -EOPNOTSUPP;
1150
1151	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
1152	jpeg_v4_0_3_core_stall_reset(ring);
1153	jpeg_v4_0_3_start_jrbc(ring);
1154	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
1155}
1156
1157static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
1158	.name = "jpeg_v4_0_3",
1159	.early_init = jpeg_v4_0_3_early_init,
1160	.sw_init = jpeg_v4_0_3_sw_init,
1161	.sw_fini = jpeg_v4_0_3_sw_fini,
1162	.hw_init = jpeg_v4_0_3_hw_init,
1163	.hw_fini = jpeg_v4_0_3_hw_fini,
1164	.suspend = jpeg_v4_0_3_suspend,
1165	.resume = jpeg_v4_0_3_resume,
1166	.is_idle = jpeg_v4_0_3_is_idle,
1167	.wait_for_idle = jpeg_v4_0_3_wait_for_idle,
1168	.set_clockgating_state = jpeg_v4_0_3_set_clockgating_state,
1169	.set_powergating_state = jpeg_v4_0_3_set_powergating_state,
1170	.dump_ip_state = amdgpu_jpeg_dump_ip_state,
1171	.print_ip_state = amdgpu_jpeg_print_ip_state,
1172};
1173
1174static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = {
1175	.type = AMDGPU_RING_TYPE_VCN_JPEG,
1176	.align_mask = 0xf,
1177	.get_rptr = jpeg_v4_0_3_dec_ring_get_rptr,
1178	.get_wptr = jpeg_v4_0_3_dec_ring_get_wptr,
1179	.set_wptr = jpeg_v4_0_3_dec_ring_set_wptr,
1180	.parse_cs = amdgpu_jpeg_dec_parse_cs,
1181	.emit_frame_size =
1182		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1183		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1184		8 + /* jpeg_v4_0_3_dec_ring_emit_vm_flush */
1185		18 + 18 + /* jpeg_v4_0_3_dec_ring_emit_fence x2 vm fence */
1186		8 + 16,
1187	.emit_ib_size = 22, /* jpeg_v4_0_3_dec_ring_emit_ib */
1188	.emit_ib = jpeg_v4_0_3_dec_ring_emit_ib,
1189	.emit_fence = jpeg_v4_0_3_dec_ring_emit_fence,
1190	.emit_vm_flush = jpeg_v4_0_3_dec_ring_emit_vm_flush,
1191	.emit_hdp_flush = jpeg_v4_0_3_ring_emit_hdp_flush,
1192	.test_ring = amdgpu_jpeg_dec_ring_test_ring,
1193	.test_ib = amdgpu_jpeg_dec_ring_test_ib,
1194	.insert_nop = jpeg_v4_0_3_dec_ring_nop,
1195	.insert_start = jpeg_v4_0_3_dec_ring_insert_start,
1196	.insert_end = jpeg_v4_0_3_dec_ring_insert_end,
1197	.pad_ib = amdgpu_ring_generic_pad_ib,
1198	.begin_use = amdgpu_jpeg_ring_begin_use,
1199	.end_use = amdgpu_jpeg_ring_end_use,
1200	.emit_wreg = jpeg_v4_0_3_dec_ring_emit_wreg,
1201	.emit_reg_wait = jpeg_v4_0_3_dec_ring_emit_reg_wait,
1202	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1203	.reset = jpeg_v4_0_3_ring_reset,
1204};
1205
1206static void jpeg_v4_0_3_set_dec_ring_funcs(struct amdgpu_device *adev)
1207{
1208	int i, j, jpeg_inst;
1209
1210	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1211		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
1212			adev->jpeg.inst[i].ring_dec[j].funcs = &jpeg_v4_0_3_dec_ring_vm_funcs;
1213			adev->jpeg.inst[i].ring_dec[j].me = i;
1214			adev->jpeg.inst[i].ring_dec[j].pipe = j;
1215		}
1216		jpeg_inst = GET_INST(JPEG, i);
1217		adev->jpeg.inst[i].aid_id =
1218			jpeg_inst / adev->jpeg.num_inst_per_aid;
1219	}
1220}
1221
1222static const struct amdgpu_irq_src_funcs jpeg_v4_0_3_irq_funcs = {
1223	.set = jpeg_v4_0_3_set_interrupt_state,
1224	.process = jpeg_v4_0_3_process_interrupt,
1225};
1226
1227static const struct amdgpu_irq_src_funcs jpeg_v4_0_3_ras_irq_funcs = {
1228	.set = jpeg_v4_0_3_set_ras_interrupt_state,
1229	.process = amdgpu_jpeg_process_poison_irq,
1230};
1231
1232static void jpeg_v4_0_3_set_irq_funcs(struct amdgpu_device *adev)
1233{
1234	int i;
1235
1236	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
1237		adev->jpeg.inst->irq.num_types += adev->jpeg.num_jpeg_rings;
1238	}
1239	adev->jpeg.inst->irq.funcs = &jpeg_v4_0_3_irq_funcs;
1240
1241	adev->jpeg.inst->ras_poison_irq.num_types = 1;
1242	adev->jpeg.inst->ras_poison_irq.funcs = &jpeg_v4_0_3_ras_irq_funcs;
1243}
1244
1245const struct amdgpu_ip_block_version jpeg_v4_0_3_ip_block = {
1246	.type = AMD_IP_BLOCK_TYPE_JPEG,
1247	.major = 4,
1248	.minor = 0,
1249	.rev = 3,
1250	.funcs = &jpeg_v4_0_3_ip_funcs,
1251};
1252
1253static const struct amdgpu_ras_err_status_reg_entry jpeg_v4_0_3_ue_reg_list[] = {
1254	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0S, regVCN_UE_ERR_STATUS_HI_JPEG0S),
1255	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0S"},
1256	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG0D, regVCN_UE_ERR_STATUS_HI_JPEG0D),
1257	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG0D"},
1258	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1S, regVCN_UE_ERR_STATUS_HI_JPEG1S),
1259	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1S"},
1260	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG1D, regVCN_UE_ERR_STATUS_HI_JPEG1D),
1261	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG1D"},
1262	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2S, regVCN_UE_ERR_STATUS_HI_JPEG2S),
1263	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2S"},
1264	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG2D, regVCN_UE_ERR_STATUS_HI_JPEG2D),
1265	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG2D"},
1266	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3S, regVCN_UE_ERR_STATUS_HI_JPEG3S),
1267	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3S"},
1268	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG3D, regVCN_UE_ERR_STATUS_HI_JPEG3D),
1269	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG3D"},
1270	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4S, regVCN_UE_ERR_STATUS_HI_JPEG4S),
1271	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4S"},
1272	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG4D, regVCN_UE_ERR_STATUS_HI_JPEG4D),
1273	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG4D"},
1274	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5S, regVCN_UE_ERR_STATUS_HI_JPEG5S),
1275	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5S"},
1276	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG5D, regVCN_UE_ERR_STATUS_HI_JPEG5D),
1277	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG5D"},
1278	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6S, regVCN_UE_ERR_STATUS_HI_JPEG6S),
1279	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6S"},
1280	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG6D, regVCN_UE_ERR_STATUS_HI_JPEG6D),
1281	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG6D"},
1282	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7S, regVCN_UE_ERR_STATUS_HI_JPEG7S),
1283	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7S"},
1284	{AMDGPU_RAS_REG_ENTRY(JPEG, 0, regVCN_UE_ERR_STATUS_LO_JPEG7D, regVCN_UE_ERR_STATUS_HI_JPEG7D),
1285	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "JPEG7D"},
1286};
1287
1288static void jpeg_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev,
1289						   uint32_t jpeg_inst,
1290						   void *ras_err_status)
1291{
1292	struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
1293
1294	/* jpeg v4_0_3 only support uncorrectable errors */
1295	amdgpu_ras_inst_query_ras_error_count(adev,
1296			jpeg_v4_0_3_ue_reg_list,
1297			ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1298			NULL, 0, GET_INST(VCN, jpeg_inst),
1299			AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
1300			&err_data->ue_count);
1301}
1302
1303static void jpeg_v4_0_3_query_ras_error_count(struct amdgpu_device *adev,
1304					      void *ras_err_status)
1305{
1306	uint32_t i;
1307
1308	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1309		dev_warn(adev->dev, "JPEG RAS is not supported\n");
1310		return;
1311	}
1312
1313	for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1314		jpeg_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status);
1315}
1316
1317static void jpeg_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev,
1318						   uint32_t jpeg_inst)
1319{
1320	amdgpu_ras_inst_reset_ras_error_count(adev,
1321			jpeg_v4_0_3_ue_reg_list,
1322			ARRAY_SIZE(jpeg_v4_0_3_ue_reg_list),
1323			GET_INST(VCN, jpeg_inst));
1324}
1325
1326static void jpeg_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev)
1327{
1328	uint32_t i;
1329
1330	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__JPEG)) {
1331		dev_warn(adev->dev, "JPEG RAS is not supported\n");
1332		return;
1333	}
1334
1335	for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
1336		jpeg_v4_0_3_inst_reset_ras_error_count(adev, i);
1337}
1338
1339static uint32_t jpeg_v4_0_3_query_poison_by_instance(struct amdgpu_device *adev,
1340		uint32_t instance, uint32_t sub_block)
1341{
1342	uint32_t poison_stat = 0, reg_value = 0;
1343
1344	switch (sub_block) {
1345	case AMDGPU_JPEG_V4_0_3_JPEG0:
1346		reg_value = RREG32_SOC15(JPEG, instance, regUVD_RAS_JPEG0_STATUS);
1347		poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_JPEG0_STATUS, POISONED_PF);
1348		break;
1349	case AMDGPU_JPEG_V4_0_3_JPEG1:
1350		reg_value = RREG32_SOC15(JPEG, instance, regUVD_RAS_JPEG1_STATUS);
1351		poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_JPEG1_STATUS, POISONED_PF);
1352		break;
1353	default:
1354		break;
1355	}
1356
1357	if (poison_stat)
1358		dev_info(adev->dev, "Poison detected in JPEG%d sub_block%d\n",
1359			instance, sub_block);
1360
1361	return poison_stat;
1362}
1363
1364static bool jpeg_v4_0_3_query_ras_poison_status(struct amdgpu_device *adev)
1365{
1366	uint32_t inst = 0, sub = 0, poison_stat = 0;
1367
1368	for (inst = 0; inst < adev->jpeg.num_jpeg_inst; inst++)
1369		for (sub = 0; sub < AMDGPU_JPEG_V4_0_3_MAX_SUB_BLOCK; sub++)
1370			poison_stat +=
1371			jpeg_v4_0_3_query_poison_by_instance(adev, inst, sub);
1372
1373	return !!poison_stat;
1374}
1375
1376static const struct amdgpu_ras_block_hw_ops jpeg_v4_0_3_ras_hw_ops = {
1377	.query_ras_error_count = jpeg_v4_0_3_query_ras_error_count,
1378	.reset_ras_error_count = jpeg_v4_0_3_reset_ras_error_count,
1379	.query_poison_status = jpeg_v4_0_3_query_ras_poison_status,
1380};
1381
1382static int jpeg_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1383				      enum aca_smu_type type, void *data)
1384{
1385	struct aca_bank_info info;
1386	u64 misc0;
1387	int ret;
1388
1389	ret = aca_bank_info_decode(bank, &info);
1390	if (ret)
1391		return ret;
1392
1393	misc0 = bank->regs[ACA_REG_IDX_MISC0];
1394	switch (type) {
1395	case ACA_SMU_TYPE_UE:
1396		bank->aca_err_type = ACA_ERROR_TYPE_UE;
1397		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
1398						     1ULL);
1399		break;
1400	case ACA_SMU_TYPE_CE:
1401		bank->aca_err_type = ACA_ERROR_TYPE_CE;
1402		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
1403						     ACA_REG__MISC0__ERRCNT(misc0));
1404		break;
1405	default:
1406		return -EINVAL;
1407	}
1408
1409	return ret;
1410}
1411
1412/* reference to smu driver if header file */
1413static int jpeg_v4_0_3_err_codes[] = {
1414	16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-7][S|D] */
1415	24, 25, 26, 27, 28, 29, 30, 31
1416};
1417
1418static bool jpeg_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
1419					 enum aca_smu_type type, void *data)
1420{
1421	u32 instlo;
1422
1423	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
1424	instlo &= GENMASK(31, 1);
1425
1426	if (instlo != mmSMNAID_AID0_MCA_SMU)
1427		return false;
1428
1429	if (aca_bank_check_error_codes(handle->adev, bank,
1430				       jpeg_v4_0_3_err_codes,
1431				       ARRAY_SIZE(jpeg_v4_0_3_err_codes)))
1432		return false;
1433
1434	return true;
1435}
1436
1437static const struct aca_bank_ops jpeg_v4_0_3_aca_bank_ops = {
1438	.aca_bank_parser = jpeg_v4_0_3_aca_bank_parser,
1439	.aca_bank_is_valid = jpeg_v4_0_3_aca_bank_is_valid,
1440};
1441
1442static const struct aca_info jpeg_v4_0_3_aca_info = {
1443	.hwip = ACA_HWIP_TYPE_SMU,
1444	.mask = ACA_ERROR_UE_MASK,
1445	.bank_ops = &jpeg_v4_0_3_aca_bank_ops,
1446};
1447
1448static int jpeg_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1449{
1450	int r;
1451
1452	r = amdgpu_ras_block_late_init(adev, ras_block);
1453	if (r)
1454		return r;
1455
1456	if (amdgpu_ras_is_supported(adev, ras_block->block) &&
1457		adev->jpeg.inst->ras_poison_irq.funcs) {
1458		r = amdgpu_irq_get(adev, &adev->jpeg.inst->ras_poison_irq, 0);
1459		if (r)
1460			goto late_fini;
1461	}
1462
1463	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__JPEG,
1464				&jpeg_v4_0_3_aca_info, NULL);
1465	if (r)
1466		goto late_fini;
1467
1468	return 0;
1469
1470late_fini:
1471	amdgpu_ras_block_late_fini(adev, ras_block);
1472
1473	return r;
1474}
1475
1476static struct amdgpu_jpeg_ras jpeg_v4_0_3_ras = {
1477	.ras_block = {
1478		.hw_ops = &jpeg_v4_0_3_ras_hw_ops,
1479		.ras_late_init = jpeg_v4_0_3_ras_late_init,
1480	},
1481};
1482
1483static void jpeg_v4_0_3_set_ras_funcs(struct amdgpu_device *adev)
1484{
1485	adev->jpeg.ras = &jpeg_v4_0_3_ras;
1486}