sdma_v4_4_2.c

   1/*
   2 * Copyright 2022 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/delay.h>
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_xcp.h"
  31#include "amdgpu_ucode.h"
  32#include "amdgpu_trace.h"
  33#include "amdgpu_reset.h"
  34
  35#include "sdma/sdma_4_4_2_offset.h"
  36#include "sdma/sdma_4_4_2_sh_mask.h"
  37
  38#include "soc15_common.h"
  39#include "soc15.h"
  40#include "vega10_sdma_pkt_open.h"
  41
  42#include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
  43#include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
  44
  45#include "amdgpu_ras.h"
  46
  47MODULE_FIRMWARE("amdgpu/sdma_4_4_2.bin");
  48MODULE_FIRMWARE("amdgpu/sdma_4_4_4.bin");
  49MODULE_FIRMWARE("amdgpu/sdma_4_4_5.bin");
  50
  51static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_4_2[] = {
  52	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS_REG),
  53	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS1_REG),
  54	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS2_REG),
  55	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS3_REG),
  56	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UCODE_CHECKSUM),
  57	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RB_RPTR_FETCH_HI),
  58	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RB_RPTR_FETCH),
  59	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_STATUS),
  60	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_STATUS),
  61	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_XNACK0),
  62	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_XNACK1),
  63	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_XNACK0),
  64	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_XNACK1),
  65	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_CNTL),
  66	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_RPTR),
  67	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_RPTR_HI),
  68	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_WPTR),
  69	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_WPTR_HI),
  70	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_OFFSET),
  71	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_BASE_LO),
  72	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_BASE_HI),
  73	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_CNTL),
  74	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_RPTR),
  75	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_SUB_REMAIN),
  76	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_DUMMY_REG),
  77	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_CNTL),
  78	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_RPTR),
  79	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_RPTR_HI),
  80	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_WPTR),
  81	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_WPTR_HI),
  82	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_OFFSET),
  83	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_BASE_LO),
  84	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_BASE_HI),
  85	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_DUMMY_REG),
  86	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_CNTL),
  87	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_RPTR),
  88	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_RPTR_HI),
  89	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_WPTR),
  90	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_WPTR_HI),
  91	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_OFFSET),
  92	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_BASE_LO),
  93	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_BASE_HI),
  94	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_DUMMY_REG),
  95	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_VM_CNTL)
  96};
  97
  98#define mmSMNAID_AID0_MCA_SMU 0x03b30400
  99
 100#define WREG32_SDMA(instance, offset, value) \
 101	WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
 102#define RREG32_SDMA(instance, offset) \
 103	RREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)))
 104
 105static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
 106static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
 107static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
 108static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
 109static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev);
 110static void sdma_v4_4_2_update_reset_mask(struct amdgpu_device *adev);
 111static int sdma_v4_4_2_stop_queue(struct amdgpu_ring *ring);
 112static int sdma_v4_4_2_restore_queue(struct amdgpu_ring *ring);
 113static int sdma_v4_4_2_soft_reset_engine(struct amdgpu_device *adev,
 114					 u32 instance_id);
 115
 116static u32 sdma_v4_4_2_get_reg_offset(struct amdgpu_device *adev,
 117		u32 instance, u32 offset)
 118{
 119	u32 dev_inst = GET_INST(SDMA0, instance);
 120
 121	return (adev->reg_offset[SDMA0_HWIP][dev_inst][0] + offset);
 122}
 123
 124static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
 125{
 126	switch (seq_num) {
 127	case 0:
 128		return SOC15_IH_CLIENTID_SDMA0;
 129	case 1:
 130		return SOC15_IH_CLIENTID_SDMA1;
 131	case 2:
 132		return SOC15_IH_CLIENTID_SDMA2;
 133	case 3:
 134		return SOC15_IH_CLIENTID_SDMA3;
 135	default:
 136		return -EINVAL;
 137	}
 138}
 139
 140static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev, unsigned client_id)
 141{
 142	switch (client_id) {
 143	case SOC15_IH_CLIENTID_SDMA0:
 144		return 0;
 145	case SOC15_IH_CLIENTID_SDMA1:
 146		return 1;
 147	case SOC15_IH_CLIENTID_SDMA2:
 148		if (amdgpu_sriov_vf(adev) && (adev->gfx.xcc_mask == 0x1))
 149			return 0;
 150		else
 151			return 2;
 152	case SOC15_IH_CLIENTID_SDMA3:
 153		if (amdgpu_sriov_vf(adev) && (adev->gfx.xcc_mask == 0x1))
 154			return 1;
 155		else
 156			return 3;
 157	default:
 158		return -EINVAL;
 159	}
 160}
 161
 162static void sdma_v4_4_2_inst_init_golden_registers(struct amdgpu_device *adev,
 163						   uint32_t inst_mask)
 164{
 165	u32 val;
 166	int i;
 167
 168	for (i = 0; i < adev->sdma.num_instances; i++) {
 169		val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG);
 170		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG, NUM_BANKS, 4);
 171		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG,
 172				    PIPE_INTERLEAVE_SIZE, 0);
 173		WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG, val);
 174
 175		val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ);
 176		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ, NUM_BANKS,
 177				    4);
 178		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ,
 179				    PIPE_INTERLEAVE_SIZE, 0);
 180		WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ, val);
 181	}
 182}
 183
 184/**
 185 * sdma_v4_4_2_init_microcode - load ucode images from disk
 186 *
 187 * @adev: amdgpu_device pointer
 188 *
 189 * Use the firmware interface to load the ucode images into
 190 * the driver (not loaded into hw).
 191 * Returns 0 on success, error on failure.
 192 */
 193static int sdma_v4_4_2_init_microcode(struct amdgpu_device *adev)
 194{
 195	int ret, i;
 196
 197	for (i = 0; i < adev->sdma.num_instances; i++) {
 198		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 2) ||
 199		    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 4) ||
 200		    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 5)) {
 201			ret = amdgpu_sdma_init_microcode(adev, 0, true);
 202			break;
 203		} else {
 204			ret = amdgpu_sdma_init_microcode(adev, i, false);
 205			if (ret)
 206				return ret;
 207		}
 208	}
 209
 210	return ret;
 211}
 212
 213/**
 214 * sdma_v4_4_2_ring_get_rptr - get the current read pointer
 215 *
 216 * @ring: amdgpu ring pointer
 217 *
 218 * Get the current rptr from the hardware.
 219 */
 220static uint64_t sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring *ring)
 221{
 222	u64 rptr;
 223
 224	/* XXX check if swapping is necessary on BE */
 225	rptr = READ_ONCE(*((u64 *)&ring->adev->wb.wb[ring->rptr_offs]));
 226
 227	DRM_DEBUG("rptr before shift == 0x%016llx\n", rptr);
 228	return rptr >> 2;
 229}
 230
 231/**
 232 * sdma_v4_4_2_ring_get_wptr - get the current write pointer
 233 *
 234 * @ring: amdgpu ring pointer
 235 *
 236 * Get the current wptr from the hardware.
 237 */
 238static uint64_t sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring *ring)
 239{
 240	struct amdgpu_device *adev = ring->adev;
 241	u64 wptr;
 242
 243	if (ring->use_doorbell) {
 244		/* XXX check if swapping is necessary on BE */
 245		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
 246		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 247	} else {
 248		wptr = RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI);
 249		wptr = wptr << 32;
 250		wptr |= RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR);
 251		DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
 252				ring->me, wptr);
 253	}
 254
 255	return wptr >> 2;
 256}
 257
 258/**
 259 * sdma_v4_4_2_ring_set_wptr - commit the write pointer
 260 *
 261 * @ring: amdgpu ring pointer
 262 *
 263 * Write the wptr back to the hardware.
 264 */
 265static void sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring *ring)
 266{
 267	struct amdgpu_device *adev = ring->adev;
 268
 269	DRM_DEBUG("Setting write pointer\n");
 270	if (ring->use_doorbell) {
 271		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
 272
 273		DRM_DEBUG("Using doorbell -- "
 274				"wptr_offs == 0x%08x "
 275				"lower_32_bits(ring->wptr) << 2 == 0x%08x "
 276				"upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
 277				ring->wptr_offs,
 278				lower_32_bits(ring->wptr << 2),
 279				upper_32_bits(ring->wptr << 2));
 280		/* XXX check if swapping is necessary on BE */
 281		WRITE_ONCE(*wb, (ring->wptr << 2));
 282		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
 283				ring->doorbell_index, ring->wptr << 2);
 284		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 285	} else {
 286		DRM_DEBUG("Not using doorbell -- "
 287				"regSDMA%i_GFX_RB_WPTR == 0x%08x "
 288				"regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
 289				ring->me,
 290				lower_32_bits(ring->wptr << 2),
 291				ring->me,
 292				upper_32_bits(ring->wptr << 2));
 293		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR,
 294			    lower_32_bits(ring->wptr << 2));
 295		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI,
 296			    upper_32_bits(ring->wptr << 2));
 297	}
 298}
 299
 300/**
 301 * sdma_v4_4_2_page_ring_get_wptr - get the current write pointer
 302 *
 303 * @ring: amdgpu ring pointer
 304 *
 305 * Get the current wptr from the hardware.
 306 */
 307static uint64_t sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring *ring)
 308{
 309	struct amdgpu_device *adev = ring->adev;
 310	u64 wptr;
 311
 312	if (ring->use_doorbell) {
 313		/* XXX check if swapping is necessary on BE */
 314		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
 315	} else {
 316		wptr = RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI);
 317		wptr = wptr << 32;
 318		wptr |= RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR);
 319	}
 320
 321	return wptr >> 2;
 322}
 323
 324/**
 325 * sdma_v4_4_2_page_ring_set_wptr - commit the write pointer
 326 *
 327 * @ring: amdgpu ring pointer
 328 *
 329 * Write the wptr back to the hardware.
 330 */
 331static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
 332{
 333	struct amdgpu_device *adev = ring->adev;
 334
 335	if (ring->use_doorbell) {
 336		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
 337
 338		/* XXX check if swapping is necessary on BE */
 339		WRITE_ONCE(*wb, (ring->wptr << 2));
 340		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 341	} else {
 342		uint64_t wptr = ring->wptr << 2;
 343
 344		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR,
 345			    lower_32_bits(wptr));
 346		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI,
 347			    upper_32_bits(wptr));
 348	}
 349}
 350
 351static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 352{
 353	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 354	int i;
 355
 356	for (i = 0; i < count; i++)
 357		if (sdma && sdma->burst_nop && (i == 0))
 358			amdgpu_ring_write(ring, ring->funcs->nop |
 359				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 360		else
 361			amdgpu_ring_write(ring, ring->funcs->nop);
 362}
 363
 364/**
 365 * sdma_v4_4_2_ring_emit_ib - Schedule an IB on the DMA engine
 366 *
 367 * @ring: amdgpu ring pointer
 368 * @job: job to retrieve vmid from
 369 * @ib: IB object to schedule
 370 * @flags: unused
 371 *
 372 * Schedule an IB in the DMA ring.
 373 */
 374static void sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring *ring,
 375				   struct amdgpu_job *job,
 376				   struct amdgpu_ib *ib,
 377				   uint32_t flags)
 378{
 379	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 380
 381	/* IB packet must end on a 8 DW boundary */
 382	sdma_v4_4_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
 383
 384	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
 385			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 386	/* base must be 32 byte aligned */
 387	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 388	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 389	amdgpu_ring_write(ring, ib->length_dw);
 390	amdgpu_ring_write(ring, 0);
 391	amdgpu_ring_write(ring, 0);
 392
 393}
 394
 395static void sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring *ring,
 396				   int mem_space, int hdp,
 397				   uint32_t addr0, uint32_t addr1,
 398				   uint32_t ref, uint32_t mask,
 399				   uint32_t inv)
 400{
 401	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
 402			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
 403			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
 404			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
 405	if (mem_space) {
 406		/* memory */
 407		amdgpu_ring_write(ring, addr0);
 408		amdgpu_ring_write(ring, addr1);
 409	} else {
 410		/* registers */
 411		amdgpu_ring_write(ring, addr0 << 2);
 412		amdgpu_ring_write(ring, addr1 << 2);
 413	}
 414	amdgpu_ring_write(ring, ref); /* reference */
 415	amdgpu_ring_write(ring, mask); /* mask */
 416	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
 417			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
 418}
 419
 420/**
 421 * sdma_v4_4_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
 422 *
 423 * @ring: amdgpu ring pointer
 424 *
 425 * Emit an hdp flush packet on the requested DMA ring.
 426 */
 427static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 428{
 429	struct amdgpu_device *adev = ring->adev;
 430	u32 ref_and_mask = 0;
 431	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 432
 433	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0
 434		       << (ring->me % adev->sdma.num_inst_per_aid);
 435
 436	sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
 437			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
 438			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
 439			       ref_and_mask, ref_and_mask, 10);
 440}
 441
 442/**
 443 * sdma_v4_4_2_ring_emit_fence - emit a fence on the DMA ring
 444 *
 445 * @ring: amdgpu ring pointer
 446 * @addr: address
 447 * @seq: sequence number
 448 * @flags: fence related flags
 449 *
 450 * Add a DMA fence packet to the ring to write
 451 * the fence seq number and DMA trap packet to generate
 452 * an interrupt if needed.
 453 */
 454static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 455				      unsigned flags)
 456{
 457	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 458	/* write the fence */
 459	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
 460	/* zero in first two bits */
 461	BUG_ON(addr & 0x3);
 462	amdgpu_ring_write(ring, lower_32_bits(addr));
 463	amdgpu_ring_write(ring, upper_32_bits(addr));
 464	amdgpu_ring_write(ring, lower_32_bits(seq));
 465
 466	/* optionally write high bits as well */
 467	if (write64bit) {
 468		addr += 4;
 469		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
 470		/* zero in first two bits */
 471		BUG_ON(addr & 0x3);
 472		amdgpu_ring_write(ring, lower_32_bits(addr));
 473		amdgpu_ring_write(ring, upper_32_bits(addr));
 474		amdgpu_ring_write(ring, upper_32_bits(seq));
 475	}
 476
 477	/* generate an interrupt */
 478	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
 479	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
 480}
 481
 482
 483/**
 484 * sdma_v4_4_2_inst_gfx_stop - stop the gfx async dma engines
 485 *
 486 * @adev: amdgpu_device pointer
 487 * @inst_mask: mask of dma engine instances to be disabled
 488 *
 489 * Stop the gfx async dma ring buffers.
 490 */
 491static void sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device *adev,
 492				      uint32_t inst_mask)
 493{
 494	struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
 495	u32 doorbell_offset, doorbell;
 496	u32 rb_cntl, ib_cntl, sdma_cntl;
 497	int i;
 498
 499	for_each_inst(i, inst_mask) {
 500		sdma[i] = &adev->sdma.instance[i].ring;
 501
 502		rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
 503		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 0);
 504		WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
 505		ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
 506		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 0);
 507		WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
 508		sdma_cntl = RREG32_SDMA(i, regSDMA_CNTL);
 509		sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, UTC_L1_ENABLE, 0);
 510		WREG32_SDMA(i, regSDMA_CNTL, sdma_cntl);
 511
 512		if (sdma[i]->use_doorbell) {
 513			doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
 514			doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
 515
 516			doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE, 0);
 517			doorbell_offset = REG_SET_FIELD(doorbell_offset,
 518					SDMA_GFX_DOORBELL_OFFSET,
 519					OFFSET, 0);
 520			WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
 521			WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
 522		}
 523	}
 524}
 525
 526/**
 527 * sdma_v4_4_2_inst_rlc_stop - stop the compute async dma engines
 528 *
 529 * @adev: amdgpu_device pointer
 530 * @inst_mask: mask of dma engine instances to be disabled
 531 *
 532 * Stop the compute async dma queues.
 533 */
 534static void sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device *adev,
 535				      uint32_t inst_mask)
 536{
 537	/* XXX todo */
 538}
 539
 540/**
 541 * sdma_v4_4_2_inst_page_stop - stop the page async dma engines
 542 *
 543 * @adev: amdgpu_device pointer
 544 * @inst_mask: mask of dma engine instances to be disabled
 545 *
 546 * Stop the page async dma ring buffers.
 547 */
 548static void sdma_v4_4_2_inst_page_stop(struct amdgpu_device *adev,
 549				       uint32_t inst_mask)
 550{
 551	u32 rb_cntl, ib_cntl;
 552	int i;
 553
 554	for_each_inst(i, inst_mask) {
 555		rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
 556		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
 557					RB_ENABLE, 0);
 558		WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
 559		ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
 560		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL,
 561					IB_ENABLE, 0);
 562		WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
 563	}
 564}
 565
 566/**
 567 * sdma_v4_4_2_inst_ctx_switch_enable - stop the async dma engines context switch
 568 *
 569 * @adev: amdgpu_device pointer
 570 * @enable: enable/disable the DMA MEs context switch.
 571 * @inst_mask: mask of dma engine instances to be enabled
 572 *
 573 * Halt or unhalt the async dma engines context switch.
 574 */
 575static void sdma_v4_4_2_inst_ctx_switch_enable(struct amdgpu_device *adev,
 576					       bool enable, uint32_t inst_mask)
 577{
 578	u32 f32_cntl, phase_quantum = 0;
 579	int i;
 580
 581	if (amdgpu_sdma_phase_quantum) {
 582		unsigned value = amdgpu_sdma_phase_quantum;
 583		unsigned unit = 0;
 584
 585		while (value > (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
 586				SDMA_PHASE0_QUANTUM__VALUE__SHIFT)) {
 587			value = (value + 1) >> 1;
 588			unit++;
 589		}
 590		if (unit > (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
 591			    SDMA_PHASE0_QUANTUM__UNIT__SHIFT)) {
 592			value = (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
 593				 SDMA_PHASE0_QUANTUM__VALUE__SHIFT);
 594			unit = (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
 595				SDMA_PHASE0_QUANTUM__UNIT__SHIFT);
 596			WARN_ONCE(1,
 597			"clamping sdma_phase_quantum to %uK clock cycles\n",
 598				  value << unit);
 599		}
 600		phase_quantum =
 601			value << SDMA_PHASE0_QUANTUM__VALUE__SHIFT |
 602			unit  << SDMA_PHASE0_QUANTUM__UNIT__SHIFT;
 603	}
 604
 605	for_each_inst(i, inst_mask) {
 606		f32_cntl = RREG32_SDMA(i, regSDMA_CNTL);
 607		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_CNTL,
 608				AUTO_CTXSW_ENABLE, enable ? 1 : 0);
 609		if (enable && amdgpu_sdma_phase_quantum) {
 610			WREG32_SDMA(i, regSDMA_PHASE0_QUANTUM, phase_quantum);
 611			WREG32_SDMA(i, regSDMA_PHASE1_QUANTUM, phase_quantum);
 612			WREG32_SDMA(i, regSDMA_PHASE2_QUANTUM, phase_quantum);
 613		}
 614		WREG32_SDMA(i, regSDMA_CNTL, f32_cntl);
 615
 616		/* Extend page fault timeout to avoid interrupt storm */
 617		WREG32_SDMA(i, regSDMA_UTCL1_TIMEOUT, 0x00800080);
 618	}
 619}
 620
 621/**
 622 * sdma_v4_4_2_inst_enable - stop the async dma engines
 623 *
 624 * @adev: amdgpu_device pointer
 625 * @enable: enable/disable the DMA MEs.
 626 * @inst_mask: mask of dma engine instances to be enabled
 627 *
 628 * Halt or unhalt the async dma engines.
 629 */
 630static void sdma_v4_4_2_inst_enable(struct amdgpu_device *adev, bool enable,
 631				    uint32_t inst_mask)
 632{
 633	u32 f32_cntl;
 634	int i;
 635
 636	if (!enable) {
 637		sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
 638		sdma_v4_4_2_inst_rlc_stop(adev, inst_mask);
 639		if (adev->sdma.has_page_queue)
 640			sdma_v4_4_2_inst_page_stop(adev, inst_mask);
 641
 642		/* SDMA FW needs to respond to FREEZE requests during reset.
 643		 * Keep it running during reset */
 644		if (!amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
 645			return;
 646	}
 647
 648	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
 649		return;
 650
 651	for_each_inst(i, inst_mask) {
 652		f32_cntl = RREG32_SDMA(i, regSDMA_F32_CNTL);
 653		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_F32_CNTL, HALT, enable ? 0 : 1);
 654		WREG32_SDMA(i, regSDMA_F32_CNTL, f32_cntl);
 655	}
 656}
 657
 658/*
 659 * sdma_v4_4_2_rb_cntl - get parameters for rb_cntl
 660 */
 661static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
 662{
 663	/* Set ring buffer size in dwords */
 664	uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
 665
 666	barrier(); /* work around https://llvm.org/pr42576 */
 667	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
 668#ifdef __BIG_ENDIAN
 669	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
 670	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
 671				RPTR_WRITEBACK_SWAP_ENABLE, 1);
 672#endif
 673	return rb_cntl;
 674}
 675
 676/**
 677 * sdma_v4_4_2_gfx_resume - setup and start the async dma engines
 678 *
 679 * @adev: amdgpu_device pointer
 680 * @i: instance to resume
 681 * @restore: used to restore wptr when restart
 682 *
 683 * Set up the gfx DMA ring buffers and enable them.
 684 * Returns 0 for success, error for failure.
 685 */
 686static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
 687{
 688	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
 689	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
 690	u32 wb_offset;
 691	u32 doorbell;
 692	u32 doorbell_offset;
 693	u64 wptr_gpu_addr;
 694	u64 rwptr;
 695
 696	wb_offset = (ring->rptr_offs * 4);
 697
 698	rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
 699	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
 700	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
 701
 702	/* set the wb address whether it's enabled or not */
 703	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_HI,
 704	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
 705	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_LO,
 706	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
 707
 708	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
 709				RPTR_WRITEBACK_ENABLE, 1);
 710
 711	WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
 712	WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
 713
 714	if (!restore)
 715		ring->wptr = 0;
 716
 717	/* before programing wptr to a less value, need set minor_ptr_update first */
 718	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
 719
 720	/* For the guilty queue, set RPTR to the current wptr to skip bad commands,
 721	 * It is not a guilty queue, restore cache_rptr and continue execution.
 722	 */
 723	if (adev->sdma.instance[i].gfx_guilty)
 724		rwptr = ring->wptr;
 725	else
 726		rwptr = ring->cached_rptr;
 727
 728	/* Initialize the ring buffer's read and write pointers */
 729	if (restore) {
 730		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, lower_32_bits(rwptr << 2));
 731		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, upper_32_bits(rwptr << 2));
 732		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, lower_32_bits(rwptr << 2));
 733		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, upper_32_bits(rwptr << 2));
 734	} else {
 735		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
 736		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
 737		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
 738		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
 739	}
 740
 741	doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
 742	doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
 743
 744	doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE,
 745				 ring->use_doorbell);
 746	doorbell_offset = REG_SET_FIELD(doorbell_offset,
 747					SDMA_GFX_DOORBELL_OFFSET,
 748					OFFSET, ring->doorbell_index);
 749	WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
 750	WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
 751
 752	sdma_v4_4_2_ring_set_wptr(ring);
 753
 754	/* set minor_ptr_update to 0 after wptr programed */
 755	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 0);
 756
 757	/* setup the wptr shadow polling */
 758	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 759	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_LO,
 760		    lower_32_bits(wptr_gpu_addr));
 761	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_HI,
 762		    upper_32_bits(wptr_gpu_addr));
 763	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL);
 764	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
 765				       SDMA_GFX_RB_WPTR_POLL_CNTL,
 766				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
 767	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
 768
 769	/* enable DMA RB */
 770	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 1);
 771	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
 772
 773	ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
 774	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 1);
 775#ifdef __BIG_ENDIAN
 776	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
 777#endif
 778	/* enable DMA IBs */
 779	WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
 780}
 781
 782/**
 783 * sdma_v4_4_2_page_resume - setup and start the async dma engines
 784 *
 785 * @adev: amdgpu_device pointer
 786 * @i: instance to resume
 787 * @restore: boolean to say restore needed or not
 788 *
 789 * Set up the page DMA ring buffers and enable them.
 790 * Returns 0 for success, error for failure.
 791 */
 792static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
 793{
 794	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
 795	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
 796	u32 wb_offset;
 797	u32 doorbell;
 798	u32 doorbell_offset;
 799	u64 wptr_gpu_addr;
 800	u64 rwptr;
 801
 802	wb_offset = (ring->rptr_offs * 4);
 803
 804	rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
 805	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
 806	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
 807
 808	/* For the guilty queue, set RPTR to the current wptr to skip bad commands,
 809	 * It is not a guilty queue, restore cache_rptr and continue execution.
 810	 */
 811	if (adev->sdma.instance[i].page_guilty)
 812		rwptr = ring->wptr;
 813	else
 814		rwptr = ring->cached_rptr;
 815
 816	/* Initialize the ring buffer's read and write pointers */
 817	if (restore) {
 818		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, lower_32_bits(rwptr << 2));
 819		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, upper_32_bits(rwptr << 2));
 820		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, lower_32_bits(rwptr << 2));
 821		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, upper_32_bits(rwptr << 2));
 822	} else {
 823		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
 824		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
 825		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
 826		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
 827	}
 828
 829	/* set the wb address whether it's enabled or not */
 830	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
 831	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
 832	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_LO,
 833	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
 834
 835	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
 836				RPTR_WRITEBACK_ENABLE, 1);
 837
 838	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
 839	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
 840
 841	if (!restore)
 842		ring->wptr = 0;
 843
 844	/* before programing wptr to a less value, need set minor_ptr_update first */
 845	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
 846
 847	doorbell = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL);
 848	doorbell_offset = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET);
 849
 850	doorbell = REG_SET_FIELD(doorbell, SDMA_PAGE_DOORBELL, ENABLE,
 851				 ring->use_doorbell);
 852	doorbell_offset = REG_SET_FIELD(doorbell_offset,
 853					SDMA_PAGE_DOORBELL_OFFSET,
 854					OFFSET, ring->doorbell_index);
 855	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL, doorbell);
 856	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET, doorbell_offset);
 857
 858	/* paging queue doorbell range is setup at sdma_v4_4_2_gfx_resume */
 859	sdma_v4_4_2_page_ring_set_wptr(ring);
 860
 861	/* set minor_ptr_update to 0 after wptr programed */
 862	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 0);
 863
 864	/* setup the wptr shadow polling */
 865	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
 866	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO,
 867		    lower_32_bits(wptr_gpu_addr));
 868	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI,
 869		    upper_32_bits(wptr_gpu_addr));
 870	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL);
 871	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
 872				       SDMA_PAGE_RB_WPTR_POLL_CNTL,
 873				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
 874	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
 875
 876	/* enable DMA RB */
 877	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL, RB_ENABLE, 1);
 878	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
 879
 880	ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
 881	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_ENABLE, 1);
 882#ifdef __BIG_ENDIAN
 883	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
 884#endif
 885	/* enable DMA IBs */
 886	WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
 887}
 888
 889static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
 890{
 891
 892}
 893
 894/**
 895 * sdma_v4_4_2_inst_rlc_resume - setup and start the async dma engines
 896 *
 897 * @adev: amdgpu_device pointer
 898 * @inst_mask: mask of dma engine instances to be enabled
 899 *
 900 * Set up the compute DMA queues and enable them.
 901 * Returns 0 for success, error for failure.
 902 */
 903static int sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device *adev,
 904				       uint32_t inst_mask)
 905{
 906	sdma_v4_4_2_init_pg(adev);
 907
 908	return 0;
 909}
 910
 911/**
 912 * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
 913 *
 914 * @adev: amdgpu_device pointer
 915 * @inst_mask: mask of dma engine instances to be enabled
 916 *
 917 * Loads the sDMA0/1 ucode.
 918 * Returns 0 for success, -EINVAL if the ucode is not available.
 919 */
 920static int sdma_v4_4_2_inst_load_microcode(struct amdgpu_device *adev,
 921					   uint32_t inst_mask)
 922{
 923	const struct sdma_firmware_header_v1_0 *hdr;
 924	const __le32 *fw_data;
 925	u32 fw_size;
 926	int i, j;
 927
 928	/* halt the MEs */
 929	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
 930
 931	for_each_inst(i, inst_mask) {
 932		if (!adev->sdma.instance[i].fw)
 933			return -EINVAL;
 934
 935		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
 936		amdgpu_ucode_print_sdma_hdr(&hdr->header);
 937		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
 938
 939		fw_data = (const __le32 *)
 940			(adev->sdma.instance[i].fw->data +
 941				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 942
 943		WREG32_SDMA(i, regSDMA_UCODE_ADDR, 0);
 944
 945		for (j = 0; j < fw_size; j++)
 946			WREG32_SDMA(i, regSDMA_UCODE_DATA,
 947				    le32_to_cpup(fw_data++));
 948
 949		WREG32_SDMA(i, regSDMA_UCODE_ADDR,
 950			    adev->sdma.instance[i].fw_version);
 951	}
 952
 953	return 0;
 954}
 955
 956/**
 957 * sdma_v4_4_2_inst_start - setup and start the async dma engines
 958 *
 959 * @adev: amdgpu_device pointer
 960 * @inst_mask: mask of dma engine instances to be enabled
 961 * @restore: boolean to say restore needed or not
 962 *
 963 * Set up the DMA engines and enable them.
 964 * Returns 0 for success, error for failure.
 965 */
 966static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
 967				  uint32_t inst_mask, bool restore)
 968{
 969	struct amdgpu_ring *ring;
 970	uint32_t tmp_mask;
 971	int i, r = 0;
 972
 973	if (amdgpu_sriov_vf(adev)) {
 974		sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
 975		sdma_v4_4_2_inst_enable(adev, false, inst_mask);
 976	} else {
 977		/* bypass sdma microcode loading on Gopher */
 978		if (!restore && adev->firmware.load_type != AMDGPU_FW_LOAD_PSP &&
 979		    adev->sdma.instance[0].fw) {
 980			r = sdma_v4_4_2_inst_load_microcode(adev, inst_mask);
 981			if (r)
 982				return r;
 983		}
 984
 985		/* unhalt the MEs */
 986		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
 987		/* enable sdma ring preemption */
 988		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
 989	}
 990
 991	/* start the gfx rings and rlc compute queues */
 992	tmp_mask = inst_mask;
 993	for_each_inst(i, tmp_mask) {
 994		uint32_t temp;
 995
 996		WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
 997		sdma_v4_4_2_gfx_resume(adev, i, restore);
 998		if (adev->sdma.has_page_queue)
 999			sdma_v4_4_2_page_resume(adev, i, restore);
1000
1001		/* set utc l1 enable flag always to 1 */
1002		temp = RREG32_SDMA(i, regSDMA_CNTL);
1003		temp = REG_SET_FIELD(temp, SDMA_CNTL, UTC_L1_ENABLE, 1);
1004		WREG32_SDMA(i, regSDMA_CNTL, temp);
1005
1006		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) < IP_VERSION(4, 4, 5)) {
1007			/* enable context empty interrupt during initialization */
1008			temp = REG_SET_FIELD(temp, SDMA_CNTL, CTXEMPTY_INT_ENABLE, 1);
1009			WREG32_SDMA(i, regSDMA_CNTL, temp);
1010		}
1011		if (!amdgpu_sriov_vf(adev)) {
1012			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
1013				/* unhalt engine */
1014				temp = RREG32_SDMA(i, regSDMA_F32_CNTL);
1015				temp = REG_SET_FIELD(temp, SDMA_F32_CNTL, HALT, 0);
1016				WREG32_SDMA(i, regSDMA_F32_CNTL, temp);
1017			}
1018		}
1019	}
1020
1021	if (amdgpu_sriov_vf(adev)) {
1022		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
1023		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
1024	} else {
1025		r = sdma_v4_4_2_inst_rlc_resume(adev, inst_mask);
1026		if (r)
1027			return r;
1028	}
1029
1030	tmp_mask = inst_mask;
1031	for_each_inst(i, tmp_mask) {
1032		ring = &adev->sdma.instance[i].ring;
1033
1034		r = amdgpu_ring_test_helper(ring);
1035		if (r)
1036			return r;
1037
1038		if (adev->sdma.has_page_queue) {
1039			struct amdgpu_ring *page = &adev->sdma.instance[i].page;
1040
1041			r = amdgpu_ring_test_helper(page);
1042			if (r)
1043				return r;
1044		}
1045	}
1046
1047	return r;
1048}
1049
1050/**
1051 * sdma_v4_4_2_ring_test_ring - simple async dma engine test
1052 *
1053 * @ring: amdgpu_ring structure holding ring information
1054 *
1055 * Test the DMA engine by writing using it to write an
1056 * value to memory.
1057 * Returns 0 for success, error for failure.
1058 */
1059static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
1060{
1061	struct amdgpu_device *adev = ring->adev;
1062	unsigned i;
1063	unsigned index;
1064	int r;
1065	u32 tmp;
1066	u64 gpu_addr;
1067
1068	r = amdgpu_device_wb_get(adev, &index);
1069	if (r)
1070		return r;
1071
1072	gpu_addr = adev->wb.gpu_addr + (index * 4);
1073	tmp = 0xCAFEDEAD;
1074	adev->wb.wb[index] = cpu_to_le32(tmp);
1075
1076	r = amdgpu_ring_alloc(ring, 5);
1077	if (r)
1078		goto error_free_wb;
1079
1080	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1081			  SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
1082	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
1083	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
1084	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
1085	amdgpu_ring_write(ring, 0xDEADBEEF);
1086	amdgpu_ring_commit(ring);
1087
1088	for (i = 0; i < adev->usec_timeout; i++) {
1089		tmp = le32_to_cpu(adev->wb.wb[index]);
1090		if (tmp == 0xDEADBEEF)
1091			break;
1092		udelay(1);
1093	}
1094
1095	if (i >= adev->usec_timeout)
1096		r = -ETIMEDOUT;
1097
1098error_free_wb:
1099	amdgpu_device_wb_free(adev, index);
1100	return r;
1101}
1102
1103/**
1104 * sdma_v4_4_2_ring_test_ib - test an IB on the DMA engine
1105 *
1106 * @ring: amdgpu_ring structure holding ring information
1107 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1108 *
1109 * Test a simple IB in the DMA ring.
1110 * Returns 0 on success, error on failure.
1111 */
1112static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1113{
1114	struct amdgpu_device *adev = ring->adev;
1115	struct amdgpu_ib ib;
1116	struct dma_fence *f = NULL;
1117	unsigned index;
1118	long r;
1119	u32 tmp = 0;
1120	u64 gpu_addr;
1121
1122	r = amdgpu_device_wb_get(adev, &index);
1123	if (r)
1124		return r;
1125
1126	gpu_addr = adev->wb.gpu_addr + (index * 4);
1127	tmp = 0xCAFEDEAD;
1128	adev->wb.wb[index] = cpu_to_le32(tmp);
1129	memset(&ib, 0, sizeof(ib));
1130	r = amdgpu_ib_get(adev, NULL, 256,
1131					AMDGPU_IB_POOL_DIRECT, &ib);
1132	if (r)
1133		goto err0;
1134
1135	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1136		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1137	ib.ptr[1] = lower_32_bits(gpu_addr);
1138	ib.ptr[2] = upper_32_bits(gpu_addr);
1139	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1140	ib.ptr[4] = 0xDEADBEEF;
1141	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1142	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1143	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1144	ib.length_dw = 8;
1145
1146	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1147	if (r)
1148		goto err1;
1149
1150	r = dma_fence_wait_timeout(f, false, timeout);
1151	if (r == 0) {
1152		r = -ETIMEDOUT;
1153		goto err1;
1154	} else if (r < 0) {
1155		goto err1;
1156	}
1157	tmp = le32_to_cpu(adev->wb.wb[index]);
1158	if (tmp == 0xDEADBEEF)
1159		r = 0;
1160	else
1161		r = -EINVAL;
1162
1163err1:
1164	amdgpu_ib_free(&ib, NULL);
1165	dma_fence_put(f);
1166err0:
1167	amdgpu_device_wb_free(adev, index);
1168	return r;
1169}
1170
1171
1172/**
1173 * sdma_v4_4_2_vm_copy_pte - update PTEs by copying them from the GART
1174 *
1175 * @ib: indirect buffer to fill with commands
1176 * @pe: addr of the page entry
1177 * @src: src addr to copy from
1178 * @count: number of page entries to update
1179 *
1180 * Update PTEs by copying them from the GART using sDMA.
1181 */
1182static void sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib *ib,
1183				  uint64_t pe, uint64_t src,
1184				  unsigned count)
1185{
1186	unsigned bytes = count * 8;
1187
1188	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1189		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1190	ib->ptr[ib->length_dw++] = bytes - 1;
1191	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1192	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1193	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1194	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1195	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1196
1197}
1198
1199/**
1200 * sdma_v4_4_2_vm_write_pte - update PTEs by writing them manually
1201 *
1202 * @ib: indirect buffer to fill with commands
1203 * @pe: addr of the page entry
1204 * @value: dst addr to write into pe
1205 * @count: number of page entries to update
1206 * @incr: increase next addr by incr bytes
1207 *
1208 * Update PTEs by writing them manually using sDMA.
1209 */
1210static void sdma_v4_4_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1211				   uint64_t value, unsigned count,
1212				   uint32_t incr)
1213{
1214	unsigned ndw = count * 2;
1215
1216	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1217		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1218	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1219	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1220	ib->ptr[ib->length_dw++] = ndw - 1;
1221	for (; ndw > 0; ndw -= 2) {
1222		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1223		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1224		value += incr;
1225	}
1226}
1227
1228/**
1229 * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
1230 *
1231 * @ib: indirect buffer to fill with commands
1232 * @pe: addr of the page entry
1233 * @addr: dst addr to write into pe
1234 * @count: number of page entries to update
1235 * @incr: increase next addr by incr bytes
1236 * @flags: access flags
1237 *
1238 * Update the page tables using sDMA.
1239 */
1240static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1241				     uint64_t pe,
1242				     uint64_t addr, unsigned count,
1243				     uint32_t incr, uint64_t flags)
1244{
1245	/* for physically contiguous pages (vram) */
1246	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1247	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1248	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1249	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1250	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1251	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1252	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1253	ib->ptr[ib->length_dw++] = incr; /* increment size */
1254	ib->ptr[ib->length_dw++] = 0;
1255	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1256}
1257
1258/**
1259 * sdma_v4_4_2_ring_pad_ib - pad the IB to the required number of dw
1260 *
1261 * @ring: amdgpu_ring structure holding ring information
1262 * @ib: indirect buffer to fill with padding
1263 */
1264static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1265{
1266	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1267	u32 pad_count;
1268	int i;
1269
1270	pad_count = (-ib->length_dw) & 7;
1271	for (i = 0; i < pad_count; i++)
1272		if (sdma && sdma->burst_nop && (i == 0))
1273			ib->ptr[ib->length_dw++] =
1274				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1275				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1276		else
1277			ib->ptr[ib->length_dw++] =
1278				SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1279}
1280
1281
1282/**
1283 * sdma_v4_4_2_ring_emit_pipeline_sync - sync the pipeline
1284 *
1285 * @ring: amdgpu_ring pointer
1286 *
1287 * Make sure all previous operations are completed (CIK).
1288 */
1289static void sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1290{
1291	uint32_t seq = ring->fence_drv.sync_seq;
1292	uint64_t addr = ring->fence_drv.gpu_addr;
1293
1294	/* wait for idle */
1295	sdma_v4_4_2_wait_reg_mem(ring, 1, 0,
1296			       addr & 0xfffffffc,
1297			       upper_32_bits(addr) & 0xffffffff,
1298			       seq, 0xffffffff, 4);
1299}
1300
1301
1302/**
1303 * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
1304 *
1305 * @ring: amdgpu_ring pointer
1306 * @vmid: vmid number to use
1307 * @pd_addr: address
1308 *
1309 * Update the page table base and flush the VM TLB
1310 * using sDMA.
1311 */
1312static void sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1313					 unsigned vmid, uint64_t pd_addr)
1314{
1315	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1316}
1317
1318static void sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring *ring,
1319				     uint32_t reg, uint32_t val)
1320{
1321	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1322			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1323	amdgpu_ring_write(ring, reg);
1324	amdgpu_ring_write(ring, val);
1325}
1326
1327static void sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1328					 uint32_t val, uint32_t mask)
1329{
1330	sdma_v4_4_2_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
1331}
1332
1333static bool sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device *adev)
1334{
1335	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1336	case IP_VERSION(4, 4, 2):
1337	case IP_VERSION(4, 4, 5):
1338		return false;
1339	default:
1340		return false;
1341	}
1342}
1343
1344static const struct amdgpu_sdma_funcs sdma_v4_4_2_sdma_funcs = {
1345	.stop_kernel_queue = &sdma_v4_4_2_stop_queue,
1346	.start_kernel_queue = &sdma_v4_4_2_restore_queue,
1347	.soft_reset_kernel_queue = &sdma_v4_4_2_soft_reset_engine,
1348};
1349
1350static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
1351{
1352	struct amdgpu_device *adev = ip_block->adev;
1353	int r;
1354
1355	r = sdma_v4_4_2_init_microcode(adev);
1356	if (r)
1357		return r;
1358
1359	/* TODO: Page queue breaks driver reload under SRIOV */
1360	if (sdma_v4_4_2_fw_support_paging_queue(adev))
1361		adev->sdma.has_page_queue = true;
1362
1363	sdma_v4_4_2_set_ring_funcs(adev);
1364	sdma_v4_4_2_set_buffer_funcs(adev);
1365	sdma_v4_4_2_set_vm_pte_funcs(adev);
1366	sdma_v4_4_2_set_irq_funcs(adev);
1367	sdma_v4_4_2_set_ras_funcs(adev);
1368	return 0;
1369}
1370
1371#if 0
1372static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1373		void *err_data,
1374		struct amdgpu_iv_entry *entry);
1375#endif
1376
1377static int sdma_v4_4_2_late_init(struct amdgpu_ip_block *ip_block)
1378{
1379	struct amdgpu_device *adev = ip_block->adev;
1380#if 0
1381	struct ras_ih_if ih_info = {
1382		.cb = sdma_v4_4_2_process_ras_data_cb,
1383	};
1384#endif
1385	if (!amdgpu_persistent_edc_harvesting_supported(adev))
1386		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__SDMA);
1387
1388	/* The initialization is done in the late_init stage to ensure that the SMU
1389	 * initialization and capability setup are completed before we check the SDMA
1390	 * reset capability
1391	 */
1392	sdma_v4_4_2_update_reset_mask(adev);
1393
1394	return 0;
1395}
1396
1397static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
1398{
1399	struct amdgpu_ring *ring;
1400	int r, i;
1401	struct amdgpu_device *adev = ip_block->adev;
1402	u32 aid_id;
1403	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1404	uint32_t *ptr;
1405
1406	/* SDMA trap event */
1407	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1408		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1409				      SDMA0_4_0__SRCID__SDMA_TRAP,
1410				      &adev->sdma.trap_irq);
1411		if (r)
1412			return r;
1413	}
1414
1415	/* SDMA SRAM ECC event */
1416	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1417		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1418				      SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
1419				      &adev->sdma.ecc_irq);
1420		if (r)
1421			return r;
1422	}
1423
1424	/* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
1425	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1426		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1427				      SDMA0_4_0__SRCID__SDMA_VM_HOLE,
1428				      &adev->sdma.vm_hole_irq);
1429		if (r)
1430			return r;
1431
1432		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1433				      SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
1434				      &adev->sdma.doorbell_invalid_irq);
1435		if (r)
1436			return r;
1437
1438		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1439				      SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
1440				      &adev->sdma.pool_timeout_irq);
1441		if (r)
1442			return r;
1443
1444		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1445				      SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
1446				      &adev->sdma.srbm_write_irq);
1447		if (r)
1448			return r;
1449
1450		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1451				      SDMA0_4_0__SRCID__SDMA_CTXEMPTY,
1452				      &adev->sdma.ctxt_empty_irq);
1453		if (r)
1454			return r;
1455	}
1456
1457	for (i = 0; i < adev->sdma.num_instances; i++) {
1458		mutex_init(&adev->sdma.instance[i].engine_reset_mutex);
1459		/* Initialize guilty flags for GFX and PAGE queues */
1460		adev->sdma.instance[i].gfx_guilty = false;
1461		adev->sdma.instance[i].page_guilty = false;
1462		adev->sdma.instance[i].funcs = &sdma_v4_4_2_sdma_funcs;
1463
1464		ring = &adev->sdma.instance[i].ring;
1465		ring->ring_obj = NULL;
1466		ring->use_doorbell = true;
1467		aid_id = adev->sdma.instance[i].aid_id;
1468
1469		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1470				ring->use_doorbell?"true":"false");
1471
1472		/* doorbell size is 2 dwords, get DWORD offset */
1473		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
1474		ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1475
1476		sprintf(ring->name, "sdma%d.%d", aid_id,
1477				i % adev->sdma.num_inst_per_aid);
1478		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1479				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1480				     AMDGPU_RING_PRIO_DEFAULT, NULL);
1481		if (r)
1482			return r;
1483
1484		if (adev->sdma.has_page_queue) {
1485			ring = &adev->sdma.instance[i].page;
1486			ring->ring_obj = NULL;
1487			ring->use_doorbell = true;
1488
1489			/* doorbell index of page queue is assigned right after
1490			 * gfx queue on the same instance
1491			 */
1492			ring->doorbell_index =
1493				(adev->doorbell_index.sdma_engine[i] + 1) << 1;
1494			ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1495
1496			sprintf(ring->name, "page%d.%d", aid_id,
1497					i % adev->sdma.num_inst_per_aid);
1498			r = amdgpu_ring_init(adev, ring, 1024,
1499					     &adev->sdma.trap_irq,
1500					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1501					     AMDGPU_RING_PRIO_DEFAULT, NULL);
1502			if (r)
1503				return r;
1504		}
1505	}
1506
1507	adev->sdma.supported_reset =
1508		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1509
1510	if (amdgpu_sdma_ras_sw_init(adev)) {
1511		dev_err(adev->dev, "fail to initialize sdma ras block\n");
1512		return -EINVAL;
1513	}
1514
1515	/* Allocate memory for SDMA IP Dump buffer */
1516	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1517	if (ptr)
1518		adev->sdma.ip_dump = ptr;
1519	else
1520		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1521
1522	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1523	if (r)
1524		return r;
1525
1526	return r;
1527}
1528
1529static int sdma_v4_4_2_sw_fini(struct amdgpu_ip_block *ip_block)
1530{
1531	struct amdgpu_device *adev = ip_block->adev;
1532	int i;
1533
1534	for (i = 0; i < adev->sdma.num_instances; i++) {
1535		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1536		if (adev->sdma.has_page_queue)
1537			amdgpu_ring_fini(&adev->sdma.instance[i].page);
1538	}
1539
1540	amdgpu_sdma_sysfs_reset_mask_fini(adev);
1541	if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 2) ||
1542	    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 4) ||
1543	    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 5))
1544		amdgpu_sdma_destroy_inst_ctx(adev, true);
1545	else
1546		amdgpu_sdma_destroy_inst_ctx(adev, false);
1547
1548	kfree(adev->sdma.ip_dump);
1549
1550	return 0;
1551}
1552
1553static int sdma_v4_4_2_hw_init(struct amdgpu_ip_block *ip_block)
1554{
1555	int r;
1556	struct amdgpu_device *adev = ip_block->adev;
1557	uint32_t inst_mask;
1558
1559	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1560	if (!amdgpu_sriov_vf(adev))
1561		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
1562
1563	r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
1564
1565	return r;
1566}
1567
1568static int sdma_v4_4_2_hw_fini(struct amdgpu_ip_block *ip_block)
1569{
1570	struct amdgpu_device *adev = ip_block->adev;
1571	uint32_t inst_mask;
1572	int i;
1573
1574	if (amdgpu_sriov_vf(adev))
1575		return 0;
1576
1577	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1578	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
1579		for (i = 0; i < adev->sdma.num_instances; i++) {
1580			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
1581				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
1582		}
1583	}
1584
1585	sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
1586	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
1587
1588	return 0;
1589}
1590
1591static int sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1592					     enum amd_clockgating_state state);
1593
1594static int sdma_v4_4_2_suspend(struct amdgpu_ip_block *ip_block)
1595{
1596	struct amdgpu_device *adev = ip_block->adev;
1597
1598	if (amdgpu_in_reset(adev))
1599		sdma_v4_4_2_set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE);
1600
1601	return sdma_v4_4_2_hw_fini(ip_block);
1602}
1603
1604static int sdma_v4_4_2_resume(struct amdgpu_ip_block *ip_block)
1605{
1606	return sdma_v4_4_2_hw_init(ip_block);
1607}
1608
1609static bool sdma_v4_4_2_is_idle(struct amdgpu_ip_block *ip_block)
1610{
1611	struct amdgpu_device *adev = ip_block->adev;
1612	u32 i;
1613
1614	for (i = 0; i < adev->sdma.num_instances; i++) {
1615		u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
1616
1617		if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
1618			return false;
1619	}
1620
1621	return true;
1622}
1623
1624static int sdma_v4_4_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
1625{
1626	unsigned i, j;
1627	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
1628	struct amdgpu_device *adev = ip_block->adev;
1629
1630	for (i = 0; i < adev->usec_timeout; i++) {
1631		for (j = 0; j < adev->sdma.num_instances; j++) {
1632			sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG);
1633			if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK))
1634				break;
1635		}
1636		if (j == adev->sdma.num_instances)
1637			return 0;
1638		udelay(1);
1639	}
1640	return -ETIMEDOUT;
1641}
1642
1643static int sdma_v4_4_2_soft_reset(struct amdgpu_ip_block *ip_block)
1644{
1645	/* todo */
1646
1647	return 0;
1648}
1649
1650static bool sdma_v4_4_2_is_queue_selected(struct amdgpu_device *adev, uint32_t instance_id, bool is_page_queue)
1651{
1652	uint32_t reg_offset = is_page_queue ? regSDMA_PAGE_CONTEXT_STATUS : regSDMA_GFX_CONTEXT_STATUS;
1653	uint32_t context_status = RREG32(sdma_v4_4_2_get_reg_offset(adev, instance_id, reg_offset));
1654
1655	/* Check if the SELECTED bit is set */
1656	return (context_status & SDMA_GFX_CONTEXT_STATUS__SELECTED_MASK) != 0;
1657}
1658
1659static int sdma_v4_4_2_reset_queue(struct amdgpu_ring *ring,
1660				   unsigned int vmid,
1661				   struct amdgpu_fence *timedout_fence)
1662{
1663	struct amdgpu_device *adev = ring->adev;
1664	u32 id = ring->me;
1665	int r;
1666
1667	amdgpu_amdkfd_suspend(adev, true);
1668	r = amdgpu_sdma_reset_engine(adev, id, false);
1669	amdgpu_amdkfd_resume(adev, true);
1670	return r;
1671}
1672
1673static int sdma_v4_4_2_stop_queue(struct amdgpu_ring *ring)
1674{
1675	struct amdgpu_device *adev = ring->adev;
1676	u32 instance_id = ring->me;
1677	u32 inst_mask;
1678	uint64_t rptr;
1679
1680	if (amdgpu_sriov_vf(adev))
1681		return -EINVAL;
1682
1683	/* Check if this queue is the guilty one */
1684	adev->sdma.instance[instance_id].gfx_guilty =
1685		sdma_v4_4_2_is_queue_selected(adev, instance_id, false);
1686	if (adev->sdma.has_page_queue)
1687		adev->sdma.instance[instance_id].page_guilty =
1688			sdma_v4_4_2_is_queue_selected(adev, instance_id, true);
1689
1690	/* Cache the rptr before reset, after the reset,
1691	* all of the registers will be reset to 0
1692	*/
1693	rptr = amdgpu_ring_get_rptr(ring);
1694	ring->cached_rptr = rptr;
1695	/* Cache the rptr for the page queue if it exists */
1696	if (adev->sdma.has_page_queue) {
1697		struct amdgpu_ring *page_ring = &adev->sdma.instance[instance_id].page;
1698		rptr = amdgpu_ring_get_rptr(page_ring);
1699		page_ring->cached_rptr = rptr;
1700	}
1701
1702	/* stop queue */
1703	inst_mask = 1 << ring->me;
1704	sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
1705	if (adev->sdma.has_page_queue)
1706		sdma_v4_4_2_inst_page_stop(adev, inst_mask);
1707
1708	return 0;
1709}
1710
1711static int sdma_v4_4_2_restore_queue(struct amdgpu_ring *ring)
1712{
1713	struct amdgpu_device *adev = ring->adev;
1714	u32 inst_mask;
1715	int i, r;
1716
1717	inst_mask = 1 << ring->me;
1718	udelay(50);
1719
1720	for (i = 0; i < adev->usec_timeout; i++) {
1721		if (!REG_GET_FIELD(RREG32_SDMA(ring->me, regSDMA_F32_CNTL), SDMA_F32_CNTL, HALT))
1722			break;
1723		udelay(1);
1724	}
1725
1726	if (i == adev->usec_timeout) {
1727		dev_err(adev->dev, "timed out waiting for SDMA%d unhalt after reset\n",
1728			ring->me);
1729		return -ETIMEDOUT;
1730	}
1731
1732	r = sdma_v4_4_2_inst_start(adev, inst_mask, true);
1733
1734	return r;
1735}
1736
1737static int sdma_v4_4_2_soft_reset_engine(struct amdgpu_device *adev,
1738					 u32 instance_id)
1739{
1740	/* For SDMA 4.x, use the existing DPM interface for backward compatibility
1741	 * we need to convert the logical instance ID to physical instance ID before reset.
1742	 */
1743	return amdgpu_dpm_reset_sdma(adev, 1 << GET_INST(SDMA0, instance_id));
1744}
1745
1746static int sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device *adev,
1747					struct amdgpu_irq_src *source,
1748					unsigned type,
1749					enum amdgpu_interrupt_state state)
1750{
1751	u32 sdma_cntl;
1752
1753	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1754	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, TRAP_ENABLE,
1755		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1756	WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1757
1758	return 0;
1759}
1760
1761static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
1762				      struct amdgpu_irq_src *source,
1763				      struct amdgpu_iv_entry *entry)
1764{
1765	uint32_t instance, i;
1766
1767	DRM_DEBUG("IH: SDMA trap\n");
1768	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1769
1770	/* Client id gives the SDMA instance in AID. To know the exact SDMA
1771	 * instance, interrupt entry gives the node id which corresponds to the AID instance.
1772	 * Match node id with the AID id associated with the SDMA instance. */
1773	for (i = instance; i < adev->sdma.num_instances;
1774	     i += adev->sdma.num_inst_per_aid) {
1775		if (adev->sdma.instance[i].aid_id ==
1776		    node_id_to_phys_map[entry->node_id])
1777			break;
1778	}
1779
1780	if (i >= adev->sdma.num_instances) {
1781		dev_WARN_ONCE(
1782			adev->dev, 1,
1783			"Couldn't find the right sdma instance in trap handler");
1784		return 0;
1785	}
1786
1787	switch (entry->ring_id) {
1788	case 0:
1789		amdgpu_fence_process(&adev->sdma.instance[i].ring);
1790		break;
1791	case 1:
1792		amdgpu_fence_process(&adev->sdma.instance[i].page);
1793		break;
1794	default:
1795		break;
1796	}
1797	return 0;
1798}
1799
1800#if 0
1801static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1802		void *err_data,
1803		struct amdgpu_iv_entry *entry)
1804{
1805	int instance;
1806
1807	/* When “Full RAS” is enabled, the per-IP interrupt sources should
1808	 * be disabled and the driver should only look for the aggregated
1809	 * interrupt via sync flood
1810	 */
1811	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
1812		goto out;
1813
1814	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1815	if (instance < 0)
1816		goto out;
1817
1818	amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
1819
1820out:
1821	return AMDGPU_RAS_SUCCESS;
1822}
1823#endif
1824
1825static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1826					      struct amdgpu_irq_src *source,
1827					      struct amdgpu_iv_entry *entry)
1828{
1829	int instance;
1830
1831	DRM_ERROR("Illegal instruction in SDMA command stream\n");
1832
1833	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1834	if (instance < 0)
1835		return 0;
1836
1837	switch (entry->ring_id) {
1838	case 0:
1839		drm_sched_fault(&adev->sdma.instance[instance].ring.sched);
1840		break;
1841	}
1842	return 0;
1843}
1844
1845static int sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device *adev,
1846					struct amdgpu_irq_src *source,
1847					unsigned type,
1848					enum amdgpu_interrupt_state state)
1849{
1850	u32 sdma_cntl;
1851
1852	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1853	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, DRAM_ECC_INT_ENABLE,
1854					state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1855	WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1856
1857	return 0;
1858}
1859
1860static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
1861					      struct amdgpu_iv_entry *entry)
1862{
1863	int instance;
1864	struct amdgpu_task_info *task_info;
1865	u64 addr;
1866
1867	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1868	if (instance < 0 || instance >= adev->sdma.num_instances) {
1869		dev_err(adev->dev, "sdma instance invalid %d\n", instance);
1870		return -EINVAL;
1871	}
1872
1873	addr = (u64)entry->src_data[0] << 12;
1874	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
1875
1876	dev_dbg_ratelimited(adev->dev,
1877			    "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u pasid:%u\n",
1878			    instance, addr, entry->src_id, entry->ring_id, entry->vmid,
1879			    entry->pasid);
1880
1881	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
1882	if (task_info) {
1883		dev_dbg_ratelimited(adev->dev, " for process %s pid %d thread %s pid %d\n",
1884				    task_info->process_name, task_info->tgid,
1885				    task_info->task.comm, task_info->task.pid);
1886		amdgpu_vm_put_task_info(task_info);
1887	}
1888
1889	return 0;
1890}
1891
1892static int sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device *adev,
1893					      struct amdgpu_irq_src *source,
1894					      struct amdgpu_iv_entry *entry)
1895{
1896	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
1897	sdma_v4_4_2_print_iv_entry(adev, entry);
1898	return 0;
1899}
1900
1901static int sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device *adev,
1902					      struct amdgpu_irq_src *source,
1903					      struct amdgpu_iv_entry *entry)
1904{
1905
1906	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
1907	sdma_v4_4_2_print_iv_entry(adev, entry);
1908	return 0;
1909}
1910
1911static int sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device *adev,
1912					      struct amdgpu_irq_src *source,
1913					      struct amdgpu_iv_entry *entry)
1914{
1915	dev_dbg_ratelimited(adev->dev,
1916		"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
1917	sdma_v4_4_2_print_iv_entry(adev, entry);
1918	return 0;
1919}
1920
1921static int sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device *adev,
1922					      struct amdgpu_irq_src *source,
1923					      struct amdgpu_iv_entry *entry)
1924{
1925	dev_dbg_ratelimited(adev->dev,
1926		"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
1927	sdma_v4_4_2_print_iv_entry(adev, entry);
1928	return 0;
1929}
1930
1931static int sdma_v4_4_2_process_ctxt_empty_irq(struct amdgpu_device *adev,
1932					      struct amdgpu_irq_src *source,
1933					      struct amdgpu_iv_entry *entry)
1934{
1935	/* There is nothing useful to be done here, only kept for debug */
1936	dev_dbg_ratelimited(adev->dev, "SDMA context empty interrupt");
1937	sdma_v4_4_2_print_iv_entry(adev, entry);
1938	return 0;
1939}
1940
1941static void sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1942	struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1943{
1944	uint32_t data, def;
1945	int i;
1946
1947	/* leave as default if it is not driver controlled */
1948	if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS))
1949		return;
1950
1951	if (enable) {
1952		for_each_inst(i, inst_mask) {
1953			/* 1-not override: enable sdma mem light sleep */
1954			def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1955			data |= SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1956			if (def != data)
1957				WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1958		}
1959	} else {
1960		for_each_inst(i, inst_mask) {
1961			/* 0-override:disable sdma mem light sleep */
1962			def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1963			data &= ~SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1964			if (def != data)
1965				WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1966		}
1967	}
1968}
1969
1970static void sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1971	struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1972{
1973	uint32_t data, def;
1974	int i;
1975
1976	/* leave as default if it is not driver controlled */
1977	if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG))
1978		return;
1979
1980	if (enable) {
1981		for_each_inst(i, inst_mask) {
1982			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1983			data &= ~(SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1984				  SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1985				  SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1986				  SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1987				  SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1988				  SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1989			if (def != data)
1990				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1991		}
1992	} else {
1993		for_each_inst(i, inst_mask) {
1994			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1995			data |= (SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1996				 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1997				 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1998				 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1999				 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
2000				 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
2001			if (def != data)
2002				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
2003		}
2004	}
2005}
2006
2007static int sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2008					  enum amd_clockgating_state state)
2009{
2010	struct amdgpu_device *adev = ip_block->adev;
2011	uint32_t inst_mask;
2012
2013	if (amdgpu_sriov_vf(adev))
2014		return 0;
2015
2016	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2017
2018	sdma_v4_4_2_inst_update_medium_grain_clock_gating(
2019		adev, state == AMD_CG_STATE_GATE, inst_mask);
2020	sdma_v4_4_2_inst_update_medium_grain_light_sleep(
2021		adev, state == AMD_CG_STATE_GATE, inst_mask);
2022	return 0;
2023}
2024
2025static int sdma_v4_4_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
2026					  enum amd_powergating_state state)
2027{
2028	return 0;
2029}
2030
2031static void sdma_v4_4_2_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2032{
2033	struct amdgpu_device *adev = ip_block->adev;
2034	int data;
2035
2036	if (amdgpu_sriov_vf(adev))
2037		*flags = 0;
2038
2039	/* AMD_CG_SUPPORT_SDMA_MGCG */
2040	data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_CLK_CTRL));
2041	if (!(data & SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK))
2042		*flags |= AMD_CG_SUPPORT_SDMA_MGCG;
2043
2044	/* AMD_CG_SUPPORT_SDMA_LS */
2045	data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_POWER_CNTL));
2046	if (data & SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
2047		*flags |= AMD_CG_SUPPORT_SDMA_LS;
2048}
2049
2050static void sdma_v4_4_2_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
2051{
2052	struct amdgpu_device *adev = ip_block->adev;
2053	int i, j;
2054	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
2055	uint32_t instance_offset;
2056
2057	if (!adev->sdma.ip_dump)
2058		return;
2059
2060	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
2061	for (i = 0; i < adev->sdma.num_instances; i++) {
2062		instance_offset = i * reg_count;
2063		drm_printf(p, "\nInstance:%d\n", i);
2064
2065		for (j = 0; j < reg_count; j++)
2066			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_4_4_2[j].reg_name,
2067				   adev->sdma.ip_dump[instance_offset + j]);
2068	}
2069}
2070
2071static void sdma_v4_4_2_dump_ip_state(struct amdgpu_ip_block *ip_block)
2072{
2073	struct amdgpu_device *adev = ip_block->adev;
2074	int i, j;
2075	uint32_t instance_offset;
2076	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
2077
2078	if (!adev->sdma.ip_dump)
2079		return;
2080
2081	for (i = 0; i < adev->sdma.num_instances; i++) {
2082		instance_offset = i * reg_count;
2083		for (j = 0; j < reg_count; j++)
2084			adev->sdma.ip_dump[instance_offset + j] =
2085				RREG32(sdma_v4_4_2_get_reg_offset(adev, i,
2086				       sdma_reg_list_4_4_2[j].reg_offset));
2087	}
2088}
2089
2090const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
2091	.name = "sdma_v4_4_2",
2092	.early_init = sdma_v4_4_2_early_init,
2093	.late_init = sdma_v4_4_2_late_init,
2094	.sw_init = sdma_v4_4_2_sw_init,
2095	.sw_fini = sdma_v4_4_2_sw_fini,
2096	.hw_init = sdma_v4_4_2_hw_init,
2097	.hw_fini = sdma_v4_4_2_hw_fini,
2098	.suspend = sdma_v4_4_2_suspend,
2099	.resume = sdma_v4_4_2_resume,
2100	.is_idle = sdma_v4_4_2_is_idle,
2101	.wait_for_idle = sdma_v4_4_2_wait_for_idle,
2102	.soft_reset = sdma_v4_4_2_soft_reset,
2103	.set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
2104	.set_powergating_state = sdma_v4_4_2_set_powergating_state,
2105	.get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
2106	.dump_ip_state = sdma_v4_4_2_dump_ip_state,
2107	.print_ip_state = sdma_v4_4_2_print_ip_state,
2108};
2109
2110static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
2111	.type = AMDGPU_RING_TYPE_SDMA,
2112	.align_mask = 0xff,
2113	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2114	.support_64bit_ptrs = true,
2115	.get_rptr = sdma_v4_4_2_ring_get_rptr,
2116	.get_wptr = sdma_v4_4_2_ring_get_wptr,
2117	.set_wptr = sdma_v4_4_2_ring_set_wptr,
2118	.emit_frame_size =
2119		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
2120		3 + /* hdp invalidate */
2121		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
2122		/* sdma_v4_4_2_ring_emit_vm_flush */
2123		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2124		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2125		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
2126	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
2127	.emit_ib = sdma_v4_4_2_ring_emit_ib,
2128	.emit_fence = sdma_v4_4_2_ring_emit_fence,
2129	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
2130	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
2131	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
2132	.test_ring = sdma_v4_4_2_ring_test_ring,
2133	.test_ib = sdma_v4_4_2_ring_test_ib,
2134	.insert_nop = sdma_v4_4_2_ring_insert_nop,
2135	.pad_ib = sdma_v4_4_2_ring_pad_ib,
2136	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
2137	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
2138	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2139	.reset = sdma_v4_4_2_reset_queue,
2140};
2141
2142static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
2143	.type = AMDGPU_RING_TYPE_SDMA,
2144	.align_mask = 0xff,
2145	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2146	.support_64bit_ptrs = true,
2147	.get_rptr = sdma_v4_4_2_ring_get_rptr,
2148	.get_wptr = sdma_v4_4_2_page_ring_get_wptr,
2149	.set_wptr = sdma_v4_4_2_page_ring_set_wptr,
2150	.emit_frame_size =
2151		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
2152		3 + /* hdp invalidate */
2153		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
2154		/* sdma_v4_4_2_ring_emit_vm_flush */
2155		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2156		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2157		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
2158	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
2159	.emit_ib = sdma_v4_4_2_ring_emit_ib,
2160	.emit_fence = sdma_v4_4_2_ring_emit_fence,
2161	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
2162	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
2163	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
2164	.test_ring = sdma_v4_4_2_ring_test_ring,
2165	.test_ib = sdma_v4_4_2_ring_test_ib,
2166	.insert_nop = sdma_v4_4_2_ring_insert_nop,
2167	.pad_ib = sdma_v4_4_2_ring_pad_ib,
2168	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
2169	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
2170	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2171	.reset = sdma_v4_4_2_reset_queue,
2172};
2173
2174static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
2175{
2176	int i, dev_inst;
2177
2178	for (i = 0; i < adev->sdma.num_instances; i++) {
2179		adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs;
2180		adev->sdma.instance[i].ring.me = i;
2181		if (adev->sdma.has_page_queue) {
2182			adev->sdma.instance[i].page.funcs =
2183				&sdma_v4_4_2_page_ring_funcs;
2184			adev->sdma.instance[i].page.me = i;
2185		}
2186
2187		dev_inst = GET_INST(SDMA0, i);
2188		/* AID to which SDMA belongs depends on physical instance */
2189		adev->sdma.instance[i].aid_id =
2190			dev_inst / adev->sdma.num_inst_per_aid;
2191	}
2192}
2193
2194static const struct amdgpu_irq_src_funcs sdma_v4_4_2_trap_irq_funcs = {
2195	.set = sdma_v4_4_2_set_trap_irq_state,
2196	.process = sdma_v4_4_2_process_trap_irq,
2197};
2198
2199static const struct amdgpu_irq_src_funcs sdma_v4_4_2_illegal_inst_irq_funcs = {
2200	.process = sdma_v4_4_2_process_illegal_inst_irq,
2201};
2202
2203static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ecc_irq_funcs = {
2204	.set = sdma_v4_4_2_set_ecc_irq_state,
2205	.process = amdgpu_sdma_process_ecc_irq,
2206};
2207
2208static const struct amdgpu_irq_src_funcs sdma_v4_4_2_vm_hole_irq_funcs = {
2209	.process = sdma_v4_4_2_process_vm_hole_irq,
2210};
2211
2212static const struct amdgpu_irq_src_funcs sdma_v4_4_2_doorbell_invalid_irq_funcs = {
2213	.process = sdma_v4_4_2_process_doorbell_invalid_irq,
2214};
2215
2216static const struct amdgpu_irq_src_funcs sdma_v4_4_2_pool_timeout_irq_funcs = {
2217	.process = sdma_v4_4_2_process_pool_timeout_irq,
2218};
2219
2220static const struct amdgpu_irq_src_funcs sdma_v4_4_2_srbm_write_irq_funcs = {
2221	.process = sdma_v4_4_2_process_srbm_write_irq,
2222};
2223
2224static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ctxt_empty_irq_funcs = {
2225	.process = sdma_v4_4_2_process_ctxt_empty_irq,
2226};
2227
2228static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
2229{
2230	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
2231	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
2232	adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
2233	adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
2234	adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
2235	adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
2236	adev->sdma.ctxt_empty_irq.num_types = adev->sdma.num_instances;
2237
2238	adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs;
2239	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs;
2240	adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs;
2241	adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs;
2242	adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs;
2243	adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs;
2244	adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs;
2245	adev->sdma.ctxt_empty_irq.funcs = &sdma_v4_4_2_ctxt_empty_irq_funcs;
2246}
2247
2248/**
2249 * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
2250 *
2251 * @ib: indirect buffer to copy to
2252 * @src_offset: src GPU address
2253 * @dst_offset: dst GPU address
2254 * @byte_count: number of bytes to xfer
2255 * @copy_flags: copy flags for the buffers
2256 *
2257 * Copy GPU buffers using the DMA engine.
2258 * Used by the amdgpu ttm implementation to move pages if
2259 * registered as the asic copy callback.
2260 */
2261static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
2262				       uint64_t src_offset,
2263				       uint64_t dst_offset,
2264				       uint32_t byte_count,
2265				       uint32_t copy_flags)
2266{
2267	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
2268		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
2269		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
2270	ib->ptr[ib->length_dw++] = byte_count - 1;
2271	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
2272	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
2273	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
2274	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2275	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2276}
2277
2278/**
2279 * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
2280 *
2281 * @ib: indirect buffer to copy to
2282 * @src_data: value to write to buffer
2283 * @dst_offset: dst GPU address
2284 * @byte_count: number of bytes to xfer
2285 *
2286 * Fill GPU buffers using the DMA engine.
2287 */
2288static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
2289				       uint32_t src_data,
2290				       uint64_t dst_offset,
2291				       uint32_t byte_count)
2292{
2293	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2294	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2295	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2296	ib->ptr[ib->length_dw++] = src_data;
2297	ib->ptr[ib->length_dw++] = byte_count - 1;
2298}
2299
2300static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
2301	.copy_max_bytes = 0x400000,
2302	.copy_num_dw = 7,
2303	.emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
2304
2305	.fill_max_bytes = 0x400000,
2306	.fill_num_dw = 5,
2307	.emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
2308};
2309
2310static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
2311{
2312	adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
2313	if (adev->sdma.has_page_queue)
2314		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
2315	else
2316		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2317}
2318
2319static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
2320	.copy_pte_num_dw = 7,
2321	.copy_pte = sdma_v4_4_2_vm_copy_pte,
2322
2323	.write_pte = sdma_v4_4_2_vm_write_pte,
2324	.set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
2325};
2326
2327static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
2328{
2329	struct drm_gpu_scheduler *sched;
2330	unsigned i;
2331
2332	adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
2333	for (i = 0; i < adev->sdma.num_instances; i++) {
2334		if (adev->sdma.has_page_queue)
2335			sched = &adev->sdma.instance[i].page.sched;
2336		else
2337			sched = &adev->sdma.instance[i].ring.sched;
2338		adev->vm_manager.vm_pte_scheds[i] = sched;
2339	}
2340	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2341}
2342
2343/**
2344 * sdma_v4_4_2_update_reset_mask - update  reset mask for SDMA
2345 * @adev: Pointer to the AMDGPU device structure
2346 *
2347 * This function update reset mask for SDMA and sets the supported
2348 * reset types based on the IP version and firmware versions.
2349 *
2350 */
2351static void sdma_v4_4_2_update_reset_mask(struct amdgpu_device *adev)
2352{
2353	/* per queue reset not supported for SRIOV */
2354	if (amdgpu_sriov_vf(adev))
2355		return;
2356
2357	/*
2358	 * the user queue relies on MEC fw and pmfw when the sdma queue do reset.
2359	 * it needs to check both of them at here to skip old mec and pmfw.
2360	 */
2361	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2362	case IP_VERSION(9, 4, 3):
2363	case IP_VERSION(9, 4, 4):
2364		if ((adev->gfx.mec_fw_version >= 0xb0) &&
2365		    amdgpu_dpm_reset_sdma_is_supported(adev) &&
2366		    !adev->debug_disable_gpu_ring_reset)
2367			adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
2368		break;
2369	case IP_VERSION(9, 5, 0):
2370		if ((adev->gfx.mec_fw_version >= 0xf) &&
2371		    amdgpu_dpm_reset_sdma_is_supported(adev) &&
2372		    !adev->debug_disable_gpu_ring_reset)
2373			adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
2374		break;
2375	default:
2376		break;
2377	}
2378
2379}
2380
2381const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block = {
2382	.type = AMD_IP_BLOCK_TYPE_SDMA,
2383	.major = 4,
2384	.minor = 4,
2385	.rev = 2,
2386	.funcs = &sdma_v4_4_2_ip_funcs,
2387};
2388
2389static int sdma_v4_4_2_xcp_resume(void *handle, uint32_t inst_mask)
2390{
2391	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2392	int r;
2393
2394	if (!amdgpu_sriov_vf(adev))
2395		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
2396
2397	r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
2398
2399	return r;
2400}
2401
2402static int sdma_v4_4_2_xcp_suspend(void *handle, uint32_t inst_mask)
2403{
2404	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2405	uint32_t tmp_mask = inst_mask;
2406	int i;
2407
2408	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2409		for_each_inst(i, tmp_mask) {
2410			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
2411				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
2412		}
2413	}
2414
2415	sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
2416	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
2417
2418	return 0;
2419}
2420
2421struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = {
2422	.suspend = &sdma_v4_4_2_xcp_suspend,
2423	.resume = &sdma_v4_4_2_xcp_resume
2424};
2425
2426static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = {
2427	{AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI),
2428	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
2429};
2430
2431static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = {
2432	{AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"},
2433	{AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"},
2434	{AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"},
2435	{AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"},
2436	{AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"},
2437	{AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"},
2438	{AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"},
2439	{AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"},
2440	{AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"},
2441	{AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"},
2442	{AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"},
2443	{AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"},
2444	{AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"},
2445	{AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"},
2446	{AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"},
2447	{AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"},
2448	{AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"},
2449	{AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"},
2450	{AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"},
2451	{AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"},
2452	{AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"},
2453	{AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"},
2454	{AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"},
2455	{AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"},
2456};
2457
2458static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev,
2459						   uint32_t sdma_inst,
2460						   void *ras_err_status)
2461{
2462	struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
2463	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2464	unsigned long ue_count = 0;
2465	struct amdgpu_smuio_mcm_config_info mcm_info = {
2466		.socket_id = adev->smuio.funcs->get_socket_id(adev),
2467		.die_id = adev->sdma.instance[sdma_inst].aid_id,
2468	};
2469
2470	/* sdma v4_4_2 doesn't support query ce counts */
2471	amdgpu_ras_inst_query_ras_error_count(adev,
2472					sdma_v4_2_2_ue_reg_list,
2473					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2474					sdma_v4_4_2_ras_memory_list,
2475					ARRAY_SIZE(sdma_v4_4_2_ras_memory_list),
2476					sdma_dev_inst,
2477					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
2478					&ue_count);
2479
2480	amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
2481}
2482
2483static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev,
2484					      void *ras_err_status)
2485{
2486	uint32_t inst_mask;
2487	int i = 0;
2488
2489	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2490	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2491		for_each_inst(i, inst_mask)
2492			sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status);
2493	} else {
2494		dev_warn(adev->dev, "SDMA RAS is not supported\n");
2495	}
2496}
2497
2498static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev,
2499						   uint32_t sdma_inst)
2500{
2501	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2502
2503	amdgpu_ras_inst_reset_ras_error_count(adev,
2504					sdma_v4_2_2_ue_reg_list,
2505					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2506					sdma_dev_inst);
2507}
2508
2509static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev)
2510{
2511	uint32_t inst_mask;
2512	int i = 0;
2513
2514	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2515	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2516		for_each_inst(i, inst_mask)
2517			sdma_v4_4_2_inst_reset_ras_error_count(adev, i);
2518	} else {
2519		dev_warn(adev->dev, "SDMA RAS is not supported\n");
2520	}
2521}
2522
2523static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
2524	.query_ras_error_count = sdma_v4_4_2_query_ras_error_count,
2525	.reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
2526};
2527
2528static int sdma_v4_4_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
2529				       enum aca_smu_type type, void *data)
2530{
2531	struct aca_bank_info info;
2532	u64 misc0;
2533	int ret;
2534
2535	ret = aca_bank_info_decode(bank, &info);
2536	if (ret)
2537		return ret;
2538
2539	misc0 = bank->regs[ACA_REG_IDX_MISC0];
2540	switch (type) {
2541	case ACA_SMU_TYPE_UE:
2542		bank->aca_err_type = ACA_ERROR_TYPE_UE;
2543		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
2544						     1ULL);
2545		break;
2546	case ACA_SMU_TYPE_CE:
2547		bank->aca_err_type = ACA_ERROR_TYPE_CE;
2548		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
2549						     ACA_REG__MISC0__ERRCNT(misc0));
2550		break;
2551	default:
2552		return -EINVAL;
2553	}
2554
2555	return ret;
2556}
2557
2558/* CODE_SDMA0 - CODE_SDMA4, reference to smu driver if header file */
2559static int sdma_v4_4_2_err_codes[] = { 33, 34, 35, 36 };
2560
2561static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
2562					  enum aca_smu_type type, void *data)
2563{
2564	u32 instlo;
2565
2566	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
2567	instlo &= GENMASK(31, 1);
2568
2569	if (instlo != mmSMNAID_AID0_MCA_SMU)
2570		return false;
2571
2572	if (aca_bank_check_error_codes(handle->adev, bank,
2573				       sdma_v4_4_2_err_codes,
2574				       ARRAY_SIZE(sdma_v4_4_2_err_codes)))
2575		return false;
2576
2577	return true;
2578}
2579
2580static const struct aca_bank_ops sdma_v4_4_2_aca_bank_ops = {
2581	.aca_bank_parser = sdma_v4_4_2_aca_bank_parser,
2582	.aca_bank_is_valid = sdma_v4_4_2_aca_bank_is_valid,
2583};
2584
2585static const struct aca_info sdma_v4_4_2_aca_info = {
2586	.hwip = ACA_HWIP_TYPE_SMU,
2587	.mask = ACA_ERROR_UE_MASK,
2588	.bank_ops = &sdma_v4_4_2_aca_bank_ops,
2589};
2590
2591static int sdma_v4_4_2_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
2592{
2593	int r;
2594
2595	r = amdgpu_sdma_ras_late_init(adev, ras_block);
2596	if (r)
2597		return r;
2598
2599	return amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__SDMA,
2600				   &sdma_v4_4_2_aca_info, NULL);
2601}
2602
2603static struct amdgpu_sdma_ras sdma_v4_4_2_ras = {
2604	.ras_block = {
2605		.hw_ops = &sdma_v4_4_2_ras_hw_ops,
2606		.ras_late_init = sdma_v4_4_2_ras_late_init,
2607	},
2608};
2609
2610static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev)
2611{
2612	adev->sdma.ras = &sdma_v4_4_2_ras;
2613}