sdma_v5_2.c

   1/*
   2 * Copyright 2019 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/delay.h>
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_ucode.h"
  31#include "amdgpu_trace.h"
  32
  33#include "gc/gc_10_3_0_offset.h"
  34#include "gc/gc_10_3_0_sh_mask.h"
  35#include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h"
  36#include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h"
  37#include "ivsrcid/sdma2/irqsrcs_sdma2_5_0.h"
  38#include "ivsrcid/sdma3/irqsrcs_sdma3_5_0.h"
  39
  40#include "soc15_common.h"
  41#include "soc15.h"
  42#include "navi10_sdma_pkt_open.h"
  43#include "nbio_v2_3.h"
  44#include "sdma_common.h"
  45#include "sdma_v5_2.h"
  46
  47MODULE_FIRMWARE("amdgpu/sienna_cichlid_sdma.bin");
  48MODULE_FIRMWARE("amdgpu/navy_flounder_sdma.bin");
  49MODULE_FIRMWARE("amdgpu/dimgrey_cavefish_sdma.bin");
  50MODULE_FIRMWARE("amdgpu/beige_goby_sdma.bin");
  51
  52MODULE_FIRMWARE("amdgpu/vangogh_sdma.bin");
  53MODULE_FIRMWARE("amdgpu/yellow_carp_sdma.bin");
  54MODULE_FIRMWARE("amdgpu/sdma_5_2_6.bin");
  55MODULE_FIRMWARE("amdgpu/sdma_5_2_7.bin");
  56
  57#define SDMA1_REG_OFFSET 0x600
  58#define SDMA3_REG_OFFSET 0x400
  59#define SDMA0_HYP_DEC_REG_START 0x5880
  60#define SDMA0_HYP_DEC_REG_END 0x5893
  61#define SDMA1_HYP_DEC_REG_OFFSET 0x20
  62
  63static const struct amdgpu_hwip_reg_entry sdma_reg_list_5_2[] = {
  64	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS_REG),
  65	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS1_REG),
  66	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS2_REG),
  67	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS3_REG),
  68	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UCODE_CHECKSUM),
  69	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH_HI),
  70	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH),
  71	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_STATUS),
  72	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_STATUS),
  73	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK0),
  74	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK1),
  75	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK0),
  76	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK1),
  77	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_CNTL),
  78	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR),
  79	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR_HI),
  80	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR),
  81	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR_HI),
  82	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_OFFSET),
  83	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_LO),
  84	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_HI),
  85	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_CNTL),
  86	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_RPTR),
  87	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_SUB_REMAIN),
  88	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_DUMMY_REG),
  89	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_CNTL),
  90	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR),
  91	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR_HI),
  92	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR),
  93	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR_HI),
  94	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_OFFSET),
  95	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_LO),
  96	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_HI),
  97	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_DUMMY_REG),
  98	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_CNTL),
  99	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR),
 100	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR_HI),
 101	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR),
 102	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR_HI),
 103	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_OFFSET),
 104	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_LO),
 105	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_HI),
 106	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_DUMMY_REG),
 107	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_INT_STATUS),
 108	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_VM_CNTL),
 109	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2)
 110};
 111
 112static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev);
 113static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev);
 114static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev);
 115static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev);
 116static int sdma_v5_2_stop_queue(struct amdgpu_ring *ring);
 117static int sdma_v5_2_restore_queue(struct amdgpu_ring *ring);
 118
 119static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
 120{
 121	u32 base;
 122
 123	if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
 124	    internal_offset <= SDMA0_HYP_DEC_REG_END) {
 125		base = adev->reg_offset[GC_HWIP][0][1];
 126		if (instance != 0)
 127			internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
 128	} else {
 129		if (instance < 2) {
 130			base = adev->reg_offset[GC_HWIP][0][0];
 131			if (instance == 1)
 132				internal_offset += SDMA1_REG_OFFSET;
 133		} else {
 134			base = adev->reg_offset[GC_HWIP][0][2];
 135			if (instance == 3)
 136				internal_offset += SDMA3_REG_OFFSET;
 137		}
 138	}
 139
 140	return base + internal_offset;
 141}
 142
 143static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring,
 144					      uint64_t addr)
 145{
 146	unsigned ret;
 147
 148	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COND_EXE));
 149	amdgpu_ring_write(ring, lower_32_bits(addr));
 150	amdgpu_ring_write(ring, upper_32_bits(addr));
 151	amdgpu_ring_write(ring, 1);
 152	/* this is the offset we need patch later */
 153	ret = ring->wptr & ring->buf_mask;
 154	/* insert dummy here and patch it later */
 155	amdgpu_ring_write(ring, 0);
 156
 157	return ret;
 158}
 159
 160/**
 161 * sdma_v5_2_ring_get_rptr - get the current read pointer
 162 *
 163 * @ring: amdgpu ring pointer
 164 *
 165 * Get the current rptr from the hardware (NAVI10+).
 166 */
 167static uint64_t sdma_v5_2_ring_get_rptr(struct amdgpu_ring *ring)
 168{
 169	u64 *rptr;
 170
 171	/* XXX check if swapping is necessary on BE */
 172	rptr = (u64 *)ring->rptr_cpu_addr;
 173
 174	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
 175	return ((*rptr) >> 2);
 176}
 177
 178/**
 179 * sdma_v5_2_ring_get_wptr - get the current write pointer
 180 *
 181 * @ring: amdgpu ring pointer
 182 *
 183 * Get the current wptr from the hardware (NAVI10+).
 184 */
 185static uint64_t sdma_v5_2_ring_get_wptr(struct amdgpu_ring *ring)
 186{
 187	struct amdgpu_device *adev = ring->adev;
 188	u64 wptr;
 189
 190	if (ring->use_doorbell) {
 191		/* XXX check if swapping is necessary on BE */
 192		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
 193		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 194	} else {
 195		wptr = RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI));
 196		wptr = wptr << 32;
 197		wptr |= RREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR));
 198		DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n", ring->me, wptr);
 199	}
 200
 201	return wptr >> 2;
 202}
 203
 204/**
 205 * sdma_v5_2_ring_set_wptr - commit the write pointer
 206 *
 207 * @ring: amdgpu ring pointer
 208 *
 209 * Write the wptr back to the hardware (NAVI10+).
 210 */
 211static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring)
 212{
 213	struct amdgpu_device *adev = ring->adev;
 214
 215	DRM_DEBUG("Setting write pointer\n");
 216	if (ring->use_doorbell) {
 217		DRM_DEBUG("Using doorbell -- "
 218				"wptr_offs == 0x%08x "
 219				"lower_32_bits(ring->wptr << 2) == 0x%08x "
 220				"upper_32_bits(ring->wptr << 2) == 0x%08x\n",
 221				ring->wptr_offs,
 222				lower_32_bits(ring->wptr << 2),
 223				upper_32_bits(ring->wptr << 2));
 224		/* XXX check if swapping is necessary on BE */
 225		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
 226			     ring->wptr << 2);
 227		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
 228				ring->doorbell_index, ring->wptr << 2);
 229		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 230		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(5, 2, 1)) {
 231			/* SDMA seems to miss doorbells sometimes when powergating kicks in.
 232			 * Updating the wptr directly will wake it. This is only safe because
 233			 * we disallow gfxoff in begin_use() and then allow it again in end_use().
 234			 */
 235			WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR),
 236			       lower_32_bits(ring->wptr << 2));
 237			WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI),
 238			       upper_32_bits(ring->wptr << 2));
 239		}
 240	} else {
 241		DRM_DEBUG("Not using doorbell -- "
 242				"mmSDMA%i_GFX_RB_WPTR == 0x%08x "
 243				"mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
 244				ring->me,
 245				lower_32_bits(ring->wptr << 2),
 246				ring->me,
 247				upper_32_bits(ring->wptr << 2));
 248		WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR),
 249			lower_32_bits(ring->wptr << 2));
 250		WREG32(sdma_v5_2_get_reg_offset(adev, ring->me, mmSDMA0_GFX_RB_WPTR_HI),
 251			upper_32_bits(ring->wptr << 2));
 252	}
 253}
 254
 255static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 256{
 257	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 258	int i;
 259
 260	for (i = 0; i < count; i++)
 261		if (sdma && sdma->burst_nop && (i == 0))
 262			amdgpu_ring_write(ring, ring->funcs->nop |
 263				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 264		else
 265			amdgpu_ring_write(ring, ring->funcs->nop);
 266}
 267
 268/**
 269 * sdma_v5_2_ring_emit_ib - Schedule an IB on the DMA engine
 270 *
 271 * @ring: amdgpu ring pointer
 272 * @job: job to retrieve vmid from
 273 * @ib: IB object to schedule
 274 * @flags: unused
 275 *
 276 * Schedule an IB in the DMA ring.
 277 */
 278static void sdma_v5_2_ring_emit_ib(struct amdgpu_ring *ring,
 279				   struct amdgpu_job *job,
 280				   struct amdgpu_ib *ib,
 281				   uint32_t flags)
 282{
 283	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 284	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
 285
 286	/* An IB packet must end on a 8 DW boundary--the next dword
 287	 * must be on a 8-dword boundary. Our IB packet below is 6
 288	 * dwords long, thus add x number of NOPs, such that, in
 289	 * modular arithmetic,
 290	 * wptr + 6 + x = 8k, k >= 0, which in C is,
 291	 * (wptr + 6 + x) % 8 = 0.
 292	 * The expression below, is a solution of x.
 293	 */
 294	sdma_v5_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
 295
 296	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
 297			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 298	/* base must be 32 byte aligned */
 299	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 300	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 301	amdgpu_ring_write(ring, ib->length_dw);
 302	amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
 303	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
 304}
 305
 306/**
 307 * sdma_v5_2_ring_emit_mem_sync - flush the IB by graphics cache rinse
 308 *
 309 * @ring: amdgpu ring pointer
 310 *
 311 * flush the IB by graphics cache rinse.
 312 */
 313static void sdma_v5_2_ring_emit_mem_sync(struct amdgpu_ring *ring)
 314{
 315	uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB |
 316			    SDMA_GCR_GLM_INV | SDMA_GCR_GL1_INV |
 317			    SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
 318			    SDMA_GCR_GLI_INV(1);
 319
 320	/* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
 321	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
 322	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
 323	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
 324			SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
 325	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
 326			SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
 327	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
 328			SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
 329}
 330
 331/**
 332 * sdma_v5_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
 333 *
 334 * @ring: amdgpu ring pointer
 335 *
 336 * Emit an hdp flush packet on the requested DMA ring.
 337 */
 338static void sdma_v5_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 339{
 340	struct amdgpu_device *adev = ring->adev;
 341	u32 ref_and_mask = 0;
 342	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 343
 344	if (ring->me > 1) {
 345		amdgpu_hdp_flush(adev, ring);
 346	} else {
 347		ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
 348
 349		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
 350				  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
 351				  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
 352		amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
 353		amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
 354		amdgpu_ring_write(ring, ref_and_mask); /* reference */
 355		amdgpu_ring_write(ring, ref_and_mask); /* mask */
 356		amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
 357				  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 358	}
 359}
 360
 361/**
 362 * sdma_v5_2_ring_emit_fence - emit a fence on the DMA ring
 363 *
 364 * @ring: amdgpu ring pointer
 365 * @addr: address
 366 * @seq: sequence number
 367 * @flags: fence related flags
 368 *
 369 * Add a DMA fence packet to the ring to write
 370 * the fence seq number and DMA trap packet to generate
 371 * an interrupt if needed.
 372 */
 373static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 374				      unsigned flags)
 375{
 376	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 377	/* write the fence */
 378	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
 379			  SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
 380	/* zero in first two bits */
 381	BUG_ON(addr & 0x3);
 382	amdgpu_ring_write(ring, lower_32_bits(addr));
 383	amdgpu_ring_write(ring, upper_32_bits(addr));
 384	amdgpu_ring_write(ring, lower_32_bits(seq));
 385
 386	/* optionally write high bits as well */
 387	if (write64bit) {
 388		addr += 4;
 389		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) |
 390				  SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
 391		/* zero in first two bits */
 392		BUG_ON(addr & 0x3);
 393		amdgpu_ring_write(ring, lower_32_bits(addr));
 394		amdgpu_ring_write(ring, upper_32_bits(addr));
 395		amdgpu_ring_write(ring, upper_32_bits(seq));
 396	}
 397
 398	if ((flags & AMDGPU_FENCE_FLAG_INT)) {
 399		/* generate an interrupt */
 400		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
 401		amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
 402	}
 403}
 404
 405
 406/**
 407 * sdma_v5_2_gfx_stop - stop the gfx async dma engines
 408 *
 409 * @adev: amdgpu_device pointer
 410 * @inst_mask: mask of dma engine instances to be disabled
 411 * Stop the gfx async dma ring buffers.
 412 */
 413static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev,  uint32_t inst_mask)
 414{
 415	u32 rb_cntl, ib_cntl;
 416	int i;
 417
 418	for_each_inst(i, inst_mask) {
 419		rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
 420		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
 421		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 422		ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
 423		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0);
 424		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
 425	}
 426}
 427
 428/**
 429 * sdma_v5_2_rlc_stop - stop the compute async dma engines
 430 *
 431 * @adev: amdgpu_device pointer
 432 *
 433 * Stop the compute async dma queues.
 434 */
 435static void sdma_v5_2_rlc_stop(struct amdgpu_device *adev)
 436{
 437	/* XXX todo */
 438}
 439
 440/**
 441 * sdma_v5_2_ctx_switch_enable - stop the async dma engines context switch
 442 *
 443 * @adev: amdgpu_device pointer
 444 * @enable: enable/disable the DMA MEs context switch.
 445 *
 446 * Halt or unhalt the async dma engines context switch.
 447 */
 448static void sdma_v5_2_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 449{
 450	u32 f32_cntl, phase_quantum = 0;
 451	int i;
 452
 453	if (amdgpu_sdma_phase_quantum) {
 454		unsigned value = amdgpu_sdma_phase_quantum;
 455		unsigned unit = 0;
 456
 457		while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
 458				SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
 459			value = (value + 1) >> 1;
 460			unit++;
 461		}
 462		if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
 463			    SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
 464			value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
 465				 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
 466			unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
 467				SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
 468			WARN_ONCE(1,
 469			"clamping sdma_phase_quantum to %uK clock cycles\n",
 470				  value << unit);
 471		}
 472		phase_quantum =
 473			value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
 474			unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
 475	}
 476
 477	for (i = 0; i < adev->sdma.num_instances; i++) {
 478		if (enable && amdgpu_sdma_phase_quantum) {
 479			WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM),
 480			       phase_quantum);
 481			WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM),
 482			       phase_quantum);
 483			WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM),
 484			       phase_quantum);
 485		}
 486
 487		if (!amdgpu_sriov_vf(adev)) {
 488			f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
 489			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 490					AUTO_CTXSW_ENABLE, enable ? 1 : 0);
 491			WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl);
 492		}
 493	}
 494
 495}
 496
 497/**
 498 * sdma_v5_2_enable - stop the async dma engines
 499 *
 500 * @adev: amdgpu_device pointer
 501 * @enable: enable/disable the DMA MEs.
 502 *
 503 * Halt or unhalt the async dma engines.
 504 */
 505static void sdma_v5_2_enable(struct amdgpu_device *adev, bool enable)
 506{
 507	u32 f32_cntl;
 508	int i;
 509	uint32_t inst_mask;
 510
 511	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
 512	if (!enable) {
 513		sdma_v5_2_gfx_stop(adev, inst_mask);
 514		sdma_v5_2_rlc_stop(adev);
 515	}
 516
 517	if (!amdgpu_sriov_vf(adev)) {
 518		for (i = 0; i < adev->sdma.num_instances; i++) {
 519			f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
 520			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
 521			WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl);
 522		}
 523	}
 524}
 525
 526/**
 527 * sdma_v5_2_gfx_resume_instance - start/restart a certain sdma engine
 528 *
 529 * @adev: amdgpu_device pointer
 530 * @i: instance
 531 * @restore: used to restore wptr when restart
 532 *
 533 * Set up the gfx DMA ring buffers and enable them. On restart, we will restore wptr and rptr.
 534 * Return 0 for success.
 535 */
 536
 537static int sdma_v5_2_gfx_resume_instance(struct amdgpu_device *adev, int i, bool restore)
 538{
 539	struct amdgpu_ring *ring;
 540	u32 rb_cntl, ib_cntl;
 541	u32 rb_bufsz;
 542	u32 doorbell;
 543	u32 doorbell_offset;
 544	u32 temp;
 545	u32 wptr_poll_cntl;
 546	u64 wptr_gpu_addr;
 547
 548	ring = &adev->sdma.instance[i].ring;
 549
 550	if (!amdgpu_sriov_vf(adev))
 551		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
 552
 553	/* Set ring buffer size in dwords */
 554	rb_bufsz = order_base_2(ring->ring_size / 4);
 555	rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
 556	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
 557#ifdef __BIG_ENDIAN
 558	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
 559	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
 560				RPTR_WRITEBACK_SWAP_ENABLE, 1);
 561#endif
 562	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 563
 564	/* Initialize the ring buffer's read and write pointers */
 565	if (restore) {
 566		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), lower_32_bits(ring->wptr << 2));
 567		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), upper_32_bits(ring->wptr << 2));
 568		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
 569		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
 570	} else {
 571		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0);
 572		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0);
 573		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0);
 574		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0);
 575	}
 576
 577	/* setup the wptr shadow polling */
 578	wptr_gpu_addr = ring->wptr_gpu_addr;
 579	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
 580	       lower_32_bits(wptr_gpu_addr));
 581	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
 582	       upper_32_bits(wptr_gpu_addr));
 583	wptr_poll_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i,
 584						 mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
 585	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
 586				       SDMA0_GFX_RB_WPTR_POLL_CNTL,
 587				       F32_POLL_ENABLE, 1);
 588	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL),
 589	       wptr_poll_cntl);
 590
 591	/* set the wb address whether it's enabled or not */
 592	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI),
 593	       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
 594	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO),
 595	       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
 596
 597	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
 598
 599	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8);
 600	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40);
 601
 602	if (!restore)
 603		ring->wptr = 0;
 604
 605	/* before programing wptr to a less value, need set minor_ptr_update first */
 606	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1);
 607
 608	if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
 609		WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
 610		WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
 611	}
 612
 613	doorbell = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL));
 614	doorbell_offset = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET));
 615
 616	if (ring->use_doorbell) {
 617		doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1);
 618		doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_GFX_DOORBELL_OFFSET,
 619				OFFSET, ring->doorbell_index);
 620	} else {
 621		doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0);
 622	}
 623	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell);
 624	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset);
 625
 626	adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
 627					      ring->doorbell_index,
 628					      adev->doorbell_index.sdma_doorbell_range);
 629
 630	if (amdgpu_sriov_vf(adev))
 631		sdma_v5_2_ring_set_wptr(ring);
 632
 633	/* set minor_ptr_update to 0 after wptr programed */
 634
 635	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0);
 636
 637	/* SRIOV VF has no control of any of registers below */
 638	if (!amdgpu_sriov_vf(adev)) {
 639		/* set utc l1 enable flag always to 1 */
 640		temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
 641		temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);
 642
 643		/* enable MCBP */
 644		temp = REG_SET_FIELD(temp, SDMA0_CNTL, MIDCMD_PREEMPT_ENABLE, 1);
 645		WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), temp);
 646
 647		/* Set up RESP_MODE to non-copy addresses */
 648		temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL));
 649		temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
 650		temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
 651		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_CNTL), temp);
 652
 653		/* program default cache read and write policy */
 654		temp = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE));
 655		/* clean read policy and write policy bits */
 656		temp &= 0xFF0FFF;
 657		temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
 658			 (CACHE_WRITE_POLICY_L2__DEFAULT << 14) |
 659			 SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK);
 660		WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UTCL1_PAGE), temp);
 661
 662		/* unhalt engine */
 663		temp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
 664		temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
 665		WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp);
 666	}
 667
 668	/* enable DMA RB */
 669	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
 670	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 671
 672	ib_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
 673	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);
 674#ifdef __BIG_ENDIAN
 675	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
 676#endif
 677	/* enable DMA IBs */
 678	WREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
 679
 680	if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
 681		sdma_v5_2_ctx_switch_enable(adev, true);
 682		sdma_v5_2_enable(adev, true);
 683	}
 684
 685	return amdgpu_ring_test_helper(ring);
 686}
 687
 688/**
 689 * sdma_v5_2_gfx_resume - setup and start the async dma engines
 690 *
 691 * @adev: amdgpu_device pointer
 692 *
 693 * Set up the gfx DMA ring buffers and enable them.
 694 * Returns 0 for success, error for failure.
 695 */
 696static int sdma_v5_2_gfx_resume(struct amdgpu_device *adev)
 697{
 698	int i, r;
 699
 700	for (i = 0; i < adev->sdma.num_instances; i++) {
 701		r = sdma_v5_2_gfx_resume_instance(adev, i, false);
 702		if (r)
 703			return r;
 704	}
 705
 706	return 0;
 707}
 708
 709/**
 710 * sdma_v5_2_rlc_resume - setup and start the async dma engines
 711 *
 712 * @adev: amdgpu_device pointer
 713 *
 714 * Set up the compute DMA queues and enable them.
 715 * Returns 0 for success, error for failure.
 716 */
 717static int sdma_v5_2_rlc_resume(struct amdgpu_device *adev)
 718{
 719	return 0;
 720}
 721
 722/**
 723 * sdma_v5_2_load_microcode - load the sDMA ME ucode
 724 *
 725 * @adev: amdgpu_device pointer
 726 *
 727 * Loads the sDMA0/1/2/3 ucode.
 728 * Returns 0 for success, -EINVAL if the ucode is not available.
 729 */
 730static int sdma_v5_2_load_microcode(struct amdgpu_device *adev)
 731{
 732	const struct sdma_firmware_header_v1_0 *hdr;
 733	const __le32 *fw_data;
 734	u32 fw_size;
 735	int i, j;
 736
 737	/* halt the MEs */
 738	sdma_v5_2_enable(adev, false);
 739
 740	for (i = 0; i < adev->sdma.num_instances; i++) {
 741		if (!adev->sdma.instance[i].fw)
 742			return -EINVAL;
 743
 744		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
 745		amdgpu_ucode_print_sdma_hdr(&hdr->header);
 746		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
 747
 748		fw_data = (const __le32 *)
 749			(adev->sdma.instance[i].fw->data +
 750				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 751
 752		WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0);
 753
 754		for (j = 0; j < fw_size; j++) {
 755			if (amdgpu_emu_mode == 1 && j % 500 == 0)
 756				msleep(1);
 757			WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
 758		}
 759
 760		WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
 761	}
 762
 763	return 0;
 764}
 765
 766static int sdma_v5_2_soft_reset_engine(struct amdgpu_device *adev, u32 instance_id)
 767{
 768	u32 grbm_soft_reset;
 769	u32 tmp;
 770
 771	grbm_soft_reset = REG_SET_FIELD(0,
 772					GRBM_SOFT_RESET, SOFT_RESET_SDMA0,
 773					1);
 774	grbm_soft_reset <<= instance_id;
 775
 776	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
 777	tmp |= grbm_soft_reset;
 778	DRM_DEBUG("GRBM_SOFT_RESET=0x%08X\n", tmp);
 779	WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
 780	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
 781
 782	udelay(50);
 783
 784	tmp &= ~grbm_soft_reset;
 785	WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
 786	tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
 787	return 0;
 788}
 789
 790static int sdma_v5_2_soft_reset(struct amdgpu_ip_block *ip_block)
 791{
 792	struct amdgpu_device *adev = ip_block->adev;
 793	int i;
 794
 795	for (i = 0; i < adev->sdma.num_instances; i++) {
 796		sdma_v5_2_soft_reset_engine(adev, i);
 797		udelay(50);
 798	}
 799
 800	return 0;
 801}
 802
 803static const struct amdgpu_sdma_funcs sdma_v5_2_sdma_funcs = {
 804	.stop_kernel_queue = &sdma_v5_2_stop_queue,
 805	.start_kernel_queue = &sdma_v5_2_restore_queue,
 806	.soft_reset_kernel_queue = &sdma_v5_2_soft_reset_engine,
 807};
 808
 809/**
 810 * sdma_v5_2_start - setup and start the async dma engines
 811 *
 812 * @adev: amdgpu_device pointer
 813 *
 814 * Set up the DMA engines and enable them.
 815 * Returns 0 for success, error for failure.
 816 */
 817static int sdma_v5_2_start(struct amdgpu_device *adev)
 818{
 819	int r = 0;
 820	struct amdgpu_ip_block *ip_block;
 821
 822	if (amdgpu_sriov_vf(adev)) {
 823		sdma_v5_2_ctx_switch_enable(adev, false);
 824		sdma_v5_2_enable(adev, false);
 825
 826		/* set RB registers */
 827		r = sdma_v5_2_gfx_resume(adev);
 828		return r;
 829	}
 830
 831	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 832		r = sdma_v5_2_load_microcode(adev);
 833		if (r)
 834			return r;
 835
 836		/* The value of mmSDMA_F32_CNTL is invalid the moment after loading fw */
 837		if (amdgpu_emu_mode == 1)
 838			msleep(1000);
 839	}
 840
 841	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_SDMA);
 842	if (!ip_block)
 843		return -EINVAL;
 844
 845	sdma_v5_2_soft_reset(ip_block);
 846	/* unhalt the MEs */
 847	sdma_v5_2_enable(adev, true);
 848	/* enable sdma ring preemption */
 849	sdma_v5_2_ctx_switch_enable(adev, true);
 850
 851	/* start the gfx rings and rlc compute queues */
 852	r = sdma_v5_2_gfx_resume(adev);
 853	if (r)
 854		return r;
 855	r = sdma_v5_2_rlc_resume(adev);
 856
 857	return r;
 858}
 859
 860static int sdma_v5_2_mqd_init(struct amdgpu_device *adev, void *mqd,
 861			      struct amdgpu_mqd_prop *prop)
 862{
 863	struct v10_sdma_mqd *m = mqd;
 864	uint64_t wb_gpu_addr;
 865
 866	m->sdmax_rlcx_rb_cntl =
 867		order_base_2(prop->queue_size / 4) << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
 868		1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 869		6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
 870		1 << SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT;
 871
 872	m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
 873	m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
 874
 875	m->sdmax_rlcx_rb_wptr_poll_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0,
 876						  mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
 877
 878	wb_gpu_addr = prop->wptr_gpu_addr;
 879	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
 880	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
 881
 882	wb_gpu_addr = prop->rptr_gpu_addr;
 883	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 884	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 885
 886	m->sdmax_rlcx_ib_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, 0,
 887							mmSDMA0_GFX_IB_CNTL));
 888
 889	m->sdmax_rlcx_doorbell_offset =
 890		prop->doorbell_index << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
 891
 892	m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_RLC0_DOORBELL, ENABLE, 1);
 893
 894	return 0;
 895}
 896
 897static void sdma_v5_2_set_mqd_funcs(struct amdgpu_device *adev)
 898{
 899	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v10_sdma_mqd);
 900	adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v5_2_mqd_init;
 901}
 902
 903/**
 904 * sdma_v5_2_ring_test_ring - simple async dma engine test
 905 *
 906 * @ring: amdgpu_ring structure holding ring information
 907 *
 908 * Test the DMA engine by writing using it to write an
 909 * value to memory.
 910 * Returns 0 for success, error for failure.
 911 */
 912static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 913{
 914	struct amdgpu_device *adev = ring->adev;
 915	unsigned i;
 916	unsigned index;
 917	int r;
 918	u32 tmp;
 919	u64 gpu_addr;
 920
 921	tmp = 0xCAFEDEAD;
 922
 923	r = amdgpu_device_wb_get(adev, &index);
 924	if (r) {
 925		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 926		return r;
 927	}
 928
 929	gpu_addr = adev->wb.gpu_addr + (index * 4);
 930	adev->wb.wb[index] = cpu_to_le32(tmp);
 931
 932	r = amdgpu_ring_alloc(ring, 20);
 933	if (r) {
 934		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
 935		amdgpu_device_wb_free(adev, index);
 936		return r;
 937	}
 938
 939	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 940			  SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
 941	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
 942	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
 943	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
 944	amdgpu_ring_write(ring, 0xDEADBEEF);
 945	amdgpu_ring_commit(ring);
 946
 947	for (i = 0; i < adev->usec_timeout; i++) {
 948		tmp = le32_to_cpu(adev->wb.wb[index]);
 949		if (tmp == 0xDEADBEEF)
 950			break;
 951		if (amdgpu_emu_mode == 1)
 952			msleep(1);
 953		else
 954			udelay(1);
 955	}
 956
 957	if (i >= adev->usec_timeout)
 958		r = -ETIMEDOUT;
 959
 960	amdgpu_device_wb_free(adev, index);
 961
 962	return r;
 963}
 964
 965/**
 966 * sdma_v5_2_ring_test_ib - test an IB on the DMA engine
 967 *
 968 * @ring: amdgpu_ring structure holding ring information
 969 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 970 *
 971 * Test a simple IB in the DMA ring.
 972 * Returns 0 on success, error on failure.
 973 */
 974static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 975{
 976	struct amdgpu_device *adev = ring->adev;
 977	struct amdgpu_ib ib;
 978	struct dma_fence *f = NULL;
 979	unsigned index;
 980	long r;
 981	u32 tmp = 0;
 982	u64 gpu_addr;
 983
 984	tmp = 0xCAFEDEAD;
 985	memset(&ib, 0, sizeof(ib));
 986
 987	r = amdgpu_device_wb_get(adev, &index);
 988	if (r) {
 989		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 990		return r;
 991	}
 992
 993	gpu_addr = adev->wb.gpu_addr + (index * 4);
 994	adev->wb.wb[index] = cpu_to_le32(tmp);
 995
 996	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
 997	if (r) {
 998		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 999		goto err0;
1000	}
1001
1002	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1003		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1004	ib.ptr[1] = lower_32_bits(gpu_addr);
1005	ib.ptr[2] = upper_32_bits(gpu_addr);
1006	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1007	ib.ptr[4] = 0xDEADBEEF;
1008	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1009	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1010	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1011	ib.length_dw = 8;
1012
1013	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1014	if (r)
1015		goto err1;
1016
1017	r = dma_fence_wait_timeout(f, false, timeout);
1018	if (r == 0) {
1019		DRM_ERROR("amdgpu: IB test timed out\n");
1020		r = -ETIMEDOUT;
1021		goto err1;
1022	} else if (r < 0) {
1023		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1024		goto err1;
1025	}
1026
1027	tmp = le32_to_cpu(adev->wb.wb[index]);
1028
1029	if (tmp == 0xDEADBEEF)
1030		r = 0;
1031	else
1032		r = -EINVAL;
1033
1034err1:
1035	amdgpu_ib_free(&ib, NULL);
1036	dma_fence_put(f);
1037err0:
1038	amdgpu_device_wb_free(adev, index);
1039	return r;
1040}
1041
1042
1043/**
1044 * sdma_v5_2_vm_copy_pte - update PTEs by copying them from the GART
1045 *
1046 * @ib: indirect buffer to fill with commands
1047 * @pe: addr of the page entry
1048 * @src: src addr to copy from
1049 * @count: number of page entries to update
1050 *
1051 * Update PTEs by copying them from the GART using sDMA.
1052 */
1053static void sdma_v5_2_vm_copy_pte(struct amdgpu_ib *ib,
1054				  uint64_t pe, uint64_t src,
1055				  unsigned count)
1056{
1057	unsigned bytes = count * 8;
1058
1059	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1060		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1061	ib->ptr[ib->length_dw++] = bytes - 1;
1062	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1063	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1064	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1065	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1066	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1067
1068}
1069
1070/**
1071 * sdma_v5_2_vm_write_pte - update PTEs by writing them manually
1072 *
1073 * @ib: indirect buffer to fill with commands
1074 * @pe: addr of the page entry
1075 * @value: dst addr to write into pe
1076 * @count: number of page entries to update
1077 * @incr: increase next addr by incr bytes
1078 *
1079 * Update PTEs by writing them manually using sDMA.
1080 */
1081static void sdma_v5_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1082				   uint64_t value, unsigned count,
1083				   uint32_t incr)
1084{
1085	unsigned ndw = count * 2;
1086
1087	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1088		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1089	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1090	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1091	ib->ptr[ib->length_dw++] = ndw - 1;
1092	for (; ndw > 0; ndw -= 2) {
1093		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1094		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1095		value += incr;
1096	}
1097}
1098
1099/**
1100 * sdma_v5_2_vm_set_pte_pde - update the page tables using sDMA
1101 *
1102 * @ib: indirect buffer to fill with commands
1103 * @pe: addr of the page entry
1104 * @addr: dst addr to write into pe
1105 * @count: number of page entries to update
1106 * @incr: increase next addr by incr bytes
1107 * @flags: access flags
1108 *
1109 * Update the page tables using sDMA.
1110 */
1111static void sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1112				     uint64_t pe,
1113				     uint64_t addr, unsigned count,
1114				     uint32_t incr, uint64_t flags)
1115{
1116	/* for physically contiguous pages (vram) */
1117	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1118	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1119	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1120	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1121	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1122	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1123	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1124	ib->ptr[ib->length_dw++] = incr; /* increment size */
1125	ib->ptr[ib->length_dw++] = 0;
1126	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1127}
1128
1129/**
1130 * sdma_v5_2_ring_pad_ib - pad the IB
1131 *
1132 * @ib: indirect buffer to fill with padding
1133 * @ring: amdgpu_ring structure holding ring information
1134 *
1135 * Pad the IB with NOPs to a boundary multiple of 8.
1136 */
1137static void sdma_v5_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1138{
1139	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1140	u32 pad_count;
1141	int i;
1142
1143	pad_count = (-ib->length_dw) & 0x7;
1144	for (i = 0; i < pad_count; i++)
1145		if (sdma && sdma->burst_nop && (i == 0))
1146			ib->ptr[ib->length_dw++] =
1147				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1148				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1149		else
1150			ib->ptr[ib->length_dw++] =
1151				SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1152}
1153
1154
1155/**
1156 * sdma_v5_2_ring_emit_pipeline_sync - sync the pipeline
1157 *
1158 * @ring: amdgpu_ring pointer
1159 *
1160 * Make sure all previous operations are completed (CIK).
1161 */
1162static void sdma_v5_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1163{
1164	uint32_t seq = ring->fence_drv.sync_seq;
1165	uint64_t addr = ring->fence_drv.gpu_addr;
1166
1167	/* wait for idle */
1168	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1169			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1170			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1171			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1172	amdgpu_ring_write(ring, addr & 0xfffffffc);
1173	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1174	amdgpu_ring_write(ring, seq); /* reference */
1175	amdgpu_ring_write(ring, 0xffffffff); /* mask */
1176	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1177			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1178}
1179
1180
1181/**
1182 * sdma_v5_2_ring_emit_vm_flush - vm flush using sDMA
1183 *
1184 * @ring: amdgpu_ring pointer
1185 * @vmid: vmid number to use
1186 * @pd_addr: address
1187 *
1188 * Update the page table base and flush the VM TLB
1189 * using sDMA.
1190 */
1191static void sdma_v5_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1192					 unsigned vmid, uint64_t pd_addr)
1193{
1194	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1195	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
1196
1197	/* Update the PD address for this VMID. */
1198	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
1199			      (hub->ctx_addr_distance * vmid),
1200			      lower_32_bits(pd_addr));
1201	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
1202			      (hub->ctx_addr_distance * vmid),
1203			      upper_32_bits(pd_addr));
1204
1205	/* Trigger invalidation. */
1206	amdgpu_ring_write(ring,
1207			  SDMA_PKT_VM_INVALIDATION_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1208			  SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP(SDMA_SUBOP_VM_INVALIDATION) |
1209			  SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID(ring->vm_inv_eng) |
1210			  SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID(0x1f));
1211	amdgpu_ring_write(ring, req);
1212	amdgpu_ring_write(ring, 0xFFFFFFFF);
1213	amdgpu_ring_write(ring,
1214			  SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK(1 << vmid) |
1215			  SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI(0x1F));
1216}
1217
1218static void sdma_v5_2_ring_emit_wreg(struct amdgpu_ring *ring,
1219				     uint32_t reg, uint32_t val)
1220{
1221	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1222			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1223	amdgpu_ring_write(ring, reg);
1224	amdgpu_ring_write(ring, val);
1225}
1226
1227static void sdma_v5_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1228					 uint32_t val, uint32_t mask)
1229{
1230	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1231			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1232			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1233	amdgpu_ring_write(ring, reg << 2);
1234	amdgpu_ring_write(ring, 0);
1235	amdgpu_ring_write(ring, val); /* reference */
1236	amdgpu_ring_write(ring, mask); /* mask */
1237	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1238			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1239}
1240
1241static void sdma_v5_2_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1242						   uint32_t reg0, uint32_t reg1,
1243						   uint32_t ref, uint32_t mask)
1244{
1245	amdgpu_ring_emit_wreg(ring, reg0, ref);
1246	/* wait for a cycle to reset vm_inv_eng*_ack */
1247	amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1248	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1249}
1250
1251static int sdma_v5_2_early_init(struct amdgpu_ip_block *ip_block)
1252{
1253	struct amdgpu_device *adev = ip_block->adev;
1254	int r;
1255
1256	r = amdgpu_sdma_init_microcode(adev, 0, true);
1257	if (r)
1258		return r;
1259
1260	sdma_v5_2_set_ring_funcs(adev);
1261	sdma_v5_2_set_buffer_funcs(adev);
1262	sdma_v5_2_set_vm_pte_funcs(adev);
1263	sdma_v5_2_set_irq_funcs(adev);
1264	sdma_v5_2_set_mqd_funcs(adev);
1265
1266	return 0;
1267}
1268
1269static unsigned sdma_v5_2_seq_to_irq_id(int seq_num)
1270{
1271	switch (seq_num) {
1272	case 0:
1273		return SOC15_IH_CLIENTID_SDMA0;
1274	case 1:
1275		return SOC15_IH_CLIENTID_SDMA1;
1276	case 2:
1277		return SOC15_IH_CLIENTID_SDMA2;
1278	case 3:
1279		return SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid;
1280	default:
1281		break;
1282	}
1283	return -EINVAL;
1284}
1285
1286static unsigned sdma_v5_2_seq_to_trap_id(int seq_num)
1287{
1288	switch (seq_num) {
1289	case 0:
1290		return SDMA0_5_0__SRCID__SDMA_TRAP;
1291	case 1:
1292		return SDMA1_5_0__SRCID__SDMA_TRAP;
1293	case 2:
1294		return SDMA2_5_0__SRCID__SDMA_TRAP;
1295	case 3:
1296		return SDMA3_5_0__SRCID__SDMA_TRAP;
1297	default:
1298		break;
1299	}
1300	return -EINVAL;
1301}
1302
1303static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block)
1304{
1305	struct amdgpu_ring *ring;
1306	int r, i;
1307	struct amdgpu_device *adev = ip_block->adev;
1308	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2);
1309	uint32_t *ptr;
1310
1311	/* SDMA trap event */
1312	for (i = 0; i < adev->sdma.num_instances; i++) {
1313		r = amdgpu_irq_add_id(adev, sdma_v5_2_seq_to_irq_id(i),
1314				      sdma_v5_2_seq_to_trap_id(i),
1315				      &adev->sdma.trap_irq);
1316		if (r)
1317			return r;
1318	}
1319
1320	for (i = 0; i < adev->sdma.num_instances; i++) {
1321		mutex_init(&adev->sdma.instance[i].engine_reset_mutex);
1322		adev->sdma.instance[i].funcs = &sdma_v5_2_sdma_funcs;
1323		ring = &adev->sdma.instance[i].ring;
1324		ring->ring_obj = NULL;
1325		ring->use_doorbell = true;
1326		ring->me = i;
1327
1328		DRM_INFO("use_doorbell being set to: [%s]\n",
1329				ring->use_doorbell?"true":"false");
1330
1331		ring->doorbell_index =
1332			(adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset
1333
1334		ring->vm_hub = AMDGPU_GFXHUB(0);
1335		sprintf(ring->name, "sdma%d", i);
1336		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1337				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1338				     AMDGPU_RING_PRIO_DEFAULT, NULL);
1339		if (r)
1340			return r;
1341	}
1342
1343	adev->sdma.supported_reset =
1344		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1345	if (!amdgpu_sriov_vf(adev) &&
1346	    !adev->debug_disable_gpu_ring_reset)
1347		adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1348
1349	/* Allocate memory for SDMA IP Dump buffer */
1350	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1351	if (ptr)
1352		adev->sdma.ip_dump = ptr;
1353	else
1354		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1355
1356	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1357	if (r)
1358		return r;
1359
1360	return r;
1361}
1362
1363static int sdma_v5_2_sw_fini(struct amdgpu_ip_block *ip_block)
1364{
1365	struct amdgpu_device *adev = ip_block->adev;
1366	int i;
1367
1368	for (i = 0; i < adev->sdma.num_instances; i++)
1369		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1370
1371	amdgpu_sdma_sysfs_reset_mask_fini(adev);
1372	amdgpu_sdma_destroy_inst_ctx(adev, true);
1373
1374	kfree(adev->sdma.ip_dump);
1375
1376	return 0;
1377}
1378
1379static int sdma_v5_2_hw_init(struct amdgpu_ip_block *ip_block)
1380{
1381	struct amdgpu_device *adev = ip_block->adev;
1382
1383	return sdma_v5_2_start(adev);
1384}
1385
1386static int sdma_v5_2_hw_fini(struct amdgpu_ip_block *ip_block)
1387{
1388	struct amdgpu_device *adev = ip_block->adev;
1389
1390	if (amdgpu_sriov_vf(adev))
1391		return 0;
1392
1393	sdma_v5_2_ctx_switch_enable(adev, false);
1394	sdma_v5_2_enable(adev, false);
1395
1396	return 0;
1397}
1398
1399static int sdma_v5_2_suspend(struct amdgpu_ip_block *ip_block)
1400{
1401	return sdma_v5_2_hw_fini(ip_block);
1402}
1403
1404static int sdma_v5_2_resume(struct amdgpu_ip_block *ip_block)
1405{
1406	return sdma_v5_2_hw_init(ip_block);
1407}
1408
1409static bool sdma_v5_2_is_idle(struct amdgpu_ip_block *ip_block)
1410{
1411	struct amdgpu_device *adev = ip_block->adev;
1412	u32 i;
1413
1414	for (i = 0; i < adev->sdma.num_instances; i++) {
1415		u32 tmp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
1416
1417		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1418			return false;
1419	}
1420
1421	return true;
1422}
1423
1424static int sdma_v5_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
1425{
1426	unsigned i;
1427	u32 sdma0, sdma1, sdma2, sdma3;
1428	struct amdgpu_device *adev = ip_block->adev;
1429
1430	for (i = 0; i < adev->usec_timeout; i++) {
1431		sdma0 = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG));
1432		sdma1 = RREG32(sdma_v5_2_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG));
1433		sdma2 = RREG32(sdma_v5_2_get_reg_offset(adev, 2, mmSDMA0_STATUS_REG));
1434		sdma3 = RREG32(sdma_v5_2_get_reg_offset(adev, 3, mmSDMA0_STATUS_REG));
1435
1436		if (sdma0 & sdma1 & sdma2 & sdma3 & SDMA0_STATUS_REG__IDLE_MASK)
1437			return 0;
1438		udelay(1);
1439	}
1440	return -ETIMEDOUT;
1441}
1442
1443static int sdma_v5_2_reset_queue(struct amdgpu_ring *ring,
1444				 unsigned int vmid,
1445				 struct amdgpu_fence *timedout_fence)
1446{
1447	struct amdgpu_device *adev = ring->adev;
1448	int r;
1449
1450	if (ring->me >= adev->sdma.num_instances) {
1451		dev_err(adev->dev, "sdma instance not found\n");
1452		return -EINVAL;
1453	}
1454
1455	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
1456
1457	amdgpu_amdkfd_suspend(adev, true);
1458	r = amdgpu_sdma_reset_engine(adev, ring->me, true);
1459	amdgpu_amdkfd_resume(adev, true);
1460	if (r)
1461		return r;
1462
1463	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
1464}
1465
1466static int sdma_v5_2_stop_queue(struct amdgpu_ring *ring)
1467{
1468	u32 f32_cntl, freeze, cntl, stat1_reg;
1469	struct amdgpu_device *adev = ring->adev;
1470	int i, j, r = 0;
1471
1472	if (amdgpu_sriov_vf(adev))
1473		return -EINVAL;
1474
1475	i = ring->me;
1476	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
1477
1478	/* stop queue */
1479	sdma_v5_2_gfx_stop(adev, 1 << i);
1480
1481	/*engine stop SDMA1_F32_CNTL.HALT to 1 and SDMAx_FREEZE freeze bit to 1 */
1482	freeze = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_FREEZE));
1483	freeze = REG_SET_FIELD(freeze, SDMA0_FREEZE, FREEZE, 1);
1484	WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_FREEZE), freeze);
1485
1486	for (j = 0; j < adev->usec_timeout; j++) {
1487		freeze = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_FREEZE));
1488
1489		if (REG_GET_FIELD(freeze, SDMA0_FREEZE, FROZEN) & 1)
1490			break;
1491		udelay(1);
1492	}
1493
1494
1495	if (j == adev->usec_timeout) {
1496		stat1_reg = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS1_REG));
1497		if ((stat1_reg & 0x3FF) != 0x3FF) {
1498			DRM_ERROR("cannot soft reset as sdma not idle\n");
1499			r = -ETIMEDOUT;
1500			goto err0;
1501		}
1502	}
1503
1504	f32_cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
1505	f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, 1);
1506	WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl);
1507
1508	cntl = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL));
1509	cntl = REG_SET_FIELD(cntl, SDMA0_CNTL, UTC_L1_ENABLE, 0);
1510	WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CNTL), cntl);
1511
1512err0:
1513	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
1514	return r;
1515}
1516
1517static int sdma_v5_2_restore_queue(struct amdgpu_ring *ring)
1518{
1519	struct amdgpu_device *adev = ring->adev;
1520	u32 inst_id = ring->me;
1521	u32 freeze;
1522	int r;
1523
1524	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
1525	/* unfreeze and unhalt */
1526	freeze = RREG32(sdma_v5_2_get_reg_offset(adev, inst_id, mmSDMA0_FREEZE));
1527	freeze = REG_SET_FIELD(freeze, SDMA0_FREEZE, FREEZE, 0);
1528	WREG32(sdma_v5_2_get_reg_offset(adev, inst_id, mmSDMA0_FREEZE), freeze);
1529
1530	r = sdma_v5_2_gfx_resume_instance(adev, inst_id, true);
1531
1532	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
1533
1534	return r;
1535}
1536
1537static int sdma_v5_2_ring_preempt_ib(struct amdgpu_ring *ring)
1538{
1539	int i, r = 0;
1540	struct amdgpu_device *adev = ring->adev;
1541	u32 index = 0;
1542	u64 sdma_gfx_preempt;
1543
1544	amdgpu_sdma_get_index_from_ring(ring, &index);
1545	sdma_gfx_preempt =
1546		sdma_v5_2_get_reg_offset(adev, index, mmSDMA0_GFX_PREEMPT);
1547
1548	/* assert preemption condition */
1549	amdgpu_ring_set_preempt_cond_exec(ring, false);
1550
1551	/* emit the trailing fence */
1552	ring->trail_seq += 1;
1553	amdgpu_ring_alloc(ring, 10);
1554	sdma_v5_2_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1555				  ring->trail_seq, 0);
1556	amdgpu_ring_commit(ring);
1557
1558	/* assert IB preemption */
1559	WREG32(sdma_gfx_preempt, 1);
1560
1561	/* poll the trailing fence */
1562	for (i = 0; i < adev->usec_timeout; i++) {
1563		if (ring->trail_seq ==
1564		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1565			break;
1566		udelay(1);
1567	}
1568
1569	if (i >= adev->usec_timeout) {
1570		r = -EINVAL;
1571		DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1572	}
1573
1574	/* deassert IB preemption */
1575	WREG32(sdma_gfx_preempt, 0);
1576
1577	/* deassert the preemption condition */
1578	amdgpu_ring_set_preempt_cond_exec(ring, true);
1579	return r;
1580}
1581
1582static int sdma_v5_2_set_trap_irq_state(struct amdgpu_device *adev,
1583					struct amdgpu_irq_src *source,
1584					unsigned type,
1585					enum amdgpu_interrupt_state state)
1586{
1587	u32 sdma_cntl;
1588	u32 reg_offset = sdma_v5_2_get_reg_offset(adev, type, mmSDMA0_CNTL);
1589
1590	if (!amdgpu_sriov_vf(adev)) {
1591		sdma_cntl = RREG32(reg_offset);
1592		sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1593			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1594		WREG32(reg_offset, sdma_cntl);
1595	}
1596
1597	return 0;
1598}
1599
1600static int sdma_v5_2_process_trap_irq(struct amdgpu_device *adev,
1601				      struct amdgpu_irq_src *source,
1602				      struct amdgpu_iv_entry *entry)
1603{
1604	uint32_t mes_queue_id = entry->src_data[0];
1605
1606	DRM_DEBUG("IH: SDMA trap\n");
1607
1608	if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
1609		struct amdgpu_mes_queue *queue;
1610
1611		mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
1612
1613		spin_lock(&adev->mes.queue_id_lock);
1614		queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
1615		if (queue) {
1616			DRM_DEBUG("process smda queue id = %d\n", mes_queue_id);
1617			amdgpu_fence_process(queue->ring);
1618		}
1619		spin_unlock(&adev->mes.queue_id_lock);
1620		return 0;
1621	}
1622
1623	switch (entry->client_id) {
1624	case SOC15_IH_CLIENTID_SDMA0:
1625		switch (entry->ring_id) {
1626		case 0:
1627			amdgpu_fence_process(&adev->sdma.instance[0].ring);
1628			break;
1629		case 1:
1630			/* XXX compute */
1631			break;
1632		case 2:
1633			/* XXX compute */
1634			break;
1635		case 3:
1636			/* XXX page queue*/
1637			break;
1638		}
1639		break;
1640	case SOC15_IH_CLIENTID_SDMA1:
1641		switch (entry->ring_id) {
1642		case 0:
1643			amdgpu_fence_process(&adev->sdma.instance[1].ring);
1644			break;
1645		case 1:
1646			/* XXX compute */
1647			break;
1648		case 2:
1649			/* XXX compute */
1650			break;
1651		case 3:
1652			/* XXX page queue*/
1653			break;
1654		}
1655		break;
1656	case SOC15_IH_CLIENTID_SDMA2:
1657		switch (entry->ring_id) {
1658		case 0:
1659			amdgpu_fence_process(&adev->sdma.instance[2].ring);
1660			break;
1661		case 1:
1662			/* XXX compute */
1663			break;
1664		case 2:
1665			/* XXX compute */
1666			break;
1667		case 3:
1668			/* XXX page queue*/
1669			break;
1670		}
1671		break;
1672	case SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid:
1673		switch (entry->ring_id) {
1674		case 0:
1675			amdgpu_fence_process(&adev->sdma.instance[3].ring);
1676			break;
1677		case 1:
1678			/* XXX compute */
1679			break;
1680		case 2:
1681			/* XXX compute */
1682			break;
1683		case 3:
1684			/* XXX page queue*/
1685			break;
1686		}
1687		break;
1688	}
1689	return 0;
1690}
1691
1692static int sdma_v5_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1693					      struct amdgpu_irq_src *source,
1694					      struct amdgpu_iv_entry *entry)
1695{
1696	return 0;
1697}
1698
1699static bool sdma_v5_2_firmware_mgcg_support(struct amdgpu_device *adev,
1700						     int i)
1701{
1702	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1703	case IP_VERSION(5, 2, 1):
1704		if (adev->sdma.instance[i].fw_version < 70)
1705			return false;
1706		break;
1707	case IP_VERSION(5, 2, 3):
1708		if (adev->sdma.instance[i].fw_version < 47)
1709			return false;
1710		break;
1711	case IP_VERSION(5, 2, 7):
1712		if (adev->sdma.instance[i].fw_version < 9)
1713			return false;
1714		break;
1715	default:
1716		return true;
1717	}
1718
1719	return true;
1720
1721}
1722
1723static void sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device *adev,
1724						       bool enable)
1725{
1726	uint32_t data, def;
1727	int i;
1728
1729	for (i = 0; i < adev->sdma.num_instances; i++) {
1730
1731		if (!sdma_v5_2_firmware_mgcg_support(adev, i))
1732			adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG;
1733
1734		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
1735			/* Enable sdma clock gating */
1736			def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
1737			data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1738				  SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1739				  SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1740				  SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1741				  SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK |
1742				  SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK);
1743			if (def != data)
1744				WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
1745		} else {
1746			/* Disable sdma clock gating */
1747			def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
1748			data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1749				 SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1750				 SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1751				 SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1752				 SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK |
1753				 SDMA0_CLK_CTRL__SOFT_OVERRIDER_REG_MASK);
1754			if (def != data)
1755				WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL), data);
1756		}
1757	}
1758}
1759
1760static void sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device *adev,
1761						      bool enable)
1762{
1763	uint32_t data, def;
1764	int i;
1765
1766	for (i = 0; i < adev->sdma.num_instances; i++) {
1767		if (adev->sdma.instance[i].fw_version < 70 &&
1768		    amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
1769			    IP_VERSION(5, 2, 1))
1770			adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS;
1771
1772		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
1773			/* Enable sdma mem light sleep */
1774			def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
1775			data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1776			if (def != data)
1777				WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
1778
1779		} else {
1780			/* Disable sdma mem light sleep */
1781			def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));
1782			data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1783			if (def != data)
1784				WREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL), data);
1785
1786		}
1787	}
1788}
1789
1790static int sdma_v5_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1791					   enum amd_clockgating_state state)
1792{
1793	struct amdgpu_device *adev = ip_block->adev;
1794
1795	if (amdgpu_sriov_vf(adev))
1796		return 0;
1797
1798	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1799	case IP_VERSION(5, 2, 0):
1800	case IP_VERSION(5, 2, 2):
1801	case IP_VERSION(5, 2, 1):
1802	case IP_VERSION(5, 2, 4):
1803	case IP_VERSION(5, 2, 5):
1804	case IP_VERSION(5, 2, 6):
1805	case IP_VERSION(5, 2, 3):
1806	case IP_VERSION(5, 2, 7):
1807		sdma_v5_2_update_medium_grain_clock_gating(adev,
1808				state == AMD_CG_STATE_GATE);
1809		sdma_v5_2_update_medium_grain_light_sleep(adev,
1810				state == AMD_CG_STATE_GATE);
1811		break;
1812	default:
1813		break;
1814	}
1815
1816	return 0;
1817}
1818
1819static int sdma_v5_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
1820					  enum amd_powergating_state state)
1821{
1822	return 0;
1823}
1824
1825static void sdma_v5_2_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
1826{
1827	struct amdgpu_device *adev = ip_block->adev;
1828	int data;
1829
1830	if (amdgpu_sriov_vf(adev))
1831		*flags = 0;
1832
1833	/* AMD_CG_SUPPORT_SDMA_MGCG */
1834	data = RREG32(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL));
1835	if (!(data & SDMA0_CLK_CTRL__CGCG_EN_OVERRIDE_MASK))
1836		*flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1837
1838	/* AMD_CG_SUPPORT_SDMA_LS */
1839	data = RREG32_KIQ(sdma_v5_2_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL));
1840	if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1841		*flags |= AMD_CG_SUPPORT_SDMA_LS;
1842}
1843
1844static void sdma_v5_2_ring_begin_use(struct amdgpu_ring *ring)
1845{
1846	struct amdgpu_device *adev = ring->adev;
1847
1848	/* SDMA 5.2.3 (RMB) FW doesn't seem to properly
1849	 * disallow GFXOFF in some cases leading to
1850	 * hangs in SDMA.  Disallow GFXOFF while SDMA is active.
1851	 * We can probably just limit this to 5.2.3,
1852	 * but it shouldn't hurt for other parts since
1853	 * this GFXOFF will be disallowed anyway when SDMA is
1854	 * active, this just makes it explicit.
1855	 * sdma_v5_2_ring_set_wptr() takes advantage of this
1856	 * to update the wptr because sometimes SDMA seems to miss
1857	 * doorbells when entering PG.  If you remove this, update
1858	 * sdma_v5_2_ring_set_wptr() as well!
1859	 */
1860	amdgpu_gfx_off_ctrl(adev, false);
1861}
1862
1863static void sdma_v5_2_ring_end_use(struct amdgpu_ring *ring)
1864{
1865	struct amdgpu_device *adev = ring->adev;
1866
1867	/* SDMA 5.2.3 (RMB) FW doesn't seem to properly
1868	 * disallow GFXOFF in some cases leading to
1869	 * hangs in SDMA.  Allow GFXOFF when SDMA is complete.
1870	 */
1871	amdgpu_gfx_off_ctrl(adev, true);
1872}
1873
1874static void sdma_v5_2_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
1875{
1876	struct amdgpu_device *adev = ip_block->adev;
1877	int i, j;
1878	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2);
1879	uint32_t instance_offset;
1880
1881	if (!adev->sdma.ip_dump)
1882		return;
1883
1884	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
1885	for (i = 0; i < adev->sdma.num_instances; i++) {
1886		instance_offset = i * reg_count;
1887		drm_printf(p, "\nInstance:%d\n", i);
1888
1889		for (j = 0; j < reg_count; j++)
1890			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_5_2[j].reg_name,
1891				   adev->sdma.ip_dump[instance_offset + j]);
1892	}
1893}
1894
1895static void sdma_v5_2_dump_ip_state(struct amdgpu_ip_block *ip_block)
1896{
1897	struct amdgpu_device *adev = ip_block->adev;
1898	int i, j;
1899	uint32_t instance_offset;
1900	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_5_2);
1901
1902	if (!adev->sdma.ip_dump)
1903		return;
1904
1905	amdgpu_gfx_off_ctrl(adev, false);
1906	for (i = 0; i < adev->sdma.num_instances; i++) {
1907		instance_offset = i * reg_count;
1908		for (j = 0; j < reg_count; j++)
1909			adev->sdma.ip_dump[instance_offset + j] =
1910				RREG32(sdma_v5_2_get_reg_offset(adev, i,
1911				       sdma_reg_list_5_2[j].reg_offset));
1912	}
1913	amdgpu_gfx_off_ctrl(adev, true);
1914}
1915
1916static const struct amd_ip_funcs sdma_v5_2_ip_funcs = {
1917	.name = "sdma_v5_2",
1918	.early_init = sdma_v5_2_early_init,
1919	.sw_init = sdma_v5_2_sw_init,
1920	.sw_fini = sdma_v5_2_sw_fini,
1921	.hw_init = sdma_v5_2_hw_init,
1922	.hw_fini = sdma_v5_2_hw_fini,
1923	.suspend = sdma_v5_2_suspend,
1924	.resume = sdma_v5_2_resume,
1925	.is_idle = sdma_v5_2_is_idle,
1926	.wait_for_idle = sdma_v5_2_wait_for_idle,
1927	.soft_reset = sdma_v5_2_soft_reset,
1928	.set_clockgating_state = sdma_v5_2_set_clockgating_state,
1929	.set_powergating_state = sdma_v5_2_set_powergating_state,
1930	.get_clockgating_state = sdma_v5_2_get_clockgating_state,
1931	.dump_ip_state = sdma_v5_2_dump_ip_state,
1932	.print_ip_state = sdma_v5_2_print_ip_state,
1933};
1934
1935static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = {
1936	.type = AMDGPU_RING_TYPE_SDMA,
1937	.align_mask = 0xf,
1938	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1939	.support_64bit_ptrs = true,
1940	.secure_submission_supported = true,
1941	.get_rptr = sdma_v5_2_ring_get_rptr,
1942	.get_wptr = sdma_v5_2_ring_get_wptr,
1943	.set_wptr = sdma_v5_2_ring_set_wptr,
1944	.emit_frame_size =
1945		5 + /* sdma_v5_2_ring_init_cond_exec */
1946		6 + /* sdma_v5_2_ring_emit_hdp_flush */
1947		3 + /* hdp_invalidate */
1948		6 + /* sdma_v5_2_ring_emit_pipeline_sync */
1949		/* sdma_v5_2_ring_emit_vm_flush */
1950		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1951		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1952		10 + 10 + 10, /* sdma_v5_2_ring_emit_fence x3 for user fence, vm fence */
1953	.emit_ib_size = 7 + 6, /* sdma_v5_2_ring_emit_ib */
1954	.emit_ib = sdma_v5_2_ring_emit_ib,
1955	.emit_mem_sync = sdma_v5_2_ring_emit_mem_sync,
1956	.emit_fence = sdma_v5_2_ring_emit_fence,
1957	.emit_pipeline_sync = sdma_v5_2_ring_emit_pipeline_sync,
1958	.emit_vm_flush = sdma_v5_2_ring_emit_vm_flush,
1959	.emit_hdp_flush = sdma_v5_2_ring_emit_hdp_flush,
1960	.test_ring = sdma_v5_2_ring_test_ring,
1961	.test_ib = sdma_v5_2_ring_test_ib,
1962	.insert_nop = sdma_v5_2_ring_insert_nop,
1963	.pad_ib = sdma_v5_2_ring_pad_ib,
1964	.begin_use = sdma_v5_2_ring_begin_use,
1965	.end_use = sdma_v5_2_ring_end_use,
1966	.emit_wreg = sdma_v5_2_ring_emit_wreg,
1967	.emit_reg_wait = sdma_v5_2_ring_emit_reg_wait,
1968	.emit_reg_write_reg_wait = sdma_v5_2_ring_emit_reg_write_reg_wait,
1969	.init_cond_exec = sdma_v5_2_ring_init_cond_exec,
1970	.preempt_ib = sdma_v5_2_ring_preempt_ib,
1971	.reset = sdma_v5_2_reset_queue,
1972};
1973
1974static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev)
1975{
1976	int i;
1977
1978	for (i = 0; i < adev->sdma.num_instances; i++) {
1979		adev->sdma.instance[i].ring.funcs = &sdma_v5_2_ring_funcs;
1980		adev->sdma.instance[i].ring.me = i;
1981	}
1982}
1983
1984static const struct amdgpu_irq_src_funcs sdma_v5_2_trap_irq_funcs = {
1985	.set = sdma_v5_2_set_trap_irq_state,
1986	.process = sdma_v5_2_process_trap_irq,
1987};
1988
1989static const struct amdgpu_irq_src_funcs sdma_v5_2_illegal_inst_irq_funcs = {
1990	.process = sdma_v5_2_process_illegal_inst_irq,
1991};
1992
1993static void sdma_v5_2_set_irq_funcs(struct amdgpu_device *adev)
1994{
1995	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1996					adev->sdma.num_instances;
1997	adev->sdma.trap_irq.funcs = &sdma_v5_2_trap_irq_funcs;
1998	adev->sdma.illegal_inst_irq.funcs = &sdma_v5_2_illegal_inst_irq_funcs;
1999}
2000
2001/**
2002 * sdma_v5_2_emit_copy_buffer - copy buffer using the sDMA engine
2003 *
2004 * @ib: indirect buffer to copy to
2005 * @src_offset: src GPU address
2006 * @dst_offset: dst GPU address
2007 * @byte_count: number of bytes to xfer
2008 * @copy_flags: copy flags for the buffers
2009 *
2010 * Copy GPU buffers using the DMA engine.
2011 * Used by the amdgpu ttm implementation to move pages if
2012 * registered as the asic copy callback.
2013 */
2014static void sdma_v5_2_emit_copy_buffer(struct amdgpu_ib *ib,
2015				       uint64_t src_offset,
2016				       uint64_t dst_offset,
2017				       uint32_t byte_count,
2018				       uint32_t copy_flags)
2019{
2020	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
2021		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
2022		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
2023	ib->ptr[ib->length_dw++] = byte_count - 1;
2024	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
2025	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
2026	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
2027	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2028	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2029}
2030
2031/**
2032 * sdma_v5_2_emit_fill_buffer - fill buffer using the sDMA engine
2033 *
2034 * @ib: indirect buffer to fill
2035 * @src_data: value to write to buffer
2036 * @dst_offset: dst GPU address
2037 * @byte_count: number of bytes to xfer
2038 *
2039 * Fill GPU buffers using the DMA engine.
2040 */
2041static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib,
2042				       uint32_t src_data,
2043				       uint64_t dst_offset,
2044				       uint32_t byte_count)
2045{
2046	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2047	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2048	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2049	ib->ptr[ib->length_dw++] = src_data;
2050	ib->ptr[ib->length_dw++] = byte_count - 1;
2051}
2052
2053static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
2054	.copy_max_bytes = 0x400000,
2055	.copy_num_dw = 7,
2056	.emit_copy_buffer = sdma_v5_2_emit_copy_buffer,
2057
2058	.fill_max_bytes = 0x400000,
2059	.fill_num_dw = 5,
2060	.emit_fill_buffer = sdma_v5_2_emit_fill_buffer,
2061};
2062
2063static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
2064{
2065	if (adev->mman.buffer_funcs == NULL) {
2066		adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs;
2067		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2068	}
2069}
2070
2071static const struct amdgpu_vm_pte_funcs sdma_v5_2_vm_pte_funcs = {
2072	.copy_pte_num_dw = 7,
2073	.copy_pte = sdma_v5_2_vm_copy_pte,
2074	.write_pte = sdma_v5_2_vm_write_pte,
2075	.set_pte_pde = sdma_v5_2_vm_set_pte_pde,
2076};
2077
2078static void sdma_v5_2_set_vm_pte_funcs(struct amdgpu_device *adev)
2079{
2080	unsigned i;
2081
2082	if (adev->vm_manager.vm_pte_funcs == NULL) {
2083		adev->vm_manager.vm_pte_funcs = &sdma_v5_2_vm_pte_funcs;
2084		for (i = 0; i < adev->sdma.num_instances; i++) {
2085			adev->vm_manager.vm_pte_scheds[i] =
2086				&adev->sdma.instance[i].ring.sched;
2087		}
2088		adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2089	}
2090}
2091
2092const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {
2093	.type = AMD_IP_BLOCK_TYPE_SDMA,
2094	.major = 5,
2095	.minor = 2,
2096	.rev = 0,
2097	.funcs = &sdma_v5_2_ip_funcs,
2098};