sdma_v7_0.c

   1/*
   2 * Copyright 2023 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/delay.h>
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28
  29#include "amdgpu.h"
  30#include "amdgpu_ucode.h"
  31#include "amdgpu_trace.h"
  32
  33#include "gc/gc_12_0_0_offset.h"
  34#include "gc/gc_12_0_0_sh_mask.h"
  35#include "hdp/hdp_6_0_0_offset.h"
  36#include "ivsrcid/gfx/irqsrcs_gfx_12_0_0.h"
  37
  38#include "soc15_common.h"
  39#include "soc15.h"
  40#include "sdma_v6_0_0_pkt_open.h"
  41#include "nbio_v4_3.h"
  42#include "sdma_common.h"
  43#include "sdma_v7_0.h"
  44#include "v12_structs.h"
  45#include "mes_userqueue.h"
  46#include "amdgpu_userq_fence.h"
  47
  48MODULE_FIRMWARE("amdgpu/sdma_7_0_0.bin");
  49MODULE_FIRMWARE("amdgpu/sdma_7_0_1.bin");
  50
  51#define SDMA1_REG_OFFSET 0x600
  52#define SDMA0_HYP_DEC_REG_START 0x5880
  53#define SDMA0_HYP_DEC_REG_END 0x589a
  54#define SDMA1_HYP_DEC_REG_OFFSET 0x20
  55
  56/*define for compression field for sdma7*/
  57#define SDMA_PKT_CONSTANT_FILL_HEADER_compress_offset 0
  58#define SDMA_PKT_CONSTANT_FILL_HEADER_compress_mask   0x00000001
  59#define SDMA_PKT_CONSTANT_FILL_HEADER_compress_shift  16
  60#define SDMA_PKT_CONSTANT_FILL_HEADER_COMPRESS(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_compress_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_compress_shift)
  61
  62static const struct amdgpu_hwip_reg_entry sdma_reg_list_7_0[] = {
  63	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS_REG),
  64	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS1_REG),
  65	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS2_REG),
  66	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS3_REG),
  67	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS4_REG),
  68	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS5_REG),
  69	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS6_REG),
  70	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UCODE_REV),
  71	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_RB_RPTR_FETCH_HI),
  72	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_RB_RPTR_FETCH),
  73	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_RD_STATUS),
  74	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_WR_STATUS),
  75	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_RD_XNACK0),
  76	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_RD_XNACK1),
  77	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_WR_XNACK0),
  78	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_WR_XNACK1),
  79	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_CNTL),
  80	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_RPTR),
  81	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_RPTR_HI),
  82	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_WPTR),
  83	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_WPTR_HI),
  84	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_OFFSET),
  85	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_BASE_LO),
  86	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_BASE_HI),
  87	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_CNTL),
  88	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_RPTR),
  89	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_SUB_REMAIN),
  90	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_DUMMY_REG),
  91	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE_STATUS0),
  92	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_CNTL),
  93	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_RPTR),
  94	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_RPTR_HI),
  95	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_WPTR),
  96	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_WPTR_HI),
  97	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_OFFSET),
  98	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_BASE_LO),
  99	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_BASE_HI),
 100	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_RPTR),
 101	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_SUB_REMAIN),
 102	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_DUMMY_REG),
 103	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_CNTL),
 104	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_RPTR),
 105	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_RPTR_HI),
 106	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_WPTR),
 107	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_WPTR_HI),
 108	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_OFFSET),
 109	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_BASE_LO),
 110	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_BASE_HI),
 111	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_RPTR),
 112	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_SUB_REMAIN),
 113	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_DUMMY_REG),
 114	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_INT_STATUS),
 115	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_VM_CNTL),
 116	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
 117	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_CHICKEN_BITS),
 118};
 119
 120static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev);
 121static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev);
 122static void sdma_v7_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 123static void sdma_v7_0_set_irq_funcs(struct amdgpu_device *adev);
 124static int sdma_v7_0_start(struct amdgpu_device *adev);
 125
 126static u32 sdma_v7_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
 127{
 128	u32 base;
 129
 130	if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
 131	    internal_offset <= SDMA0_HYP_DEC_REG_END) {
 132		base = adev->reg_offset[GC_HWIP][0][1];
 133		if (instance != 0)
 134			internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
 135	} else {
 136		base = adev->reg_offset[GC_HWIP][0][0];
 137		if (instance == 1)
 138			internal_offset += SDMA1_REG_OFFSET;
 139	}
 140
 141	return base + internal_offset;
 142}
 143
 144static unsigned sdma_v7_0_ring_init_cond_exec(struct amdgpu_ring *ring,
 145					      uint64_t addr)
 146{
 147	unsigned ret;
 148
 149	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COND_EXE));
 150	amdgpu_ring_write(ring, lower_32_bits(addr));
 151	amdgpu_ring_write(ring, upper_32_bits(addr));
 152	amdgpu_ring_write(ring, 1);
 153	/* this is the offset we need patch later */
 154	ret = ring->wptr & ring->buf_mask;
 155	/* insert dummy here and patch it later */
 156	amdgpu_ring_write(ring, 0);
 157
 158	return ret;
 159}
 160
 161/**
 162 * sdma_v7_0_ring_get_rptr - get the current read pointer
 163 *
 164 * @ring: amdgpu ring pointer
 165 *
 166 * Get the current rptr from the hardware.
 167 */
 168static uint64_t sdma_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
 169{
 170	u64 *rptr;
 171
 172	/* XXX check if swapping is necessary on BE */
 173	rptr = (u64 *)ring->rptr_cpu_addr;
 174
 175	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
 176	return ((*rptr) >> 2);
 177}
 178
 179/**
 180 * sdma_v7_0_ring_get_wptr - get the current write pointer
 181 *
 182 * @ring: amdgpu ring pointer
 183 *
 184 * Get the current wptr from the hardware.
 185 */
 186static uint64_t sdma_v7_0_ring_get_wptr(struct amdgpu_ring *ring)
 187{
 188	u64 wptr = 0;
 189
 190	if (ring->use_doorbell) {
 191		/* XXX check if swapping is necessary on BE */
 192		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
 193		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 194	}
 195
 196	return wptr >> 2;
 197}
 198
 199/**
 200 * sdma_v7_0_ring_set_wptr - commit the write pointer
 201 *
 202 * @ring: amdgpu ring pointer
 203 *
 204 * Write the wptr back to the hardware.
 205 */
 206static void sdma_v7_0_ring_set_wptr(struct amdgpu_ring *ring)
 207{
 208	struct amdgpu_device *adev = ring->adev;
 209
 210	DRM_DEBUG("Setting write pointer\n");
 211
 212	if (ring->use_doorbell) {
 213		DRM_DEBUG("Using doorbell -- "
 214			  "wptr_offs == 0x%08x "
 215			  "lower_32_bits(ring->wptr) << 2 == 0x%08x "
 216			  "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
 217			  ring->wptr_offs,
 218			  lower_32_bits(ring->wptr << 2),
 219			  upper_32_bits(ring->wptr << 2));
 220		/* XXX check if swapping is necessary on BE */
 221		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
 222			     ring->wptr << 2);
 223		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
 224			  ring->doorbell_index, ring->wptr << 2);
 225		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
 226	} else {
 227		DRM_DEBUG("Not using doorbell -- "
 228			  "regSDMA%i_GFX_RB_WPTR == 0x%08x "
 229			  "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
 230			  ring->me,
 231			  lower_32_bits(ring->wptr << 2),
 232			  ring->me,
 233			  upper_32_bits(ring->wptr << 2));
 234		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev,
 235							     ring->me,
 236							     regSDMA0_QUEUE0_RB_WPTR),
 237				lower_32_bits(ring->wptr << 2));
 238		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev,
 239							     ring->me,
 240							     regSDMA0_QUEUE0_RB_WPTR_HI),
 241				upper_32_bits(ring->wptr << 2));
 242	}
 243}
 244
 245static void sdma_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 246{
 247	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 248	int i;
 249
 250	for (i = 0; i < count; i++)
 251		if (sdma && sdma->burst_nop && (i == 0))
 252			amdgpu_ring_write(ring, ring->funcs->nop |
 253				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 254		else
 255			amdgpu_ring_write(ring, ring->funcs->nop);
 256}
 257
 258/**
 259 * sdma_v7_0_ring_emit_ib - Schedule an IB on the DMA engine
 260 *
 261 * @ring: amdgpu ring pointer
 262 * @job: job to retrieve vmid from
 263 * @ib: IB object to schedule
 264 * @flags: unused
 265 *
 266 * Schedule an IB in the DMA ring.
 267 */
 268static void sdma_v7_0_ring_emit_ib(struct amdgpu_ring *ring,
 269				   struct amdgpu_job *job,
 270				   struct amdgpu_ib *ib,
 271				   uint32_t flags)
 272{
 273	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 274	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
 275
 276	/* An IB packet must end on a 8 DW boundary--the next dword
 277	 * must be on a 8-dword boundary. Our IB packet below is 6
 278	 * dwords long, thus add x number of NOPs, such that, in
 279	 * modular arithmetic,
 280	 * wptr + 6 + x = 8k, k >= 0, which in C is,
 281	 * (wptr + 6 + x) % 8 = 0.
 282	 * The expression below, is a solution of x.
 283	 */
 284	sdma_v7_0_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
 285
 286	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_INDIRECT) |
 287			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 288	/* base must be 32 byte aligned */
 289	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 290	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 291	amdgpu_ring_write(ring, ib->length_dw);
 292	amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
 293	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
 294}
 295
 296/**
 297 * sdma_v7_0_ring_emit_mem_sync - flush the IB by graphics cache rinse
 298 *
 299 * @ring: amdgpu ring pointer
 300 *
 301 * flush the IB by graphics cache rinse.
 302 */
 303static void sdma_v7_0_ring_emit_mem_sync(struct amdgpu_ring *ring)
 304{
 305	uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
 306		SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
 307		SDMA_GCR_GLI_INV(1);
 308
 309	/* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
 310	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_GCR_REQ));
 311	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
 312	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
 313			  SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
 314	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
 315			  SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
 316	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
 317			  SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
 318}
 319
 320
 321/**
 322 * sdma_v7_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
 323 *
 324 * @ring: amdgpu ring pointer
 325 *
 326 * Emit an hdp flush packet on the requested DMA ring.
 327 */
 328static void sdma_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 329{
 330	struct amdgpu_device *adev = ring->adev;
 331	u32 ref_and_mask = 0;
 332	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
 333
 334	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
 335
 336	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
 337			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
 338			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
 339	amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
 340	amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
 341	amdgpu_ring_write(ring, ref_and_mask); /* reference */
 342	amdgpu_ring_write(ring, ref_and_mask); /* mask */
 343	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
 344			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 345}
 346
 347/**
 348 * sdma_v7_0_ring_emit_fence - emit a fence on the DMA ring
 349 *
 350 * @ring: amdgpu ring pointer
 351 * @addr: address
 352 * @seq: fence seq number
 353 * @flags: fence flags
 354 *
 355 * Add a DMA fence packet to the ring to write
 356 * the fence seq number and DMA trap packet to generate
 357 * an interrupt if needed.
 358 */
 359static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 360				      unsigned flags)
 361{
 362	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 363	/* write the fence */
 364	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
 365			  SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
 366	/* zero in first two bits */
 367	BUG_ON(addr & 0x3);
 368	amdgpu_ring_write(ring, lower_32_bits(addr));
 369	amdgpu_ring_write(ring, upper_32_bits(addr));
 370	amdgpu_ring_write(ring, lower_32_bits(seq));
 371
 372	/* optionally write high bits as well */
 373	if (write64bit) {
 374		addr += 4;
 375		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
 376				  SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
 377		/* zero in first two bits */
 378		BUG_ON(addr & 0x3);
 379		amdgpu_ring_write(ring, lower_32_bits(addr));
 380		amdgpu_ring_write(ring, upper_32_bits(addr));
 381		amdgpu_ring_write(ring, upper_32_bits(seq));
 382	}
 383
 384	if (flags & AMDGPU_FENCE_FLAG_INT) {
 385		/* generate an interrupt */
 386		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_TRAP));
 387		amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
 388	}
 389}
 390
 391/**
 392 * sdma_v7_0_gfx_stop - stop the gfx async dma engines
 393 *
 394 * @adev: amdgpu_device pointer
 395 *
 396 * Stop the gfx async dma ring buffers.
 397 */
 398static void sdma_v7_0_gfx_stop(struct amdgpu_device *adev)
 399{
 400	u32 rb_cntl, ib_cntl;
 401	int i;
 402
 403	for (i = 0; i < adev->sdma.num_instances; i++) {
 404		rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
 405		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 0);
 406		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
 407		ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
 408		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 0);
 409		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
 410	}
 411}
 412
 413/**
 414 * sdma_v7_0_rlc_stop - stop the compute async dma engines
 415 *
 416 * @adev: amdgpu_device pointer
 417 *
 418 * Stop the compute async dma queues.
 419 */
 420static void sdma_v7_0_rlc_stop(struct amdgpu_device *adev)
 421{
 422	/* XXX todo */
 423}
 424
 425/**
 426 * sdma_v7_0_ctx_switch_enable - stop the async dma engines context switch
 427 *
 428 * @adev: amdgpu_device pointer
 429 * @enable: enable/disable the DMA MEs context switch.
 430 *
 431 * Halt or unhalt the async dma engines context switch.
 432 */
 433static void sdma_v7_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 434{
 435}
 436
 437/**
 438 * sdma_v7_0_enable - stop the async dma engines
 439 *
 440 * @adev: amdgpu_device pointer
 441 * @enable: enable/disable the DMA MEs.
 442 *
 443 * Halt or unhalt the async dma engines.
 444 */
 445static void sdma_v7_0_enable(struct amdgpu_device *adev, bool enable)
 446{
 447	u32 mcu_cntl;
 448	int i;
 449
 450	if (!enable) {
 451		sdma_v7_0_gfx_stop(adev);
 452		sdma_v7_0_rlc_stop(adev);
 453	}
 454
 455	if (amdgpu_sriov_vf(adev))
 456		return;
 457
 458	for (i = 0; i < adev->sdma.num_instances; i++) {
 459		mcu_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL));
 460		mcu_cntl = REG_SET_FIELD(mcu_cntl, SDMA0_MCU_CNTL, HALT, enable ? 0 : 1);
 461		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL), mcu_cntl);
 462	}
 463}
 464
 465/**
 466 * sdma_v7_0_gfx_resume_instance - start/restart a certain sdma engine
 467 *
 468 * @adev: amdgpu_device pointer
 469 * @i: instance
 470 * @restore: used to restore wptr when restart
 471 *
 472 * Set up the gfx DMA ring buffers and enable them. On restart, we will restore wptr and rptr.
 473 * Return 0 for success.
 474 */
 475static int sdma_v7_0_gfx_resume_instance(struct amdgpu_device *adev, int i, bool restore)
 476{
 477	struct amdgpu_ring *ring;
 478	u32 rb_cntl, ib_cntl;
 479	u32 rb_bufsz;
 480	u32 doorbell;
 481	u32 doorbell_offset;
 482	u32 temp;
 483	u64 wptr_gpu_addr;
 484	int r;
 485
 486	ring = &adev->sdma.instance[i].ring;
 487
 488	/* Set ring buffer size in dwords */
 489	rb_bufsz = order_base_2(ring->ring_size / 4);
 490	rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
 491	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
 492#ifdef __BIG_ENDIAN
 493	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SWAP_ENABLE, 1);
 494	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL,
 495				RPTR_WRITEBACK_SWAP_ENABLE, 1);
 496#endif
 497	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_PRIV, 1);
 498	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
 499
 500	/* Initialize the ring buffer's read and write pointers */
 501	if (restore) {
 502		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR), lower_32_bits(ring->wptr << 2));
 503		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_HI), upper_32_bits(ring->wptr << 2));
 504		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr << 2));
 505		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
 506	} else {
 507		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR), 0);
 508		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_HI), 0);
 509		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), 0);
 510		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), 0);
 511	}
 512	/* setup the wptr shadow polling */
 513	wptr_gpu_addr = ring->wptr_gpu_addr;
 514	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO),
 515	       lower_32_bits(wptr_gpu_addr));
 516	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI),
 517	       upper_32_bits(wptr_gpu_addr));
 518
 519	/* set the wb address whether it's enabled or not */
 520	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_HI),
 521	       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
 522	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_LO),
 523	       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
 524
 525	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
 526	if (amdgpu_sriov_vf(adev))
 527		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 1);
 528	else
 529		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
 530
 531	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, MCU_WPTR_POLL_ENABLE, 1);
 532
 533	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
 534	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
 535
 536	if (!restore)
 537		ring->wptr = 0;
 538
 539	/* before programing wptr to a less value, need set minor_ptr_update first */
 540	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 1);
 541
 542	if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
 543		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
 544		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
 545	}
 546
 547	doorbell = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL));
 548	doorbell_offset = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET));
 549
 550	if (ring->use_doorbell) {
 551		doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
 552		doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_QUEUE0_DOORBELL_OFFSET,
 553				OFFSET, ring->doorbell_index);
 554	} else {
 555		doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 0);
 556	}
 557	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL), doorbell);
 558	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
 559
 560	if (i == 0)
 561		adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
 562					      ring->doorbell_index,
 563					      adev->doorbell_index.sdma_doorbell_range * adev->sdma.num_instances);
 564
 565	if (amdgpu_sriov_vf(adev))
 566		sdma_v7_0_ring_set_wptr(ring);
 567
 568	/* set minor_ptr_update to 0 after wptr programed */
 569	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 0);
 570
 571	/* Set up sdma hang watchdog */
 572	temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL));
 573	/* 100ms per unit */
 574	temp = REG_SET_FIELD(temp, SDMA0_WATCHDOG_CNTL, QUEUE_HANG_COUNT,
 575			     max(adev->usec_timeout/100000, 1));
 576	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL), temp);
 577
 578	/* Set up RESP_MODE to non-copy addresses */
 579	temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL));
 580	temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
 581	temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
 582	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL), temp);
 583
 584	/* program default cache read and write policy */
 585	temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE));
 586	/* clean read policy and write policy bits */
 587	temp &= 0xFF0FFF;
 588	temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
 589		 (CACHE_WRITE_POLICY_L2__DEFAULT << 14));
 590	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE), temp);
 591
 592	if (!amdgpu_sriov_vf(adev)) {
 593		/* unhalt engine */
 594		temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL));
 595		temp = REG_SET_FIELD(temp, SDMA0_MCU_CNTL, HALT, 0);
 596		temp = REG_SET_FIELD(temp, SDMA0_MCU_CNTL, RESET, 0);
 597		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL), temp);
 598	}
 599
 600	/* enable DMA RB */
 601	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 1);
 602	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
 603
 604	ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
 605	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 1);
 606#ifdef __BIG_ENDIAN
 607	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_SWAP_ENABLE, 1);
 608#endif
 609	/* enable DMA IBs */
 610	WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
 611	ring->sched.ready = true;
 612
 613	if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
 614		sdma_v7_0_ctx_switch_enable(adev, true);
 615		sdma_v7_0_enable(adev, true);
 616	}
 617
 618	r = amdgpu_ring_test_helper(ring);
 619	if (r)
 620		ring->sched.ready = false;
 621
 622	return r;
 623}
 624
 625/**
 626 * sdma_v7_0_gfx_resume - setup and start the async dma engines
 627 *
 628 * @adev: amdgpu_device pointer
 629 *
 630 * Set up the gfx DMA ring buffers and enable them.
 631 * Returns 0 for success, error for failure.
 632 */
 633static int sdma_v7_0_gfx_resume(struct amdgpu_device *adev)
 634{
 635	int i, r;
 636
 637	for (i = 0; i < adev->sdma.num_instances; i++) {
 638		r = sdma_v7_0_gfx_resume_instance(adev, i, false);
 639		if (r)
 640			return r;
 641	}
 642
 643	return 0;
 644
 645}
 646
 647/**
 648 * sdma_v7_0_rlc_resume - setup and start the async dma engines
 649 *
 650 * @adev: amdgpu_device pointer
 651 *
 652 * Set up the compute DMA queues and enable them.
 653 * Returns 0 for success, error for failure.
 654 */
 655static int sdma_v7_0_rlc_resume(struct amdgpu_device *adev)
 656{
 657	return 0;
 658}
 659
 660static void sdma_v12_0_free_ucode_buffer(struct amdgpu_device *adev)
 661{
 662	int i;
 663
 664	for (i = 0; i < adev->sdma.num_instances; i++) {
 665		amdgpu_bo_free_kernel(&adev->sdma.instance[i].sdma_fw_obj,
 666				      &adev->sdma.instance[i].sdma_fw_gpu_addr,
 667				      (void **)&adev->sdma.instance[i].sdma_fw_ptr);
 668	}
 669}
 670
 671/**
 672 * sdma_v7_0_load_microcode - load the sDMA ME ucode
 673 *
 674 * @adev: amdgpu_device pointer
 675 *
 676 * Loads the sDMA0/1 ucode.
 677 * Returns 0 for success, -EINVAL if the ucode is not available.
 678 */
 679static int sdma_v7_0_load_microcode(struct amdgpu_device *adev)
 680{
 681	const struct sdma_firmware_header_v3_0 *hdr;
 682	const __le32 *fw_data;
 683	u32 fw_size;
 684	uint32_t tmp, sdma_status, ic_op_cntl;
 685	int i, r, j;
 686
 687	/* halt the MEs */
 688	sdma_v7_0_enable(adev, false);
 689
 690	if (!adev->sdma.instance[0].fw)
 691		return -EINVAL;
 692
 693	hdr = (const struct sdma_firmware_header_v3_0 *)
 694		adev->sdma.instance[0].fw->data;
 695	amdgpu_ucode_print_sdma_hdr(&hdr->header);
 696
 697	fw_data = (const __le32 *)(adev->sdma.instance[0].fw->data +
 698			le32_to_cpu(hdr->ucode_offset_bytes));
 699	fw_size = le32_to_cpu(hdr->ucode_size_bytes);
 700
 701	for (i = 0; i < adev->sdma.num_instances; i++) {
 702		r = amdgpu_bo_create_reserved(adev, fw_size,
 703					      PAGE_SIZE,
 704					      AMDGPU_GEM_DOMAIN_VRAM,
 705					      &adev->sdma.instance[i].sdma_fw_obj,
 706					      &adev->sdma.instance[i].sdma_fw_gpu_addr,
 707					      (void **)&adev->sdma.instance[i].sdma_fw_ptr);
 708		if (r) {
 709			dev_err(adev->dev, "(%d) failed to create sdma ucode bo\n", r);
 710			return r;
 711		}
 712
 713		memcpy(adev->sdma.instance[i].sdma_fw_ptr, fw_data, fw_size);
 714
 715		amdgpu_bo_kunmap(adev->sdma.instance[i].sdma_fw_obj);
 716		amdgpu_bo_unreserve(adev->sdma.instance[i].sdma_fw_obj);
 717
 718		tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_CNTL));
 719		tmp = REG_SET_FIELD(tmp, SDMA0_IC_CNTL, GPA, 0);
 720		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_CNTL), tmp);
 721
 722		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_BASE_LO),
 723			lower_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
 724		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_BASE_HI),
 725			upper_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
 726
 727		tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_OP_CNTL));
 728		tmp = REG_SET_FIELD(tmp, SDMA0_IC_OP_CNTL, PRIME_ICACHE, 1);
 729		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_OP_CNTL), tmp);
 730
 731		/* Wait for sdma ucode init complete */
 732		for (j = 0; j < adev->usec_timeout; j++) {
 733			ic_op_cntl = RREG32_SOC15_IP(GC,
 734					sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_OP_CNTL));
 735			sdma_status = RREG32_SOC15_IP(GC,
 736					sdma_v7_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
 737			if ((REG_GET_FIELD(ic_op_cntl, SDMA0_IC_OP_CNTL, ICACHE_PRIMED) == 1) &&
 738			    (REG_GET_FIELD(sdma_status, SDMA0_STATUS_REG, UCODE_INIT_DONE) == 1))
 739				break;
 740			udelay(1);
 741		}
 742
 743		if (j >= adev->usec_timeout) {
 744			dev_err(adev->dev, "failed to init sdma ucode\n");
 745			return -EINVAL;
 746		}
 747	}
 748
 749	return 0;
 750}
 751
 752static int sdma_v7_0_soft_reset(struct amdgpu_ip_block *ip_block)
 753{
 754	struct amdgpu_device *adev = ip_block->adev;
 755	u32 tmp;
 756	int i;
 757
 758	sdma_v7_0_gfx_stop(adev);
 759
 760	for (i = 0; i < adev->sdma.num_instances; i++) {
 761		//tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_FREEZE));
 762		//tmp |= SDMA0_FREEZE__FREEZE_MASK;
 763		//WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_FREEZE), tmp);
 764		tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL));
 765		tmp |= SDMA0_MCU_CNTL__HALT_MASK;
 766		tmp |= SDMA0_MCU_CNTL__RESET_MASK;
 767		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL), tmp);
 768
 769		WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_PREEMPT), 0);
 770
 771		udelay(100);
 772
 773		tmp = GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK << i;
 774		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
 775		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
 776
 777		udelay(100);
 778
 779		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, 0);
 780		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
 781
 782		udelay(100);
 783	}
 784
 785	return sdma_v7_0_start(adev);
 786}
 787
 788static bool sdma_v7_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
 789{
 790	struct amdgpu_device *adev = ip_block->adev;
 791	struct amdgpu_ring *ring;
 792	int i, r;
 793	long tmo = msecs_to_jiffies(1000);
 794
 795	for (i = 0; i < adev->sdma.num_instances; i++) {
 796		ring = &adev->sdma.instance[i].ring;
 797		r = amdgpu_ring_test_ib(ring, tmo);
 798		if (r)
 799			return true;
 800	}
 801
 802	return false;
 803}
 804
 805static int sdma_v7_0_reset_queue(struct amdgpu_ring *ring,
 806				 unsigned int vmid,
 807				 struct amdgpu_fence *timedout_fence)
 808{
 809	struct amdgpu_device *adev = ring->adev;
 810	int r;
 811
 812	if (ring->me >= adev->sdma.num_instances) {
 813		dev_err(adev->dev, "sdma instance not found\n");
 814		return -EINVAL;
 815	}
 816
 817	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
 818
 819	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true);
 820	if (r)
 821		return r;
 822
 823	r = sdma_v7_0_gfx_resume_instance(adev, ring->me, true);
 824	if (r)
 825		return r;
 826
 827	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
 828}
 829
 830/**
 831 * sdma_v7_0_start - setup and start the async dma engines
 832 *
 833 * @adev: amdgpu_device pointer
 834 *
 835 * Set up the DMA engines and enable them.
 836 * Returns 0 for success, error for failure.
 837 */
 838static int sdma_v7_0_start(struct amdgpu_device *adev)
 839{
 840	int r = 0;
 841
 842	if (amdgpu_sriov_vf(adev)) {
 843		sdma_v7_0_ctx_switch_enable(adev, false);
 844		sdma_v7_0_enable(adev, false);
 845
 846		/* set RB registers */
 847		r = sdma_v7_0_gfx_resume(adev);
 848		return r;
 849	}
 850
 851	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 852		r = sdma_v7_0_load_microcode(adev);
 853		if (r) {
 854			sdma_v12_0_free_ucode_buffer(adev);
 855			return r;
 856		}
 857
 858		if (amdgpu_emu_mode == 1)
 859			msleep(1000);
 860	}
 861
 862	/* unhalt the MEs */
 863	sdma_v7_0_enable(adev, true);
 864	/* enable sdma ring preemption */
 865	sdma_v7_0_ctx_switch_enable(adev, true);
 866
 867	/* start the gfx rings and rlc compute queues */
 868	r = sdma_v7_0_gfx_resume(adev);
 869	if (r)
 870		return r;
 871	r = sdma_v7_0_rlc_resume(adev);
 872
 873	return r;
 874}
 875
 876static int sdma_v7_0_mqd_init(struct amdgpu_device *adev, void *mqd,
 877			      struct amdgpu_mqd_prop *prop)
 878{
 879	struct v12_sdma_mqd *m = mqd;
 880	uint64_t wb_gpu_addr;
 881
 882	m->sdmax_rlcx_rb_cntl =
 883		order_base_2(prop->queue_size / 4) << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
 884		1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 885		4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
 886		1 << SDMA0_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
 887
 888	m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
 889	m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
 890
 891	wb_gpu_addr = prop->wptr_gpu_addr;
 892	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
 893	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
 894
 895	wb_gpu_addr = prop->rptr_gpu_addr;
 896	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
 897	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
 898
 899	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, 0,
 900							regSDMA0_QUEUE0_IB_CNTL));
 901
 902	m->sdmax_rlcx_doorbell_offset =
 903		prop->doorbell_index << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
 904
 905	m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
 906
 907	m->sdmax_rlcx_doorbell_log = 0;
 908	m->sdmax_rlcx_rb_aql_cntl = 0x4000;	//regSDMA0_QUEUE0_RB_AQL_CNTL_DEFAULT;
 909	m->sdmax_rlcx_dummy_reg = 0xf;	//regSDMA0_QUEUE0_DUMMY_REG_DEFAULT;
 910
 911	m->sdmax_rlcx_csa_addr_lo = lower_32_bits(prop->csa_addr);
 912	m->sdmax_rlcx_csa_addr_hi = upper_32_bits(prop->csa_addr);
 913
 914	m->sdmax_rlcx_mcu_dbg0 = lower_32_bits(prop->fence_address);
 915	m->sdmax_rlcx_mcu_dbg1 = upper_32_bits(prop->fence_address);
 916
 917	return 0;
 918}
 919
 920static void sdma_v7_0_set_mqd_funcs(struct amdgpu_device *adev)
 921{
 922	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v12_sdma_mqd);
 923	adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v7_0_mqd_init;
 924}
 925
 926/**
 927 * sdma_v7_0_ring_test_ring - simple async dma engine test
 928 *
 929 * @ring: amdgpu_ring structure holding ring information
 930 *
 931 * Test the DMA engine by writing using it to write an
 932 * value to memory.
 933 * Returns 0 for success, error for failure.
 934 */
 935static int sdma_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 936{
 937	struct amdgpu_device *adev = ring->adev;
 938	unsigned i;
 939	unsigned index;
 940	int r;
 941	u32 tmp;
 942	u64 gpu_addr;
 943
 944	tmp = 0xCAFEDEAD;
 945
 946	r = amdgpu_device_wb_get(adev, &index);
 947	if (r) {
 948		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 949		return r;
 950	}
 951
 952	gpu_addr = adev->wb.gpu_addr + (index * 4);
 953	adev->wb.wb[index] = cpu_to_le32(tmp);
 954
 955	r = amdgpu_ring_alloc(ring, 5);
 956	if (r) {
 957		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
 958		amdgpu_device_wb_free(adev, index);
 959		return r;
 960	}
 961
 962	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
 963			  SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
 964	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
 965	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
 966	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
 967	amdgpu_ring_write(ring, 0xDEADBEEF);
 968	amdgpu_ring_commit(ring);
 969
 970	for (i = 0; i < adev->usec_timeout; i++) {
 971		tmp = le32_to_cpu(adev->wb.wb[index]);
 972		if (tmp == 0xDEADBEEF)
 973			break;
 974		if (amdgpu_emu_mode == 1)
 975			msleep(1);
 976		else
 977			udelay(1);
 978	}
 979
 980	if (i >= adev->usec_timeout)
 981		r = -ETIMEDOUT;
 982
 983	amdgpu_device_wb_free(adev, index);
 984
 985	return r;
 986}
 987
 988/**
 989 * sdma_v7_0_ring_test_ib - test an IB on the DMA engine
 990 *
 991 * @ring: amdgpu_ring structure holding ring information
 992 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 993 *
 994 * Test a simple IB in the DMA ring.
 995 * Returns 0 on success, error on failure.
 996 */
 997static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 998{
 999	struct amdgpu_device *adev = ring->adev;
1000	struct amdgpu_ib ib;
1001	struct dma_fence *f = NULL;
1002	unsigned index;
1003	long r;
1004	u32 tmp = 0;
1005	u64 gpu_addr;
1006
1007	tmp = 0xCAFEDEAD;
1008	memset(&ib, 0, sizeof(ib));
1009
1010	r = amdgpu_device_wb_get(adev, &index);
1011	if (r) {
1012		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
1013		return r;
1014	}
1015
1016	gpu_addr = adev->wb.gpu_addr + (index * 4);
1017	adev->wb.wb[index] = cpu_to_le32(tmp);
1018
1019	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
1020	if (r) {
1021		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
1022		goto err0;
1023	}
1024
1025	ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1026		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1027	ib.ptr[1] = lower_32_bits(gpu_addr);
1028	ib.ptr[2] = upper_32_bits(gpu_addr);
1029	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1030	ib.ptr[4] = 0xDEADBEEF;
1031	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1032	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1033	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1034	ib.length_dw = 8;
1035
1036	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1037	if (r)
1038		goto err1;
1039
1040	r = dma_fence_wait_timeout(f, false, timeout);
1041	if (r == 0) {
1042		DRM_ERROR("amdgpu: IB test timed out\n");
1043		r = -ETIMEDOUT;
1044		goto err1;
1045	} else if (r < 0) {
1046		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1047		goto err1;
1048	}
1049
1050	tmp = le32_to_cpu(adev->wb.wb[index]);
1051
1052	if (tmp == 0xDEADBEEF)
1053		r = 0;
1054	else
1055		r = -EINVAL;
1056
1057err1:
1058	amdgpu_ib_free(&ib, NULL);
1059	dma_fence_put(f);
1060err0:
1061	amdgpu_device_wb_free(adev, index);
1062	return r;
1063}
1064
1065
1066/**
1067 * sdma_v7_0_vm_copy_pte - update PTEs by copying them from the GART
1068 *
1069 * @ib: indirect buffer to fill with commands
1070 * @pe: addr of the page entry
1071 * @src: src addr to copy from
1072 * @count: number of page entries to update
1073 *
1074 * Update PTEs by copying them from the GART using sDMA.
1075 */
1076static void sdma_v7_0_vm_copy_pte(struct amdgpu_ib *ib,
1077				  uint64_t pe, uint64_t src,
1078				  unsigned count)
1079{
1080	unsigned bytes = count * 8;
1081
1082	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1083		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1084		SDMA_PKT_COPY_LINEAR_HEADER_CPV(1);
1085
1086	ib->ptr[ib->length_dw++] = bytes - 1;
1087	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1088	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1089	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1090	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1091	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1092	ib->ptr[ib->length_dw++] = 0;
1093
1094}
1095
1096/**
1097 * sdma_v7_0_vm_write_pte - update PTEs by writing them manually
1098 *
1099 * @ib: indirect buffer to fill with commands
1100 * @pe: addr of the page entry
1101 * @value: dst addr to write into pe
1102 * @count: number of page entries to update
1103 * @incr: increase next addr by incr bytes
1104 *
1105 * Update PTEs by writing them manually using sDMA.
1106 */
1107static void sdma_v7_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1108				   uint64_t value, unsigned count,
1109				   uint32_t incr)
1110{
1111	unsigned ndw = count * 2;
1112
1113	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1114		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1115	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1116	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1117	ib->ptr[ib->length_dw++] = ndw - 1;
1118	for (; ndw > 0; ndw -= 2) {
1119		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1120		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1121		value += incr;
1122	}
1123}
1124
1125/**
1126 * sdma_v7_0_vm_set_pte_pde - update the page tables using sDMA
1127 *
1128 * @ib: indirect buffer to fill with commands
1129 * @pe: addr of the page entry
1130 * @addr: dst addr to write into pe
1131 * @count: number of page entries to update
1132 * @incr: increase next addr by incr bytes
1133 * @flags: access flags
1134 *
1135 * Update the page tables using sDMA.
1136 */
1137static void sdma_v7_0_vm_set_pte_pde(struct amdgpu_ib *ib,
1138				     uint64_t pe,
1139				     uint64_t addr, unsigned count,
1140				     uint32_t incr, uint64_t flags)
1141{
1142	/* for physically contiguous pages (vram) */
1143	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
1144	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1145	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1146	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1147	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1148	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1149	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1150	ib->ptr[ib->length_dw++] = incr; /* increment size */
1151	ib->ptr[ib->length_dw++] = 0;
1152	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1153}
1154
1155/**
1156 * sdma_v7_0_ring_pad_ib - pad the IB
1157 *
1158 * @ring: amdgpu ring pointer
1159 * @ib: indirect buffer to fill with padding
1160 *
1161 * Pad the IB with NOPs to a boundary multiple of 8.
1162 */
1163static void sdma_v7_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1164{
1165	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1166	u32 pad_count;
1167	int i;
1168
1169	pad_count = (-ib->length_dw) & 0x7;
1170	for (i = 0; i < pad_count; i++)
1171		if (sdma && sdma->burst_nop && (i == 0))
1172			ib->ptr[ib->length_dw++] =
1173				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
1174				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1175		else
1176			ib->ptr[ib->length_dw++] =
1177				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP);
1178}
1179
1180/**
1181 * sdma_v7_0_ring_emit_pipeline_sync - sync the pipeline
1182 *
1183 * @ring: amdgpu_ring pointer
1184 *
1185 * Make sure all previous operations are completed (CIK).
1186 */
1187static void sdma_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1188{
1189	uint32_t seq = ring->fence_drv.sync_seq;
1190	uint64_t addr = ring->fence_drv.gpu_addr;
1191
1192	/* wait for idle */
1193	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1194			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1195			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1196			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1197	amdgpu_ring_write(ring, addr & 0xfffffffc);
1198	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1199	amdgpu_ring_write(ring, seq); /* reference */
1200	amdgpu_ring_write(ring, 0xffffffff); /* mask */
1201	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1202			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1203}
1204
1205/**
1206 * sdma_v7_0_ring_emit_vm_flush - vm flush using sDMA
1207 *
1208 * @ring: amdgpu_ring pointer
1209 * @vmid: vmid number to use
1210 * @pd_addr: address
1211 *
1212 * Update the page table base and flush the VM TLB
1213 * using sDMA.
1214 */
1215static void sdma_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1216					 unsigned vmid, uint64_t pd_addr)
1217{
1218	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1219}
1220
1221static void sdma_v7_0_ring_emit_wreg(struct amdgpu_ring *ring,
1222				     uint32_t reg, uint32_t val)
1223{
1224	/* SRBM WRITE command will not support on sdma v7.
1225	 * Use Register WRITE command instead, which OPCODE is same as SRBM WRITE
1226	 */
1227	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_SRBM_WRITE));
1228	amdgpu_ring_write(ring, reg << 2);
1229	amdgpu_ring_write(ring, val);
1230}
1231
1232static void sdma_v7_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1233					 uint32_t val, uint32_t mask)
1234{
1235	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1236			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1237			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1238	amdgpu_ring_write(ring, reg << 2);
1239	amdgpu_ring_write(ring, 0);
1240	amdgpu_ring_write(ring, val); /* reference */
1241	amdgpu_ring_write(ring, mask); /* mask */
1242	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1243			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1244}
1245
1246static void sdma_v7_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1247						   uint32_t reg0, uint32_t reg1,
1248						   uint32_t ref, uint32_t mask)
1249{
1250	amdgpu_ring_emit_wreg(ring, reg0, ref);
1251	/* wait for a cycle to reset vm_inv_eng*_ack */
1252	amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1253	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1254}
1255
1256static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
1257{
1258	struct amdgpu_device *adev = ip_block->adev;
1259	int r;
1260
1261	switch (amdgpu_user_queue) {
1262	case -1:
1263	case 0:
1264	default:
1265		adev->sdma.no_user_submission = false;
1266		adev->sdma.disable_uq = true;
1267		break;
1268	case 1:
1269		adev->sdma.no_user_submission = false;
1270		adev->sdma.disable_uq = false;
1271		break;
1272	case 2:
1273		adev->sdma.no_user_submission = true;
1274		adev->sdma.disable_uq = false;
1275		break;
1276	}
1277
1278	r = amdgpu_sdma_init_microcode(adev, 0, true);
1279	if (r) {
1280		DRM_ERROR("Failed to init sdma firmware!\n");
1281		return r;
1282	}
1283
1284	sdma_v7_0_set_ring_funcs(adev);
1285	sdma_v7_0_set_buffer_funcs(adev);
1286	sdma_v7_0_set_vm_pte_funcs(adev);
1287	sdma_v7_0_set_irq_funcs(adev);
1288	sdma_v7_0_set_mqd_funcs(adev);
1289
1290	return 0;
1291}
1292
1293static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
1294{
1295	struct amdgpu_ring *ring;
1296	int r, i;
1297	struct amdgpu_device *adev = ip_block->adev;
1298	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_0);
1299	uint32_t *ptr;
1300
1301	/* SDMA trap event */
1302	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1303			      GFX_12_0_0__SRCID__SDMA_TRAP,
1304			      &adev->sdma.trap_irq);
1305	if (r)
1306		return r;
1307
1308	/* SDMA user fence event */
1309	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1310			      GFX_12_0_0__SRCID__SDMA_FENCE,
1311			      &adev->sdma.fence_irq);
1312	if (r)
1313		return r;
1314
1315	for (i = 0; i < adev->sdma.num_instances; i++) {
1316		ring = &adev->sdma.instance[i].ring;
1317		ring->ring_obj = NULL;
1318		ring->use_doorbell = true;
1319		ring->me = i;
1320		ring->no_user_submission = adev->sdma.no_user_submission;
1321
1322		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1323				ring->use_doorbell?"true":"false");
1324
1325		ring->doorbell_index =
1326			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
1327
1328		ring->vm_hub = AMDGPU_GFXHUB(0);
1329		sprintf(ring->name, "sdma%d", i);
1330		r = amdgpu_ring_init(adev, ring, 1024,
1331				     &adev->sdma.trap_irq,
1332				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1333				     AMDGPU_RING_PRIO_DEFAULT, NULL);
1334		if (r)
1335			return r;
1336	}
1337
1338	adev->sdma.supported_reset =
1339		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1340	if (!amdgpu_sriov_vf(adev) &&
1341	    !adev->debug_disable_gpu_ring_reset)
1342		adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1343
1344	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1345	if (r)
1346		return r;
1347	/* Allocate memory for SDMA IP Dump buffer */
1348	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1349	if (ptr)
1350		adev->sdma.ip_dump = ptr;
1351	else
1352		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1353
1354	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1355	case IP_VERSION(7, 0, 0):
1356	case IP_VERSION(7, 0, 1):
1357		if ((adev->sdma.instance[0].fw_version >= 7966358) && !adev->sdma.disable_uq)
1358			adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs;
1359		break;
1360	default:
1361		break;
1362	}
1363
1364	return r;
1365}
1366
1367static int sdma_v7_0_sw_fini(struct amdgpu_ip_block *ip_block)
1368{
1369	struct amdgpu_device *adev = ip_block->adev;
1370	int i;
1371
1372	for (i = 0; i < adev->sdma.num_instances; i++)
1373		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1374
1375	amdgpu_sdma_sysfs_reset_mask_fini(adev);
1376	amdgpu_sdma_destroy_inst_ctx(adev, true);
1377
1378	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)
1379		sdma_v12_0_free_ucode_buffer(adev);
1380
1381	kfree(adev->sdma.ip_dump);
1382
1383	return 0;
1384}
1385
1386static int sdma_v7_0_set_userq_trap_interrupts(struct amdgpu_device *adev,
1387					       bool enable)
1388{
1389	unsigned int irq_type;
1390	int i, r;
1391
1392	if (adev->userq_funcs[AMDGPU_HW_IP_DMA]) {
1393		for (i = 0; i < adev->sdma.num_instances; i++) {
1394			irq_type = AMDGPU_SDMA_IRQ_INSTANCE0 + i;
1395			if (enable)
1396				r = amdgpu_irq_get(adev, &adev->sdma.trap_irq,
1397						   irq_type);
1398			else
1399				r = amdgpu_irq_put(adev, &adev->sdma.trap_irq,
1400						   irq_type);
1401			if (r)
1402				return r;
1403		}
1404	}
1405
1406	return 0;
1407}
1408
1409static int sdma_v7_0_hw_init(struct amdgpu_ip_block *ip_block)
1410{
1411	struct amdgpu_device *adev = ip_block->adev;
1412	int r;
1413
1414	r = sdma_v7_0_start(adev);
1415	if (r)
1416		return r;
1417
1418	return sdma_v7_0_set_userq_trap_interrupts(adev, true);
1419}
1420
1421static int sdma_v7_0_hw_fini(struct amdgpu_ip_block *ip_block)
1422{
1423	struct amdgpu_device *adev = ip_block->adev;
1424
1425	if (amdgpu_sriov_vf(adev))
1426		return 0;
1427
1428	sdma_v7_0_ctx_switch_enable(adev, false);
1429	sdma_v7_0_enable(adev, false);
1430	sdma_v7_0_set_userq_trap_interrupts(adev, false);
1431
1432	return 0;
1433}
1434
1435static int sdma_v7_0_suspend(struct amdgpu_ip_block *ip_block)
1436{
1437	return sdma_v7_0_hw_fini(ip_block);
1438}
1439
1440static int sdma_v7_0_resume(struct amdgpu_ip_block *ip_block)
1441{
1442	return sdma_v7_0_hw_init(ip_block);
1443}
1444
1445static bool sdma_v7_0_is_idle(struct amdgpu_ip_block *ip_block)
1446{
1447	struct amdgpu_device *adev = ip_block->adev;
1448	u32 i;
1449
1450	for (i = 0; i < adev->sdma.num_instances; i++) {
1451		u32 tmp = RREG32(sdma_v7_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
1452
1453		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1454			return false;
1455	}
1456
1457	return true;
1458}
1459
1460static int sdma_v7_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
1461{
1462	unsigned i;
1463	u32 sdma0, sdma1;
1464	struct amdgpu_device *adev = ip_block->adev;
1465
1466	for (i = 0; i < adev->usec_timeout; i++) {
1467		sdma0 = RREG32(sdma_v7_0_get_reg_offset(adev, 0, regSDMA0_STATUS_REG));
1468		sdma1 = RREG32(sdma_v7_0_get_reg_offset(adev, 1, regSDMA0_STATUS_REG));
1469
1470		if (sdma0 & sdma1 & SDMA0_STATUS_REG__IDLE_MASK)
1471			return 0;
1472		udelay(1);
1473	}
1474	return -ETIMEDOUT;
1475}
1476
1477static int sdma_v7_0_ring_preempt_ib(struct amdgpu_ring *ring)
1478{
1479	int i, r = 0;
1480	struct amdgpu_device *adev = ring->adev;
1481	u32 index = 0;
1482	u64 sdma_gfx_preempt;
1483
1484	amdgpu_sdma_get_index_from_ring(ring, &index);
1485	sdma_gfx_preempt =
1486		sdma_v7_0_get_reg_offset(adev, index, regSDMA0_QUEUE0_PREEMPT);
1487
1488	/* assert preemption condition */
1489	amdgpu_ring_set_preempt_cond_exec(ring, false);
1490
1491	/* emit the trailing fence */
1492	ring->trail_seq += 1;
1493	r = amdgpu_ring_alloc(ring, 10);
1494	if (r) {
1495		DRM_ERROR("ring %d failed to be allocated \n", ring->idx);
1496		return r;
1497	}
1498	sdma_v7_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1499				  ring->trail_seq, 0);
1500	amdgpu_ring_commit(ring);
1501
1502	/* assert IB preemption */
1503	WREG32(sdma_gfx_preempt, 1);
1504
1505	/* poll the trailing fence */
1506	for (i = 0; i < adev->usec_timeout; i++) {
1507		if (ring->trail_seq ==
1508		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1509			break;
1510		udelay(1);
1511	}
1512
1513	if (i >= adev->usec_timeout) {
1514		r = -EINVAL;
1515		DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1516	}
1517
1518	/* deassert IB preemption */
1519	WREG32(sdma_gfx_preempt, 0);
1520
1521	/* deassert the preemption condition */
1522	amdgpu_ring_set_preempt_cond_exec(ring, true);
1523	return r;
1524}
1525
1526static int sdma_v7_0_set_trap_irq_state(struct amdgpu_device *adev,
1527					struct amdgpu_irq_src *source,
1528					unsigned type,
1529					enum amdgpu_interrupt_state state)
1530{
1531	u32 sdma_cntl;
1532
1533	u32 reg_offset = sdma_v7_0_get_reg_offset(adev, type, regSDMA0_CNTL);
1534
1535	sdma_cntl = RREG32(reg_offset);
1536	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1537		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1538	WREG32(reg_offset, sdma_cntl);
1539
1540	return 0;
1541}
1542
1543static int sdma_v7_0_process_trap_irq(struct amdgpu_device *adev,
1544				      struct amdgpu_irq_src *source,
1545				      struct amdgpu_iv_entry *entry)
1546{
1547	int instances, queue;
1548
1549	DRM_DEBUG("IH: SDMA trap\n");
1550
1551	queue = entry->ring_id & 0xf;
1552	instances = (entry->ring_id & 0xf0) >> 4;
1553	if (instances > 1) {
1554		DRM_ERROR("IH: wrong ring_ID detected, as wrong sdma instance\n");
1555		return -EINVAL;
1556	}
1557
1558	switch (entry->client_id) {
1559	case SOC21_IH_CLIENTID_GFX:
1560		switch (queue) {
1561		case 0:
1562			amdgpu_fence_process(&adev->sdma.instance[instances].ring);
1563			break;
1564		default:
1565			break;
1566		}
1567		break;
1568	}
1569	return 0;
1570}
1571
1572static int sdma_v7_0_process_fence_irq(struct amdgpu_device *adev,
1573				       struct amdgpu_irq_src *source,
1574				       struct amdgpu_iv_entry *entry)
1575{
1576	u32 doorbell_offset = entry->src_data[0];
1577
1578	if (adev->enable_mes && doorbell_offset) {
1579		struct amdgpu_userq_fence_driver *fence_drv = NULL;
1580		struct xarray *xa = &adev->userq_xa;
1581		unsigned long flags;
1582
1583		doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
1584
1585		xa_lock_irqsave(xa, flags);
1586		fence_drv = xa_load(xa, doorbell_offset);
1587		if (fence_drv)
1588			amdgpu_userq_fence_driver_process(fence_drv);
1589		xa_unlock_irqrestore(xa, flags);
1590	}
1591
1592	return 0;
1593}
1594
1595static int sdma_v7_0_process_illegal_inst_irq(struct amdgpu_device *adev,
1596					      struct amdgpu_irq_src *source,
1597					      struct amdgpu_iv_entry *entry)
1598{
1599	return 0;
1600}
1601
1602static int sdma_v7_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1603					   enum amd_clockgating_state state)
1604{
1605	return 0;
1606}
1607
1608static int sdma_v7_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
1609					  enum amd_powergating_state state)
1610{
1611	return 0;
1612}
1613
1614static void sdma_v7_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
1615{
1616}
1617
1618static void sdma_v7_0_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
1619{
1620	struct amdgpu_device *adev = ip_block->adev;
1621	int i, j;
1622	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_0);
1623	uint32_t instance_offset;
1624
1625	if (!adev->sdma.ip_dump)
1626		return;
1627
1628	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
1629	for (i = 0; i < adev->sdma.num_instances; i++) {
1630		instance_offset = i * reg_count;
1631		drm_printf(p, "\nInstance:%d\n", i);
1632
1633		for (j = 0; j < reg_count; j++)
1634			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_7_0[j].reg_name,
1635				   adev->sdma.ip_dump[instance_offset + j]);
1636	}
1637}
1638
1639static void sdma_v7_0_dump_ip_state(struct amdgpu_ip_block *ip_block)
1640{
1641	struct amdgpu_device *adev = ip_block->adev;
1642	int i, j;
1643	uint32_t instance_offset;
1644	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_0);
1645
1646	if (!adev->sdma.ip_dump)
1647		return;
1648
1649	amdgpu_gfx_off_ctrl(adev, false);
1650	for (i = 0; i < adev->sdma.num_instances; i++) {
1651		instance_offset = i * reg_count;
1652		for (j = 0; j < reg_count; j++)
1653			adev->sdma.ip_dump[instance_offset + j] =
1654				RREG32(sdma_v7_0_get_reg_offset(adev, i,
1655				       sdma_reg_list_7_0[j].reg_offset));
1656	}
1657	amdgpu_gfx_off_ctrl(adev, true);
1658}
1659
1660const struct amd_ip_funcs sdma_v7_0_ip_funcs = {
1661	.name = "sdma_v7_0",
1662	.early_init = sdma_v7_0_early_init,
1663	.late_init = NULL,
1664	.sw_init = sdma_v7_0_sw_init,
1665	.sw_fini = sdma_v7_0_sw_fini,
1666	.hw_init = sdma_v7_0_hw_init,
1667	.hw_fini = sdma_v7_0_hw_fini,
1668	.suspend = sdma_v7_0_suspend,
1669	.resume = sdma_v7_0_resume,
1670	.is_idle = sdma_v7_0_is_idle,
1671	.wait_for_idle = sdma_v7_0_wait_for_idle,
1672	.soft_reset = sdma_v7_0_soft_reset,
1673	.check_soft_reset = sdma_v7_0_check_soft_reset,
1674	.set_clockgating_state = sdma_v7_0_set_clockgating_state,
1675	.set_powergating_state = sdma_v7_0_set_powergating_state,
1676	.get_clockgating_state = sdma_v7_0_get_clockgating_state,
1677	.dump_ip_state = sdma_v7_0_dump_ip_state,
1678	.print_ip_state = sdma_v7_0_print_ip_state,
1679};
1680
1681static const struct amdgpu_ring_funcs sdma_v7_0_ring_funcs = {
1682	.type = AMDGPU_RING_TYPE_SDMA,
1683	.align_mask = 0xf,
1684	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1685	.support_64bit_ptrs = true,
1686	.secure_submission_supported = true,
1687	.get_rptr = sdma_v7_0_ring_get_rptr,
1688	.get_wptr = sdma_v7_0_ring_get_wptr,
1689	.set_wptr = sdma_v7_0_ring_set_wptr,
1690	.emit_frame_size =
1691		5 + /* sdma_v7_0_ring_init_cond_exec */
1692		6 + /* sdma_v7_0_ring_emit_hdp_flush */
1693		6 + /* sdma_v7_0_ring_emit_pipeline_sync */
1694		/* sdma_v7_0_ring_emit_vm_flush */
1695		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1696		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1697		10 + 10 + 10, /* sdma_v7_0_ring_emit_fence x3 for user fence, vm fence */
1698	.emit_ib_size = 5 + 7 + 6, /* sdma_v7_0_ring_emit_ib */
1699	.emit_ib = sdma_v7_0_ring_emit_ib,
1700	.emit_mem_sync = sdma_v7_0_ring_emit_mem_sync,
1701	.emit_fence = sdma_v7_0_ring_emit_fence,
1702	.emit_pipeline_sync = sdma_v7_0_ring_emit_pipeline_sync,
1703	.emit_vm_flush = sdma_v7_0_ring_emit_vm_flush,
1704	.emit_hdp_flush = sdma_v7_0_ring_emit_hdp_flush,
1705	.test_ring = sdma_v7_0_ring_test_ring,
1706	.test_ib = sdma_v7_0_ring_test_ib,
1707	.insert_nop = sdma_v7_0_ring_insert_nop,
1708	.pad_ib = sdma_v7_0_ring_pad_ib,
1709	.emit_wreg = sdma_v7_0_ring_emit_wreg,
1710	.emit_reg_wait = sdma_v7_0_ring_emit_reg_wait,
1711	.emit_reg_write_reg_wait = sdma_v7_0_ring_emit_reg_write_reg_wait,
1712	.init_cond_exec = sdma_v7_0_ring_init_cond_exec,
1713	.preempt_ib = sdma_v7_0_ring_preempt_ib,
1714	.reset = sdma_v7_0_reset_queue,
1715};
1716
1717static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev)
1718{
1719	int i;
1720
1721	for (i = 0; i < adev->sdma.num_instances; i++) {
1722		adev->sdma.instance[i].ring.funcs = &sdma_v7_0_ring_funcs;
1723		adev->sdma.instance[i].ring.me = i;
1724	}
1725}
1726
1727static const struct amdgpu_irq_src_funcs sdma_v7_0_trap_irq_funcs = {
1728	.set = sdma_v7_0_set_trap_irq_state,
1729	.process = sdma_v7_0_process_trap_irq,
1730};
1731
1732static const struct amdgpu_irq_src_funcs sdma_v7_0_fence_irq_funcs = {
1733	.process = sdma_v7_0_process_fence_irq,
1734};
1735
1736static const struct amdgpu_irq_src_funcs sdma_v7_0_illegal_inst_irq_funcs = {
1737	.process = sdma_v7_0_process_illegal_inst_irq,
1738};
1739
1740static void sdma_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1741{
1742	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1743					adev->sdma.num_instances;
1744	adev->sdma.trap_irq.funcs = &sdma_v7_0_trap_irq_funcs;
1745	adev->sdma.fence_irq.funcs = &sdma_v7_0_fence_irq_funcs;
1746	adev->sdma.illegal_inst_irq.funcs = &sdma_v7_0_illegal_inst_irq_funcs;
1747}
1748
1749/**
1750 * sdma_v7_0_emit_copy_buffer - copy buffer using the sDMA engine
1751 *
1752 * @ib: indirect buffer to fill with commands
1753 * @src_offset: src GPU address
1754 * @dst_offset: dst GPU address
1755 * @byte_count: number of bytes to xfer
1756 * @copy_flags: copy flags for the buffers
1757 *
1758 * Copy GPU buffers using the DMA engine.
1759 * Used by the amdgpu ttm implementation to move pages if
1760 * registered as the asic copy callback.
1761 */
1762static void sdma_v7_0_emit_copy_buffer(struct amdgpu_ib *ib,
1763				       uint64_t src_offset,
1764				       uint64_t dst_offset,
1765				       uint32_t byte_count,
1766				       uint32_t copy_flags)
1767{
1768	uint32_t num_type, data_format, max_com, write_cm;
1769
1770	max_com = AMDGPU_COPY_FLAGS_GET(copy_flags, MAX_COMPRESSED);
1771	data_format = AMDGPU_COPY_FLAGS_GET(copy_flags, DATA_FORMAT);
1772	num_type = AMDGPU_COPY_FLAGS_GET(copy_flags, NUMBER_TYPE);
1773	write_cm = AMDGPU_COPY_FLAGS_GET(copy_flags, WRITE_COMPRESS_DISABLE) ? 2 : 1;
1774
1775	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1776		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1777		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0) |
1778		SDMA_PKT_COPY_LINEAR_HEADER_CPV(1);
1779
1780	ib->ptr[ib->length_dw++] = byte_count - 1;
1781	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1782	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1783	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1784	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1785	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1786
1787	if ((copy_flags & (AMDGPU_COPY_FLAGS_READ_DECOMPRESSED | AMDGPU_COPY_FLAGS_WRITE_COMPRESSED)))
1788		ib->ptr[ib->length_dw++] = SDMA_DCC_DATA_FORMAT(data_format) | SDMA_DCC_NUM_TYPE(num_type) |
1789			((copy_flags & AMDGPU_COPY_FLAGS_READ_DECOMPRESSED) ? SDMA_DCC_READ_CM(2) : 0) |
1790			((copy_flags & AMDGPU_COPY_FLAGS_WRITE_COMPRESSED) ? SDMA_DCC_WRITE_CM(write_cm) : 0) |
1791			SDMA_DCC_MAX_COM(max_com) | SDMA_DCC_MAX_UCOM(1);
1792	else
1793		ib->ptr[ib->length_dw++] = 0;
1794}
1795
1796/**
1797 * sdma_v7_0_emit_fill_buffer - fill buffer using the sDMA engine
1798 *
1799 * @ib: indirect buffer to fill
1800 * @src_data: value to write to buffer
1801 * @dst_offset: dst GPU address
1802 * @byte_count: number of bytes to xfer
1803 *
1804 * Fill GPU buffers using the DMA engine.
1805 */
1806static void sdma_v7_0_emit_fill_buffer(struct amdgpu_ib *ib,
1807				       uint32_t src_data,
1808				       uint64_t dst_offset,
1809				       uint32_t byte_count)
1810{
1811	ib->ptr[ib->length_dw++] = SDMA_PKT_CONSTANT_FILL_HEADER_OP(SDMA_OP_CONST_FILL) |
1812		SDMA_PKT_CONSTANT_FILL_HEADER_COMPRESS(1);
1813	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1814	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1815	ib->ptr[ib->length_dw++] = src_data;
1816	ib->ptr[ib->length_dw++] = byte_count - 1;
1817}
1818
1819static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = {
1820	.copy_max_bytes = 0x400000,
1821	.copy_num_dw = 8,
1822	.emit_copy_buffer = sdma_v7_0_emit_copy_buffer,
1823	.fill_max_bytes = 0x400000,
1824	.fill_num_dw = 5,
1825	.emit_fill_buffer = sdma_v7_0_emit_fill_buffer,
1826};
1827
1828static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev)
1829{
1830	adev->mman.buffer_funcs = &sdma_v7_0_buffer_funcs;
1831	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1832}
1833
1834static const struct amdgpu_vm_pte_funcs sdma_v7_0_vm_pte_funcs = {
1835	.copy_pte_num_dw = 8,
1836	.copy_pte = sdma_v7_0_vm_copy_pte,
1837	.write_pte = sdma_v7_0_vm_write_pte,
1838	.set_pte_pde = sdma_v7_0_vm_set_pte_pde,
1839};
1840
1841static void sdma_v7_0_set_vm_pte_funcs(struct amdgpu_device *adev)
1842{
1843	unsigned i;
1844
1845	adev->vm_manager.vm_pte_funcs = &sdma_v7_0_vm_pte_funcs;
1846	for (i = 0; i < adev->sdma.num_instances; i++) {
1847		adev->vm_manager.vm_pte_scheds[i] =
1848			&adev->sdma.instance[i].ring.sched;
1849	}
1850	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
1851}
1852
1853const struct amdgpu_ip_block_version sdma_v7_0_ip_block = {
1854	.type = AMD_IP_BLOCK_TYPE_SDMA,
1855	.major = 7,
1856	.minor = 0,
1857	.rev = 0,
1858	.funcs = &sdma_v7_0_ip_funcs,
1859};