cik_sdma.c

   1/*
   2 * Copyright 2013 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27
  28#include "amdgpu.h"
  29#include "amdgpu_ucode.h"
  30#include "amdgpu_trace.h"
  31#include "cikd.h"
  32#include "cik.h"
  33
  34#include "bif/bif_4_1_d.h"
  35#include "bif/bif_4_1_sh_mask.h"
  36
  37#include "gca/gfx_7_2_d.h"
  38#include "gca/gfx_7_2_enum.h"
  39#include "gca/gfx_7_2_sh_mask.h"
  40
  41#include "gmc/gmc_7_1_d.h"
  42#include "gmc/gmc_7_1_sh_mask.h"
  43
  44#include "oss/oss_2_0_d.h"
  45#include "oss/oss_2_0_sh_mask.h"
  46
  47static const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
  48{
  49	SDMA0_REGISTER_OFFSET,
  50	SDMA1_REGISTER_OFFSET
  51};
  52
  53static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev);
  54static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev);
  55static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);
  56static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev);
  57static int cik_sdma_soft_reset(struct amdgpu_ip_block *ip_block);
  58
  59u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev);
  60
  61MODULE_FIRMWARE("amdgpu/bonaire_sdma.bin");
  62MODULE_FIRMWARE("amdgpu/bonaire_sdma1.bin");
  63MODULE_FIRMWARE("amdgpu/hawaii_sdma.bin");
  64MODULE_FIRMWARE("amdgpu/hawaii_sdma1.bin");
  65MODULE_FIRMWARE("amdgpu/kaveri_sdma.bin");
  66MODULE_FIRMWARE("amdgpu/kaveri_sdma1.bin");
  67MODULE_FIRMWARE("amdgpu/kabini_sdma.bin");
  68MODULE_FIRMWARE("amdgpu/kabini_sdma1.bin");
  69MODULE_FIRMWARE("amdgpu/mullins_sdma.bin");
  70MODULE_FIRMWARE("amdgpu/mullins_sdma1.bin");
  71
  72static void cik_sdma_free_microcode(struct amdgpu_device *adev)
  73{
  74	int i;
  75
  76	for (i = 0; i < adev->sdma.num_instances; i++)
  77		amdgpu_ucode_release(&adev->sdma.instance[i].fw);
  78}
  79
  80/*
  81 * sDMA - System DMA
  82 * Starting with CIK, the GPU has new asynchronous
  83 * DMA engines.  These engines are used for compute
  84 * and gfx.  There are two DMA engines (SDMA0, SDMA1)
  85 * and each one supports 1 ring buffer used for gfx
  86 * and 2 queues used for compute.
  87 *
  88 * The programming model is very similar to the CP
  89 * (ring buffer, IBs, etc.), but sDMA has it's own
  90 * packet format that is different from the PM4 format
  91 * used by the CP. sDMA supports copying data, writing
  92 * embedded data, solid fills, and a number of other
  93 * things.  It also has support for tiling/detiling of
  94 * buffers.
  95 */
  96
  97/**
  98 * cik_sdma_init_microcode - load ucode images from disk
  99 *
 100 * @adev: amdgpu_device pointer
 101 *
 102 * Use the firmware interface to load the ucode images into
 103 * the driver (not loaded into hw).
 104 * Returns 0 on success, error on failure.
 105 */
 106static int cik_sdma_init_microcode(struct amdgpu_device *adev)
 107{
 108	const char *chip_name;
 109	int err = 0, i;
 110
 111	DRM_DEBUG("\n");
 112
 113	switch (adev->asic_type) {
 114	case CHIP_BONAIRE:
 115		chip_name = "bonaire";
 116		break;
 117	case CHIP_HAWAII:
 118		chip_name = "hawaii";
 119		break;
 120	case CHIP_KAVERI:
 121		chip_name = "kaveri";
 122		break;
 123	case CHIP_KABINI:
 124		chip_name = "kabini";
 125		break;
 126	case CHIP_MULLINS:
 127		chip_name = "mullins";
 128		break;
 129	default: BUG();
 130	}
 131
 132	for (i = 0; i < adev->sdma.num_instances; i++) {
 133		if (i == 0)
 134			err = amdgpu_ucode_request(adev, &adev->sdma.instance[i].fw,
 135						   AMDGPU_UCODE_REQUIRED,
 136						   "amdgpu/%s_sdma.bin", chip_name);
 137		else
 138			err = amdgpu_ucode_request(adev, &adev->sdma.instance[i].fw,
 139						   AMDGPU_UCODE_REQUIRED,
 140						   "amdgpu/%s_sdma1.bin", chip_name);
 141		if (err)
 142			goto out;
 143	}
 144out:
 145	if (err) {
 146		pr_err("cik_sdma: Failed to load firmware \"%s_sdma%s.bin\"\n",
 147		       chip_name, i == 0 ? "" : "1");
 148		for (i = 0; i < adev->sdma.num_instances; i++)
 149			amdgpu_ucode_release(&adev->sdma.instance[i].fw);
 150	}
 151	return err;
 152}
 153
 154/**
 155 * cik_sdma_ring_get_rptr - get the current read pointer
 156 *
 157 * @ring: amdgpu ring pointer
 158 *
 159 * Get the current rptr from the hardware (CIK+).
 160 */
 161static uint64_t cik_sdma_ring_get_rptr(struct amdgpu_ring *ring)
 162{
 163	u32 rptr;
 164
 165	rptr = *ring->rptr_cpu_addr;
 166
 167	return (rptr & 0x3fffc) >> 2;
 168}
 169
 170/**
 171 * cik_sdma_ring_get_wptr - get the current write pointer
 172 *
 173 * @ring: amdgpu ring pointer
 174 *
 175 * Get the current wptr from the hardware (CIK+).
 176 */
 177static uint64_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring)
 178{
 179	struct amdgpu_device *adev = ring->adev;
 180
 181	return (RREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me]) & 0x3fffc) >> 2;
 182}
 183
 184/**
 185 * cik_sdma_ring_set_wptr - commit the write pointer
 186 *
 187 * @ring: amdgpu ring pointer
 188 *
 189 * Write the wptr back to the hardware (CIK+).
 190 */
 191static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring)
 192{
 193	struct amdgpu_device *adev = ring->adev;
 194
 195	WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[ring->me],
 196	       (ring->wptr << 2) & 0x3fffc);
 197}
 198
 199static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 200{
 201	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 202	int i;
 203
 204	for (i = 0; i < count; i++)
 205		if (sdma && sdma->burst_nop && (i == 0))
 206			amdgpu_ring_write(ring, ring->funcs->nop |
 207					  SDMA_NOP_COUNT(count - 1));
 208		else
 209			amdgpu_ring_write(ring, ring->funcs->nop);
 210}
 211
 212/**
 213 * cik_sdma_ring_emit_ib - Schedule an IB on the DMA engine
 214 *
 215 * @ring: amdgpu ring pointer
 216 * @job: job to retrive vmid from
 217 * @ib: IB object to schedule
 218 * @flags: unused
 219 *
 220 * Schedule an IB in the DMA ring (CIK).
 221 */
 222static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,
 223				  struct amdgpu_job *job,
 224				  struct amdgpu_ib *ib,
 225				  uint32_t flags)
 226{
 227	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 228	u32 extra_bits = vmid & 0xf;
 229
 230	/* IB packet must end on a 8 DW boundary */
 231	cik_sdma_ring_insert_nop(ring, (4 - lower_32_bits(ring->wptr)) & 7);
 232
 233	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));
 234	amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */
 235	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);
 236	amdgpu_ring_write(ring, ib->length_dw);
 237
 238}
 239
 240/**
 241 * cik_sdma_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
 242 *
 243 * @ring: amdgpu ring pointer
 244 *
 245 * Emit an hdp flush packet on the requested DMA ring.
 246 */
 247static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 248{
 249	u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
 250			  SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
 251	u32 ref_and_mask;
 252
 253	if (ring->me == 0)
 254		ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA0_MASK;
 255	else
 256		ref_and_mask = GPU_HDP_FLUSH_DONE__SDMA1_MASK;
 257
 258	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
 259	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2);
 260	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2);
 261	amdgpu_ring_write(ring, ref_and_mask); /* reference */
 262	amdgpu_ring_write(ring, ref_and_mask); /* mask */
 263	amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
 264}
 265
 266/**
 267 * cik_sdma_ring_emit_fence - emit a fence on the DMA ring
 268 *
 269 * @ring: amdgpu ring pointer
 270 * @addr: address
 271 * @seq: sequence number
 272 * @flags: fence related flags
 273 *
 274 * Add a DMA fence packet to the ring to write
 275 * the fence seq number and DMA trap packet to generate
 276 * an interrupt if needed (CIK).
 277 */
 278static void cik_sdma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 279				     unsigned flags)
 280{
 281	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
 282	/* write the fence */
 283	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
 284	amdgpu_ring_write(ring, lower_32_bits(addr));
 285	amdgpu_ring_write(ring, upper_32_bits(addr));
 286	amdgpu_ring_write(ring, lower_32_bits(seq));
 287
 288	/* optionally write high bits as well */
 289	if (write64bit) {
 290		addr += 4;
 291		amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
 292		amdgpu_ring_write(ring, lower_32_bits(addr));
 293		amdgpu_ring_write(ring, upper_32_bits(addr));
 294		amdgpu_ring_write(ring, upper_32_bits(seq));
 295	}
 296
 297	/* generate an interrupt */
 298	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
 299}
 300
 301/**
 302 * cik_sdma_gfx_stop - stop the gfx async dma engines
 303 *
 304 * @adev: amdgpu_device pointer
 305 *
 306 * Stop the gfx async dma ring buffers (CIK).
 307 */
 308static void cik_sdma_gfx_stop(struct amdgpu_device *adev)
 309{
 310	u32 rb_cntl;
 311	int i;
 312
 313	for (i = 0; i < adev->sdma.num_instances; i++) {
 314		rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]);
 315		rb_cntl &= ~SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK;
 316		WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
 317		WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], 0);
 318	}
 319}
 320
 321/**
 322 * cik_sdma_rlc_stop - stop the compute async dma engines
 323 *
 324 * @adev: amdgpu_device pointer
 325 *
 326 * Stop the compute async dma queues (CIK).
 327 */
 328static void cik_sdma_rlc_stop(struct amdgpu_device *adev)
 329{
 330	/* XXX todo */
 331}
 332
 333/**
 334 * cik_ctx_switch_enable - stop the async dma engines context switch
 335 *
 336 * @adev: amdgpu_device pointer
 337 * @enable: enable/disable the DMA MEs context switch.
 338 *
 339 * Halt or unhalt the async dma engines context switch (VI).
 340 */
 341static void cik_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 342{
 343	u32 f32_cntl, phase_quantum = 0;
 344	int i;
 345
 346	if (amdgpu_sdma_phase_quantum) {
 347		unsigned value = amdgpu_sdma_phase_quantum;
 348		unsigned unit = 0;
 349
 350		while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
 351				SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
 352			value = (value + 1) >> 1;
 353			unit++;
 354		}
 355		if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
 356			    SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
 357			value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
 358				 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
 359			unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
 360				SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
 361			WARN_ONCE(1,
 362			"clamping sdma_phase_quantum to %uK clock cycles\n",
 363				  value << unit);
 364		}
 365		phase_quantum =
 366			value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
 367			unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
 368	}
 369
 370	for (i = 0; i < adev->sdma.num_instances; i++) {
 371		f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]);
 372		if (enable) {
 373			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 374					AUTO_CTXSW_ENABLE, 1);
 375			if (amdgpu_sdma_phase_quantum) {
 376				WREG32(mmSDMA0_PHASE0_QUANTUM + sdma_offsets[i],
 377				       phase_quantum);
 378				WREG32(mmSDMA0_PHASE1_QUANTUM + sdma_offsets[i],
 379				       phase_quantum);
 380			}
 381		} else {
 382			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 383					AUTO_CTXSW_ENABLE, 0);
 384		}
 385
 386		WREG32(mmSDMA0_CNTL + sdma_offsets[i], f32_cntl);
 387	}
 388}
 389
 390/**
 391 * cik_sdma_enable - stop the async dma engines
 392 *
 393 * @adev: amdgpu_device pointer
 394 * @enable: enable/disable the DMA MEs.
 395 *
 396 * Halt or unhalt the async dma engines (CIK).
 397 */
 398static void cik_sdma_enable(struct amdgpu_device *adev, bool enable)
 399{
 400	u32 me_cntl;
 401	int i;
 402
 403	if (!enable) {
 404		cik_sdma_gfx_stop(adev);
 405		cik_sdma_rlc_stop(adev);
 406	}
 407
 408	for (i = 0; i < adev->sdma.num_instances; i++) {
 409		me_cntl = RREG32(mmSDMA0_F32_CNTL + sdma_offsets[i]);
 410		if (enable)
 411			me_cntl &= ~SDMA0_F32_CNTL__HALT_MASK;
 412		else
 413			me_cntl |= SDMA0_F32_CNTL__HALT_MASK;
 414		WREG32(mmSDMA0_F32_CNTL + sdma_offsets[i], me_cntl);
 415	}
 416}
 417
 418/**
 419 * cik_sdma_gfx_resume - setup and start the async dma engines
 420 *
 421 * @adev: amdgpu_device pointer
 422 *
 423 * Set up the gfx DMA ring buffers and enable them (CIK).
 424 * Returns 0 for success, error for failure.
 425 */
 426static int cik_sdma_gfx_resume(struct amdgpu_device *adev)
 427{
 428	struct amdgpu_ring *ring;
 429	u32 rb_cntl, ib_cntl;
 430	u32 rb_bufsz;
 431	int i, j, r;
 432
 433	for (i = 0; i < adev->sdma.num_instances; i++) {
 434		ring = &adev->sdma.instance[i].ring;
 435
 436		mutex_lock(&adev->srbm_mutex);
 437		for (j = 0; j < 16; j++) {
 438			cik_srbm_select(adev, 0, 0, 0, j);
 439			/* SDMA GFX */
 440			WREG32(mmSDMA0_GFX_VIRTUAL_ADDR + sdma_offsets[i], 0);
 441			WREG32(mmSDMA0_GFX_APE1_CNTL + sdma_offsets[i], 0);
 442			/* XXX SDMA RLC - todo */
 443		}
 444		cik_srbm_select(adev, 0, 0, 0, 0);
 445		mutex_unlock(&adev->srbm_mutex);
 446
 447		WREG32(mmSDMA0_TILING_CONFIG + sdma_offsets[i],
 448		       adev->gfx.config.gb_addr_config & 0x70);
 449
 450		WREG32(mmSDMA0_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0);
 451		WREG32(mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0);
 452
 453		/* Set ring buffer size in dwords */
 454		rb_bufsz = order_base_2(ring->ring_size / 4);
 455		rb_cntl = rb_bufsz << 1;
 456#ifdef __BIG_ENDIAN
 457		rb_cntl |= SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK |
 458			SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK;
 459#endif
 460		WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
 461
 462		/* Initialize the ring buffer's read and write pointers */
 463		WREG32(mmSDMA0_GFX_RB_RPTR + sdma_offsets[i], 0);
 464		WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], 0);
 465		WREG32(mmSDMA0_GFX_IB_RPTR + sdma_offsets[i], 0);
 466		WREG32(mmSDMA0_GFX_IB_OFFSET + sdma_offsets[i], 0);
 467
 468		/* set the wb address whether it's enabled or not */
 469		WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i],
 470		       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
 471		WREG32(mmSDMA0_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i],
 472		       ((ring->rptr_gpu_addr) & 0xFFFFFFFC));
 473
 474		rb_cntl |= SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK;
 475
 476		WREG32(mmSDMA0_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8);
 477		WREG32(mmSDMA0_GFX_RB_BASE_HI + sdma_offsets[i], ring->gpu_addr >> 40);
 478
 479		ring->wptr = 0;
 480		WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], ring->wptr << 2);
 481
 482		/* enable DMA RB */
 483		WREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i],
 484		       rb_cntl | SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK);
 485
 486		ib_cntl = SDMA0_GFX_IB_CNTL__IB_ENABLE_MASK;
 487#ifdef __BIG_ENDIAN
 488		ib_cntl |= SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK;
 489#endif
 490		/* enable DMA IBs */
 491		WREG32(mmSDMA0_GFX_IB_CNTL + sdma_offsets[i], ib_cntl);
 492	}
 493
 494	cik_sdma_enable(adev, true);
 495
 496	for (i = 0; i < adev->sdma.num_instances; i++) {
 497		ring = &adev->sdma.instance[i].ring;
 498		r = amdgpu_ring_test_helper(ring);
 499		if (r)
 500			return r;
 501	}
 502
 503	return 0;
 504}
 505
 506/**
 507 * cik_sdma_rlc_resume - setup and start the async dma engines
 508 *
 509 * @adev: amdgpu_device pointer
 510 *
 511 * Set up the compute DMA queues and enable them (CIK).
 512 * Returns 0 for success, error for failure.
 513 */
 514static int cik_sdma_rlc_resume(struct amdgpu_device *adev)
 515{
 516	/* XXX todo */
 517	return 0;
 518}
 519
 520/**
 521 * cik_sdma_load_microcode - load the sDMA ME ucode
 522 *
 523 * @adev: amdgpu_device pointer
 524 *
 525 * Loads the sDMA0/1 ucode.
 526 * Returns 0 for success, -EINVAL if the ucode is not available.
 527 */
 528static int cik_sdma_load_microcode(struct amdgpu_device *adev)
 529{
 530	const struct sdma_firmware_header_v1_0 *hdr;
 531	const __le32 *fw_data;
 532	u32 fw_size;
 533	int i, j;
 534
 535	/* halt the MEs */
 536	cik_sdma_enable(adev, false);
 537
 538	for (i = 0; i < adev->sdma.num_instances; i++) {
 539		if (!adev->sdma.instance[i].fw)
 540			return -EINVAL;
 541		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
 542		amdgpu_ucode_print_sdma_hdr(&hdr->header);
 543		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
 544		adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
 545		adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
 546		if (adev->sdma.instance[i].feature_version >= 20)
 547			adev->sdma.instance[i].burst_nop = true;
 548		fw_data = (const __le32 *)
 549			(adev->sdma.instance[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 550		WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0);
 551		for (j = 0; j < fw_size; j++)
 552			WREG32(mmSDMA0_UCODE_DATA + sdma_offsets[i], le32_to_cpup(fw_data++));
 553		WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version);
 554	}
 555
 556	return 0;
 557}
 558
 559/**
 560 * cik_sdma_start - setup and start the async dma engines
 561 *
 562 * @adev: amdgpu_device pointer
 563 *
 564 * Set up the DMA engines and enable them (CIK).
 565 * Returns 0 for success, error for failure.
 566 */
 567static int cik_sdma_start(struct amdgpu_device *adev)
 568{
 569	int r;
 570
 571	r = cik_sdma_load_microcode(adev);
 572	if (r)
 573		return r;
 574
 575	/* halt the engine before programing */
 576	cik_sdma_enable(adev, false);
 577	/* enable sdma ring preemption */
 578	cik_ctx_switch_enable(adev, true);
 579
 580	/* start the gfx rings and rlc compute queues */
 581	r = cik_sdma_gfx_resume(adev);
 582	if (r)
 583		return r;
 584	r = cik_sdma_rlc_resume(adev);
 585	if (r)
 586		return r;
 587
 588	return 0;
 589}
 590
 591/**
 592 * cik_sdma_ring_test_ring - simple async dma engine test
 593 *
 594 * @ring: amdgpu_ring structure holding ring information
 595 *
 596 * Test the DMA engine by writing using it to write an
 597 * value to memory. (CIK).
 598 * Returns 0 for success, error for failure.
 599 */
 600static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 601{
 602	struct amdgpu_device *adev = ring->adev;
 603	unsigned i;
 604	unsigned index;
 605	int r;
 606	u32 tmp;
 607	u64 gpu_addr;
 608
 609	r = amdgpu_device_wb_get(adev, &index);
 610	if (r)
 611		return r;
 612
 613	gpu_addr = adev->wb.gpu_addr + (index * 4);
 614	tmp = 0xCAFEDEAD;
 615	adev->wb.wb[index] = cpu_to_le32(tmp);
 616
 617	r = amdgpu_ring_alloc(ring, 5);
 618	if (r)
 619		goto error_free_wb;
 620
 621	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
 622	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
 623	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
 624	amdgpu_ring_write(ring, 1); /* number of DWs to follow */
 625	amdgpu_ring_write(ring, 0xDEADBEEF);
 626	amdgpu_ring_commit(ring);
 627
 628	for (i = 0; i < adev->usec_timeout; i++) {
 629		tmp = le32_to_cpu(adev->wb.wb[index]);
 630		if (tmp == 0xDEADBEEF)
 631			break;
 632		udelay(1);
 633	}
 634
 635	if (i >= adev->usec_timeout)
 636		r = -ETIMEDOUT;
 637
 638error_free_wb:
 639	amdgpu_device_wb_free(adev, index);
 640	return r;
 641}
 642
 643/**
 644 * cik_sdma_ring_test_ib - test an IB on the DMA engine
 645 *
 646 * @ring: amdgpu_ring structure holding ring information
 647 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 648 *
 649 * Test a simple IB in the DMA ring (CIK).
 650 * Returns 0 on success, error on failure.
 651 */
 652static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 653{
 654	struct amdgpu_device *adev = ring->adev;
 655	struct amdgpu_ib ib;
 656	struct dma_fence *f = NULL;
 657	unsigned index;
 658	u32 tmp = 0;
 659	u64 gpu_addr;
 660	long r;
 661
 662	r = amdgpu_device_wb_get(adev, &index);
 663	if (r)
 664		return r;
 665
 666	gpu_addr = adev->wb.gpu_addr + (index * 4);
 667	tmp = 0xCAFEDEAD;
 668	adev->wb.wb[index] = cpu_to_le32(tmp);
 669	memset(&ib, 0, sizeof(ib));
 670	r = amdgpu_ib_get(adev, NULL, 256,
 671					AMDGPU_IB_POOL_DIRECT, &ib);
 672	if (r)
 673		goto err0;
 674
 675	ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE,
 676				SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
 677	ib.ptr[1] = lower_32_bits(gpu_addr);
 678	ib.ptr[2] = upper_32_bits(gpu_addr);
 679	ib.ptr[3] = 1;
 680	ib.ptr[4] = 0xDEADBEEF;
 681	ib.length_dw = 5;
 682	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
 683	if (r)
 684		goto err1;
 685
 686	r = dma_fence_wait_timeout(f, false, timeout);
 687	if (r == 0) {
 688		r = -ETIMEDOUT;
 689		goto err1;
 690	} else if (r < 0) {
 691		goto err1;
 692	}
 693	tmp = le32_to_cpu(adev->wb.wb[index]);
 694	if (tmp == 0xDEADBEEF)
 695		r = 0;
 696	else
 697		r = -EINVAL;
 698
 699err1:
 700	amdgpu_ib_free(&ib, NULL);
 701	dma_fence_put(f);
 702err0:
 703	amdgpu_device_wb_free(adev, index);
 704	return r;
 705}
 706
 707/**
 708 * cik_sdma_vm_copy_pte - update PTEs by copying them from the GART
 709 *
 710 * @ib: indirect buffer to fill with commands
 711 * @pe: addr of the page entry
 712 * @src: src addr to copy from
 713 * @count: number of page entries to update
 714 *
 715 * Update PTEs by copying them from the GART using sDMA (CIK).
 716 */
 717static void cik_sdma_vm_copy_pte(struct amdgpu_ib *ib,
 718				 uint64_t pe, uint64_t src,
 719				 unsigned count)
 720{
 721	unsigned bytes = count * 8;
 722
 723	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY,
 724		SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
 725	ib->ptr[ib->length_dw++] = bytes;
 726	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
 727	ib->ptr[ib->length_dw++] = lower_32_bits(src);
 728	ib->ptr[ib->length_dw++] = upper_32_bits(src);
 729	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
 730	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 731}
 732
 733/**
 734 * cik_sdma_vm_write_pte - update PTEs by writing them manually
 735 *
 736 * @ib: indirect buffer to fill with commands
 737 * @pe: addr of the page entry
 738 * @value: dst addr to write into pe
 739 * @count: number of page entries to update
 740 * @incr: increase next addr by incr bytes
 741 *
 742 * Update PTEs by writing them manually using sDMA (CIK).
 743 */
 744static void cik_sdma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
 745				  uint64_t value, unsigned count,
 746				  uint32_t incr)
 747{
 748	unsigned ndw = count * 2;
 749
 750	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_WRITE,
 751		SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
 752	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
 753	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 754	ib->ptr[ib->length_dw++] = ndw;
 755	for (; ndw > 0; ndw -= 2) {
 756		ib->ptr[ib->length_dw++] = lower_32_bits(value);
 757		ib->ptr[ib->length_dw++] = upper_32_bits(value);
 758		value += incr;
 759	}
 760}
 761
 762/**
 763 * cik_sdma_vm_set_pte_pde - update the page tables using sDMA
 764 *
 765 * @ib: indirect buffer to fill with commands
 766 * @pe: addr of the page entry
 767 * @addr: dst addr to write into pe
 768 * @count: number of page entries to update
 769 * @incr: increase next addr by incr bytes
 770 * @flags: access flags
 771 *
 772 * Update the page tables using sDMA (CIK).
 773 */
 774static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,
 775				    uint64_t addr, unsigned count,
 776				    uint32_t incr, uint64_t flags)
 777{
 778	/* for physically contiguous pages (vram) */
 779	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
 780	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
 781	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 782	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
 783	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
 784	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
 785	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
 786	ib->ptr[ib->length_dw++] = incr; /* increment size */
 787	ib->ptr[ib->length_dw++] = 0;
 788	ib->ptr[ib->length_dw++] = count; /* number of entries */
 789}
 790
 791/**
 792 * cik_sdma_ring_pad_ib - pad the IB to the required number of dw
 793 *
 794 * @ring: amdgpu_ring structure holding ring information
 795 * @ib: indirect buffer to fill with padding
 796 *
 797 */
 798static void cik_sdma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 799{
 800	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
 801	u32 pad_count;
 802	int i;
 803
 804	pad_count = (-ib->length_dw) & 7;
 805	for (i = 0; i < pad_count; i++)
 806		if (sdma && sdma->burst_nop && (i == 0))
 807			ib->ptr[ib->length_dw++] =
 808					SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0) |
 809					SDMA_NOP_COUNT(pad_count - 1);
 810		else
 811			ib->ptr[ib->length_dw++] =
 812					SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0);
 813}
 814
 815/**
 816 * cik_sdma_ring_emit_pipeline_sync - sync the pipeline
 817 *
 818 * @ring: amdgpu_ring pointer
 819 *
 820 * Make sure all previous operations are completed (CIK).
 821 */
 822static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 823{
 824	uint32_t seq = ring->fence_drv.sync_seq;
 825	uint64_t addr = ring->fence_drv.gpu_addr;
 826
 827	/* wait for idle */
 828	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0,
 829					    SDMA_POLL_REG_MEM_EXTRA_OP(0) |
 830					    SDMA_POLL_REG_MEM_EXTRA_FUNC(3) | /* equal */
 831					    SDMA_POLL_REG_MEM_EXTRA_M));
 832	amdgpu_ring_write(ring, addr & 0xfffffffc);
 833	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
 834	amdgpu_ring_write(ring, seq); /* reference */
 835	amdgpu_ring_write(ring, 0xffffffff); /* mask */
 836	amdgpu_ring_write(ring, (0xfff << 16) | 4); /* retry count, poll interval */
 837}
 838
 839/**
 840 * cik_sdma_ring_emit_vm_flush - cik vm flush using sDMA
 841 *
 842 * @ring: amdgpu_ring pointer
 843 * @vmid: vmid number to use
 844 * @pd_addr: address
 845 *
 846 * Update the page table base and flush the VM TLB
 847 * using sDMA (CIK).
 848 */
 849static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
 850					unsigned vmid, uint64_t pd_addr)
 851{
 852	u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
 853			  SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
 854
 855	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 856
 857	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
 858	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
 859	amdgpu_ring_write(ring, 0);
 860	amdgpu_ring_write(ring, 0); /* reference */
 861	amdgpu_ring_write(ring, 0); /* mask */
 862	amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
 863}
 864
 865static void cik_sdma_ring_emit_wreg(struct amdgpu_ring *ring,
 866				    uint32_t reg, uint32_t val)
 867{
 868	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 869	amdgpu_ring_write(ring, reg);
 870	amdgpu_ring_write(ring, val);
 871}
 872
 873static void cik_enable_sdma_mgcg(struct amdgpu_device *adev,
 874				 bool enable)
 875{
 876	u32 orig, data;
 877
 878	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
 879		WREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
 880		WREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
 881	} else {
 882		orig = data = RREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
 883		data |= 0xff000000;
 884		if (data != orig)
 885			WREG32(mmSDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
 886
 887		orig = data = RREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
 888		data |= 0xff000000;
 889		if (data != orig)
 890			WREG32(mmSDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
 891	}
 892}
 893
 894static void cik_enable_sdma_mgls(struct amdgpu_device *adev,
 895				 bool enable)
 896{
 897	u32 orig, data;
 898
 899	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
 900		orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
 901		data |= 0x100;
 902		if (orig != data)
 903			WREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
 904
 905		orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
 906		data |= 0x100;
 907		if (orig != data)
 908			WREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
 909	} else {
 910		orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
 911		data &= ~0x100;
 912		if (orig != data)
 913			WREG32(mmSDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
 914
 915		orig = data = RREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
 916		data &= ~0x100;
 917		if (orig != data)
 918			WREG32(mmSDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
 919	}
 920}
 921
 922static int cik_sdma_early_init(struct amdgpu_ip_block *ip_block)
 923{
 924	struct amdgpu_device *adev = ip_block->adev;
 925	int r;
 926
 927	adev->sdma.num_instances = SDMA_MAX_INSTANCE;
 928
 929	r = cik_sdma_init_microcode(adev);
 930	if (r)
 931		return r;
 932
 933	cik_sdma_set_ring_funcs(adev);
 934	cik_sdma_set_irq_funcs(adev);
 935	cik_sdma_set_buffer_funcs(adev);
 936	cik_sdma_set_vm_pte_funcs(adev);
 937
 938	return 0;
 939}
 940
 941static int cik_sdma_sw_init(struct amdgpu_ip_block *ip_block)
 942{
 943	struct amdgpu_ring *ring;
 944	struct amdgpu_device *adev = ip_block->adev;
 945	int r, i;
 946
 947	/* SDMA trap event */
 948	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224,
 949			      &adev->sdma.trap_irq);
 950	if (r)
 951		return r;
 952
 953	/* SDMA Privileged inst */
 954	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 241,
 955			      &adev->sdma.illegal_inst_irq);
 956	if (r)
 957		return r;
 958
 959	/* SDMA Privileged inst */
 960	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 247,
 961			      &adev->sdma.illegal_inst_irq);
 962	if (r)
 963		return r;
 964
 965	for (i = 0; i < adev->sdma.num_instances; i++) {
 966		ring = &adev->sdma.instance[i].ring;
 967		ring->ring_obj = NULL;
 968		sprintf(ring->name, "sdma%d", i);
 969		r = amdgpu_ring_init(adev, ring, 1024,
 970				     &adev->sdma.trap_irq,
 971				     (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :
 972				     AMDGPU_SDMA_IRQ_INSTANCE1,
 973				     AMDGPU_RING_PRIO_DEFAULT, NULL);
 974		if (r)
 975			return r;
 976	}
 977
 978	return r;
 979}
 980
 981static int cik_sdma_sw_fini(struct amdgpu_ip_block *ip_block)
 982{
 983	struct amdgpu_device *adev = ip_block->adev;
 984	int i;
 985
 986	for (i = 0; i < adev->sdma.num_instances; i++)
 987		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 988
 989	cik_sdma_free_microcode(adev);
 990	return 0;
 991}
 992
 993static int cik_sdma_hw_init(struct amdgpu_ip_block *ip_block)
 994{
 995	struct amdgpu_device *adev = ip_block->adev;
 996
 997	return cik_sdma_start(adev);
 998}
 999
1000static int cik_sdma_hw_fini(struct amdgpu_ip_block *ip_block)
1001{
1002	struct amdgpu_device *adev = ip_block->adev;
1003
1004	cik_ctx_switch_enable(adev, false);
1005	cik_sdma_enable(adev, false);
1006
1007	return 0;
1008}
1009
1010static int cik_sdma_suspend(struct amdgpu_ip_block *ip_block)
1011{
1012	return cik_sdma_hw_fini(ip_block);
1013}
1014
1015static int cik_sdma_resume(struct amdgpu_ip_block *ip_block)
1016{
1017	cik_sdma_soft_reset(ip_block);
1018
1019	return cik_sdma_hw_init(ip_block);
1020}
1021
1022static bool cik_sdma_is_idle(struct amdgpu_ip_block *ip_block)
1023{
1024	struct amdgpu_device *adev = ip_block->adev;
1025	u32 tmp = RREG32(mmSRBM_STATUS2);
1026
1027	if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK |
1028				SRBM_STATUS2__SDMA1_BUSY_MASK))
1029	    return false;
1030
1031	return true;
1032}
1033
1034static int cik_sdma_wait_for_idle(struct amdgpu_ip_block *ip_block)
1035{
1036	unsigned i;
1037	struct amdgpu_device *adev = ip_block->adev;
1038
1039	for (i = 0; i < adev->usec_timeout; i++) {
1040		if (cik_sdma_is_idle(ip_block))
1041			return 0;
1042		udelay(1);
1043	}
1044	return -ETIMEDOUT;
1045}
1046
1047static int cik_sdma_soft_reset(struct amdgpu_ip_block *ip_block)
1048{
1049	u32 srbm_soft_reset = 0;
1050	struct amdgpu_device *adev = ip_block->adev;
1051	u32 tmp;
1052
1053	/* sdma0 */
1054	tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1055	tmp |= SDMA0_F32_CNTL__HALT_MASK;
1056	WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1057	srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;
1058
1059	/* sdma1 */
1060	tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1061	tmp |= SDMA0_F32_CNTL__HALT_MASK;
1062	WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1063	srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
1064
1065	if (srbm_soft_reset) {
1066		tmp = RREG32(mmSRBM_SOFT_RESET);
1067		tmp |= srbm_soft_reset;
1068		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1069		WREG32(mmSRBM_SOFT_RESET, tmp);
1070		tmp = RREG32(mmSRBM_SOFT_RESET);
1071
1072		udelay(50);
1073
1074		tmp &= ~srbm_soft_reset;
1075		WREG32(mmSRBM_SOFT_RESET, tmp);
1076		tmp = RREG32(mmSRBM_SOFT_RESET);
1077
1078		/* Wait a little for things to settle down */
1079		udelay(50);
1080	}
1081
1082	return 0;
1083}
1084
1085static int cik_sdma_set_trap_irq_state(struct amdgpu_device *adev,
1086				       struct amdgpu_irq_src *src,
1087				       unsigned type,
1088				       enum amdgpu_interrupt_state state)
1089{
1090	u32 sdma_cntl;
1091
1092	switch (type) {
1093	case AMDGPU_SDMA_IRQ_INSTANCE0:
1094		switch (state) {
1095		case AMDGPU_IRQ_STATE_DISABLE:
1096			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);
1097			sdma_cntl &= ~SDMA0_CNTL__TRAP_ENABLE_MASK;
1098			WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl);
1099			break;
1100		case AMDGPU_IRQ_STATE_ENABLE:
1101			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);
1102			sdma_cntl |= SDMA0_CNTL__TRAP_ENABLE_MASK;
1103			WREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET, sdma_cntl);
1104			break;
1105		default:
1106			break;
1107		}
1108		break;
1109	case AMDGPU_SDMA_IRQ_INSTANCE1:
1110		switch (state) {
1111		case AMDGPU_IRQ_STATE_DISABLE:
1112			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);
1113			sdma_cntl &= ~SDMA0_CNTL__TRAP_ENABLE_MASK;
1114			WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl);
1115			break;
1116		case AMDGPU_IRQ_STATE_ENABLE:
1117			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);
1118			sdma_cntl |= SDMA0_CNTL__TRAP_ENABLE_MASK;
1119			WREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET, sdma_cntl);
1120			break;
1121		default:
1122			break;
1123		}
1124		break;
1125	default:
1126		break;
1127	}
1128	return 0;
1129}
1130
1131static int cik_sdma_process_trap_irq(struct amdgpu_device *adev,
1132				     struct amdgpu_irq_src *source,
1133				     struct amdgpu_iv_entry *entry)
1134{
1135	u8 instance_id, queue_id;
1136
1137	instance_id = (entry->ring_id & 0x3) >> 0;
1138	queue_id = (entry->ring_id & 0xc) >> 2;
1139	DRM_DEBUG("IH: SDMA trap\n");
1140	switch (instance_id) {
1141	case 0:
1142		switch (queue_id) {
1143		case 0:
1144			amdgpu_fence_process(&adev->sdma.instance[0].ring);
1145			break;
1146		case 1:
1147			/* XXX compute */
1148			break;
1149		case 2:
1150			/* XXX compute */
1151			break;
1152		}
1153		break;
1154	case 1:
1155		switch (queue_id) {
1156		case 0:
1157			amdgpu_fence_process(&adev->sdma.instance[1].ring);
1158			break;
1159		case 1:
1160			/* XXX compute */
1161			break;
1162		case 2:
1163			/* XXX compute */
1164			break;
1165		}
1166		break;
1167	}
1168
1169	return 0;
1170}
1171
1172static int cik_sdma_process_illegal_inst_irq(struct amdgpu_device *adev,
1173					     struct amdgpu_irq_src *source,
1174					     struct amdgpu_iv_entry *entry)
1175{
1176	u8 instance_id;
1177
1178	DRM_ERROR("Illegal instruction in SDMA command stream\n");
1179	instance_id = (entry->ring_id & 0x3) >> 0;
1180	drm_sched_fault(&adev->sdma.instance[instance_id].ring.sched);
1181	return 0;
1182}
1183
1184static int cik_sdma_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1185					  enum amd_clockgating_state state)
1186{
1187	bool gate = false;
1188	struct amdgpu_device *adev = ip_block->adev;
1189
1190	if (state == AMD_CG_STATE_GATE)
1191		gate = true;
1192
1193	cik_enable_sdma_mgcg(adev, gate);
1194	cik_enable_sdma_mgls(adev, gate);
1195
1196	return 0;
1197}
1198
1199static int cik_sdma_set_powergating_state(struct amdgpu_ip_block *ip_block,
1200					  enum amd_powergating_state state)
1201{
1202	return 0;
1203}
1204
1205static const struct amd_ip_funcs cik_sdma_ip_funcs = {
1206	.name = "cik_sdma",
1207	.early_init = cik_sdma_early_init,
1208	.sw_init = cik_sdma_sw_init,
1209	.sw_fini = cik_sdma_sw_fini,
1210	.hw_init = cik_sdma_hw_init,
1211	.hw_fini = cik_sdma_hw_fini,
1212	.suspend = cik_sdma_suspend,
1213	.resume = cik_sdma_resume,
1214	.is_idle = cik_sdma_is_idle,
1215	.wait_for_idle = cik_sdma_wait_for_idle,
1216	.soft_reset = cik_sdma_soft_reset,
1217	.set_clockgating_state = cik_sdma_set_clockgating_state,
1218	.set_powergating_state = cik_sdma_set_powergating_state,
1219};
1220
1221static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
1222	.type = AMDGPU_RING_TYPE_SDMA,
1223	.align_mask = 0xf,
1224	.nop = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0),
1225	.support_64bit_ptrs = false,
1226	.get_rptr = cik_sdma_ring_get_rptr,
1227	.get_wptr = cik_sdma_ring_get_wptr,
1228	.set_wptr = cik_sdma_ring_set_wptr,
1229	.emit_frame_size =
1230		6 + /* cik_sdma_ring_emit_hdp_flush */
1231		3 + /* hdp invalidate */
1232		6 + /* cik_sdma_ring_emit_pipeline_sync */
1233		CIK_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* cik_sdma_ring_emit_vm_flush */
1234		9 + 9 + 9, /* cik_sdma_ring_emit_fence x3 for user fence, vm fence */
1235	.emit_ib_size = 7 + 4, /* cik_sdma_ring_emit_ib */
1236	.emit_ib = cik_sdma_ring_emit_ib,
1237	.emit_fence = cik_sdma_ring_emit_fence,
1238	.emit_pipeline_sync = cik_sdma_ring_emit_pipeline_sync,
1239	.emit_vm_flush = cik_sdma_ring_emit_vm_flush,
1240	.emit_hdp_flush = cik_sdma_ring_emit_hdp_flush,
1241	.test_ring = cik_sdma_ring_test_ring,
1242	.test_ib = cik_sdma_ring_test_ib,
1243	.insert_nop = cik_sdma_ring_insert_nop,
1244	.pad_ib = cik_sdma_ring_pad_ib,
1245	.emit_wreg = cik_sdma_ring_emit_wreg,
1246};
1247
1248static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)
1249{
1250	int i;
1251
1252	for (i = 0; i < adev->sdma.num_instances; i++) {
1253		adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs;
1254		adev->sdma.instance[i].ring.me = i;
1255	}
1256}
1257
1258static const struct amdgpu_irq_src_funcs cik_sdma_trap_irq_funcs = {
1259	.set = cik_sdma_set_trap_irq_state,
1260	.process = cik_sdma_process_trap_irq,
1261};
1262
1263static const struct amdgpu_irq_src_funcs cik_sdma_illegal_inst_irq_funcs = {
1264	.process = cik_sdma_process_illegal_inst_irq,
1265};
1266
1267static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev)
1268{
1269	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
1270	adev->sdma.trap_irq.funcs = &cik_sdma_trap_irq_funcs;
1271	adev->sdma.illegal_inst_irq.funcs = &cik_sdma_illegal_inst_irq_funcs;
1272}
1273
1274/**
1275 * cik_sdma_emit_copy_buffer - copy buffer using the sDMA engine
1276 *
1277 * @ib: indirect buffer to copy to
1278 * @src_offset: src GPU address
1279 * @dst_offset: dst GPU address
1280 * @byte_count: number of bytes to xfer
1281 * @copy_flags: unused
1282 *
1283 * Copy GPU buffers using the DMA engine (CIK).
1284 * Used by the amdgpu ttm implementation to move pages if
1285 * registered as the asic copy callback.
1286 */
1287static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib,
1288				      uint64_t src_offset,
1289				      uint64_t dst_offset,
1290				      uint32_t byte_count,
1291				      uint32_t copy_flags)
1292{
1293	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0);
1294	ib->ptr[ib->length_dw++] = byte_count;
1295	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1296	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1297	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1298	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1299	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1300}
1301
1302/**
1303 * cik_sdma_emit_fill_buffer - fill buffer using the sDMA engine
1304 *
1305 * @ib: indirect buffer to fill
1306 * @src_data: value to write to buffer
1307 * @dst_offset: dst GPU address
1308 * @byte_count: number of bytes to xfer
1309 *
1310 * Fill GPU buffers using the DMA engine (CIK).
1311 */
1312static void cik_sdma_emit_fill_buffer(struct amdgpu_ib *ib,
1313				      uint32_t src_data,
1314				      uint64_t dst_offset,
1315				      uint32_t byte_count)
1316{
1317	ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0);
1318	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1319	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1320	ib->ptr[ib->length_dw++] = src_data;
1321	ib->ptr[ib->length_dw++] = byte_count;
1322}
1323
1324static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = {
1325	.copy_max_bytes = 0x1fffff,
1326	.copy_num_dw = 7,
1327	.emit_copy_buffer = cik_sdma_emit_copy_buffer,
1328
1329	.fill_max_bytes = 0x1fffff,
1330	.fill_num_dw = 5,
1331	.emit_fill_buffer = cik_sdma_emit_fill_buffer,
1332};
1333
1334static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
1335{
1336	adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
1337	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
1338}
1339
1340static const struct amdgpu_vm_pte_funcs cik_sdma_vm_pte_funcs = {
1341	.copy_pte_num_dw = 7,
1342	.copy_pte = cik_sdma_vm_copy_pte,
1343
1344	.write_pte = cik_sdma_vm_write_pte,
1345	.set_pte_pde = cik_sdma_vm_set_pte_pde,
1346};
1347
1348static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev)
1349{
1350	unsigned i;
1351
1352	adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;
1353	for (i = 0; i < adev->sdma.num_instances; i++) {
1354		adev->vm_manager.vm_pte_scheds[i] =
1355			&adev->sdma.instance[i].ring.sched;
1356	}
1357	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
1358}
1359
1360const struct amdgpu_ip_block_version cik_sdma_ip_block =
1361{
1362	.type = AMD_IP_BLOCK_TYPE_SDMA,
1363	.major = 2,
1364	.minor = 0,
1365	.rev = 0,
1366	.funcs = &cik_sdma_ip_funcs,
1367};