File Index
Files and directories
Sign in
FILE

mes_v12_0.c

File preview with syntax highlighting.

linux-6.19.14/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c C 
   1/*
   2 * Copyright 2023 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 */
  23
  24#include <linux/firmware.h>
  25#include <linux/module.h>
  26#include "amdgpu.h"
  27#include "gfx_v12_0.h"
  28#include "soc15_common.h"
  29#include "soc21.h"
  30#include "gc/gc_12_0_0_offset.h"
  31#include "gc/gc_12_0_0_sh_mask.h"
  32#include "gc/gc_11_0_0_default.h"
  33#include "v12_structs.h"
  34#include "mes_v12_api_def.h"
  35
  36MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes.bin");
  37MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes1.bin");
  38MODULE_FIRMWARE("amdgpu/gc_12_0_0_uni_mes.bin");
  39MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes.bin");
  40MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes1.bin");
  41MODULE_FIRMWARE("amdgpu/gc_12_0_1_uni_mes.bin");
  42
  43static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block);
  44static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block);
  45static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev);
  46static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev);
  47
  48#define MES_EOP_SIZE   2048
  49
  50#define MES12_HUNG_DB_OFFSET_ARRAY_SIZE 8 /* [0:3] = db offset [4:7] hqd info */
  51#define MES12_HUNG_HQD_INFO_OFFSET	4
  52
  53static void mes_v12_0_ring_set_wptr(struct amdgpu_ring *ring)
  54{
  55	struct amdgpu_device *adev = ring->adev;
  56
  57	if (ring->use_doorbell) {
  58		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
  59			     ring->wptr);
  60		WDOORBELL64(ring->doorbell_index, ring->wptr);
  61	} else {
  62		BUG();
  63	}
  64}
  65
  66static u64 mes_v12_0_ring_get_rptr(struct amdgpu_ring *ring)
  67{
  68	return *ring->rptr_cpu_addr;
  69}
  70
  71static u64 mes_v12_0_ring_get_wptr(struct amdgpu_ring *ring)
  72{
  73	u64 wptr;
  74
  75	if (ring->use_doorbell)
  76		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
  77	else
  78		BUG();
  79	return wptr;
  80}
  81
  82static const struct amdgpu_ring_funcs mes_v12_0_ring_funcs = {
  83	.type = AMDGPU_RING_TYPE_MES,
  84	.align_mask = 1,
  85	.nop = 0,
  86	.support_64bit_ptrs = true,
  87	.get_rptr = mes_v12_0_ring_get_rptr,
  88	.get_wptr = mes_v12_0_ring_get_wptr,
  89	.set_wptr = mes_v12_0_ring_set_wptr,
  90	.insert_nop = amdgpu_ring_insert_nop,
  91};
  92
  93static const char *mes_v12_0_opcodes[] = {
  94	"SET_HW_RSRC",
  95	"SET_SCHEDULING_CONFIG",
  96	"ADD_QUEUE",
  97	"REMOVE_QUEUE",
  98	"PERFORM_YIELD",
  99	"SET_GANG_PRIORITY_LEVEL",
 100	"SUSPEND",
 101	"RESUME",
 102	"RESET",
 103	"SET_LOG_BUFFER",
 104	"CHANGE_GANG_PRORITY",
 105	"QUERY_SCHEDULER_STATUS",
 106	"unused",
 107	"SET_DEBUG_VMID",
 108	"MISC",
 109	"UPDATE_ROOT_PAGE_TABLE",
 110	"AMD_LOG",
 111	"SET_SE_MODE",
 112	"SET_GANG_SUBMIT",
 113	"SET_HW_RSRC_1",
 114	"INVALIDATE_TLBS",
 115};
 116
 117static const char *mes_v12_0_misc_opcodes[] = {
 118	"WRITE_REG",
 119	"INV_GART",
 120	"QUERY_STATUS",
 121	"READ_REG",
 122	"WAIT_REG_MEM",
 123	"SET_SHADER_DEBUGGER",
 124	"NOTIFY_WORK_ON_UNMAPPED_QUEUE",
 125	"NOTIFY_TO_UNMAP_PROCESSES",
 126};
 127
 128static const char *mes_v12_0_get_op_string(union MESAPI__MISC *x_pkt)
 129{
 130	const char *op_str = NULL;
 131
 132	if (x_pkt->header.opcode < ARRAY_SIZE(mes_v12_0_opcodes))
 133		op_str = mes_v12_0_opcodes[x_pkt->header.opcode];
 134
 135	return op_str;
 136}
 137
 138static const char *mes_v12_0_get_misc_op_string(union MESAPI__MISC *x_pkt)
 139{
 140	const char *op_str = NULL;
 141
 142	if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
 143	    (x_pkt->opcode < ARRAY_SIZE(mes_v12_0_misc_opcodes)))
 144		op_str = mes_v12_0_misc_opcodes[x_pkt->opcode];
 145
 146	return op_str;
 147}
 148
 149static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 150					    int pipe, void *pkt, int size,
 151					    int api_status_off)
 152{
 153	union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 154	signed long timeout = 2100000; /* 2100 ms */
 155	struct amdgpu_device *adev = mes->adev;
 156	struct amdgpu_ring *ring = &mes->ring[pipe];
 157	spinlock_t *ring_lock = &mes->ring_lock[pipe];
 158	struct MES_API_STATUS *api_status;
 159	union MESAPI__MISC *x_pkt = pkt;
 160	const char *op_str, *misc_op_str;
 161	unsigned long flags;
 162	u64 status_gpu_addr;
 163	u32 seq, status_offset;
 164	u64 *status_ptr;
 165	signed long r;
 166	int ret;
 167
 168	if (x_pkt->header.opcode >= MES_SCH_API_MAX)
 169		return -EINVAL;
 170
 171	if (amdgpu_emu_mode) {
 172		timeout *= 100;
 173	} else if (amdgpu_sriov_vf(adev)) {
 174		/* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
 175		timeout = 15 * 600 * 1000;
 176	}
 177
 178	ret = amdgpu_device_wb_get(adev, &status_offset);
 179	if (ret)
 180		return ret;
 181
 182	status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
 183	status_ptr = (u64 *)&adev->wb.wb[status_offset];
 184	*status_ptr = 0;
 185
 186	spin_lock_irqsave(ring_lock, flags);
 187	r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
 188	if (r)
 189		goto error_unlock_free;
 190
 191	seq = ++ring->fence_drv.sync_seq;
 192	r = amdgpu_fence_wait_polling(ring,
 193				      seq - ring->fence_drv.num_fences_mask,
 194				      timeout);
 195	if (r < 1)
 196		goto error_undo;
 197
 198	api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
 199	api_status->api_completion_fence_addr = status_gpu_addr;
 200	api_status->api_completion_fence_value = 1;
 201
 202	amdgpu_ring_write_multiple(ring, pkt, size / 4);
 203
 204	memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 205	mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 206	mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 207	mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 208	mes_status_pkt.api_status.api_completion_fence_addr =
 209		ring->fence_drv.gpu_addr;
 210	mes_status_pkt.api_status.api_completion_fence_value = seq;
 211
 212	amdgpu_ring_write_multiple(ring, &mes_status_pkt,
 213				   sizeof(mes_status_pkt) / 4);
 214
 215	amdgpu_ring_commit(ring);
 216	spin_unlock_irqrestore(ring_lock, flags);
 217
 218	op_str = mes_v12_0_get_op_string(x_pkt);
 219	misc_op_str = mes_v12_0_get_misc_op_string(x_pkt);
 220
 221	if (misc_op_str)
 222		dev_dbg(adev->dev, "MES(%d) msg=%s (%s) was emitted\n",
 223			pipe, op_str, misc_op_str);
 224	else if (op_str)
 225		dev_dbg(adev->dev, "MES(%d) msg=%s was emitted\n",
 226			pipe, op_str);
 227	else
 228		dev_dbg(adev->dev, "MES(%d) msg=%d was emitted\n",
 229			pipe, x_pkt->header.opcode);
 230
 231	r = amdgpu_fence_wait_polling(ring, seq, timeout);
 232
 233	/*
 234	 * status_ptr[31:0] == 0 (fail) or status_ptr[63:0] == 1 (success).
 235	 * If status_ptr[31:0] == 0 then status_ptr[63:32] will have debug error information.
 236	 */
 237	if (r < 1 || !(lower_32_bits(*status_ptr))) {
 238
 239		if (misc_op_str)
 240			dev_err(adev->dev, "MES(%d) failed to respond to msg=%s (%s)\n",
 241				pipe, op_str, misc_op_str);
 242		else if (op_str)
 243			dev_err(adev->dev, "MES(%d) failed to respond to msg=%s\n",
 244				pipe, op_str);
 245		else
 246			dev_err(adev->dev, "MES(%d) failed to respond to msg=%d\n",
 247				pipe, x_pkt->header.opcode);
 248
 249		while (halt_if_hws_hang)
 250			schedule();
 251
 252		r = -ETIMEDOUT;
 253		goto error_wb_free;
 254	}
 255
 256	amdgpu_device_wb_free(adev, status_offset);
 257	return 0;
 258
 259error_undo:
 260	dev_err(adev->dev, "MES ring buffer is full.\n");
 261	amdgpu_ring_undo(ring);
 262
 263error_unlock_free:
 264	spin_unlock_irqrestore(ring_lock, flags);
 265
 266error_wb_free:
 267	amdgpu_device_wb_free(adev, status_offset);
 268	return r;
 269}
 270
 271static int convert_to_mes_queue_type(int queue_type)
 272{
 273	if (queue_type == AMDGPU_RING_TYPE_GFX)
 274		return MES_QUEUE_TYPE_GFX;
 275	else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
 276		return MES_QUEUE_TYPE_COMPUTE;
 277	else if (queue_type == AMDGPU_RING_TYPE_SDMA)
 278		return MES_QUEUE_TYPE_SDMA;
 279	else if (queue_type == AMDGPU_RING_TYPE_MES)
 280		return MES_QUEUE_TYPE_SCHQ;
 281	else
 282		BUG();
 283	return -1;
 284}
 285
 286static int convert_to_mes_priority_level(int priority_level)
 287{
 288	switch (priority_level) {
 289	case AMDGPU_MES_PRIORITY_LEVEL_LOW:
 290		return AMD_PRIORITY_LEVEL_LOW;
 291	case AMDGPU_MES_PRIORITY_LEVEL_NORMAL:
 292	default:
 293		return AMD_PRIORITY_LEVEL_NORMAL;
 294	case AMDGPU_MES_PRIORITY_LEVEL_MEDIUM:
 295		return AMD_PRIORITY_LEVEL_MEDIUM;
 296	case AMDGPU_MES_PRIORITY_LEVEL_HIGH:
 297		return AMD_PRIORITY_LEVEL_HIGH;
 298	case AMDGPU_MES_PRIORITY_LEVEL_REALTIME:
 299		return AMD_PRIORITY_LEVEL_REALTIME;
 300	}
 301}
 302
 303static int mes_v12_0_add_hw_queue(struct amdgpu_mes *mes,
 304				  struct mes_add_queue_input *input)
 305{
 306	struct amdgpu_device *adev = mes->adev;
 307	union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 308	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
 309	uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
 310
 311	memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 312
 313	mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 314	mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 315	mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 316
 317	mes_add_queue_pkt.process_id = input->process_id;
 318	mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
 319	mes_add_queue_pkt.process_va_start = input->process_va_start;
 320	mes_add_queue_pkt.process_va_end = input->process_va_end;
 321	mes_add_queue_pkt.process_quantum = input->process_quantum;
 322	mes_add_queue_pkt.process_context_addr = input->process_context_addr;
 323	mes_add_queue_pkt.gang_quantum = input->gang_quantum;
 324	mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
 325	mes_add_queue_pkt.inprocess_gang_priority =
 326		convert_to_mes_priority_level(input->inprocess_gang_priority);
 327	mes_add_queue_pkt.gang_global_priority_level =
 328		convert_to_mes_priority_level(input->gang_global_priority_level);
 329	mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 330	mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 331
 332	mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
 333
 334	mes_add_queue_pkt.queue_type =
 335		convert_to_mes_queue_type(input->queue_type);
 336	mes_add_queue_pkt.paging = input->paging;
 337	mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
 338	mes_add_queue_pkt.gws_base = input->gws_base;
 339	mes_add_queue_pkt.gws_size = input->gws_size;
 340	mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
 341	mes_add_queue_pkt.tma_addr = input->tma_addr;
 342	mes_add_queue_pkt.trap_en = input->trap_en;
 343	mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
 344	mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
 345
 346	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 347	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 348	mes_add_queue_pkt.gds_size = input->queue_size;
 349
 350	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 351	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 352	mes_add_queue_pkt.gds_size = input->queue_size;
 353
 354	return mes_v12_0_submit_pkt_and_poll_completion(mes,
 355			AMDGPU_MES_SCHED_PIPE,
 356			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 357			offsetof(union MESAPI__ADD_QUEUE, api_status));
 358}
 359
 360static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes,
 361				     struct mes_remove_queue_input *input)
 362{
 363	union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 364	uint32_t mes_rev = mes->sched_version & AMDGPU_MES_VERSION_MASK;
 365
 366	memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 367
 368	mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 369	mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 370	mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 371
 372	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 373	mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
 374
 375	if (mes_rev >= 0x5a)
 376		mes_remove_queue_pkt.remove_queue_after_reset = input->remove_queue_after_reset;
 377
 378	return mes_v12_0_submit_pkt_and_poll_completion(mes,
 379			AMDGPU_MES_SCHED_PIPE,
 380			&mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 381			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 382}
 383
 384int gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device *adev,
 385				      bool req)
 386{
 387	u32 i, tmp, val;
 388
 389	for (i = 0; i < adev->usec_timeout; i++) {
 390		/* Request with MeId=2, PipeId=0 */
 391		tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
 392		tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
 393		WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
 394
 395		val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
 396		if (req) {
 397			if (val == tmp)
 398				break;
 399		} else {
 400			tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
 401					    REQUEST, 1);
 402
 403			/* unlocked or locked by firmware */
 404			if (val != tmp)
 405				break;
 406		}
 407		udelay(1);
 408	}
 409
 410	if (i >= adev->usec_timeout)
 411		return -EINVAL;
 412
 413	return 0;
 414}
 415
 416static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
 417				      uint32_t me_id, uint32_t pipe_id,
 418				      uint32_t queue_id, uint32_t vmid)
 419{
 420	struct amdgpu_device *adev = mes->adev;
 421	uint32_t value, reg;
 422	int i, r = 0;
 423
 424	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
 425
 426	if (queue_type == AMDGPU_RING_TYPE_GFX) {
 427		dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n",
 428			 me_id, pipe_id, queue_id, vmid);
 429
 430		mutex_lock(&adev->gfx.reset_sem_mutex);
 431		gfx_v12_0_request_gfx_index_mutex(adev, true);
 432		/* all se allow writes */
 433		WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX,
 434			     (uint32_t)(0x1 << GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT));
 435		value = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
 436		if (pipe_id == 0)
 437			value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE0_QUEUES, 1 << queue_id);
 438		else
 439			value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE1_QUEUES, 1 << queue_id);
 440		WREG32_SOC15(GC, 0, regCP_VMID_RESET, value);
 441		gfx_v12_0_request_gfx_index_mutex(adev, false);
 442		mutex_unlock(&adev->gfx.reset_sem_mutex);
 443
 444		mutex_lock(&adev->srbm_mutex);
 445		soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 446		/* wait till dequeue take effects */
 447		for (i = 0; i < adev->usec_timeout; i++) {
 448			if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))
 449				break;
 450			udelay(1);
 451		}
 452		if (i >= adev->usec_timeout) {
 453			dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n");
 454			r = -ETIMEDOUT;
 455		}
 456
 457		soc21_grbm_select(adev, 0, 0, 0, 0);
 458		mutex_unlock(&adev->srbm_mutex);
 459	} else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 460		dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",
 461			 me_id, pipe_id, queue_id);
 462		mutex_lock(&adev->srbm_mutex);
 463		soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 464		WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
 465		WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
 466
 467		/* wait till dequeue take effects */
 468		for (i = 0; i < adev->usec_timeout; i++) {
 469			if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
 470				break;
 471			udelay(1);
 472		}
 473		if (i >= adev->usec_timeout) {
 474			dev_err(adev->dev, "failed to wait on hqd deactivate\n");
 475			r = -ETIMEDOUT;
 476		}
 477		soc21_grbm_select(adev, 0, 0, 0, 0);
 478		mutex_unlock(&adev->srbm_mutex);
 479	} else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
 480		dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n",
 481			 me_id, pipe_id, queue_id);
 482		switch (me_id) {
 483		case 1:
 484			reg = SOC15_REG_OFFSET(GC, 0, regSDMA1_QUEUE_RESET_REQ);
 485			break;
 486		case 0:
 487		default:
 488			reg = SOC15_REG_OFFSET(GC, 0, regSDMA0_QUEUE_RESET_REQ);
 489			break;
 490		}
 491
 492		value = 1 << queue_id;
 493		WREG32(reg, value);
 494		/* wait for queue reset done */
 495		for (i = 0; i < adev->usec_timeout; i++) {
 496			if (!(RREG32(reg) & value))
 497				break;
 498			udelay(1);
 499		}
 500		if (i >= adev->usec_timeout) {
 501			dev_err(adev->dev, "failed to wait on sdma queue reset done\n");
 502			r = -ETIMEDOUT;
 503		}
 504	}
 505
 506	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 507	return r;
 508}
 509
 510static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,
 511				      struct mes_map_legacy_queue_input *input)
 512{
 513	union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 514	int pipe;
 515
 516	memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 517
 518	mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 519	mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 520	mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 521
 522	mes_add_queue_pkt.pipe_id = input->pipe_id;
 523	mes_add_queue_pkt.queue_id = input->queue_id;
 524	mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 525	mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 526	mes_add_queue_pkt.wptr_addr = input->wptr_addr;
 527	mes_add_queue_pkt.queue_type =
 528		convert_to_mes_queue_type(input->queue_type);
 529	mes_add_queue_pkt.map_legacy_kq = 1;
 530
 531	if (mes->adev->enable_uni_mes)
 532		pipe = AMDGPU_MES_KIQ_PIPE;
 533	else
 534		pipe = AMDGPU_MES_SCHED_PIPE;
 535
 536	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 537			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 538			offsetof(union MESAPI__ADD_QUEUE, api_status));
 539}
 540
 541static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 542			struct mes_unmap_legacy_queue_input *input)
 543{
 544	union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 545	int pipe;
 546
 547	memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 548
 549	mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 550	mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 551	mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 552
 553	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 554	mes_remove_queue_pkt.gang_context_addr = 0;
 555
 556	mes_remove_queue_pkt.pipe_id = input->pipe_id;
 557	mes_remove_queue_pkt.queue_id = input->queue_id;
 558
 559	if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
 560		mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
 561		mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
 562		mes_remove_queue_pkt.tf_data =
 563			lower_32_bits(input->trail_fence_data);
 564	} else {
 565		mes_remove_queue_pkt.unmap_legacy_queue = 1;
 566		mes_remove_queue_pkt.queue_type =
 567			convert_to_mes_queue_type(input->queue_type);
 568	}
 569
 570	if (mes->adev->enable_uni_mes)
 571		pipe = AMDGPU_MES_KIQ_PIPE;
 572	else
 573		pipe = AMDGPU_MES_SCHED_PIPE;
 574
 575	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 576			&mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 577			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 578}
 579
 580static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes,
 581				  struct mes_suspend_gang_input *input)
 582{
 583	union MESAPI__SUSPEND mes_suspend_gang_pkt;
 584
 585	memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));
 586
 587	mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
 588	mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;
 589	mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 590
 591	mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;
 592	mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;
 593	mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;
 594	mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;
 595
 596	return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE,
 597			&mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),
 598			offsetof(union MESAPI__SUSPEND, api_status));
 599}
 600
 601static int mes_v12_0_resume_gang(struct amdgpu_mes *mes,
 602				 struct mes_resume_gang_input *input)
 603{
 604	union MESAPI__RESUME mes_resume_gang_pkt;
 605
 606	memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));
 607
 608	mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
 609	mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;
 610	mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 611
 612	mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;
 613	mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;
 614
 615	return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE,
 616			&mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),
 617			offsetof(union MESAPI__RESUME, api_status));
 618}
 619
 620static int mes_v12_0_query_sched_status(struct amdgpu_mes *mes, int pipe)
 621{
 622	union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 623
 624	memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 625
 626	mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 627	mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 628	mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 629
 630	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 631			&mes_status_pkt, sizeof(mes_status_pkt),
 632			offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
 633}
 634
 635static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
 636			     struct mes_misc_op_input *input)
 637{
 638	union MESAPI__MISC misc_pkt;
 639	int pipe;
 640
 641	if (mes->adev->enable_uni_mes)
 642		pipe = AMDGPU_MES_KIQ_PIPE;
 643	else
 644		pipe = AMDGPU_MES_SCHED_PIPE;
 645
 646	memset(&misc_pkt, 0, sizeof(misc_pkt));
 647
 648	misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
 649	misc_pkt.header.opcode = MES_SCH_API_MISC;
 650	misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 651
 652	switch (input->op) {
 653	case MES_MISC_OP_READ_REG:
 654		misc_pkt.opcode = MESAPI_MISC__READ_REG;
 655		misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 656		misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
 657		break;
 658	case MES_MISC_OP_WRITE_REG:
 659		misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
 660		misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
 661		misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
 662		break;
 663	case MES_MISC_OP_WRM_REG_WAIT:
 664		misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 665		misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
 666		misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 667		misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 668		misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 669		misc_pkt.wait_reg_mem.reg_offset2 = 0;
 670		break;
 671	case MES_MISC_OP_WRM_REG_WR_WAIT:
 672		misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 673		misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
 674		misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 675		misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 676		misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 677		misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
 678		break;
 679	case MES_MISC_OP_SET_SHADER_DEBUGGER:
 680		pipe = AMDGPU_MES_SCHED_PIPE;
 681		misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
 682		misc_pkt.set_shader_debugger.process_context_addr =
 683				input->set_shader_debugger.process_context_addr;
 684		misc_pkt.set_shader_debugger.flags.u32all =
 685				input->set_shader_debugger.flags.u32all;
 686		misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
 687				input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
 688		memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
 689				input->set_shader_debugger.tcp_watch_cntl,
 690				sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
 691		misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
 692		break;
 693	case MES_MISC_OP_CHANGE_CONFIG:
 694		misc_pkt.opcode = MESAPI_MISC__CHANGE_CONFIG;
 695		misc_pkt.change_config.opcode =
 696				MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS;
 697		misc_pkt.change_config.option.bits.limit_single_process =
 698				input->change_config.option.limit_single_process;
 699		break;
 700
 701	default:
 702		DRM_ERROR("unsupported misc op (%d) \n", input->op);
 703		return -EINVAL;
 704	}
 705
 706	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 707			&misc_pkt, sizeof(misc_pkt),
 708			offsetof(union MESAPI__MISC, api_status));
 709}
 710
 711static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
 712{
 713	union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
 714
 715	memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
 716
 717	mes_set_hw_res_1_pkt.header.type = MES_API_TYPE_SCHEDULER;
 718	mes_set_hw_res_1_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
 719	mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 720	mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 0xa;
 721	mes_set_hw_res_1_pkt.cleaner_shader_fence_mc_addr =
 722		mes->resource_1_gpu_addr[pipe];
 723
 724	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 725			&mes_set_hw_res_1_pkt, sizeof(mes_set_hw_res_1_pkt),
 726			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 727}
 728
 729static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 730{
 731	int i;
 732	struct amdgpu_device *adev = mes->adev;
 733	union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
 734	uint32_t mes_rev = (pipe == AMDGPU_MES_SCHED_PIPE) ?
 735		(mes->sched_version & AMDGPU_MES_VERSION_MASK) :
 736		(mes->kiq_version & AMDGPU_MES_VERSION_MASK);
 737
 738	memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
 739
 740	mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
 741	mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
 742	mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 743
 744	if (pipe == AMDGPU_MES_SCHED_PIPE) {
 745		mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
 746		mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
 747		mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
 748		mes_set_hw_res_pkt.paging_vmid = 0;
 749
 750		for (i = 0; i < MAX_COMPUTE_PIPES; i++)
 751			mes_set_hw_res_pkt.compute_hqd_mask[i] =
 752				mes->compute_hqd_mask[i];
 753
 754		for (i = 0; i < MAX_GFX_PIPES; i++)
 755			mes_set_hw_res_pkt.gfx_hqd_mask[i] =
 756				mes->gfx_hqd_mask[i];
 757
 758		for (i = 0; i < MAX_SDMA_PIPES; i++)
 759			mes_set_hw_res_pkt.sdma_hqd_mask[i] =
 760				mes->sdma_hqd_mask[i];
 761
 762		for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
 763			mes_set_hw_res_pkt.aggregated_doorbells[i] =
 764				mes->aggregated_doorbells[i];
 765	}
 766
 767	mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr =
 768		mes->sch_ctx_gpu_addr[pipe];
 769	mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
 770		mes->query_status_fence_gpu_addr[pipe];
 771
 772	for (i = 0; i < 5; i++) {
 773		mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
 774		mes_set_hw_res_pkt.mmhub_base[i] =
 775				adev->reg_offset[MMHUB_HWIP][0][i];
 776		mes_set_hw_res_pkt.osssys_base[i] =
 777		adev->reg_offset[OSSSYS_HWIP][0][i];
 778	}
 779
 780	mes_set_hw_res_pkt.disable_reset = 1;
 781	mes_set_hw_res_pkt.disable_mes_log = 1;
 782	mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 783	mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 784	mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
 785
 786	/*
 787	 * Keep oversubscribe timer for sdma . When we have unmapped doorbell
 788	 * handling support, other queue will not use the oversubscribe timer.
 789	 * handling  mode - 0: disabled; 1: basic version; 2: basic+ version
 790	 */
 791	mes_set_hw_res_pkt.oversubscription_timer = mes_rev < 0x8b ? 0 : 50;
 792	mes_set_hw_res_pkt.unmapped_doorbell_handling = 1;
 793
 794	if (amdgpu_mes_log_enable) {
 795		mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
 796		mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = mes->event_log_gpu_addr +
 797				pipe * (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE);
 798	}
 799
 800	if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE)
 801		mes_set_hw_res_pkt.limit_single_process = 1;
 802
 803	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 804			&mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
 805			offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
 806}
 807
 808static void mes_v12_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
 809{
 810	struct amdgpu_device *adev = mes->adev;
 811	uint32_t data;
 812
 813	data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1);
 814	data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
 815		  CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
 816		  CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
 817	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
 818		CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
 819	data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
 820	WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data);
 821
 822	data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2);
 823	data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
 824		  CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
 825		  CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
 826	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
 827		CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
 828	data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
 829	WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data);
 830
 831	data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3);
 832	data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
 833		  CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
 834		  CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
 835	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
 836		CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
 837	data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
 838	WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data);
 839
 840	data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4);
 841	data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
 842		  CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
 843		  CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
 844	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
 845		CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
 846	data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
 847	WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data);
 848
 849	data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5);
 850	data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
 851		  CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
 852		  CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
 853	data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
 854		CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
 855	data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
 856	WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data);
 857
 858	data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
 859	WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data);
 860}
 861
 862
 863static void mes_v12_0_enable_unmapped_doorbell_handling(
 864		struct amdgpu_mes *mes, bool enable)
 865{
 866	struct amdgpu_device *adev = mes->adev;
 867	uint32_t data = RREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL);
 868
 869	/*
 870	 * The default PROC_LSB settng is 0xc which means doorbell
 871	 * addr[16:12] gives the doorbell page number. For kfd, each
 872	 * process will use 2 pages of doorbell, we need to change the
 873	 * setting to 0xd
 874	 */
 875	data &= ~CP_UNMAPPED_DOORBELL__PROC_LSB_MASK;
 876	data |= 0xd <<  CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT;
 877
 878	data |= (enable ? 1 : 0) << CP_UNMAPPED_DOORBELL__ENABLE__SHIFT;
 879
 880	WREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL, data);
 881}
 882
 883static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
 884				    struct mes_reset_queue_input *input)
 885{
 886	union MESAPI__RESET mes_reset_queue_pkt;
 887	int pipe;
 888
 889	if (input->use_mmio)
 890		return mes_v12_0_reset_queue_mmio(mes, input->queue_type,
 891						  input->me_id, input->pipe_id,
 892						  input->queue_id, input->vmid);
 893
 894	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 895
 896	mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 897	mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 898	mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 899
 900	mes_reset_queue_pkt.queue_type =
 901		convert_to_mes_queue_type(input->queue_type);
 902
 903	if (input->legacy_gfx) {
 904		mes_reset_queue_pkt.reset_legacy_gfx = 1;
 905		mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
 906		mes_reset_queue_pkt.queue_id_lp = input->queue_id;
 907		mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
 908		mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
 909		mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
 910		mes_reset_queue_pkt.vmid_id_lp = input->vmid;
 911	} else {
 912		mes_reset_queue_pkt.reset_queue_only = 1;
 913		mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 914	}
 915
 916	if (input->is_kq)
 917		pipe = AMDGPU_MES_KIQ_PIPE;
 918	else
 919		pipe = AMDGPU_MES_SCHED_PIPE;
 920
 921	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 922			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 923			offsetof(union MESAPI__RESET, api_status));
 924}
 925
 926static int mes_v12_0_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
 927						  struct mes_detect_and_reset_queue_input *input)
 928{
 929	union MESAPI__RESET mes_reset_queue_pkt;
 930
 931	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 932
 933	mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 934	mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 935	mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 936
 937	mes_reset_queue_pkt.queue_type =
 938		convert_to_mes_queue_type(input->queue_type);
 939	mes_reset_queue_pkt.doorbell_offset_addr =
 940		mes->hung_queue_db_array_gpu_addr;
 941
 942	if (input->detect_only)
 943		mes_reset_queue_pkt.hang_detect_only = 1;
 944	else
 945		mes_reset_queue_pkt.hang_detect_then_reset = 1;
 946
 947	return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_SCHED_PIPE,
 948			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 949			offsetof(union MESAPI__RESET, api_status));
 950}
 951
 952static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
 953{
 954	/*
 955	 * MES doesn't support invalidate gc_hub on slave xcc individually
 956	 * master xcc will invalidate all gc_hub for the partition
 957	 */
 958	if (AMDGPU_IS_GFXHUB(id))
 959		return 0;
 960	else if (AMDGPU_IS_MMHUB0(id))
 961		return 1;
 962	else
 963		return -EINVAL;
 964
 965}
 966
 967static int mes_v12_0_inv_tlbs_pasid(struct amdgpu_mes *mes,
 968				    struct mes_inv_tlbs_pasid_input *input)
 969{
 970	union MESAPI__INV_TLBS mes_inv_tlbs;
 971	int ret;
 972
 973	memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
 974
 975	mes_inv_tlbs.header.type = MES_API_TYPE_SCHEDULER;
 976	mes_inv_tlbs.header.opcode = MES_SCH_API_INV_TLBS;
 977	mes_inv_tlbs.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 978
 979	mes_inv_tlbs.invalidate_tlbs.inv_sel = 0;
 980	mes_inv_tlbs.invalidate_tlbs.flush_type = input->flush_type;
 981	mes_inv_tlbs.invalidate_tlbs.inv_sel_id = input->pasid;
 982
 983	/*convert amdgpu_mes_hub_id to mes expected hub_id */
 984	ret = mes_v12_inv_tlb_convert_hub_id(input->hub_id);
 985	if (ret < 0)
 986		return -EINVAL;
 987	mes_inv_tlbs.invalidate_tlbs.hub_id = ret;
 988	return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_KIQ_PIPE,
 989			&mes_inv_tlbs, sizeof(mes_inv_tlbs),
 990			offsetof(union MESAPI__INV_TLBS, api_status));
 991
 992}
 993
 994static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
 995	.add_hw_queue = mes_v12_0_add_hw_queue,
 996	.remove_hw_queue = mes_v12_0_remove_hw_queue,
 997	.map_legacy_queue = mes_v12_0_map_legacy_queue,
 998	.unmap_legacy_queue = mes_v12_0_unmap_legacy_queue,
 999	.suspend_gang = mes_v12_0_suspend_gang,
1000	.resume_gang = mes_v12_0_resume_gang,
1001	.misc_op = mes_v12_0_misc_op,
1002	.reset_hw_queue = mes_v12_0_reset_hw_queue,
1003	.invalidate_tlbs_pasid = mes_v12_0_inv_tlbs_pasid,
1004	.detect_and_reset_hung_queues = mes_v12_0_detect_and_reset_hung_queues,
1005};
1006
1007static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
1008					   enum amdgpu_mes_pipe pipe)
1009{
1010	int r;
1011	const struct mes_firmware_header_v1_0 *mes_hdr;
1012	const __le32 *fw_data;
1013	unsigned fw_size;
1014
1015	mes_hdr = (const struct mes_firmware_header_v1_0 *)
1016		adev->mes.fw[pipe]->data;
1017
1018	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1019		   le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1020	fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1021
1022	r = amdgpu_bo_create_reserved(adev, fw_size,
1023				      PAGE_SIZE,
1024				      AMDGPU_GEM_DOMAIN_VRAM,
1025				      &adev->mes.ucode_fw_obj[pipe],
1026				      &adev->mes.ucode_fw_gpu_addr[pipe],
1027				      (void **)&adev->mes.ucode_fw_ptr[pipe]);
1028	if (r) {
1029		dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
1030		return r;
1031	}
1032
1033	memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
1034
1035	amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
1036	amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
1037
1038	return 0;
1039}
1040
1041static int mes_v12_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
1042						enum amdgpu_mes_pipe pipe)
1043{
1044	int r;
1045	const struct mes_firmware_header_v1_0 *mes_hdr;
1046	const __le32 *fw_data;
1047	unsigned fw_size;
1048
1049	mes_hdr = (const struct mes_firmware_header_v1_0 *)
1050		adev->mes.fw[pipe]->data;
1051
1052	fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1053		   le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1054	fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1055
1056	r = amdgpu_bo_create_reserved(adev, fw_size,
1057				      64 * 1024,
1058				      AMDGPU_GEM_DOMAIN_VRAM,
1059				      &adev->mes.data_fw_obj[pipe],
1060				      &adev->mes.data_fw_gpu_addr[pipe],
1061				      (void **)&adev->mes.data_fw_ptr[pipe]);
1062	if (r) {
1063		dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
1064		return r;
1065	}
1066
1067	memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
1068
1069	amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
1070	amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
1071
1072	return 0;
1073}
1074
1075static void mes_v12_0_free_ucode_buffers(struct amdgpu_device *adev,
1076					 enum amdgpu_mes_pipe pipe)
1077{
1078	amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
1079			      &adev->mes.data_fw_gpu_addr[pipe],
1080			      (void **)&adev->mes.data_fw_ptr[pipe]);
1081
1082	amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
1083			      &adev->mes.ucode_fw_gpu_addr[pipe],
1084			      (void **)&adev->mes.ucode_fw_ptr[pipe]);
1085}
1086
1087static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
1088{
1089	uint64_t ucode_addr;
1090	uint32_t pipe, data = 0;
1091
1092	if (enable) {
1093		mutex_lock(&adev->srbm_mutex);
1094		for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1095			soc21_grbm_select(adev, 3, pipe, 0, 0);
1096			if (amdgpu_mes_log_enable) {
1097				u32 log_size = AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE;
1098				/* In case uni mes is not enabled, only program for pipe 0 */
1099				if (adev->mes.event_log_size >= (pipe + 1) * log_size) {
1100					WREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_LO,
1101						     lower_32_bits(adev->mes.event_log_gpu_addr +
1102						     pipe * log_size + AMDGPU_MES_LOG_BUFFER_SIZE));
1103					WREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_HI,
1104						     upper_32_bits(adev->mes.event_log_gpu_addr +
1105						     pipe * log_size + AMDGPU_MES_LOG_BUFFER_SIZE));
1106					dev_info(adev->dev, "Setup CP MES MSCRATCH address : 0x%x. 0x%x\n",
1107						 RREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_HI),
1108						 RREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_LO));
1109				}
1110			}
1111
1112			data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1113			if (pipe == 0)
1114				data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1115			else
1116				data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1117			WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1118
1119			ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1120			WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1121				     lower_32_bits(ucode_addr));
1122			WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1123				     upper_32_bits(ucode_addr));
1124
1125			/* unhalt MES and activate one pipe each loop */
1126			data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
1127			if (pipe)
1128				data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 1);
1129			dev_info(adev->dev, "program CP_MES_CNTL : 0x%x\n", data);
1130
1131			WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1132
1133		}
1134		soc21_grbm_select(adev, 0, 0, 0, 0);
1135		mutex_unlock(&adev->srbm_mutex);
1136
1137		if (amdgpu_emu_mode)
1138			msleep(100);
1139		else if (adev->enable_uni_mes)
1140			udelay(500);
1141		else
1142			udelay(50);
1143	} else {
1144		data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1145		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
1146		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
1147		data = REG_SET_FIELD(data, CP_MES_CNTL,
1148				     MES_INVALIDATE_ICACHE, 1);
1149		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1150		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1151		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
1152		WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1153	}
1154}
1155
1156static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev)
1157{
1158	uint64_t ucode_addr;
1159	int pipe;
1160
1161	mes_v12_0_enable(adev, false);
1162
1163	mutex_lock(&adev->srbm_mutex);
1164	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1165		/* me=3, queue=0 */
1166		soc21_grbm_select(adev, 3, pipe, 0, 0);
1167
1168		/* set ucode start address */
1169		ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1170		WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1171				lower_32_bits(ucode_addr));
1172		WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1173				upper_32_bits(ucode_addr));
1174
1175		soc21_grbm_select(adev, 0, 0, 0, 0);
1176	}
1177	mutex_unlock(&adev->srbm_mutex);
1178}
1179
1180/* This function is for backdoor MES firmware */
1181static int mes_v12_0_load_microcode(struct amdgpu_device *adev,
1182				    enum amdgpu_mes_pipe pipe, bool prime_icache)
1183{
1184	int r;
1185	uint32_t data;
1186
1187	mes_v12_0_enable(adev, false);
1188
1189	if (!adev->mes.fw[pipe])
1190		return -EINVAL;
1191
1192	r = mes_v12_0_allocate_ucode_buffer(adev, pipe);
1193	if (r)
1194		return r;
1195
1196	r = mes_v12_0_allocate_ucode_data_buffer(adev, pipe);
1197	if (r) {
1198		mes_v12_0_free_ucode_buffers(adev, pipe);
1199		return r;
1200	}
1201
1202	mutex_lock(&adev->srbm_mutex);
1203	/* me=3, pipe=0, queue=0 */
1204	soc21_grbm_select(adev, 3, pipe, 0, 0);
1205
1206	WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
1207
1208	/* set ucode fimrware address */
1209	WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
1210		     lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1211	WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
1212		     upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1213
1214	/* set ucode instruction cache boundary to 2M-1 */
1215	WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
1216
1217	/* set ucode data firmware address */
1218	WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
1219		     lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1220	WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
1221		     upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1222
1223	/* Set data cache boundary CP_MES_MDBOUND_LO */
1224	WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF);
1225
1226	if (prime_icache) {
1227		/* invalidate ICACHE */
1228		data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1229		data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
1230		data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1231		WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1232
1233		/* prime the ICACHE. */
1234		data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1235		data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
1236		WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1237	}
1238
1239	soc21_grbm_select(adev, 0, 0, 0, 0);
1240	mutex_unlock(&adev->srbm_mutex);
1241
1242	return 0;
1243}
1244
1245static int mes_v12_0_allocate_eop_buf(struct amdgpu_device *adev,
1246				      enum amdgpu_mes_pipe pipe)
1247{
1248	int r;
1249	u32 *eop;
1250
1251	r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
1252			      AMDGPU_GEM_DOMAIN_GTT,
1253			      &adev->mes.eop_gpu_obj[pipe],
1254			      &adev->mes.eop_gpu_addr[pipe],
1255			      (void **)&eop);
1256	if (r) {
1257		dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
1258		return r;
1259	}
1260
1261	memset(eop, 0,
1262	       adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
1263
1264	amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
1265	amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
1266
1267	return 0;
1268}
1269
1270static int mes_v12_0_mqd_init(struct amdgpu_ring *ring)
1271{
1272	struct v12_compute_mqd *mqd = ring->mqd_ptr;
1273	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1274	uint32_t tmp;
1275
1276	mqd->header = 0xC0310800;
1277	mqd->compute_pipelinestat_enable = 0x00000001;
1278	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1279	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1280	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1281	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1282	mqd->compute_misc_reserved = 0x00000007;
1283
1284	eop_base_addr = ring->eop_gpu_addr >> 8;
1285
1286	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1287	tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
1288	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1289			(order_base_2(MES_EOP_SIZE / 4) - 1));
1290
1291	mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
1292	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1293	mqd->cp_hqd_eop_control = tmp;
1294
1295	/* disable the queue if it's active */
1296	ring->wptr = 0;
1297	mqd->cp_hqd_pq_rptr = 0;
1298	mqd->cp_hqd_pq_wptr_lo = 0;
1299	mqd->cp_hqd_pq_wptr_hi = 0;
1300
1301	/* set the pointer to the MQD */
1302	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1303	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1304
1305	/* set MQD vmid to 0 */
1306	tmp = regCP_MQD_CONTROL_DEFAULT;
1307	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1308	mqd->cp_mqd_control = tmp;
1309
1310	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1311	hqd_gpu_addr = ring->gpu_addr >> 8;
1312	mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
1313	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1314
1315	/* set the wb address whether it's enabled or not */
1316	wb_gpu_addr = ring->rptr_gpu_addr;
1317	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1318	mqd->cp_hqd_pq_rptr_report_addr_hi =
1319		upper_32_bits(wb_gpu_addr) & 0xffff;
1320
1321	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1322	wb_gpu_addr = ring->wptr_gpu_addr;
1323	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
1324	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1325
1326	/* set up the HQD, this is similar to CP_RB0_CNTL */
1327	tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
1328	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1329			    (order_base_2(ring->ring_size / 4) - 1));
1330	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1331			    ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1332	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
1333	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
1334	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1335	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1336	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
1337	mqd->cp_hqd_pq_control = tmp;
1338
1339	/* enable doorbell */
1340	tmp = 0;
1341	if (ring->use_doorbell) {
1342		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1343				    DOORBELL_OFFSET, ring->doorbell_index);
1344		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1345				    DOORBELL_EN, 1);
1346		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1347				    DOORBELL_SOURCE, 0);
1348		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1349				    DOORBELL_HIT, 0);
1350	} else {
1351		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1352				    DOORBELL_EN, 0);
1353	}
1354	mqd->cp_hqd_pq_doorbell_control = tmp;
1355
1356	mqd->cp_hqd_vmid = 0;
1357	/* activate the queue */
1358	mqd->cp_hqd_active = 1;
1359
1360	tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
1361	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
1362			    PRELOAD_SIZE, 0x55);
1363	mqd->cp_hqd_persistent_state = tmp;
1364
1365	mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
1366	mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
1367	mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
1368
1369	/*
1370	 * Set CP_HQD_GFX_CONTROL.DB_UPDATED_MSG_EN[15] to enable unmapped
1371	 * doorbell handling. This is a reserved CP internal register can
1372	 * not be accesss by others
1373	 */
1374	mqd->reserved_184 = BIT(15);
1375
1376	return 0;
1377}
1378
1379static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring)
1380{
1381	struct v12_compute_mqd *mqd = ring->mqd_ptr;
1382	struct amdgpu_device *adev = ring->adev;
1383	uint32_t data = 0;
1384
1385	mutex_lock(&adev->srbm_mutex);
1386	soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
1387
1388	/* set CP_HQD_VMID.VMID = 0. */
1389	data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
1390	data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
1391	WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
1392
1393	/* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
1394	data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1395	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1396			     DOORBELL_EN, 0);
1397	WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1398
1399	/* set CP_MQD_BASE_ADDR/HI with the MQD base address */
1400	WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
1401	WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
1402
1403	/* set CP_MQD_CONTROL.VMID=0 */
1404	data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
1405	data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
1406	WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
1407
1408	/* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
1409	WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
1410	WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
1411
1412	/* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
1413	WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
1414		     mqd->cp_hqd_pq_rptr_report_addr_lo);
1415	WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1416		     mqd->cp_hqd_pq_rptr_report_addr_hi);
1417
1418	/* set CP_HQD_PQ_CONTROL */
1419	WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
1420
1421	/* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
1422	WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
1423		     mqd->cp_hqd_pq_wptr_poll_addr_lo);
1424	WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1425		     mqd->cp_hqd_pq_wptr_poll_addr_hi);
1426
1427	/* set CP_HQD_PQ_DOORBELL_CONTROL */
1428	WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
1429		     mqd->cp_hqd_pq_doorbell_control);
1430
1431	/* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
1432	WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
1433
1434	/* set CP_HQD_ACTIVE.ACTIVE=1 */
1435	WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
1436
1437	soc21_grbm_select(adev, 0, 0, 0, 0);
1438	mutex_unlock(&adev->srbm_mutex);
1439}
1440
1441static int mes_v12_0_kiq_enable_queue(struct amdgpu_device *adev)
1442{
1443	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1444	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
1445	int r;
1446
1447	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
1448		return -EINVAL;
1449
1450	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
1451	if (r) {
1452		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
1453		return r;
1454	}
1455
1456	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);
1457
1458	r = amdgpu_ring_test_ring(kiq_ring);
1459	if (r) {
1460		DRM_ERROR("kfq enable failed\n");
1461		kiq_ring->sched.ready = false;
1462	}
1463	return r;
1464}
1465
1466static int mes_v12_0_queue_init(struct amdgpu_device *adev,
1467				enum amdgpu_mes_pipe pipe)
1468{
1469	struct amdgpu_ring *ring;
1470	int r;
1471
1472	if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1473		ring = &adev->gfx.kiq[0].ring;
1474	else
1475		ring = &adev->mes.ring[pipe];
1476
1477	if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
1478	    (amdgpu_in_reset(adev) || adev->in_suspend)) {
1479		*(ring->wptr_cpu_addr) = 0;
1480		*(ring->rptr_cpu_addr) = 0;
1481		amdgpu_ring_clear_ring(ring);
1482	}
1483
1484	r = mes_v12_0_mqd_init(ring);
1485	if (r)
1486		return r;
1487
1488	if (pipe == AMDGPU_MES_SCHED_PIPE) {
1489		if (adev->enable_uni_mes)
1490			r = amdgpu_mes_map_legacy_queue(adev, ring);
1491		else
1492			r = mes_v12_0_kiq_enable_queue(adev);
1493		if (r)
1494			return r;
1495	} else {
1496		mes_v12_0_queue_init_register(ring);
1497	}
1498
1499	if (((pipe == AMDGPU_MES_SCHED_PIPE) && !adev->mes.sched_version) ||
1500	    ((pipe == AMDGPU_MES_KIQ_PIPE) && !adev->mes.kiq_version)) {
1501		/* get MES scheduler/KIQ versions */
1502		mutex_lock(&adev->srbm_mutex);
1503		soc21_grbm_select(adev, 3, pipe, 0, 0);
1504
1505		if (pipe == AMDGPU_MES_SCHED_PIPE)
1506			adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1507		else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
1508			adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1509
1510		soc21_grbm_select(adev, 0, 0, 0, 0);
1511		mutex_unlock(&adev->srbm_mutex);
1512	}
1513
1514	return 0;
1515}
1516
1517static int mes_v12_0_ring_init(struct amdgpu_device *adev, int pipe)
1518{
1519	struct amdgpu_ring *ring;
1520
1521	ring = &adev->mes.ring[pipe];
1522
1523	ring->funcs = &mes_v12_0_ring_funcs;
1524
1525	ring->me = 3;
1526	ring->pipe = pipe;
1527	ring->queue = 0;
1528
1529	ring->ring_obj = NULL;
1530	ring->use_doorbell = true;
1531	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[pipe];
1532	ring->no_scheduler = true;
1533	sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1534
1535	if (pipe == AMDGPU_MES_SCHED_PIPE)
1536		ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
1537	else
1538		ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1539
1540	return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1541				AMDGPU_RING_PRIO_DEFAULT, NULL);
1542}
1543
1544static int mes_v12_0_kiq_ring_init(struct amdgpu_device *adev)
1545{
1546	struct amdgpu_ring *ring;
1547
1548	spin_lock_init(&adev->gfx.kiq[0].ring_lock);
1549
1550	ring = &adev->gfx.kiq[0].ring;
1551
1552	ring->me = 3;
1553	ring->pipe = 1;
1554	ring->queue = 0;
1555
1556	ring->adev = NULL;
1557	ring->ring_obj = NULL;
1558	ring->use_doorbell = true;
1559	ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1560	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
1561	ring->no_scheduler = true;
1562	sprintf(ring->name, "mes_kiq_%d.%d.%d",
1563		ring->me, ring->pipe, ring->queue);
1564
1565	return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1566				AMDGPU_RING_PRIO_DEFAULT, NULL);
1567}
1568
1569static int mes_v12_0_mqd_sw_init(struct amdgpu_device *adev,
1570				 enum amdgpu_mes_pipe pipe)
1571{
1572	int r, mqd_size = sizeof(struct v12_compute_mqd);
1573	struct amdgpu_ring *ring;
1574
1575	if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1576		ring = &adev->gfx.kiq[0].ring;
1577	else
1578		ring = &adev->mes.ring[pipe];
1579
1580	if (ring->mqd_obj)
1581		return 0;
1582
1583	r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1584				    AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1585				    &ring->mqd_gpu_addr, &ring->mqd_ptr);
1586	if (r) {
1587		dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1588		return r;
1589	}
1590
1591	memset(ring->mqd_ptr, 0, mqd_size);
1592
1593	/* prepare MQD backup */
1594	adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1595	if (!adev->mes.mqd_backup[pipe])
1596		dev_warn(adev->dev,
1597			 "no memory to create MQD backup for ring %s\n",
1598			 ring->name);
1599
1600	return 0;
1601}
1602
1603static int mes_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
1604{
1605	struct amdgpu_device *adev = ip_block->adev;
1606	int pipe, r;
1607
1608	adev->mes.funcs = &mes_v12_0_funcs;
1609	adev->mes.kiq_hw_init = &mes_v12_0_kiq_hw_init;
1610	adev->mes.kiq_hw_fini = &mes_v12_0_kiq_hw_fini;
1611	adev->mes.enable_legacy_queue_map = true;
1612
1613	adev->mes.event_log_size = adev->enable_uni_mes ?
1614		(AMDGPU_MAX_MES_PIPES * (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE)) :
1615		(AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE);
1616	r = amdgpu_mes_init(adev);
1617	if (r)
1618		return r;
1619
1620	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1621		r = mes_v12_0_allocate_eop_buf(adev, pipe);
1622		if (r)
1623			return r;
1624
1625		r = mes_v12_0_mqd_sw_init(adev, pipe);
1626		if (r)
1627			return r;
1628
1629		if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE) {
1630			r = mes_v12_0_kiq_ring_init(adev);
1631		}
1632		else {
1633			r = mes_v12_0_ring_init(adev, pipe);
1634			if (r)
1635				return r;
1636			r = amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1637						    AMDGPU_GEM_DOMAIN_VRAM,
1638						    &adev->mes.resource_1[pipe],
1639						    &adev->mes.resource_1_gpu_addr[pipe],
1640						    &adev->mes.resource_1_addr[pipe]);
1641			if (r) {
1642				dev_err(adev->dev, "(%d) failed to create mes resource_1 bo pipe[%d]\n", r, pipe);
1643				return r;
1644			}
1645		}
1646	}
1647
1648	return 0;
1649}
1650
1651static int mes_v12_0_sw_fini(struct amdgpu_ip_block *ip_block)
1652{
1653	struct amdgpu_device *adev = ip_block->adev;
1654	int pipe;
1655
1656	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1657		amdgpu_bo_free_kernel(&adev->mes.resource_1[pipe],
1658				      &adev->mes.resource_1_gpu_addr[pipe],
1659				      &adev->mes.resource_1_addr[pipe]);
1660
1661		kfree(adev->mes.mqd_backup[pipe]);
1662
1663		amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1664				      &adev->mes.eop_gpu_addr[pipe],
1665				      NULL);
1666		amdgpu_ucode_release(&adev->mes.fw[pipe]);
1667
1668		if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
1669			amdgpu_bo_free_kernel(&adev->mes.ring[pipe].mqd_obj,
1670					      &adev->mes.ring[pipe].mqd_gpu_addr,
1671					      &adev->mes.ring[pipe].mqd_ptr);
1672			amdgpu_ring_fini(&adev->mes.ring[pipe]);
1673		}
1674	}
1675
1676	if (!adev->enable_uni_mes) {
1677		amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj,
1678				      &adev->gfx.kiq[0].ring.mqd_gpu_addr,
1679				      &adev->gfx.kiq[0].ring.mqd_ptr);
1680		amdgpu_ring_fini(&adev->gfx.kiq[0].ring);
1681	}
1682
1683	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1684		mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1685		mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1686	}
1687
1688	amdgpu_mes_fini(adev);
1689	return 0;
1690}
1691
1692static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev)
1693{
1694	uint32_t data;
1695	int i;
1696
1697	mutex_lock(&adev->srbm_mutex);
1698	soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
1699
1700	/* disable the queue if it's active */
1701	if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
1702		WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
1703		for (i = 0; i < adev->usec_timeout; i++) {
1704			if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
1705				break;
1706			udelay(1);
1707		}
1708	}
1709	data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1710	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1711				DOORBELL_EN, 0);
1712	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1713				DOORBELL_HIT, 1);
1714	WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1715
1716	WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1717
1718	WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);
1719	WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
1720	WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
1721
1722	soc21_grbm_select(adev, 0, 0, 0, 0);
1723	mutex_unlock(&adev->srbm_mutex);
1724
1725	adev->mes.ring[0].sched.ready = false;
1726}
1727
1728static void mes_v12_0_kiq_setting(struct amdgpu_ring *ring)
1729{
1730	uint32_t tmp;
1731	struct amdgpu_device *adev = ring->adev;
1732
1733	/* tell RLC which is KIQ queue */
1734	tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1735	tmp &= 0xffffff00;
1736	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1737	WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
1738}
1739
1740static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev)
1741{
1742	int r = 0;
1743	struct amdgpu_ip_block *ip_block;
1744
1745	if (adev->enable_uni_mes)
1746		mes_v12_0_kiq_setting(&adev->mes.ring[AMDGPU_MES_KIQ_PIPE]);
1747	else
1748		mes_v12_0_kiq_setting(&adev->gfx.kiq[0].ring);
1749
1750	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1751
1752		r = mes_v12_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1753		if (r) {
1754			DRM_ERROR("failed to load MES fw, r=%d\n", r);
1755			return r;
1756		}
1757
1758		r = mes_v12_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1759		if (r) {
1760			DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1761			return r;
1762		}
1763
1764		mes_v12_0_set_ucode_start_addr(adev);
1765
1766	} else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1767		mes_v12_0_set_ucode_start_addr(adev);
1768
1769	mes_v12_0_enable(adev, true);
1770
1771	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_MES);
1772	if (unlikely(!ip_block)) {
1773		dev_err(adev->dev, "Failed to get MES handle\n");
1774		return -EINVAL;
1775	}
1776
1777	r = mes_v12_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1778	if (r)
1779		goto failure;
1780
1781	if (adev->enable_uni_mes) {
1782		r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1783		if (r)
1784			goto failure;
1785
1786		mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1787	}
1788
1789	if (adev->mes.enable_legacy_queue_map) {
1790		r = mes_v12_0_hw_init(ip_block);
1791		if (r)
1792			goto failure;
1793	}
1794
1795	return r;
1796
1797failure:
1798	mes_v12_0_hw_fini(ip_block);
1799	return r;
1800}
1801
1802static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev)
1803{
1804	if (adev->mes.ring[0].sched.ready) {
1805		if (adev->enable_uni_mes)
1806			amdgpu_mes_unmap_legacy_queue(adev,
1807				      &adev->mes.ring[AMDGPU_MES_SCHED_PIPE],
1808				      RESET_QUEUES, 0, 0);
1809		else
1810			mes_v12_0_kiq_dequeue_sched(adev);
1811
1812		adev->mes.ring[0].sched.ready = false;
1813	}
1814
1815	mes_v12_0_enable(adev, false);
1816
1817	return 0;
1818}
1819
1820static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block)
1821{
1822	int r;
1823	struct amdgpu_device *adev = ip_block->adev;
1824
1825	if (adev->mes.ring[0].sched.ready)
1826		goto out;
1827
1828	if (!adev->enable_mes_kiq) {
1829		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1830			r = mes_v12_0_load_microcode(adev,
1831					     AMDGPU_MES_SCHED_PIPE, true);
1832			if (r) {
1833				DRM_ERROR("failed to MES fw, r=%d\n", r);
1834				return r;
1835			}
1836
1837			mes_v12_0_set_ucode_start_addr(adev);
1838
1839		} else if (adev->firmware.load_type ==
1840			   AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1841
1842			mes_v12_0_set_ucode_start_addr(adev);
1843		}
1844
1845		mes_v12_0_enable(adev, true);
1846	}
1847
1848	/* Enable the MES to handle doorbell ring on unmapped queue */
1849	mes_v12_0_enable_unmapped_doorbell_handling(&adev->mes, true);
1850
1851	r = mes_v12_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1852	if (r)
1853		goto failure;
1854
1855	r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1856	if (r)
1857		goto failure;
1858
1859	if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x4b)
1860		mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1861
1862	mes_v12_0_init_aggregated_doorbell(&adev->mes);
1863
1864	r = mes_v12_0_query_sched_status(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1865	if (r) {
1866		DRM_ERROR("MES is busy\n");
1867		goto failure;
1868	}
1869
1870	r = amdgpu_mes_update_enforce_isolation(adev);
1871	if (r)
1872		goto failure;
1873
1874out:
1875	/*
1876	 * Disable KIQ ring usage from the driver once MES is enabled.
1877	 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1878	 * with MES enabled.
1879	 */
1880	adev->gfx.kiq[0].ring.sched.ready = false;
1881	adev->mes.ring[0].sched.ready = true;
1882
1883	return 0;
1884
1885failure:
1886	mes_v12_0_hw_fini(ip_block);
1887	return r;
1888}
1889
1890static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
1891{
1892	return 0;
1893}
1894
1895static int mes_v12_0_suspend(struct amdgpu_ip_block *ip_block)
1896{
1897	return mes_v12_0_hw_fini(ip_block);
1898}
1899
1900static int mes_v12_0_resume(struct amdgpu_ip_block *ip_block)
1901{
1902	return mes_v12_0_hw_init(ip_block);
1903}
1904
1905static int mes_v12_0_early_init(struct amdgpu_ip_block *ip_block)
1906{
1907	struct amdgpu_device *adev = ip_block->adev;
1908	int pipe, r;
1909
1910	adev->mes.hung_queue_db_array_size = MES12_HUNG_DB_OFFSET_ARRAY_SIZE;
1911	adev->mes.hung_queue_hqd_info_offset = MES12_HUNG_HQD_INFO_OFFSET;
1912
1913	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1914		r = amdgpu_mes_init_microcode(adev, pipe);
1915		if (r)
1916			return r;
1917	}
1918
1919	return 0;
1920}
1921
1922static const struct amd_ip_funcs mes_v12_0_ip_funcs = {
1923	.name = "mes_v12_0",
1924	.early_init = mes_v12_0_early_init,
1925	.late_init = NULL,
1926	.sw_init = mes_v12_0_sw_init,
1927	.sw_fini = mes_v12_0_sw_fini,
1928	.hw_init = mes_v12_0_hw_init,
1929	.hw_fini = mes_v12_0_hw_fini,
1930	.suspend = mes_v12_0_suspend,
1931	.resume = mes_v12_0_resume,
1932};
1933
1934const struct amdgpu_ip_block_version mes_v12_0_ip_block = {
1935	.type = AMD_IP_BLOCK_TYPE_MES,
1936	.major = 12,
1937	.minor = 0,
1938	.rev = 0,
1939	.funcs = &mes_v12_0_ip_funcs,
1940};
~/ Read-only file preview.