kfd_device_queue_manager.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/ratelimit.h>
  26#include <linux/printk.h>
  27#include <linux/slab.h>
  28#include <linux/list.h>
  29#include <linux/types.h>
  30#include <linux/bitops.h>
  31#include <linux/sched.h>
  32#include "kfd_priv.h"
  33#include "kfd_device_queue_manager.h"
  34#include "kfd_mqd_manager.h"
  35#include "cik_regs.h"
  36#include "kfd_kernel_queue.h"
  37#include "amdgpu_amdkfd.h"
  38#include "amdgpu_reset.h"
  39#include "amdgpu_sdma.h"
  40#include "mes_v11_api_def.h"
  41#include "kfd_debug.h"
  42
  43/* Size of the per-pipe EOP queue */
  44#define CIK_HPD_EOP_BYTES_LOG2 11
  45#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
  46/* See unmap_queues_cpsch() */
  47#define USE_DEFAULT_GRACE_PERIOD 0xffffffff
  48
  49static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
  50				  u32 pasid, unsigned int vmid);
  51
  52static int execute_queues_cpsch(struct device_queue_manager *dqm,
  53				enum kfd_unmap_queues_filter filter,
  54				uint32_t filter_param,
  55				uint32_t grace_period);
  56static int unmap_queues_cpsch(struct device_queue_manager *dqm,
  57				enum kfd_unmap_queues_filter filter,
  58				uint32_t filter_param,
  59				uint32_t grace_period,
  60				bool reset);
  61
  62static int map_queues_cpsch(struct device_queue_manager *dqm);
  63
  64static void deallocate_sdma_queue(struct device_queue_manager *dqm,
  65				struct queue *q);
  66
  67static inline void deallocate_hqd(struct device_queue_manager *dqm,
  68				struct queue *q);
  69static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
  70static int allocate_sdma_queue(struct device_queue_manager *dqm,
  71				struct queue *q, const uint32_t *restore_sdma_id);
  72
  73static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
  74
  75static inline
  76enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
  77{
  78	if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
  79		return KFD_MQD_TYPE_SDMA;
  80	return KFD_MQD_TYPE_CP;
  81}
  82
  83static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
  84{
  85	int i;
  86	int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
  87		+ pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
  88
  89	/* queue is available for KFD usage if bit is 1 */
  90	for (i = 0; i <  dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
  91		if (test_bit(pipe_offset + i,
  92			      dqm->dev->kfd->shared_resources.cp_queue_bitmap))
  93			return true;
  94	return false;
  95}
  96
  97unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
  98{
  99	return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,
 100				AMDGPU_MAX_QUEUES);
 101}
 102
 103unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
 104{
 105	return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
 106}
 107
 108unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
 109{
 110	return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
 111}
 112
 113static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
 114{
 115	return kfd_get_num_sdma_engines(dqm->dev) +
 116		kfd_get_num_xgmi_sdma_engines(dqm->dev);
 117}
 118
 119unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
 120{
 121	return kfd_get_num_sdma_engines(dqm->dev) *
 122		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
 123}
 124
 125unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
 126{
 127	return kfd_get_num_xgmi_sdma_engines(dqm->dev) *
 128		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
 129}
 130
 131static void init_sdma_bitmaps(struct device_queue_manager *dqm)
 132{
 133	bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 134	bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm));
 135
 136	bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 137	bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));
 138
 139	/* Mask out the reserved queues */
 140	bitmap_andnot(dqm->sdma_bitmap, dqm->sdma_bitmap,
 141		      dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
 142		      KFD_MAX_SDMA_QUEUES);
 143}
 144
 145void program_sh_mem_settings(struct device_queue_manager *dqm,
 146					struct qcm_process_device *qpd)
 147{
 148	uint32_t xcc_mask = dqm->dev->xcc_mask;
 149	int xcc_id;
 150
 151	for_each_inst(xcc_id, xcc_mask)
 152		dqm->dev->kfd2kgd->program_sh_mem_settings(
 153			dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,
 154			qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,
 155			qpd->sh_mem_bases, xcc_id);
 156}
 157
 158static void kfd_hws_hang(struct device_queue_manager *dqm)
 159{
 160	struct device_process_node *cur;
 161	struct qcm_process_device *qpd;
 162	struct queue *q;
 163
 164	/* Mark all device queues as reset. */
 165	list_for_each_entry(cur, &dqm->queues, list) {
 166		qpd = cur->qpd;
 167		list_for_each_entry(q, &qpd->queues_list, list) {
 168			struct kfd_process_device *pdd = qpd_to_pdd(qpd);
 169
 170			pdd->has_reset_queue = true;
 171		}
 172	}
 173
 174	/*
 175	 * Issue a GPU reset if HWS is unresponsive
 176	 */
 177	amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
 178}
 179
 180static int convert_to_mes_queue_type(int queue_type)
 181{
 182	int mes_queue_type;
 183
 184	switch (queue_type) {
 185	case KFD_QUEUE_TYPE_COMPUTE:
 186		mes_queue_type = MES_QUEUE_TYPE_COMPUTE;
 187		break;
 188	case KFD_QUEUE_TYPE_SDMA:
 189		mes_queue_type = MES_QUEUE_TYPE_SDMA;
 190		break;
 191	default:
 192		WARN(1, "Invalid queue type %d", queue_type);
 193		mes_queue_type = -EINVAL;
 194		break;
 195	}
 196
 197	return mes_queue_type;
 198}
 199
 200static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 201			 struct qcm_process_device *qpd)
 202{
 203	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 204	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
 205	struct mes_add_queue_input queue_input;
 206	int r, queue_type;
 207	uint64_t wptr_addr_off;
 208
 209	if (!dqm->sched_running || dqm->sched_halt)
 210		return 0;
 211	if (!down_read_trylock(&adev->reset_domain->sem))
 212		return -EIO;
 213
 214	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
 215	queue_input.process_id = pdd->pasid;
 216	queue_input.page_table_base_addr =  qpd->page_table_base;
 217	queue_input.process_va_start = 0;
 218	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
 219	/* MES unit for quantum is 100ns */
 220	queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM;  /* Equivalent to 10ms. */
 221	queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;
 222	queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */
 223	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
 224	queue_input.inprocess_gang_priority = q->properties.priority;
 225	queue_input.gang_global_priority_level =
 226					AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
 227	queue_input.doorbell_offset = q->properties.doorbell_off;
 228	queue_input.mqd_addr = q->gart_mqd_addr;
 229	queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;
 230
 231	wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);
 232	queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off;
 233
 234	queue_input.is_kfd_process = 1;
 235	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
 236	queue_input.queue_size = q->properties.queue_size >> 2;
 237
 238	queue_input.paging = false;
 239	queue_input.tba_addr = qpd->tba_addr;
 240	queue_input.tma_addr = qpd->tma_addr;
 241	queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device);
 242	queue_input.skip_process_ctx_clear =
 243		qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&
 244						(qpd->pqm->process->debug_trap_enabled ||
 245						 kfd_dbg_has_ttmps_always_setup(q->device));
 246
 247	queue_type = convert_to_mes_queue_type(q->properties.type);
 248	if (queue_type < 0) {
 249		dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",
 250			q->properties.type);
 251		up_read(&adev->reset_domain->sem);
 252		return -EINVAL;
 253	}
 254	queue_input.queue_type = (uint32_t)queue_type;
 255
 256	queue_input.exclusively_scheduled = q->properties.is_gws;
 257
 258	amdgpu_mes_lock(&adev->mes);
 259	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
 260	amdgpu_mes_unlock(&adev->mes);
 261	up_read(&adev->reset_domain->sem);
 262	if (r) {
 263		dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",
 264			q->properties.doorbell_off);
 265		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 266		kfd_hws_hang(dqm);
 267	}
 268
 269	return r;
 270}
 271
 272static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 273			struct qcm_process_device *qpd)
 274{
 275	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 276	int r;
 277	struct mes_remove_queue_input queue_input;
 278
 279	if (!dqm->sched_running || dqm->sched_halt)
 280		return 0;
 281	if (!down_read_trylock(&adev->reset_domain->sem))
 282		return -EIO;
 283
 284	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
 285	queue_input.doorbell_offset = q->properties.doorbell_off;
 286	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
 287
 288	amdgpu_mes_lock(&adev->mes);
 289	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
 290	amdgpu_mes_unlock(&adev->mes);
 291	up_read(&adev->reset_domain->sem);
 292
 293	if (r) {
 294		dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
 295			q->properties.doorbell_off);
 296		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 297		kfd_hws_hang(dqm);
 298	}
 299
 300	return r;
 301}
 302
 303static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
 304{
 305	struct device_process_node *cur;
 306	struct device *dev = dqm->dev->adev->dev;
 307	struct qcm_process_device *qpd;
 308	struct queue *q;
 309	int retval = 0;
 310
 311	list_for_each_entry(cur, &dqm->queues, list) {
 312		qpd = cur->qpd;
 313		list_for_each_entry(q, &qpd->queues_list, list) {
 314			if (q->properties.is_active) {
 315				retval = remove_queue_mes(dqm, q, qpd);
 316				if (retval) {
 317					dev_err(dev, "%s: Failed to remove queue %d for dev %d",
 318						__func__,
 319						q->properties.queue_id,
 320						dqm->dev->id);
 321					return retval;
 322				}
 323			}
 324		}
 325	}
 326
 327	return retval;
 328}
 329
 330static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
 331{
 332	struct device_process_node *cur;
 333	struct device *dev = dqm->dev->adev->dev;
 334	struct qcm_process_device *qpd;
 335	struct queue *q;
 336	int retval = 0;
 337
 338	list_for_each_entry(cur, &dqm->queues, list) {
 339		qpd = cur->qpd;
 340		list_for_each_entry(q, &qpd->queues_list, list) {
 341			if (!q->properties.is_active)
 342				continue;
 343			retval = add_queue_mes(dqm, q, qpd);
 344			if (retval) {
 345				dev_err(dev, "%s: Failed to add queue %d for dev %d",
 346					__func__,
 347					q->properties.queue_id,
 348					dqm->dev->id);
 349				return retval;
 350			}
 351		}
 352	}
 353
 354	return retval;
 355}
 356
 357static int suspend_all_queues_mes(struct device_queue_manager *dqm)
 358{
 359	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 360	int r = 0;
 361
 362	if (!down_read_trylock(&adev->reset_domain->sem))
 363		return -EIO;
 364
 365	r = amdgpu_mes_suspend(adev);
 366	up_read(&adev->reset_domain->sem);
 367
 368	if (r) {
 369		dev_err(adev->dev, "failed to suspend gangs from MES\n");
 370		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 371		kfd_hws_hang(dqm);
 372	}
 373
 374	return r;
 375}
 376
 377static int resume_all_queues_mes(struct device_queue_manager *dqm)
 378{
 379	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 380	int r = 0;
 381
 382	if (!down_read_trylock(&adev->reset_domain->sem))
 383		return -EIO;
 384
 385	r = amdgpu_mes_resume(adev);
 386	up_read(&adev->reset_domain->sem);
 387
 388	if (r) {
 389		dev_err(adev->dev, "failed to resume gangs from MES\n");
 390		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 391		kfd_hws_hang(dqm);
 392	}
 393
 394	return r;
 395}
 396
 397static void increment_queue_count(struct device_queue_manager *dqm,
 398				  struct qcm_process_device *qpd,
 399				  struct queue *q)
 400{
 401	dqm->active_queue_count++;
 402	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 403	    q->properties.type == KFD_QUEUE_TYPE_DIQ)
 404		dqm->active_cp_queue_count++;
 405
 406	if (q->properties.is_gws) {
 407		dqm->gws_queue_count++;
 408		qpd->mapped_gws_queue = true;
 409	}
 410}
 411
 412static void decrement_queue_count(struct device_queue_manager *dqm,
 413				  struct qcm_process_device *qpd,
 414				  struct queue *q)
 415{
 416	dqm->active_queue_count--;
 417	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 418	    q->properties.type == KFD_QUEUE_TYPE_DIQ)
 419		dqm->active_cp_queue_count--;
 420
 421	if (q->properties.is_gws) {
 422		dqm->gws_queue_count--;
 423		qpd->mapped_gws_queue = false;
 424	}
 425}
 426
 427/*
 428 * Allocate a doorbell ID to this queue.
 429 * If doorbell_id is passed in, make sure requested ID is valid then allocate it.
 430 */
 431static int allocate_doorbell(struct qcm_process_device *qpd,
 432			     struct queue *q,
 433			     uint32_t const *restore_id)
 434{
 435	struct kfd_node *dev = qpd->dqm->dev;
 436
 437	if (!KFD_IS_SOC15(dev)) {
 438		/* On pre-SOC15 chips we need to use the queue ID to
 439		 * preserve the user mode ABI.
 440		 */
 441
 442		if (restore_id && *restore_id != q->properties.queue_id)
 443			return -EINVAL;
 444
 445		q->doorbell_id = q->properties.queue_id;
 446	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 447			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
 448		/* For SDMA queues on SOC15 with 8-byte doorbell, use static
 449		 * doorbell assignments based on the engine and queue id.
 450		 * The doobell index distance between RLC (2*i) and (2*i+1)
 451		 * for a SDMA engine is 512.
 452		 */
 453
 454		uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
 455
 456		/*
 457		 * q->properties.sdma_engine_id corresponds to the virtual
 458		 * sdma engine number. However, for doorbell allocation,
 459		 * we need the physical sdma engine id in order to get the
 460		 * correct doorbell offset.
 461		 */
 462		uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id *
 463					       get_num_all_sdma_engines(qpd->dqm) +
 464					       q->properties.sdma_engine_id]
 465						+ (q->properties.sdma_queue_id & 1)
 466						* KFD_QUEUE_DOORBELL_MIRROR_OFFSET
 467						+ (q->properties.sdma_queue_id >> 1);
 468
 469		if (restore_id && *restore_id != valid_id)
 470			return -EINVAL;
 471		q->doorbell_id = valid_id;
 472	} else {
 473		/* For CP queues on SOC15 */
 474		if (restore_id) {
 475			/* make sure that ID is free  */
 476			if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
 477				return -EINVAL;
 478
 479			q->doorbell_id = *restore_id;
 480		} else {
 481			/* or reserve a free doorbell ID */
 482			unsigned int found;
 483
 484			found = find_first_zero_bit(qpd->doorbell_bitmap,
 485						    KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
 486			if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
 487				pr_debug("No doorbells available");
 488				return -EBUSY;
 489			}
 490			set_bit(found, qpd->doorbell_bitmap);
 491			q->doorbell_id = found;
 492		}
 493	}
 494
 495	q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev,
 496								  qpd->proc_doorbells,
 497								  q->doorbell_id,
 498								  dev->kfd->device_info.doorbell_size);
 499	return 0;
 500}
 501
 502static void deallocate_doorbell(struct qcm_process_device *qpd,
 503				struct queue *q)
 504{
 505	unsigned int old;
 506	struct kfd_node *dev = qpd->dqm->dev;
 507
 508	if (!KFD_IS_SOC15(dev) ||
 509	    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 510	    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
 511		return;
 512
 513	old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
 514	WARN_ON(!old);
 515}
 516
 517static void program_trap_handler_settings(struct device_queue_manager *dqm,
 518				struct qcm_process_device *qpd)
 519{
 520	uint32_t xcc_mask = dqm->dev->xcc_mask;
 521	int xcc_id;
 522
 523	if (dqm->dev->kfd2kgd->program_trap_handler_settings)
 524		for_each_inst(xcc_id, xcc_mask)
 525			dqm->dev->kfd2kgd->program_trap_handler_settings(
 526				dqm->dev->adev, qpd->vmid, qpd->tba_addr,
 527				qpd->tma_addr, xcc_id);
 528}
 529
 530static int allocate_vmid(struct device_queue_manager *dqm,
 531			struct qcm_process_device *qpd,
 532			struct queue *q)
 533{
 534	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
 535	struct device *dev = dqm->dev->adev->dev;
 536	int allocated_vmid = -1, i;
 537
 538	for (i = dqm->dev->vm_info.first_vmid_kfd;
 539			i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
 540		if (!dqm->vmid_pasid[i]) {
 541			allocated_vmid = i;
 542			break;
 543		}
 544	}
 545
 546	if (allocated_vmid < 0) {
 547		dev_err(dev, "no more vmid to allocate\n");
 548		return -ENOSPC;
 549	}
 550
 551	pr_debug("vmid allocated: %d\n", allocated_vmid);
 552
 553	dqm->vmid_pasid[allocated_vmid] = pdd->pasid;
 554
 555	set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid);
 556
 557	qpd->vmid = allocated_vmid;
 558	q->properties.vmid = allocated_vmid;
 559
 560	program_sh_mem_settings(dqm, qpd);
 561
 562	if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
 563		program_trap_handler_settings(dqm, qpd);
 564
 565	/* qpd->page_table_base is set earlier when register_process()
 566	 * is called, i.e. when the first queue is created.
 567	 */
 568	dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
 569			qpd->vmid,
 570			qpd->page_table_base);
 571	/* invalidate the VM context after pasid and vmid mapping is set up */
 572	kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
 573
 574	if (dqm->dev->kfd2kgd->set_scratch_backing_va)
 575		dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
 576				qpd->sh_hidden_private_base, qpd->vmid);
 577
 578	return 0;
 579}
 580
 581static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
 582				struct qcm_process_device *qpd)
 583{
 584	const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
 585	int ret;
 586
 587	if (!qpd->ib_kaddr)
 588		return -ENOMEM;
 589
 590	ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
 591	if (ret)
 592		return ret;
 593
 594	return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,
 595				qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
 596				pmf->release_mem_size / sizeof(uint32_t));
 597}
 598
 599static void deallocate_vmid(struct device_queue_manager *dqm,
 600				struct qcm_process_device *qpd,
 601				struct queue *q)
 602{
 603	struct device *dev = dqm->dev->adev->dev;
 604
 605	/* On GFX v7, CP doesn't flush TC at dequeue */
 606	if (q->device->adev->asic_type == CHIP_HAWAII)
 607		if (flush_texture_cache_nocpsch(q->device, qpd))
 608			dev_err(dev, "Failed to flush TC\n");
 609
 610	kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
 611
 612	/* Release the vmid mapping */
 613	set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
 614	dqm->vmid_pasid[qpd->vmid] = 0;
 615
 616	qpd->vmid = 0;
 617	q->properties.vmid = 0;
 618}
 619
 620static int create_queue_nocpsch(struct device_queue_manager *dqm,
 621				struct queue *q,
 622				struct qcm_process_device *qpd,
 623				const struct kfd_criu_queue_priv_data *qd,
 624				const void *restore_mqd, const void *restore_ctl_stack)
 625{
 626	struct mqd_manager *mqd_mgr;
 627	int retval;
 628
 629	dqm_lock(dqm);
 630
 631	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 632		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 633				dqm->total_queue_count);
 634		retval = -EPERM;
 635		goto out_unlock;
 636	}
 637
 638	if (list_empty(&qpd->queues_list)) {
 639		retval = allocate_vmid(dqm, qpd, q);
 640		if (retval)
 641			goto out_unlock;
 642	}
 643	q->properties.vmid = qpd->vmid;
 644	/*
 645	 * Eviction state logic: mark all queues as evicted, even ones
 646	 * not currently active. Restoring inactive queues later only
 647	 * updates the is_evicted flag but is a no-op otherwise.
 648	 */
 649	q->properties.is_evicted = !!qpd->evicted;
 650
 651	q->properties.tba_addr = qpd->tba_addr;
 652	q->properties.tma_addr = qpd->tma_addr;
 653
 654	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 655			q->properties.type)];
 656	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 657		retval = allocate_hqd(dqm, q);
 658		if (retval)
 659			goto deallocate_vmid;
 660		pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
 661			q->pipe, q->queue);
 662	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 663		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
 664		retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
 665		if (retval)
 666			goto deallocate_vmid;
 667		dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
 668	}
 669
 670	retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
 671	if (retval)
 672		goto out_deallocate_hqd;
 673
 674	/* Temporarily release dqm lock to avoid a circular lock dependency */
 675	dqm_unlock(dqm);
 676	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
 677	dqm_lock(dqm);
 678
 679	if (!q->mqd_mem_obj) {
 680		retval = -ENOMEM;
 681		goto out_deallocate_doorbell;
 682	}
 683
 684	if (qd)
 685		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
 686				     &q->properties, restore_mqd, restore_ctl_stack,
 687				     qd->ctl_stack_size);
 688	else
 689		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
 690					&q->gart_mqd_addr, &q->properties);
 691
 692	if (q->properties.is_active) {
 693		if (!dqm->sched_running) {
 694			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
 695			goto add_queue_to_list;
 696		}
 697
 698		if (WARN(q->process->mm != current->mm,
 699					"should only run in user thread"))
 700			retval = -EFAULT;
 701		else
 702			retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
 703					q->queue, &q->properties, current->mm);
 704		if (retval)
 705			goto out_free_mqd;
 706	}
 707
 708add_queue_to_list:
 709	list_add(&q->list, &qpd->queues_list);
 710	qpd->queue_count++;
 711	if (q->properties.is_active)
 712		increment_queue_count(dqm, qpd, q);
 713
 714	/*
 715	 * Unconditionally increment this counter, regardless of the queue's
 716	 * type or whether the queue is active.
 717	 */
 718	dqm->total_queue_count++;
 719	pr_debug("Total of %d queues are accountable so far\n",
 720			dqm->total_queue_count);
 721	goto out_unlock;
 722
 723out_free_mqd:
 724	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 725out_deallocate_doorbell:
 726	deallocate_doorbell(qpd, q);
 727out_deallocate_hqd:
 728	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 729		deallocate_hqd(dqm, q);
 730	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 731		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
 732		deallocate_sdma_queue(dqm, q);
 733deallocate_vmid:
 734	if (list_empty(&qpd->queues_list))
 735		deallocate_vmid(dqm, qpd, q);
 736out_unlock:
 737	dqm_unlock(dqm);
 738	return retval;
 739}
 740
 741static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 742{
 743	bool set;
 744	int pipe, bit, i;
 745
 746	set = false;
 747
 748	for (pipe = dqm->next_pipe_to_allocate, i = 0;
 749			i < get_pipes_per_mec(dqm);
 750			pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
 751
 752		if (!is_pipe_enabled(dqm, 0, pipe))
 753			continue;
 754
 755		if (dqm->allocated_queues[pipe] != 0) {
 756			bit = ffs(dqm->allocated_queues[pipe]) - 1;
 757			dqm->allocated_queues[pipe] &= ~(1 << bit);
 758			q->pipe = pipe;
 759			q->queue = bit;
 760			set = true;
 761			break;
 762		}
 763	}
 764
 765	if (!set)
 766		return -EBUSY;
 767
 768	pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
 769	/* horizontal hqd allocation */
 770	dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
 771
 772	return 0;
 773}
 774
 775static inline void deallocate_hqd(struct device_queue_manager *dqm,
 776				struct queue *q)
 777{
 778	dqm->allocated_queues[q->pipe] |= (1 << q->queue);
 779}
 780
 781#define SQ_IND_CMD_CMD_KILL		0x00000003
 782#define SQ_IND_CMD_MODE_BROADCAST	0x00000001
 783
 784static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
 785{
 786	int status = 0;
 787	unsigned int vmid;
 788	uint16_t queried_pasid;
 789	union SQ_CMD_BITS reg_sq_cmd;
 790	union GRBM_GFX_INDEX_BITS reg_gfx_index;
 791	struct kfd_process_device *pdd;
 792	int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
 793	int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
 794	uint32_t xcc_mask = dev->xcc_mask;
 795	int xcc_id;
 796
 797	reg_sq_cmd.u32All = 0;
 798	reg_gfx_index.u32All = 0;
 799
 800	pr_debug("Killing all process wavefronts\n");
 801
 802	if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {
 803		dev_err(dev->adev->dev, "no vmid pasid mapping supported\n");
 804		return -EOPNOTSUPP;
 805	}
 806
 807	/* taking the VMID for that process on the safe way using PDD */
 808	pdd = kfd_get_process_device_data(dev, p);
 809	if (!pdd)
 810		return -EFAULT;
 811
 812	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
 813	 * ATC_VMID15_PASID_MAPPING
 814	 * to check which VMID the current process is mapped to.
 815	 */
 816
 817	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
 818		status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
 819				(dev->adev, vmid, &queried_pasid);
 820
 821		if (status && queried_pasid == pdd->pasid) {
 822			pr_debug("Killing wave fronts of vmid %d and process pid %d\n",
 823					vmid, p->lead_thread->pid);
 824			break;
 825		}
 826	}
 827
 828	if (vmid > last_vmid_to_scan) {
 829		dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n",
 830				p->lead_thread->pid);
 831		return -EFAULT;
 832	}
 833
 834	reg_gfx_index.bits.sh_broadcast_writes = 1;
 835	reg_gfx_index.bits.se_broadcast_writes = 1;
 836	reg_gfx_index.bits.instance_broadcast_writes = 1;
 837	reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
 838	reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
 839	reg_sq_cmd.bits.vm_id = vmid;
 840
 841	for_each_inst(xcc_id, xcc_mask)
 842		dev->kfd2kgd->wave_control_execute(
 843			dev->adev, reg_gfx_index.u32All,
 844			reg_sq_cmd.u32All, xcc_id);
 845
 846	return 0;
 847}
 848
 849/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
 850 * to avoid asynchronized access
 851 */
 852static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
 853				struct qcm_process_device *qpd,
 854				struct queue *q)
 855{
 856	int retval;
 857	struct mqd_manager *mqd_mgr;
 858
 859	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 860			q->properties.type)];
 861
 862	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 863		deallocate_hqd(dqm, q);
 864	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 865		deallocate_sdma_queue(dqm, q);
 866	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
 867		deallocate_sdma_queue(dqm, q);
 868	else {
 869		pr_debug("q->properties.type %d is invalid\n",
 870				q->properties.type);
 871		return -EINVAL;
 872	}
 873	dqm->total_queue_count--;
 874
 875	deallocate_doorbell(qpd, q);
 876
 877	if (!dqm->sched_running) {
 878		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
 879		return 0;
 880	}
 881
 882	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 883				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 884				KFD_UNMAP_LATENCY_MS,
 885				q->pipe, q->queue);
 886	if (retval == -ETIME)
 887		qpd->reset_wavefronts = true;
 888
 889	list_del(&q->list);
 890	if (list_empty(&qpd->queues_list)) {
 891		if (qpd->reset_wavefronts) {
 892			pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
 893					dqm->dev);
 894			/* dbgdev_wave_reset_wavefronts has to be called before
 895			 * deallocate_vmid(), i.e. when vmid is still in use.
 896			 */
 897			dbgdev_wave_reset_wavefronts(dqm->dev,
 898					qpd->pqm->process);
 899			qpd->reset_wavefronts = false;
 900		}
 901
 902		deallocate_vmid(dqm, qpd, q);
 903	}
 904	qpd->queue_count--;
 905	if (q->properties.is_active)
 906		decrement_queue_count(dqm, qpd, q);
 907
 908	return retval;
 909}
 910
 911static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 912				struct qcm_process_device *qpd,
 913				struct queue *q)
 914{
 915	int retval;
 916	uint64_t sdma_val = 0;
 917	struct device *dev = dqm->dev->adev->dev;
 918	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
 919	struct mqd_manager *mqd_mgr =
 920		dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
 921
 922	/* Get the SDMA queue stats */
 923	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
 924	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
 925		retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
 926							&sdma_val);
 927		if (retval)
 928			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
 929				q->properties.queue_id);
 930	}
 931
 932	dqm_lock(dqm);
 933	retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
 934	if (!retval)
 935		pdd->sdma_past_activity_counter += sdma_val;
 936	dqm_unlock(dqm);
 937
 938	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 939
 940	return retval;
 941}
 942
 943static int update_queue(struct device_queue_manager *dqm, struct queue *q,
 944			struct mqd_update_info *minfo)
 945{
 946	int retval = 0;
 947	struct device *dev = dqm->dev->adev->dev;
 948	struct mqd_manager *mqd_mgr;
 949	struct kfd_process_device *pdd;
 950	bool prev_active = false;
 951
 952	dqm_lock(dqm);
 953	pdd = kfd_get_process_device_data(q->device, q->process);
 954	if (!pdd) {
 955		retval = -ENODEV;
 956		goto out_unlock;
 957	}
 958	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 959			q->properties.type)];
 960
 961	/* Save previous activity state for counters */
 962	prev_active = q->properties.is_active;
 963
 964	/* Make sure the queue is unmapped before updating the MQD */
 965	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
 966		if (!dqm->dev->kfd->shared_resources.enable_mes)
 967			retval = unmap_queues_cpsch(dqm,
 968						    KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);
 969		else if (prev_active)
 970			retval = remove_queue_mes(dqm, q, &pdd->qpd);
 971
 972		/* queue is reset so inaccessable  */
 973		if (pdd->has_reset_queue) {
 974			retval = -EACCES;
 975			goto out_unlock;
 976		}
 977
 978		if (retval) {
 979			dev_err(dev, "unmap queue failed\n");
 980			goto out_unlock;
 981		}
 982	} else if (prev_active &&
 983		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 984		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
 985		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
 986
 987		if (!dqm->sched_running) {
 988			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
 989			goto out_unlock;
 990		}
 991
 992		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 993				(dqm->dev->kfd->cwsr_enabled ?
 994				 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
 995				 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
 996				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
 997		if (retval) {
 998			dev_err(dev, "destroy mqd failed\n");
 999			goto out_unlock;
1000		}
1001	}
1002
1003	mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
1004
1005	/*
1006	 * check active state vs. the previous state and modify
1007	 * counter accordingly. map_queues_cpsch uses the
1008	 * dqm->active_queue_count to determine whether a new runlist must be
1009	 * uploaded.
1010	 */
1011	if (q->properties.is_active && !prev_active) {
1012		increment_queue_count(dqm, &pdd->qpd, q);
1013	} else if (!q->properties.is_active && prev_active) {
1014		decrement_queue_count(dqm, &pdd->qpd, q);
1015	} else if (q->gws && !q->properties.is_gws) {
1016		if (q->properties.is_active) {
1017			dqm->gws_queue_count++;
1018			pdd->qpd.mapped_gws_queue = true;
1019		}
1020		q->properties.is_gws = true;
1021	} else if (!q->gws && q->properties.is_gws) {
1022		if (q->properties.is_active) {
1023			dqm->gws_queue_count--;
1024			pdd->qpd.mapped_gws_queue = false;
1025		}
1026		q->properties.is_gws = false;
1027	}
1028
1029	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
1030		if (!dqm->dev->kfd->shared_resources.enable_mes)
1031			retval = map_queues_cpsch(dqm);
1032		else if (q->properties.is_active)
1033			retval = add_queue_mes(dqm, q, &pdd->qpd);
1034	} else if (q->properties.is_active &&
1035		 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1036		  q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1037		  q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1038		if (WARN(q->process->mm != current->mm,
1039			 "should only run in user thread"))
1040			retval = -EFAULT;
1041		else
1042			retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
1043						   q->pipe, q->queue,
1044						   &q->properties, current->mm);
1045	}
1046
1047out_unlock:
1048	dqm_unlock(dqm);
1049	return retval;
1050}
1051
1052/* suspend_single_queue does not lock the dqm like the
1053 * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should
1054 * lock the dqm before calling, and unlock after calling.
1055 *
1056 * The reason we don't lock the dqm is because this function may be
1057 * called on multiple queues in a loop, so rather than locking/unlocking
1058 * multiple times, we will just keep the dqm locked for all of the calls.
1059 */
1060static int suspend_single_queue(struct device_queue_manager *dqm,
1061				      struct kfd_process_device *pdd,
1062				      struct queue *q)
1063{
1064	bool is_new;
1065
1066	if (q->properties.is_suspended)
1067		return 0;
1068
1069	pr_debug("Suspending process pid %d queue [%i]\n",
1070			pdd->process->lead_thread->pid,
1071			q->properties.queue_id);
1072
1073	is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);
1074
1075	if (is_new || q->properties.is_being_destroyed) {
1076		pr_debug("Suspend: skip %s queue id %i\n",
1077				is_new ? "new" : "destroyed",
1078				q->properties.queue_id);
1079		return -EBUSY;
1080	}
1081
1082	q->properties.is_suspended = true;
1083	if (q->properties.is_active) {
1084		if (dqm->dev->kfd->shared_resources.enable_mes) {
1085			int r = remove_queue_mes(dqm, q, &pdd->qpd);
1086
1087			if (r)
1088				return r;
1089		}
1090
1091		decrement_queue_count(dqm, &pdd->qpd, q);
1092		q->properties.is_active = false;
1093	}
1094
1095	return 0;
1096}
1097
1098/* resume_single_queue does not lock the dqm like the functions
1099 * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should
1100 * lock the dqm before calling, and unlock after calling.
1101 *
1102 * The reason we don't lock the dqm is because this function may be
1103 * called on multiple queues in a loop, so rather than locking/unlocking
1104 * multiple times, we will just keep the dqm locked for all of the calls.
1105 */
1106static int resume_single_queue(struct device_queue_manager *dqm,
1107				      struct qcm_process_device *qpd,
1108				      struct queue *q)
1109{
1110	struct kfd_process_device *pdd;
1111
1112	if (!q->properties.is_suspended)
1113		return 0;
1114
1115	pdd = qpd_to_pdd(qpd);
1116
1117	pr_debug("Restoring from suspend process pid %d queue [%i]\n",
1118			    pdd->process->lead_thread->pid,
1119			    q->properties.queue_id);
1120
1121	q->properties.is_suspended = false;
1122
1123	if (QUEUE_IS_ACTIVE(q->properties)) {
1124		if (dqm->dev->kfd->shared_resources.enable_mes) {
1125			int r = add_queue_mes(dqm, q, &pdd->qpd);
1126
1127			if (r)
1128				return r;
1129		}
1130
1131		q->properties.is_active = true;
1132		increment_queue_count(dqm, qpd, q);
1133	}
1134
1135	return 0;
1136}
1137
1138static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
1139					struct qcm_process_device *qpd)
1140{
1141	struct queue *q;
1142	struct mqd_manager *mqd_mgr;
1143	struct kfd_process_device *pdd;
1144	int retval, ret = 0;
1145
1146	dqm_lock(dqm);
1147	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1148		goto out;
1149
1150	pdd = qpd_to_pdd(qpd);
1151	pr_debug_ratelimited("Evicting process pid %d queues\n",
1152			    pdd->process->lead_thread->pid);
1153
1154	pdd->last_evict_timestamp = get_jiffies_64();
1155	/* Mark all queues as evicted. Deactivate all active queues on
1156	 * the qpd.
1157	 */
1158	list_for_each_entry(q, &qpd->queues_list, list) {
1159		q->properties.is_evicted = true;
1160		if (!q->properties.is_active)
1161			continue;
1162
1163		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1164				q->properties.type)];
1165		q->properties.is_active = false;
1166		decrement_queue_count(dqm, qpd, q);
1167
1168		if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
1169			continue;
1170
1171		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1172				(dqm->dev->kfd->cwsr_enabled ?
1173				 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1174				 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1175				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1176		if (retval && !ret)
1177			/* Return the first error, but keep going to
1178			 * maintain a consistent eviction state
1179			 */
1180			ret = retval;
1181	}
1182
1183out:
1184	dqm_unlock(dqm);
1185	return ret;
1186}
1187
1188static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
1189				      struct qcm_process_device *qpd)
1190{
1191	struct queue *q;
1192	struct device *dev = dqm->dev->adev->dev;
1193	struct kfd_process_device *pdd;
1194	int retval = 0;
1195
1196	dqm_lock(dqm);
1197	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1198		goto out;
1199
1200	pdd = qpd_to_pdd(qpd);
1201
1202	/* The debugger creates processes that temporarily have not acquired
1203	 * all VMs for all devices and has no VMs itself.
1204	 * Skip queue eviction on process eviction.
1205	 */
1206	if (!pdd->drm_priv)
1207		goto out;
1208
1209	pr_debug_ratelimited("Evicting process pid %d queues\n",
1210			    pdd->process->lead_thread->pid);
1211
1212	if (dqm->dev->kfd->shared_resources.enable_mes)
1213		pdd->last_evict_timestamp = get_jiffies_64();
1214
1215	/* Mark all queues as evicted. Deactivate all active queues on
1216	 * the qpd.
1217	 */
1218	list_for_each_entry(q, &qpd->queues_list, list) {
1219		q->properties.is_evicted = true;
1220		if (!q->properties.is_active)
1221			continue;
1222
1223		q->properties.is_active = false;
1224		decrement_queue_count(dqm, qpd, q);
1225
1226		if (dqm->dev->kfd->shared_resources.enable_mes) {
1227			retval = remove_queue_mes(dqm, q, qpd);
1228			if (retval) {
1229				dev_err(dev, "Failed to evict queue %d\n",
1230					q->properties.queue_id);
1231				goto out;
1232			}
1233		}
1234	}
1235
1236	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1237		pdd->last_evict_timestamp = get_jiffies_64();
1238		retval = execute_queues_cpsch(dqm,
1239					      qpd->is_debug ?
1240					      KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
1241					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1242					      USE_DEFAULT_GRACE_PERIOD);
1243	}
1244
1245out:
1246	dqm_unlock(dqm);
1247	return retval;
1248}
1249
1250static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
1251					  struct qcm_process_device *qpd)
1252{
1253	struct mm_struct *mm = NULL;
1254	struct queue *q;
1255	struct mqd_manager *mqd_mgr;
1256	struct kfd_process_device *pdd;
1257	uint64_t pd_base;
1258	uint64_t eviction_duration;
1259	int retval, ret = 0;
1260
1261	pdd = qpd_to_pdd(qpd);
1262	/* Retrieve PD base */
1263	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1264
1265	dqm_lock(dqm);
1266	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1267		goto out;
1268	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1269		qpd->evicted--;
1270		goto out;
1271	}
1272
1273	pr_debug_ratelimited("Restoring process pid %d queues\n",
1274			    pdd->process->lead_thread->pid);
1275
1276	/* Update PD Base in QPD */
1277	qpd->page_table_base = pd_base;
1278	pr_debug("Updated PD address to 0x%llx\n", pd_base);
1279
1280	if (!list_empty(&qpd->queues_list)) {
1281		dqm->dev->kfd2kgd->set_vm_context_page_table_base(
1282				dqm->dev->adev,
1283				qpd->vmid,
1284				qpd->page_table_base);
1285		kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
1286	}
1287
1288	/* Take a safe reference to the mm_struct, which may otherwise
1289	 * disappear even while the kfd_process is still referenced.
1290	 */
1291	mm = get_task_mm(pdd->process->lead_thread);
1292	if (!mm) {
1293		ret = -EFAULT;
1294		goto out;
1295	}
1296
1297	/* Remove the eviction flags. Activate queues that are not
1298	 * inactive for other reasons.
1299	 */
1300	list_for_each_entry(q, &qpd->queues_list, list) {
1301		q->properties.is_evicted = false;
1302		if (!QUEUE_IS_ACTIVE(q->properties))
1303			continue;
1304
1305		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1306				q->properties.type)];
1307		q->properties.is_active = true;
1308		increment_queue_count(dqm, qpd, q);
1309
1310		if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
1311			continue;
1312
1313		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1314				       q->queue, &q->properties, mm);
1315		if (retval && !ret)
1316			/* Return the first error, but keep going to
1317			 * maintain a consistent eviction state
1318			 */
1319			ret = retval;
1320	}
1321	qpd->evicted = 0;
1322	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1323	atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1324out:
1325	if (mm)
1326		mmput(mm);
1327	dqm_unlock(dqm);
1328	return ret;
1329}
1330
1331static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
1332					struct qcm_process_device *qpd)
1333{
1334	struct queue *q;
1335	struct device *dev = dqm->dev->adev->dev;
1336	struct kfd_process_device *pdd;
1337	uint64_t eviction_duration;
1338	int retval = 0;
1339
1340	pdd = qpd_to_pdd(qpd);
1341
1342	dqm_lock(dqm);
1343	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1344		goto out;
1345	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1346		qpd->evicted--;
1347		goto out;
1348	}
1349
1350	/* The debugger creates processes that temporarily have not acquired
1351	 * all VMs for all devices and has no VMs itself.
1352	 * Skip queue restore on process restore.
1353	 */
1354	if (!pdd->drm_priv)
1355		goto vm_not_acquired;
1356
1357	pr_debug_ratelimited("Restoring process pid %d queues\n",
1358			    pdd->process->lead_thread->pid);
1359
1360	/* Update PD Base in QPD */
1361	qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1362	pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);
1363
1364	/* activate all active queues on the qpd */
1365	list_for_each_entry(q, &qpd->queues_list, list) {
1366		q->properties.is_evicted = false;
1367		if (!QUEUE_IS_ACTIVE(q->properties))
1368			continue;
1369
1370		q->properties.is_active = true;
1371		increment_queue_count(dqm, &pdd->qpd, q);
1372
1373		if (dqm->dev->kfd->shared_resources.enable_mes) {
1374			retval = add_queue_mes(dqm, q, qpd);
1375			if (retval) {
1376				dev_err(dev, "Failed to restore queue %d\n",
1377					q->properties.queue_id);
1378				goto out;
1379			}
1380		}
1381	}
1382	if (!dqm->dev->kfd->shared_resources.enable_mes)
1383		retval = execute_queues_cpsch(dqm,
1384					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1385	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1386	atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1387vm_not_acquired:
1388	qpd->evicted = 0;
1389out:
1390	dqm_unlock(dqm);
1391	return retval;
1392}
1393
1394static int register_process(struct device_queue_manager *dqm,
1395					struct qcm_process_device *qpd)
1396{
1397	struct device_process_node *n;
1398	struct kfd_process_device *pdd;
1399	uint64_t pd_base;
1400	int retval;
1401
1402	n = kzalloc(sizeof(*n), GFP_KERNEL);
1403	if (!n)
1404		return -ENOMEM;
1405
1406	n->qpd = qpd;
1407
1408	pdd = qpd_to_pdd(qpd);
1409	/* Retrieve PD base */
1410	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1411
1412	dqm_lock(dqm);
1413	list_add(&n->list, &dqm->queues);
1414
1415	/* Update PD Base in QPD */
1416	qpd->page_table_base = pd_base;
1417	pr_debug("Updated PD address to 0x%llx\n", pd_base);
1418
1419	retval = dqm->asic_ops.update_qpd(dqm, qpd);
1420
1421	dqm->processes_count++;
1422
1423	dqm_unlock(dqm);
1424
1425	/* Outside the DQM lock because under the DQM lock we can't do
1426	 * reclaim or take other locks that others hold while reclaiming.
1427	 */
1428	kfd_inc_compute_active(dqm->dev);
1429
1430	return retval;
1431}
1432
1433static int unregister_process(struct device_queue_manager *dqm,
1434					struct qcm_process_device *qpd)
1435{
1436	int retval;
1437	struct device_process_node *cur, *next;
1438
1439	pr_debug("qpd->queues_list is %s\n",
1440			list_empty(&qpd->queues_list) ? "empty" : "not empty");
1441
1442	retval = 0;
1443	dqm_lock(dqm);
1444
1445	list_for_each_entry_safe(cur, next, &dqm->queues, list) {
1446		if (qpd == cur->qpd) {
1447			list_del(&cur->list);
1448			kfree(cur);
1449			dqm->processes_count--;
1450			goto out;
1451		}
1452	}
1453	/* qpd not found in dqm list */
1454	retval = 1;
1455out:
1456	dqm_unlock(dqm);
1457
1458	/* Outside the DQM lock because under the DQM lock we can't do
1459	 * reclaim or take other locks that others hold while reclaiming.
1460	 */
1461	if (!retval)
1462		kfd_dec_compute_active(dqm->dev);
1463
1464	return retval;
1465}
1466
1467static int
1468set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
1469			unsigned int vmid)
1470{
1471	uint32_t xcc_mask = dqm->dev->xcc_mask;
1472	int xcc_id, ret;
1473
1474	for_each_inst(xcc_id, xcc_mask) {
1475		ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
1476			dqm->dev->adev, pasid, vmid, xcc_id);
1477		if (ret)
1478			break;
1479	}
1480
1481	return ret;
1482}
1483
1484static void init_interrupts(struct device_queue_manager *dqm)
1485{
1486	uint32_t xcc_mask = dqm->dev->xcc_mask;
1487	unsigned int i, xcc_id;
1488
1489	for_each_inst(xcc_id, xcc_mask) {
1490		for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {
1491			if (is_pipe_enabled(dqm, 0, i)) {
1492				dqm->dev->kfd2kgd->init_interrupts(
1493					dqm->dev->adev, i, xcc_id);
1494			}
1495		}
1496	}
1497}
1498
1499static int initialize_nocpsch(struct device_queue_manager *dqm)
1500{
1501	int pipe, queue;
1502
1503	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1504
1505	dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
1506					sizeof(unsigned int), GFP_KERNEL);
1507	if (!dqm->allocated_queues)
1508		return -ENOMEM;
1509
1510	mutex_init(&dqm->lock_hidden);
1511	INIT_LIST_HEAD(&dqm->queues);
1512	dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
1513	dqm->active_cp_queue_count = 0;
1514	dqm->gws_queue_count = 0;
1515
1516	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1517		int pipe_offset = pipe * get_queues_per_pipe(dqm);
1518
1519		for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
1520			if (test_bit(pipe_offset + queue,
1521				     dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1522				dqm->allocated_queues[pipe] |= 1 << queue;
1523	}
1524
1525	memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
1526
1527	init_sdma_bitmaps(dqm);
1528
1529	return 0;
1530}
1531
1532static void uninitialize(struct device_queue_manager *dqm)
1533{
1534	int i;
1535
1536	WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
1537
1538	kfree(dqm->allocated_queues);
1539	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
1540		kfree(dqm->mqd_mgrs[i]);
1541	mutex_destroy(&dqm->lock_hidden);
1542}
1543
1544static int start_nocpsch(struct device_queue_manager *dqm)
1545{
1546	int r = 0;
1547
1548	pr_info("SW scheduler is used");
1549	init_interrupts(dqm);
1550
1551	if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1552		r = pm_init(&dqm->packet_mgr, dqm);
1553	if (!r)
1554		dqm->sched_running = true;
1555
1556	return r;
1557}
1558
1559static int stop_nocpsch(struct device_queue_manager *dqm)
1560{
1561	dqm_lock(dqm);
1562	if (!dqm->sched_running) {
1563		dqm_unlock(dqm);
1564		return 0;
1565	}
1566
1567	if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1568		pm_uninit(&dqm->packet_mgr);
1569	dqm->sched_running = false;
1570	dqm_unlock(dqm);
1571
1572	return 0;
1573}
1574
1575static int allocate_sdma_queue(struct device_queue_manager *dqm,
1576				struct queue *q, const uint32_t *restore_sdma_id)
1577{
1578	struct device *dev = dqm->dev->adev->dev;
1579	int bit;
1580
1581	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1582		if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) {
1583			dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n",
1584				 get_num_sdma_queues(dqm));
1585			return -ENOMEM;
1586		}
1587
1588		if (restore_sdma_id) {
1589			/* Re-use existing sdma_id */
1590			if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
1591				dev_err(dev, "SDMA queue already in use\n");
1592				return -EBUSY;
1593			}
1594			clear_bit(*restore_sdma_id, dqm->sdma_bitmap);
1595			q->sdma_id = *restore_sdma_id;
1596		} else {
1597			/* Find first available sdma_id */
1598			bit = find_first_bit(dqm->sdma_bitmap,
1599					     get_num_sdma_queues(dqm));
1600			clear_bit(bit, dqm->sdma_bitmap);
1601			q->sdma_id = bit;
1602		}
1603
1604		q->properties.sdma_engine_id =
1605			q->sdma_id % kfd_get_num_sdma_engines(dqm->dev);
1606		q->properties.sdma_queue_id = q->sdma_id /
1607				kfd_get_num_sdma_engines(dqm->dev);
1608	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1609		if (bitmap_empty(dqm->xgmi_sdma_bitmap, get_num_xgmi_sdma_queues(dqm))) {
1610			dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n",
1611				 get_num_xgmi_sdma_queues(dqm));
1612			return -ENOMEM;
1613		}
1614		if (restore_sdma_id) {
1615			/* Re-use existing sdma_id */
1616			if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
1617				dev_err(dev, "SDMA queue already in use\n");
1618				return -EBUSY;
1619			}
1620			clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap);
1621			q->sdma_id = *restore_sdma_id;
1622		} else {
1623			bit = find_first_bit(dqm->xgmi_sdma_bitmap,
1624					     get_num_xgmi_sdma_queues(dqm));
1625			clear_bit(bit, dqm->xgmi_sdma_bitmap);
1626			q->sdma_id = bit;
1627		}
1628		/* sdma_engine_id is sdma id including
1629		 * both PCIe-optimized SDMAs and XGMI-
1630		 * optimized SDMAs. The calculation below
1631		 * assumes the first N engines are always
1632		 * PCIe-optimized ones
1633		 */
1634		q->properties.sdma_engine_id =
1635			kfd_get_num_sdma_engines(dqm->dev) +
1636			q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev);
1637		q->properties.sdma_queue_id = q->sdma_id /
1638			kfd_get_num_xgmi_sdma_engines(dqm->dev);
1639	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1640		int i, num_queues, num_engines, eng_offset = 0, start_engine;
1641		bool free_bit_found = false, is_xgmi = false;
1642
1643		if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) {
1644			num_queues = get_num_sdma_queues(dqm);
1645			num_engines = kfd_get_num_sdma_engines(dqm->dev);
1646			q->properties.type = KFD_QUEUE_TYPE_SDMA;
1647		} else {
1648			num_queues = get_num_xgmi_sdma_queues(dqm);
1649			num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev);
1650			eng_offset = kfd_get_num_sdma_engines(dqm->dev);
1651			q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI;
1652			is_xgmi = true;
1653		}
1654
1655		/* Scan available bit based on target engine ID. */
1656		start_engine = q->properties.sdma_engine_id - eng_offset;
1657		for (i = start_engine; i < num_queues; i += num_engines) {
1658
1659			if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap))
1660				continue;
1661
1662			clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap);
1663			q->sdma_id = i;
1664			q->properties.sdma_queue_id = q->sdma_id / num_engines;
1665			free_bit_found = true;
1666			break;
1667		}
1668
1669		if (!free_bit_found) {
1670			dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n",
1671				 q->properties.sdma_engine_id, num_queues);
1672			return -ENOMEM;
1673		}
1674	}
1675
1676	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1677	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
1678
1679	return 0;
1680}
1681
1682static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1683				struct queue *q)
1684{
1685	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1686		if (q->sdma_id >= get_num_sdma_queues(dqm))
1687			return;
1688		set_bit(q->sdma_id, dqm->sdma_bitmap);
1689	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1690		if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1691			return;
1692		set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap);
1693	}
1694}
1695
1696/*
1697 * Device Queue Manager implementation for cp scheduler
1698 */
1699
1700static int set_sched_resources(struct device_queue_manager *dqm)
1701{
1702	int i, mec;
1703	struct scheduling_resources res;
1704	struct device *dev = dqm->dev->adev->dev;
1705
1706	res.vmid_mask = dqm->dev->compute_vmid_bitmap;
1707
1708	res.queue_mask = 0;
1709	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
1710		mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
1711			/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;
1712
1713		if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1714			continue;
1715
1716		/* only acquire queues from the first MEC */
1717		if (mec > 0)
1718			continue;
1719
1720		/* This situation may be hit in the future if a new HW
1721		 * generation exposes more than 64 queues. If so, the
1722		 * definition of res.queue_mask needs updating
1723		 */
1724		if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
1725			dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i);
1726			break;
1727		}
1728
1729		res.queue_mask |= 1ull
1730			<< amdgpu_queue_mask_bit_to_set_resource_bit(
1731				dqm->dev->adev, i);
1732	}
1733	res.gws_mask = ~0ull;
1734	res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
1735
1736	pr_debug("Scheduling resources:\n"
1737			"vmid mask: 0x%8X\n"
1738			"queue mask: 0x%8llX\n",
1739			res.vmid_mask, res.queue_mask);
1740
1741	return pm_send_set_resources(&dqm->packet_mgr, &res);
1742}
1743
1744static int initialize_cpsch(struct device_queue_manager *dqm)
1745{
1746	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1747
1748	mutex_init(&dqm->lock_hidden);
1749	INIT_LIST_HEAD(&dqm->queues);
1750	dqm->active_queue_count = dqm->processes_count = 0;
1751	dqm->active_cp_queue_count = 0;
1752	dqm->gws_queue_count = 0;
1753	dqm->active_runlist = false;
1754	dqm->trap_debug_vmid = 0;
1755
1756	init_sdma_bitmaps(dqm);
1757
1758	update_dqm_wait_times(dqm);
1759	return 0;
1760}
1761
1762/* halt_cpsch:
1763 * Unmap queues so the schedule doesn't continue remaining jobs in the queue.
1764 * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch
1765 * is called.
1766 */
1767static int halt_cpsch(struct device_queue_manager *dqm)
1768{
1769	int ret = 0;
1770
1771	dqm_lock(dqm);
1772	if (!dqm->sched_running) {
1773		dqm_unlock(dqm);
1774		return 0;
1775	}
1776
1777	WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n");
1778
1779	if (!dqm->is_hws_hang) {
1780		if (!dqm->dev->kfd->shared_resources.enable_mes)
1781			ret = unmap_queues_cpsch(dqm,
1782						 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
1783				USE_DEFAULT_GRACE_PERIOD, false);
1784		else
1785			ret = remove_all_kfd_queues_mes(dqm);
1786	}
1787	dqm->sched_halt = true;
1788	dqm_unlock(dqm);
1789
1790	return ret;
1791}
1792
1793/* unhalt_cpsch
1794 * Unset dqm->sched_halt and map queues back to runlist
1795 */
1796static int unhalt_cpsch(struct device_queue_manager *dqm)
1797{
1798	int ret = 0;
1799
1800	dqm_lock(dqm);
1801	if (!dqm->sched_running || !dqm->sched_halt) {
1802		WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");
1803		dqm_unlock(dqm);
1804		return 0;
1805	}
1806	dqm->sched_halt = false;
1807	if (!dqm->dev->kfd->shared_resources.enable_mes)
1808		ret = execute_queues_cpsch(dqm,
1809					   KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
1810			0, USE_DEFAULT_GRACE_PERIOD);
1811	else
1812		ret = add_all_kfd_queues_mes(dqm);
1813
1814	dqm_unlock(dqm);
1815
1816	return ret;
1817}
1818
1819static int start_cpsch(struct device_queue_manager *dqm)
1820{
1821	struct device *dev = dqm->dev->adev->dev;
1822	int retval, num_hw_queue_slots;
1823
1824	retval = 0;
1825
1826	dqm_lock(dqm);
1827
1828	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1829		retval = pm_init(&dqm->packet_mgr, dqm);
1830		if (retval)
1831			goto fail_packet_manager_init;
1832
1833		retval = set_sched_resources(dqm);
1834		if (retval)
1835			goto fail_set_sched_resources;
1836	}
1837	pr_debug("Allocating fence memory\n");
1838
1839	/* allocate fence memory on the gart */
1840	retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1841					&dqm->fence_mem);
1842
1843	if (retval)
1844		goto fail_allocate_vidmem;
1845
1846	dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
1847	dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
1848
1849	init_interrupts(dqm);
1850
1851	/* clear hang status when driver try to start the hw scheduler */
1852	dqm->sched_running = true;
1853
1854	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1855		if (pm_config_dequeue_wait_counts(&dqm->packet_mgr,
1856				KFD_DEQUEUE_WAIT_INIT, 0 /* unused */))
1857			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
1858		execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1859	}
1860
1861	/* setup per-queue reset detection buffer  */
1862	num_hw_queue_slots =  dqm->dev->kfd->shared_resources.num_queue_per_pipe *
1863			      dqm->dev->kfd->shared_resources.num_pipe_per_mec *
1864			      NUM_XCC(dqm->dev->xcc_mask);
1865
1866	dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info);
1867	dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL);
1868
1869	if (!dqm->detect_hang_info) {
1870		retval = -ENOMEM;
1871		goto fail_detect_hang_buffer;
1872	}
1873
1874	dqm_unlock(dqm);
1875
1876	return 0;
1877fail_detect_hang_buffer:
1878	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
1879fail_allocate_vidmem:
1880fail_set_sched_resources:
1881	if (!dqm->dev->kfd->shared_resources.enable_mes)
1882		pm_uninit(&dqm->packet_mgr);
1883fail_packet_manager_init:
1884	dqm_unlock(dqm);
1885	return retval;
1886}
1887
1888static int stop_cpsch(struct device_queue_manager *dqm)
1889{
1890	int ret = 0;
1891
1892	dqm_lock(dqm);
1893	if (!dqm->sched_running) {
1894		dqm_unlock(dqm);
1895		return 0;
1896	}
1897
1898	if (!dqm->dev->kfd->shared_resources.enable_mes)
1899		ret = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
1900								0, USE_DEFAULT_GRACE_PERIOD, false);
1901	else
1902		ret = remove_all_kfd_queues_mes(dqm);
1903
1904	dqm->sched_running = false;
1905
1906	if (!dqm->dev->kfd->shared_resources.enable_mes)
1907		pm_release_ib(&dqm->packet_mgr);
1908
1909	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
1910	if (!dqm->dev->kfd->shared_resources.enable_mes)
1911		pm_uninit(&dqm->packet_mgr);
1912	kfree(dqm->detect_hang_info);
1913	dqm->detect_hang_info = NULL;
1914	dqm_unlock(dqm);
1915
1916	return ret;
1917}
1918
1919static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1920					struct kernel_queue *kq,
1921					struct qcm_process_device *qpd)
1922{
1923	dqm_lock(dqm);
1924	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1925		pr_warn("Can't create new kernel queue because %d queues were already created\n",
1926				dqm->total_queue_count);
1927		dqm_unlock(dqm);
1928		return -EPERM;
1929	}
1930
1931	/*
1932	 * Unconditionally increment this counter, regardless of the queue's
1933	 * type or whether the queue is active.
1934	 */
1935	dqm->total_queue_count++;
1936	pr_debug("Total of %d queues are accountable so far\n",
1937			dqm->total_queue_count);
1938
1939	list_add(&kq->list, &qpd->priv_queue_list);
1940	increment_queue_count(dqm, qpd, kq->queue);
1941	qpd->is_debug = true;
1942	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1943			USE_DEFAULT_GRACE_PERIOD);
1944	dqm_unlock(dqm);
1945
1946	return 0;
1947}
1948
1949static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1950					struct kernel_queue *kq,
1951					struct qcm_process_device *qpd)
1952{
1953	dqm_lock(dqm);
1954	list_del(&kq->list);
1955	decrement_queue_count(dqm, qpd, kq->queue);
1956	qpd->is_debug = false;
1957	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
1958			USE_DEFAULT_GRACE_PERIOD);
1959	/*
1960	 * Unconditionally decrement this counter, regardless of the queue's
1961	 * type.
1962	 */
1963	dqm->total_queue_count--;
1964	pr_debug("Total of %d queues are accountable so far\n",
1965			dqm->total_queue_count);
1966	dqm_unlock(dqm);
1967}
1968
1969static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1970			struct qcm_process_device *qpd,
1971			const struct kfd_criu_queue_priv_data *qd,
1972			const void *restore_mqd, const void *restore_ctl_stack)
1973{
1974	int retval;
1975	struct mqd_manager *mqd_mgr;
1976
1977	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1978		pr_warn("Can't create new usermode queue because %d queues were already created\n",
1979				dqm->total_queue_count);
1980		retval = -EPERM;
1981		goto out;
1982	}
1983
1984	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1985		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI ||
1986		q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1987		dqm_lock(dqm);
1988		retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
1989		dqm_unlock(dqm);
1990		if (retval)
1991			goto out;
1992	}
1993
1994	retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
1995	if (retval)
1996		goto out_deallocate_sdma_queue;
1997
1998	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1999			q->properties.type)];
2000
2001	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2002		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2003		dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
2004	q->properties.tba_addr = qpd->tba_addr;
2005	q->properties.tma_addr = qpd->tma_addr;
2006	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
2007	if (!q->mqd_mem_obj) {
2008		retval = -ENOMEM;
2009		goto out_deallocate_doorbell;
2010	}
2011
2012	dqm_lock(dqm);
2013	/*
2014	 * Eviction state logic: mark all queues as evicted, even ones
2015	 * not currently active. Restoring inactive queues later only
2016	 * updates the is_evicted flag but is a no-op otherwise.
2017	 */
2018	q->properties.is_evicted = !!qpd->evicted;
2019	q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&
2020				  kfd_dbg_has_cwsr_workaround(q->device);
2021
2022	if (qd)
2023		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
2024				     &q->properties, restore_mqd, restore_ctl_stack,
2025				     qd->ctl_stack_size);
2026	else
2027		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
2028					&q->gart_mqd_addr, &q->properties);
2029
2030	list_add(&q->list, &qpd->queues_list);
2031	qpd->queue_count++;
2032
2033	if (q->properties.is_active) {
2034		increment_queue_count(dqm, qpd, q);
2035
2036		if (!dqm->dev->kfd->shared_resources.enable_mes)
2037			retval = execute_queues_cpsch(dqm,
2038					KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
2039		else
2040			retval = add_queue_mes(dqm, q, qpd);
2041		if (retval)
2042			goto cleanup_queue;
2043	}
2044
2045	/*
2046	 * Unconditionally increment this counter, regardless of the queue's
2047	 * type or whether the queue is active.
2048	 */
2049	dqm->total_queue_count++;
2050
2051	pr_debug("Total of %d queues are accountable so far\n",
2052			dqm->total_queue_count);
2053
2054	dqm_unlock(dqm);
2055	return retval;
2056
2057cleanup_queue:
2058	qpd->queue_count--;
2059	list_del(&q->list);
2060	if (q->properties.is_active)
2061		decrement_queue_count(dqm, qpd, q);
2062	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2063	dqm_unlock(dqm);
2064out_deallocate_doorbell:
2065	deallocate_doorbell(qpd, q);
2066out_deallocate_sdma_queue:
2067	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2068		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
2069		dqm_lock(dqm);
2070		deallocate_sdma_queue(dqm, q);
2071		dqm_unlock(dqm);
2072	}
2073out:
2074	return retval;
2075}
2076
2077int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,
2078			      uint64_t fence_value,
2079			      unsigned int timeout_ms)
2080{
2081	unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
2082	struct device *dev = dqm->dev->adev->dev;
2083	uint64_t *fence_addr = dqm->fence_addr;
2084
2085	while (*fence_addr != fence_value) {
2086		/* Fatal err detected, this response won't come */
2087		if (amdgpu_amdkfd_is_fed(dqm->dev->adev) ||
2088		    amdgpu_in_reset(dqm->dev->adev))
2089			return -EIO;
2090
2091		if (time_after(jiffies, end_jiffies)) {
2092			dev_err(dev, "qcm fence wait loop timeout expired\n");
2093			/* In HWS case, this is used to halt the driver thread
2094			 * in order not to mess up CP states before doing
2095			 * scandumps for FW debugging.
2096			 */
2097			while (halt_if_hws_hang)
2098				schedule();
2099
2100			return -ETIME;
2101		}
2102		schedule();
2103	}
2104
2105	return 0;
2106}
2107
2108/* dqm->lock mutex has to be locked before calling this function */
2109static int map_queues_cpsch(struct device_queue_manager *dqm)
2110{
2111	struct device *dev = dqm->dev->adev->dev;
2112	int retval;
2113
2114	if (!dqm->sched_running || dqm->sched_halt)
2115		return 0;
2116	if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
2117		return 0;
2118	if (dqm->active_runlist)
2119		return 0;
2120
2121	retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues);
2122	pr_debug("%s sent runlist\n", __func__);
2123	if (retval) {
2124		dev_err(dev, "failed to execute runlist\n");
2125		return retval;
2126	}
2127	dqm->active_runlist = true;
2128
2129	return retval;
2130}
2131
2132static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
2133			       struct qcm_process_device *qpd)
2134{
2135	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2136
2137	dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n",
2138		q->properties.queue_id, pdd->process->lead_thread->pid);
2139
2140	pdd->has_reset_queue = true;
2141	if (q->properties.is_active) {
2142		q->properties.is_active = false;
2143		decrement_queue_count(dqm, qpd, q);
2144	}
2145}
2146
2147static int detect_queue_hang(struct device_queue_manager *dqm)
2148{
2149	int i;
2150
2151	/* detect should be used only in dqm locked queue reset */
2152	if (WARN_ON(dqm->detect_hang_count > 0))
2153		return 0;
2154
2155	memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size);
2156
2157	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
2158		uint32_t mec, pipe, queue;
2159		int xcc_id;
2160
2161		mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
2162			/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;
2163
2164		if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
2165			continue;
2166
2167		amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue);
2168
2169		for_each_inst(xcc_id, dqm->dev->xcc_mask) {
2170			uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr(
2171						dqm->dev->adev, pipe, queue, xcc_id);
2172			struct dqm_detect_hang_info hang_info;
2173
2174			if (!queue_addr)
2175				continue;
2176
2177			hang_info.pipe_id = pipe;
2178			hang_info.queue_id = queue;
2179			hang_info.xcc_id = xcc_id;
2180			hang_info.queue_address = queue_addr;
2181
2182			dqm->detect_hang_info[dqm->detect_hang_count] = hang_info;
2183			dqm->detect_hang_count++;
2184		}
2185	}
2186
2187	return dqm->detect_hang_count;
2188}
2189
2190static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address)
2191{
2192	struct device_process_node *cur;
2193	struct qcm_process_device *qpd;
2194	struct queue *q;
2195
2196	list_for_each_entry(cur, &dqm->queues, list) {
2197		qpd = cur->qpd;
2198		list_for_each_entry(q, &qpd->queues_list, list) {
2199			if (queue_address == q->properties.queue_address)
2200				return q;
2201		}
2202	}
2203
2204	return NULL;
2205}
2206
2207static int reset_hung_queues(struct device_queue_manager *dqm)
2208{
2209	int r = 0, reset_count = 0, i;
2210
2211	if (!dqm->detect_hang_info || dqm->is_hws_hang)
2212		return -EIO;
2213
2214	/* assume dqm locked. */
2215	if (!detect_queue_hang(dqm))
2216		return -ENOTRECOVERABLE;
2217
2218	for (i = 0; i < dqm->detect_hang_count; i++) {
2219		struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i];
2220		struct queue *q = find_queue_by_address(dqm, hang_info.queue_address);
2221		struct kfd_process_device *pdd;
2222		uint64_t queue_addr = 0;
2223
2224		if (!q) {
2225			r = -ENOTRECOVERABLE;
2226			goto reset_fail;
2227		}
2228
2229		pdd = kfd_get_process_device_data(dqm->dev, q->process);
2230		if (!pdd) {
2231			r = -ENOTRECOVERABLE;
2232			goto reset_fail;
2233		}
2234
2235		queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev,
2236				hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id,
2237				KFD_UNMAP_LATENCY_MS);
2238
2239		/* either reset failed or we reset an unexpected queue. */
2240		if (queue_addr != q->properties.queue_address) {
2241			r = -ENOTRECOVERABLE;
2242			goto reset_fail;
2243		}
2244
2245		set_queue_as_reset(dqm, q, &pdd->qpd);
2246		reset_count++;
2247	}
2248
2249	if (reset_count == dqm->detect_hang_count)
2250		kfd_signal_reset_event(dqm->dev);
2251	else
2252		r = -ENOTRECOVERABLE;
2253
2254reset_fail:
2255	dqm->detect_hang_count = 0;
2256
2257	return r;
2258}
2259
2260static bool sdma_has_hang(struct device_queue_manager *dqm)
2261{
2262	int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2263	int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2264	int num_queues_per_eng =  dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2265	int i, j;
2266
2267	for (i = engine_start; i < engine_end; i++) {
2268		for (j = 0; j < num_queues_per_eng; j++) {
2269			if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j))
2270				continue;
2271
2272			return true;
2273		}
2274	}
2275
2276	return false;
2277}
2278
2279static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm,
2280				    uint32_t doorbell_off)
2281{
2282	struct device_process_node *cur;
2283	struct qcm_process_device *qpd;
2284	struct queue *q;
2285
2286	list_for_each_entry(cur, &dqm->queues, list) {
2287		qpd = cur->qpd;
2288		list_for_each_entry(q, &qpd->queues_list, list) {
2289			if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2290			     q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
2291			     q->properties.doorbell_off == doorbell_off) {
2292				set_queue_as_reset(dqm, q, qpd);
2293				return true;
2294			}
2295		}
2296	}
2297
2298	return false;
2299}
2300
2301static int reset_hung_queues_sdma(struct device_queue_manager *dqm)
2302{
2303	int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2304	int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2305	int num_queues_per_eng =  dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2306	int r = 0, i, j;
2307
2308	if (dqm->is_hws_hang)
2309		return -EIO;
2310
2311	/* Scan for hung HW queues and reset engine. */
2312	dqm->detect_hang_count = 0;
2313	for (i = engine_start; i < engine_end; i++) {
2314		for (j = 0; j < num_queues_per_eng; j++) {
2315			uint32_t doorbell_off =
2316				dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j);
2317
2318			if (!doorbell_off)
2319				continue;
2320
2321			/* Reset engine and check. */
2322			if (amdgpu_sdma_reset_engine(dqm->dev->adev, i, false) ||
2323			    dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) ||
2324			    !set_sdma_queue_as_reset(dqm, doorbell_off)) {
2325				r = -ENOTRECOVERABLE;
2326				goto reset_fail;
2327			}
2328
2329			/* Should only expect one queue active per engine */
2330			dqm->detect_hang_count++;
2331			break;
2332		}
2333	}
2334
2335	/* Signal process reset */
2336	if (dqm->detect_hang_count)
2337		kfd_signal_reset_event(dqm->dev);
2338	else
2339		r = -ENOTRECOVERABLE;
2340
2341reset_fail:
2342	dqm->detect_hang_count = 0;
2343
2344	return r;
2345}
2346
2347static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma)
2348{
2349	struct amdgpu_device *adev = dqm->dev->adev;
2350
2351	while (halt_if_hws_hang)
2352		schedule();
2353
2354	if (adev->debug_disable_gpu_ring_reset) {
2355		dev_info_once(adev->dev,
2356			      "%s queue hung, but ring reset disabled",
2357			      is_sdma ? "sdma" : "compute");
2358
2359		return -EPERM;
2360	}
2361	if (!amdgpu_gpu_recovery)
2362		return -ENOTRECOVERABLE;
2363
2364	return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm);
2365}
2366
2367/* dqm->lock mutex has to be locked before calling this function
2368 *
2369 * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time
2370 *   for context switch latency. Lower values are used by debugger
2371 *   since context switching are triggered at high frequency.
2372 *   This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE
2373 *
2374 */
2375static int unmap_queues_cpsch(struct device_queue_manager *dqm,
2376				enum kfd_unmap_queues_filter filter,
2377				uint32_t filter_param,
2378				uint32_t grace_period,
2379				bool reset)
2380{
2381	struct device *dev = dqm->dev->adev->dev;
2382	struct mqd_manager *mqd_mgr;
2383	int retval;
2384
2385	if (!dqm->sched_running)
2386		return 0;
2387	if (!dqm->active_runlist)
2388		return 0;
2389	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))
2390		return -EIO;
2391
2392	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2393		retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2394				KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period);
2395		if (retval)
2396			goto out;
2397	}
2398
2399	retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset);
2400	if (retval)
2401		goto out;
2402
2403	*dqm->fence_addr = KFD_FENCE_INIT;
2404	mb();
2405	pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr,
2406				KFD_FENCE_COMPLETED);
2407	/* should be timed out */
2408	retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED,
2409					   queue_preemption_timeout_ms);
2410	if (retval) {
2411		dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
2412		kfd_hws_hang(dqm);
2413		goto out;
2414	}
2415
2416	/* In the current MEC firmware implementation, if compute queue
2417	 * doesn't response to the preemption request in time, HIQ will
2418	 * abandon the unmap request without returning any timeout error
2419	 * to driver. Instead, MEC firmware will log the doorbell of the
2420	 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
2421	 * To make sure the queue unmap was successful, driver need to
2422	 * check those fields
2423	 */
2424	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
2425	if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) &&
2426	    reset_queues_on_hws_hang(dqm, false))
2427		goto reset_fail;
2428
2429	/* Check for SDMA hang and attempt SDMA reset */
2430	if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true))
2431		goto reset_fail;
2432
2433	/* We need to reset the grace period value for this device */
2434	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2435		if (pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2436				KFD_DEQUEUE_WAIT_RESET, 0 /* unused */))
2437			dev_err(dev, "Failed to reset grace period\n");
2438	}
2439
2440	pm_release_ib(&dqm->packet_mgr);
2441	dqm->active_runlist = false;
2442out:
2443	up_read(&dqm->dev->adev->reset_domain->sem);
2444	return retval;
2445
2446reset_fail:
2447	dqm->is_hws_hang = true;
2448	kfd_hws_hang(dqm);
2449	up_read(&dqm->dev->adev->reset_domain->sem);
2450	return -ETIME;
2451}
2452
2453/* only for compute queue */
2454static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid)
2455{
2456	int retval;
2457
2458	dqm_lock(dqm);
2459
2460	retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,
2461			pasid, USE_DEFAULT_GRACE_PERIOD, true);
2462
2463	dqm_unlock(dqm);
2464	return retval;
2465}
2466
2467/* dqm->lock mutex has to be locked before calling this function */
2468static int execute_queues_cpsch(struct device_queue_manager *dqm,
2469				enum kfd_unmap_queues_filter filter,
2470				uint32_t filter_param,
2471				uint32_t grace_period)
2472{
2473	int retval;
2474
2475	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))
2476		return -EIO;
2477	retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false);
2478	if (!retval)
2479		retval = map_queues_cpsch(dqm);
2480	up_read(&dqm->dev->adev->reset_domain->sem);
2481	return retval;
2482}
2483
2484static int wait_on_destroy_queue(struct device_queue_manager *dqm,
2485				 struct queue *q)
2486{
2487	struct kfd_process_device *pdd = kfd_get_process_device_data(q->device,
2488								q->process);
2489	int ret = 0;
2490
2491	if (WARN_ON(!pdd))
2492		return ret;
2493
2494	if (pdd->qpd.is_debug)
2495		return ret;
2496
2497	q->properties.is_being_destroyed = true;
2498
2499	if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {
2500		dqm_unlock(dqm);
2501		mutex_unlock(&q->process->mutex);
2502		ret = wait_event_interruptible(dqm->destroy_wait,
2503						!q->properties.is_suspended);
2504
2505		mutex_lock(&q->process->mutex);
2506		dqm_lock(dqm);
2507	}
2508
2509	return ret;
2510}
2511
2512static int destroy_queue_cpsch(struct device_queue_manager *dqm,
2513				struct qcm_process_device *qpd,
2514				struct queue *q)
2515{
2516	int retval;
2517	struct mqd_manager *mqd_mgr;
2518	uint64_t sdma_val = 0;
2519	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2520	struct device *dev = dqm->dev->adev->dev;
2521
2522	/* Get the SDMA queue stats */
2523	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2524	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2525		retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
2526							&sdma_val);
2527		if (retval)
2528			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
2529				q->properties.queue_id);
2530	}
2531
2532	/* remove queue from list to prevent rescheduling after preemption */
2533	dqm_lock(dqm);
2534
2535	retval = wait_on_destroy_queue(dqm, q);
2536
2537	if (retval) {
2538		dqm_unlock(dqm);
2539		return retval;
2540	}
2541
2542	if (qpd->is_debug) {
2543		/*
2544		 * error, currently we do not allow to destroy a queue
2545		 * of a currently debugged process
2546		 */
2547		retval = -EBUSY;
2548		goto failed_try_destroy_debugged_queue;
2549
2550	}
2551
2552	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2553			q->properties.type)];
2554
2555	deallocate_doorbell(qpd, q);
2556
2557	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2558	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2559		deallocate_sdma_queue(dqm, q);
2560		pdd->sdma_past_activity_counter += sdma_val;
2561	}
2562
2563	if (q->properties.is_active) {
2564		decrement_queue_count(dqm, qpd, q);
2565		q->properties.is_active = false;
2566		if (!dqm->dev->kfd->shared_resources.enable_mes) {
2567			retval = execute_queues_cpsch(dqm,
2568						      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
2569						      USE_DEFAULT_GRACE_PERIOD);
2570			if (retval == -ETIME)
2571				qpd->reset_wavefronts = true;
2572		} else {
2573			retval = remove_queue_mes(dqm, q, qpd);
2574		}
2575	}
2576	list_del(&q->list);
2577	qpd->queue_count--;
2578
2579	/*
2580	 * Unconditionally decrement this counter, regardless of the queue's
2581	 * type
2582	 */
2583	dqm->total_queue_count--;
2584	pr_debug("Total of %d queues are accountable so far\n",
2585			dqm->total_queue_count);
2586
2587	dqm_unlock(dqm);
2588
2589	/*
2590	 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid
2591	 * circular locking
2592	 */
2593	kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),
2594				qpd->pqm->process, q->device,
2595				-1, false, NULL, 0);
2596
2597	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2598
2599	return retval;
2600
2601failed_try_destroy_debugged_queue:
2602
2603	dqm_unlock(dqm);
2604	return retval;
2605}
2606
2607static bool set_cache_memory_policy(struct device_queue_manager *dqm,
2608				   struct qcm_process_device *qpd,
2609				   enum cache_policy default_policy,
2610				   enum cache_policy alternate_policy,
2611				   void __user *alternate_aperture_base,
2612				   uint64_t alternate_aperture_size,
2613				   u32 misc_process_properties)
2614{
2615	bool retval = true;
2616
2617	if (!dqm->asic_ops.set_cache_memory_policy)
2618		return retval;
2619
2620	dqm_lock(dqm);
2621
2622	retval = dqm->asic_ops.set_cache_memory_policy(
2623			dqm,
2624			qpd,
2625			default_policy,
2626			alternate_policy,
2627			alternate_aperture_base,
2628			alternate_aperture_size,
2629			misc_process_properties);
2630
2631	if (retval)
2632		goto out;
2633
2634	if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
2635		program_sh_mem_settings(dqm, qpd);
2636
2637	pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
2638		qpd->sh_mem_config, qpd->sh_mem_ape1_base,
2639		qpd->sh_mem_ape1_limit);
2640
2641out:
2642	dqm_unlock(dqm);
2643	return retval;
2644}
2645
2646static int process_termination_nocpsch(struct device_queue_manager *dqm,
2647		struct qcm_process_device *qpd)
2648{
2649	struct queue *q;
2650	struct device_process_node *cur, *next_dpn;
2651	int retval = 0;
2652	bool found = false;
2653
2654	dqm_lock(dqm);
2655
2656	/* Clear all user mode queues */
2657	while (!list_empty(&qpd->queues_list)) {
2658		struct mqd_manager *mqd_mgr;
2659		int ret;
2660
2661		q = list_first_entry(&qpd->queues_list, struct queue, list);
2662		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2663				q->properties.type)];
2664		ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
2665		if (ret)
2666			retval = ret;
2667		dqm_unlock(dqm);
2668		mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2669		dqm_lock(dqm);
2670	}
2671
2672	/* Unregister process */
2673	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2674		if (qpd == cur->qpd) {
2675			list_del(&cur->list);
2676			kfree(cur);
2677			dqm->processes_count--;
2678			found = true;
2679			break;
2680		}
2681	}
2682
2683	dqm_unlock(dqm);
2684
2685	/* Outside the DQM lock because under the DQM lock we can't do
2686	 * reclaim or take other locks that others hold while reclaiming.
2687	 */
2688	if (found)
2689		kfd_dec_compute_active(dqm->dev);
2690
2691	return retval;
2692}
2693
2694static int get_wave_state(struct device_queue_manager *dqm,
2695			  struct queue *q,
2696			  void __user *ctl_stack,
2697			  u32 *ctl_stack_used_size,
2698			  u32 *save_area_used_size)
2699{
2700	struct mqd_manager *mqd_mgr;
2701
2702	dqm_lock(dqm);
2703
2704	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
2705
2706	if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
2707	    q->properties.is_active || !q->device->kfd->cwsr_enabled ||
2708	    !mqd_mgr->get_wave_state) {
2709		dqm_unlock(dqm);
2710		return -EINVAL;
2711	}
2712
2713	dqm_unlock(dqm);
2714
2715	/*
2716	 * get_wave_state is outside the dqm lock to prevent circular locking
2717	 * and the queue should be protected against destruction by the process
2718	 * lock.
2719	 */
2720	return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,
2721			ctl_stack, ctl_stack_used_size, save_area_used_size);
2722}
2723
2724static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
2725			const struct queue *q,
2726			u32 *mqd_size,
2727			u32 *ctl_stack_size)
2728{
2729	struct mqd_manager *mqd_mgr;
2730	enum KFD_MQD_TYPE mqd_type =
2731			get_mqd_type_from_queue_type(q->properties.type);
2732
2733	dqm_lock(dqm);
2734	mqd_mgr = dqm->mqd_mgrs[mqd_type];
2735	*mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask);
2736	*ctl_stack_size = 0;
2737
2738	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
2739		mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
2740
2741	dqm_unlock(dqm);
2742}
2743
2744static int checkpoint_mqd(struct device_queue_manager *dqm,
2745			  const struct queue *q,
2746			  void *mqd,
2747			  void *ctl_stack)
2748{
2749	struct mqd_manager *mqd_mgr;
2750	int r = 0;
2751	enum KFD_MQD_TYPE mqd_type =
2752			get_mqd_type_from_queue_type(q->properties.type);
2753
2754	dqm_lock(dqm);
2755
2756	if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
2757		r = -EINVAL;
2758		goto dqm_unlock;
2759	}
2760
2761	mqd_mgr = dqm->mqd_mgrs[mqd_type];
2762	if (!mqd_mgr->checkpoint_mqd) {
2763		r = -EOPNOTSUPP;
2764		goto dqm_unlock;
2765	}
2766
2767	mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
2768
2769dqm_unlock:
2770	dqm_unlock(dqm);
2771	return r;
2772}
2773
2774static int process_termination_cpsch(struct device_queue_manager *dqm,
2775		struct qcm_process_device *qpd)
2776{
2777	int retval;
2778	struct queue *q;
2779	struct device *dev = dqm->dev->adev->dev;
2780	struct kernel_queue *kq, *kq_next;
2781	struct mqd_manager *mqd_mgr;
2782	struct device_process_node *cur, *next_dpn;
2783	enum kfd_unmap_queues_filter filter =
2784		KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
2785	bool found = false;
2786
2787	retval = 0;
2788
2789	dqm_lock(dqm);
2790
2791	/* Clean all kernel queues */
2792	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
2793		list_del(&kq->list);
2794		decrement_queue_count(dqm, qpd, kq->queue);
2795		qpd->is_debug = false;
2796		dqm->total_queue_count--;
2797		filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
2798	}
2799
2800	/* Clear all user mode queues */
2801	list_for_each_entry(q, &qpd->queues_list, list) {
2802		if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
2803			deallocate_sdma_queue(dqm, q);
2804		else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2805			deallocate_sdma_queue(dqm, q);
2806
2807		if (q->properties.is_active) {
2808			decrement_queue_count(dqm, qpd, q);
2809
2810			if (dqm->dev->kfd->shared_resources.enable_mes) {
2811				retval = remove_queue_mes(dqm, q, qpd);
2812				if (retval)
2813					dev_err(dev, "Failed to remove queue %d\n",
2814						q->properties.queue_id);
2815			}
2816		}
2817
2818		dqm->total_queue_count--;
2819	}
2820
2821	/* Unregister process */
2822	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2823		if (qpd == cur->qpd) {
2824			list_del(&cur->list);
2825			kfree(cur);
2826			dqm->processes_count--;
2827			found = true;
2828			break;
2829		}
2830	}
2831
2832	if (!dqm->dev->kfd->shared_resources.enable_mes)
2833		retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD);
2834
2835	if ((retval || qpd->reset_wavefronts) &&
2836	    down_read_trylock(&dqm->dev->adev->reset_domain->sem)) {
2837		pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
2838		dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
2839		qpd->reset_wavefronts = false;
2840		up_read(&dqm->dev->adev->reset_domain->sem);
2841	}
2842
2843	/* Lastly, free mqd resources.
2844	 * Do free_mqd() after dqm_unlock to avoid circular locking.
2845	 */
2846	while (!list_empty(&qpd->queues_list)) {
2847		q = list_first_entry(&qpd->queues_list, struct queue, list);
2848		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2849				q->properties.type)];
2850		list_del(&q->list);
2851		qpd->queue_count--;
2852		dqm_unlock(dqm);
2853		mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2854		dqm_lock(dqm);
2855	}
2856	dqm_unlock(dqm);
2857
2858	/* Outside the DQM lock because under the DQM lock we can't do
2859	 * reclaim or take other locks that others hold while reclaiming.
2860	 */
2861	if (found)
2862		kfd_dec_compute_active(dqm->dev);
2863
2864	return retval;
2865}
2866
2867static int init_mqd_managers(struct device_queue_manager *dqm)
2868{
2869	int i, j;
2870	struct device *dev = dqm->dev->adev->dev;
2871	struct mqd_manager *mqd_mgr;
2872
2873	for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
2874		mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
2875		if (!mqd_mgr) {
2876			dev_err(dev, "mqd manager [%d] initialization failed\n", i);
2877			goto out_free;
2878		}
2879		dqm->mqd_mgrs[i] = mqd_mgr;
2880	}
2881
2882	return 0;
2883
2884out_free:
2885	for (j = 0; j < i; j++) {
2886		kfree(dqm->mqd_mgrs[j]);
2887		dqm->mqd_mgrs[j] = NULL;
2888	}
2889
2890	return -ENOMEM;
2891}
2892
2893/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
2894static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
2895{
2896	int retval;
2897	struct kfd_node *dev = dqm->dev;
2898	struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
2899	uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
2900		get_num_all_sdma_engines(dqm) *
2901		dev->kfd->device_info.num_sdma_queues_per_engine +
2902		(dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *
2903		NUM_XCC(dqm->dev->xcc_mask));
2904
2905	retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,
2906		&(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
2907		(void *)&(mem_obj->cpu_ptr), false);
2908
2909	return retval;
2910}
2911
2912static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
2913				    struct kfd_mem_obj *mqd)
2914{
2915	WARN(!mqd, "No hiq sdma mqd trunk to free");
2916
2917	amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem);
2918}
2919
2920struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
2921{
2922	struct device_queue_manager *dqm;
2923
2924	pr_debug("Loading device queue manager\n");
2925
2926	dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
2927	if (!dqm)
2928		return NULL;
2929
2930	switch (dev->adev->asic_type) {
2931	/* HWS is not available on Hawaii. */
2932	case CHIP_HAWAII:
2933	/* HWS depends on CWSR for timely dequeue. CWSR is not
2934	 * available on Tonga.
2935	 *
2936	 * FIXME: This argument also applies to Kaveri.
2937	 */
2938	case CHIP_TONGA:
2939		dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
2940		break;
2941	default:
2942		dqm->sched_policy = sched_policy;
2943		break;
2944	}
2945
2946	dqm->dev = dev;
2947	switch (dqm->sched_policy) {
2948	case KFD_SCHED_POLICY_HWS:
2949	case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
2950		/* initialize dqm for cp scheduling */
2951		dqm->ops.create_queue = create_queue_cpsch;
2952		dqm->ops.initialize = initialize_cpsch;
2953		dqm->ops.start = start_cpsch;
2954		dqm->ops.stop = stop_cpsch;
2955		dqm->ops.halt = halt_cpsch;
2956		dqm->ops.unhalt = unhalt_cpsch;
2957		dqm->ops.destroy_queue = destroy_queue_cpsch;
2958		dqm->ops.update_queue = update_queue;
2959		dqm->ops.register_process = register_process;
2960		dqm->ops.unregister_process = unregister_process;
2961		dqm->ops.uninitialize = uninitialize;
2962		dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
2963		dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
2964		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2965		dqm->ops.process_termination = process_termination_cpsch;
2966		dqm->ops.evict_process_queues = evict_process_queues_cpsch;
2967		dqm->ops.restore_process_queues = restore_process_queues_cpsch;
2968		dqm->ops.get_wave_state = get_wave_state;
2969		dqm->ops.reset_queues = reset_queues_cpsch;
2970		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2971		dqm->ops.checkpoint_mqd = checkpoint_mqd;
2972		break;
2973	case KFD_SCHED_POLICY_NO_HWS:
2974		/* initialize dqm for no cp scheduling */
2975		dqm->ops.start = start_nocpsch;
2976		dqm->ops.stop = stop_nocpsch;
2977		dqm->ops.create_queue = create_queue_nocpsch;
2978		dqm->ops.destroy_queue = destroy_queue_nocpsch;
2979		dqm->ops.update_queue = update_queue;
2980		dqm->ops.register_process = register_process;
2981		dqm->ops.unregister_process = unregister_process;
2982		dqm->ops.initialize = initialize_nocpsch;
2983		dqm->ops.uninitialize = uninitialize;
2984		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2985		dqm->ops.process_termination = process_termination_nocpsch;
2986		dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
2987		dqm->ops.restore_process_queues =
2988			restore_process_queues_nocpsch;
2989		dqm->ops.get_wave_state = get_wave_state;
2990		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2991		dqm->ops.checkpoint_mqd = checkpoint_mqd;
2992		break;
2993	default:
2994		dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
2995		goto out_free;
2996	}
2997
2998	switch (dev->adev->asic_type) {
2999	case CHIP_KAVERI:
3000	case CHIP_HAWAII:
3001		device_queue_manager_init_cik(&dqm->asic_ops);
3002		break;
3003
3004	case CHIP_CARRIZO:
3005	case CHIP_TONGA:
3006	case CHIP_FIJI:
3007	case CHIP_POLARIS10:
3008	case CHIP_POLARIS11:
3009	case CHIP_POLARIS12:
3010	case CHIP_VEGAM:
3011		device_queue_manager_init_vi(&dqm->asic_ops);
3012		break;
3013
3014	default:
3015		if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
3016			device_queue_manager_init_v12(&dqm->asic_ops);
3017		else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
3018			device_queue_manager_init_v11(&dqm->asic_ops);
3019		else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
3020			device_queue_manager_init_v10(&dqm->asic_ops);
3021		else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
3022			device_queue_manager_init_v9(&dqm->asic_ops);
3023		else {
3024			WARN(1, "Unexpected ASIC family %u",
3025			     dev->adev->asic_type);
3026			goto out_free;
3027		}
3028	}
3029
3030	if (init_mqd_managers(dqm))
3031		goto out_free;
3032
3033	if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
3034		dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n");
3035		goto out_free;
3036	}
3037
3038	if (!dqm->ops.initialize(dqm)) {
3039		init_waitqueue_head(&dqm->destroy_wait);
3040		return dqm;
3041	}
3042
3043	if (!dev->kfd->shared_resources.enable_mes)
3044		deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd);
3045
3046out_free:
3047	kfree(dqm);
3048	return NULL;
3049}
3050
3051void device_queue_manager_uninit(struct device_queue_manager *dqm)
3052{
3053	dqm->ops.stop(dqm);
3054	dqm->ops.uninitialize(dqm);
3055	if (!dqm->dev->kfd->shared_resources.enable_mes)
3056		deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
3057	kfree(dqm);
3058}
3059
3060int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)
3061{
3062	struct kfd_process_device *pdd = NULL;
3063	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd);
3064	struct device_queue_manager *dqm = knode->dqm;
3065	struct device *dev = dqm->dev->adev->dev;
3066	struct qcm_process_device *qpd;
3067	struct queue *q = NULL;
3068	int ret = 0;
3069
3070	if (!pdd)
3071		return -EINVAL;
3072
3073	dqm_lock(dqm);
3074
3075	if (pdd) {
3076		qpd = &pdd->qpd;
3077
3078		list_for_each_entry(q, &qpd->queues_list, list) {
3079			if (q->doorbell_id == doorbell_id && q->properties.is_active) {
3080				ret = suspend_all_queues_mes(dqm);
3081				if (ret) {
3082					dev_err(dev, "Suspending all queues failed");
3083					goto out;
3084				}
3085
3086				q->properties.is_evicted = true;
3087				q->properties.is_active = false;
3088				decrement_queue_count(dqm, qpd, q);
3089
3090				ret = remove_queue_mes(dqm, q, qpd);
3091				if (ret) {
3092					dev_err(dev, "Removing bad queue failed");
3093					goto out;
3094				}
3095
3096				ret = resume_all_queues_mes(dqm);
3097				if (ret)
3098					dev_err(dev, "Resuming all queues failed");
3099
3100				break;
3101			}
3102		}
3103	}
3104
3105out:
3106	dqm_unlock(dqm);
3107	kfd_unref_process(p);
3108	return ret;
3109}
3110
3111int kfd_evict_process_device(struct kfd_process_device *pdd)
3112{
3113	struct device_queue_manager *dqm;
3114	struct kfd_process *p;
3115
3116	p = pdd->process;
3117	dqm = pdd->dev->dqm;
3118
3119	WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
3120
3121	return dqm->ops.evict_process_queues(dqm, &pdd->qpd);
3122}
3123
3124int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
3125				struct qcm_process_device *qpd)
3126{
3127	int r;
3128	struct device *dev = dqm->dev->adev->dev;
3129	int updated_vmid_mask;
3130
3131	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3132		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3133		return -EINVAL;
3134	}
3135
3136	dqm_lock(dqm);
3137
3138	if (dqm->trap_debug_vmid != 0) {
3139		dev_err(dev, "Trap debug id already reserved\n");
3140		r = -EBUSY;
3141		goto out_unlock;
3142	}
3143
3144	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
3145			USE_DEFAULT_GRACE_PERIOD, false);
3146	if (r)
3147		goto out_unlock;
3148
3149	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3150	updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);
3151
3152	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3153	dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;
3154	r = set_sched_resources(dqm);
3155	if (r)
3156		goto out_unlock;
3157
3158	r = map_queues_cpsch(dqm);
3159	if (r)
3160		goto out_unlock;
3161
3162	pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);
3163
3164out_unlock:
3165	dqm_unlock(dqm);
3166	return r;
3167}
3168
3169/*
3170 * Releases vmid for the trap debugger
3171 */
3172int release_debug_trap_vmid(struct device_queue_manager *dqm,
3173			struct qcm_process_device *qpd)
3174{
3175	struct device *dev = dqm->dev->adev->dev;
3176	int r;
3177	int updated_vmid_mask;
3178	uint32_t trap_debug_vmid;
3179
3180	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3181		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3182		return -EINVAL;
3183	}
3184
3185	dqm_lock(dqm);
3186	trap_debug_vmid = dqm->trap_debug_vmid;
3187	if (dqm->trap_debug_vmid == 0) {
3188		dev_err(dev, "Trap debug id is not reserved\n");
3189		r = -EINVAL;
3190		goto out_unlock;
3191	}
3192
3193	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
3194			USE_DEFAULT_GRACE_PERIOD, false);
3195	if (r)
3196		goto out_unlock;
3197
3198	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3199	updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);
3200
3201	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3202	dqm->trap_debug_vmid = 0;
3203	r = set_sched_resources(dqm);
3204	if (r)
3205		goto out_unlock;
3206
3207	r = map_queues_cpsch(dqm);
3208	if (r)
3209		goto out_unlock;
3210
3211	pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);
3212
3213out_unlock:
3214	dqm_unlock(dqm);
3215	return r;
3216}
3217
3218#define QUEUE_NOT_FOUND		-1
3219/* invalidate queue operation in array */
3220static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)
3221{
3222	int i;
3223
3224	for (i = 0; i < num_queues; i++)
3225		queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;
3226}
3227
3228/* find queue index in array */
3229static int q_array_get_index(unsigned int queue_id,
3230		uint32_t num_queues,
3231		uint32_t *queue_ids)
3232{
3233	int i;
3234
3235	for (i = 0; i < num_queues; i++)
3236		if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))
3237			return i;
3238
3239	return QUEUE_NOT_FOUND;
3240}
3241
3242struct copy_context_work_handler_workarea {
3243	struct work_struct copy_context_work;
3244	struct kfd_process *p;
3245};
3246
3247static void copy_context_work_handler(struct work_struct *work)
3248{
3249	struct copy_context_work_handler_workarea *workarea;
3250	struct mqd_manager *mqd_mgr;
3251	struct queue *q;
3252	struct mm_struct *mm;
3253	struct kfd_process *p;
3254	uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;
3255	int i;
3256
3257	workarea = container_of(work,
3258			struct copy_context_work_handler_workarea,
3259			copy_context_work);
3260
3261	p = workarea->p;
3262	mm = get_task_mm(p->lead_thread);
3263
3264	if (!mm)
3265		return;
3266
3267	kthread_use_mm(mm);
3268	for (i = 0; i < p->n_pdds; i++) {
3269		struct kfd_process_device *pdd = p->pdds[i];
3270		struct device_queue_manager *dqm = pdd->dev->dqm;
3271		struct qcm_process_device *qpd = &pdd->qpd;
3272
3273		list_for_each_entry(q, &qpd->queues_list, list) {
3274			if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
3275				continue;
3276
3277			mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
3278
3279			/* We ignore the return value from get_wave_state
3280			 * because
3281			 * i) right now, it always returns 0, and
3282			 * ii) if we hit an error, we would continue to the
3283			 *      next queue anyway.
3284			 */
3285			mqd_mgr->get_wave_state(mqd_mgr,
3286					q->mqd,
3287					&q->properties,
3288					(void __user *)	q->properties.ctx_save_restore_area_address,
3289					&tmp_ctl_stack_used_size,
3290					&tmp_save_area_used_size);
3291		}
3292	}
3293	kthread_unuse_mm(mm);
3294	mmput(mm);
3295}
3296
3297static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
3298{
3299	size_t array_size = num_queues * sizeof(uint32_t);
3300
3301	if (!usr_queue_id_array)
3302		return NULL;
3303
3304	return memdup_user(usr_queue_id_array, array_size);
3305}
3306
3307int resume_queues(struct kfd_process *p,
3308		uint32_t num_queues,
3309		uint32_t *usr_queue_id_array)
3310{
3311	uint32_t *queue_ids = NULL;
3312	int total_resumed = 0;
3313	int i;
3314
3315	if (usr_queue_id_array) {
3316		queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3317
3318		if (IS_ERR(queue_ids))
3319			return PTR_ERR(queue_ids);
3320
3321		/* mask all queues as invalid.  unmask per successful request */
3322		q_array_invalidate(num_queues, queue_ids);
3323	}
3324
3325	for (i = 0; i < p->n_pdds; i++) {
3326		struct kfd_process_device *pdd = p->pdds[i];
3327		struct device_queue_manager *dqm = pdd->dev->dqm;
3328		struct device *dev = dqm->dev->adev->dev;
3329		struct qcm_process_device *qpd = &pdd->qpd;
3330		struct queue *q;
3331		int r, per_device_resumed = 0;
3332
3333		dqm_lock(dqm);
3334
3335		/* unmask queues that resume or already resumed as valid */
3336		list_for_each_entry(q, &qpd->queues_list, list) {
3337			int q_idx = QUEUE_NOT_FOUND;
3338
3339			if (queue_ids)
3340				q_idx = q_array_get_index(
3341						q->properties.queue_id,
3342						num_queues,
3343						queue_ids);
3344
3345			if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {
3346				int err = resume_single_queue(dqm, &pdd->qpd, q);
3347
3348				if (queue_ids) {
3349					if (!err) {
3350						queue_ids[q_idx] &=
3351							~KFD_DBG_QUEUE_INVALID_MASK;
3352					} else {
3353						queue_ids[q_idx] |=
3354							KFD_DBG_QUEUE_ERROR_MASK;
3355						break;
3356					}
3357				}
3358
3359				if (dqm->dev->kfd->shared_resources.enable_mes) {
3360					wake_up_all(&dqm->destroy_wait);
3361					if (!err)
3362						total_resumed++;
3363				} else {
3364					per_device_resumed++;
3365				}
3366			}
3367		}
3368
3369		if (!per_device_resumed) {
3370			dqm_unlock(dqm);
3371			continue;
3372		}
3373
3374		r = execute_queues_cpsch(dqm,
3375					KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
3376					0,
3377					USE_DEFAULT_GRACE_PERIOD);
3378		if (r) {
3379			dev_err(dev, "Failed to resume process queues\n");
3380			if (queue_ids) {
3381				list_for_each_entry(q, &qpd->queues_list, list) {
3382					int q_idx = q_array_get_index(
3383							q->properties.queue_id,
3384							num_queues,
3385							queue_ids);
3386
3387					/* mask queue as error on resume fail */
3388					if (q_idx != QUEUE_NOT_FOUND)
3389						queue_ids[q_idx] |=
3390							KFD_DBG_QUEUE_ERROR_MASK;
3391				}
3392			}
3393		} else {
3394			wake_up_all(&dqm->destroy_wait);
3395			total_resumed += per_device_resumed;
3396		}
3397
3398		dqm_unlock(dqm);
3399	}
3400
3401	if (queue_ids) {
3402		if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
3403				num_queues * sizeof(uint32_t)))
3404			pr_err("copy_to_user failed on queue resume\n");
3405
3406		kfree(queue_ids);
3407	}
3408
3409	return total_resumed;
3410}
3411
3412int suspend_queues(struct kfd_process *p,
3413			uint32_t num_queues,
3414			uint32_t grace_period,
3415			uint64_t exception_clear_mask,
3416			uint32_t *usr_queue_id_array)
3417{
3418	uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3419	int total_suspended = 0;
3420	int i;
3421
3422	if (IS_ERR(queue_ids))
3423		return PTR_ERR(queue_ids);
3424
3425	/* mask all queues as invalid.  umask on successful request */
3426	q_array_invalidate(num_queues, queue_ids);
3427
3428	for (i = 0; i < p->n_pdds; i++) {
3429		struct kfd_process_device *pdd = p->pdds[i];
3430		struct device_queue_manager *dqm = pdd->dev->dqm;
3431		struct device *dev = dqm->dev->adev->dev;
3432		struct qcm_process_device *qpd = &pdd->qpd;
3433		struct queue *q;
3434		int r, per_device_suspended = 0;
3435
3436		mutex_lock(&p->event_mutex);
3437		dqm_lock(dqm);
3438
3439		/* unmask queues that suspend or already suspended */
3440		list_for_each_entry(q, &qpd->queues_list, list) {
3441			int q_idx = q_array_get_index(q->properties.queue_id,
3442							num_queues,
3443							queue_ids);
3444
3445			if (q_idx != QUEUE_NOT_FOUND) {
3446				int err = suspend_single_queue(dqm, pdd, q);
3447				bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;
3448
3449				if (!err) {
3450					queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;
3451					if (exception_clear_mask && is_mes)
3452						q->properties.exception_status &=
3453							~exception_clear_mask;
3454
3455					if (is_mes)
3456						total_suspended++;
3457					else
3458						per_device_suspended++;
3459				} else if (err != -EBUSY) {
3460					r = err;
3461					queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3462					break;
3463				}
3464			}
3465		}
3466
3467		if (!per_device_suspended) {
3468			dqm_unlock(dqm);
3469			mutex_unlock(&p->event_mutex);
3470			if (total_suspended)
3471				amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev);
3472			continue;
3473		}
3474
3475		r = execute_queues_cpsch(dqm,
3476			KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
3477			grace_period);
3478
3479		if (r)
3480			dev_err(dev, "Failed to suspend process queues.\n");
3481		else
3482			total_suspended += per_device_suspended;
3483
3484		list_for_each_entry(q, &qpd->queues_list, list) {
3485			int q_idx = q_array_get_index(q->properties.queue_id,
3486						num_queues, queue_ids);
3487
3488			if (q_idx == QUEUE_NOT_FOUND)
3489				continue;
3490
3491			/* mask queue as error on suspend fail */
3492			if (r)
3493				queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3494			else if (exception_clear_mask)
3495				q->properties.exception_status &=
3496							~exception_clear_mask;
3497		}
3498
3499		dqm_unlock(dqm);
3500		mutex_unlock(&p->event_mutex);
3501		amdgpu_device_flush_hdp(dqm->dev->adev, NULL);
3502	}
3503
3504	if (total_suspended) {
3505		struct copy_context_work_handler_workarea copy_context_worker;
3506
3507		INIT_WORK_ONSTACK(
3508				&copy_context_worker.copy_context_work,
3509				copy_context_work_handler);
3510
3511		copy_context_worker.p = p;
3512
3513		schedule_work(&copy_context_worker.copy_context_work);
3514
3515
3516		flush_work(&copy_context_worker.copy_context_work);
3517		destroy_work_on_stack(&copy_context_worker.copy_context_work);
3518	}
3519
3520	if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
3521			num_queues * sizeof(uint32_t)))
3522		pr_err("copy_to_user failed on queue suspend\n");
3523
3524	kfree(queue_ids);
3525
3526	return total_suspended;
3527}
3528
3529static uint32_t set_queue_type_for_user(struct queue_properties *q_props)
3530{
3531	switch (q_props->type) {
3532	case KFD_QUEUE_TYPE_COMPUTE:
3533		return q_props->format == KFD_QUEUE_FORMAT_PM4
3534					? KFD_IOC_QUEUE_TYPE_COMPUTE
3535					: KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;
3536	case KFD_QUEUE_TYPE_SDMA:
3537		return KFD_IOC_QUEUE_TYPE_SDMA;
3538	case KFD_QUEUE_TYPE_SDMA_XGMI:
3539		return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;
3540	default:
3541		WARN_ONCE(true, "queue type not recognized!");
3542		return 0xffffffff;
3543	};
3544}
3545
3546void set_queue_snapshot_entry(struct queue *q,
3547			      uint64_t exception_clear_mask,
3548			      struct kfd_queue_snapshot_entry *qss_entry)
3549{
3550	qss_entry->ring_base_address = q->properties.queue_address;
3551	qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;
3552	qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;
3553	qss_entry->ctx_save_restore_address =
3554				q->properties.ctx_save_restore_area_address;
3555	qss_entry->ctx_save_restore_area_size =
3556				q->properties.ctx_save_restore_area_size;
3557	qss_entry->exception_status = q->properties.exception_status;
3558	qss_entry->queue_id = q->properties.queue_id;
3559	qss_entry->gpu_id = q->device->id;
3560	qss_entry->ring_size = (uint32_t)q->properties.queue_size;
3561	qss_entry->queue_type = set_queue_type_for_user(&q->properties);
3562	q->properties.exception_status &= ~exception_clear_mask;
3563}
3564
3565int debug_lock_and_unmap(struct device_queue_manager *dqm)
3566{
3567	struct device *dev = dqm->dev->adev->dev;
3568	int r;
3569
3570	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3571		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3572		return -EINVAL;
3573	}
3574
3575	if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
3576		return 0;
3577
3578	dqm_lock(dqm);
3579
3580	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false);
3581	if (r)
3582		dqm_unlock(dqm);
3583
3584	return r;
3585}
3586
3587int debug_map_and_unlock(struct device_queue_manager *dqm)
3588{
3589	struct device *dev = dqm->dev->adev->dev;
3590	int r;
3591
3592	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3593		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3594		return -EINVAL;
3595	}
3596
3597	if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
3598		return 0;
3599
3600	r = map_queues_cpsch(dqm);
3601
3602	dqm_unlock(dqm);
3603
3604	return r;
3605}
3606
3607int debug_refresh_runlist(struct device_queue_manager *dqm)
3608{
3609	int r = debug_lock_and_unmap(dqm);
3610
3611	if (r)
3612		return r;
3613
3614	return debug_map_and_unlock(dqm);
3615}
3616
3617bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
3618				 struct qcm_process_device *qpd,
3619				 int doorbell_off, u32 *queue_format)
3620{
3621	struct queue *q;
3622	bool r = false;
3623
3624	if (!queue_format)
3625		return r;
3626
3627	dqm_lock(dqm);
3628
3629	list_for_each_entry(q, &qpd->queues_list, list) {
3630		if (q->properties.doorbell_off == doorbell_off) {
3631			*queue_format = q->properties.format;
3632			r = true;
3633			goto out;
3634		}
3635	}
3636
3637out:
3638	dqm_unlock(dqm);
3639	return r;
3640}
3641#if defined(CONFIG_DEBUG_FS)
3642
3643static void seq_reg_dump(struct seq_file *m,
3644			 uint32_t (*dump)[2], uint32_t n_regs)
3645{
3646	uint32_t i, count;
3647
3648	for (i = 0, count = 0; i < n_regs; i++) {
3649		if (count == 0 ||
3650		    dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
3651			seq_printf(m, "%s    %08x: %08x",
3652				   i ? "\n" : "",
3653				   dump[i][0], dump[i][1]);
3654			count = 7;
3655		} else {
3656			seq_printf(m, " %08x", dump[i][1]);
3657			count--;
3658		}
3659	}
3660
3661	seq_puts(m, "\n");
3662}
3663
3664int dqm_debugfs_hqds(struct seq_file *m, void *data)
3665{
3666	struct device_queue_manager *dqm = data;
3667	uint32_t xcc_mask = dqm->dev->xcc_mask;
3668	uint32_t (*dump)[2], n_regs;
3669	int pipe, queue;
3670	int r = 0, xcc_id;
3671	uint32_t sdma_engine_start;
3672
3673	if (!dqm->sched_running) {
3674		seq_puts(m, " Device is stopped\n");
3675		return 0;
3676	}
3677
3678	for_each_inst(xcc_id, xcc_mask) {
3679		r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3680						KFD_CIK_HIQ_PIPE,
3681						KFD_CIK_HIQ_QUEUE, &dump,
3682						&n_regs, xcc_id);
3683		if (!r) {
3684			seq_printf(
3685				m,
3686				"   Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",
3687				xcc_id,
3688				KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,
3689				KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),
3690				KFD_CIK_HIQ_QUEUE);
3691			seq_reg_dump(m, dump, n_regs);
3692
3693			kfree(dump);
3694		}
3695
3696		for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
3697			int pipe_offset = pipe * get_queues_per_pipe(dqm);
3698
3699			for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
3700				if (!test_bit(pipe_offset + queue,
3701				      dqm->dev->kfd->shared_resources.cp_queue_bitmap))
3702					continue;
3703
3704				r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3705								pipe, queue,
3706								&dump, &n_regs,
3707								xcc_id);
3708				if (r)
3709					break;
3710
3711				seq_printf(m,
3712					   " Inst %d,  CP Pipe %d, Queue %d\n",
3713					   xcc_id, pipe, queue);
3714				seq_reg_dump(m, dump, n_regs);
3715
3716				kfree(dump);
3717			}
3718		}
3719	}
3720
3721	sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
3722	for (pipe = sdma_engine_start;
3723	     pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));
3724	     pipe++) {
3725		for (queue = 0;
3726		     queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
3727		     queue++) {
3728			r = dqm->dev->kfd2kgd->hqd_sdma_dump(
3729				dqm->dev->adev, pipe, queue, &dump, &n_regs);
3730			if (r)
3731				break;
3732
3733			seq_printf(m, "  SDMA Engine %d, RLC %d\n",
3734				  pipe, queue);
3735			seq_reg_dump(m, dump, n_regs);
3736
3737			kfree(dump);
3738		}
3739	}
3740
3741	return r;
3742}
3743
3744int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
3745{
3746	int r = 0;
3747
3748	dqm_lock(dqm);
3749	r = pm_debugfs_hang_hws(&dqm->packet_mgr);
3750	if (r) {
3751		dqm_unlock(dqm);
3752		return r;
3753	}
3754	dqm->active_runlist = true;
3755	r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
3756				0, USE_DEFAULT_GRACE_PERIOD);
3757	dqm_unlock(dqm);
3758
3759	return r;
3760}
3761
3762#endif