kfd_debug.c

   1/*
   2 * Copyright 2023 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22
  23#include "kfd_debug.h"
  24#include "kfd_device_queue_manager.h"
  25#include "kfd_topology.h"
  26#include <linux/file.h>
  27#include <uapi/linux/kfd_ioctl.h>
  28#include <uapi/linux/kfd_sysfs.h>
  29
  30#define MAX_WATCH_ADDRESSES	4
  31
  32int kfd_dbg_ev_query_debug_event(struct kfd_process *process,
  33		      unsigned int *queue_id,
  34		      unsigned int *gpu_id,
  35		      uint64_t exception_clear_mask,
  36		      uint64_t *event_status)
  37{
  38	struct process_queue_manager *pqm;
  39	struct process_queue_node *pqn;
  40	int i;
  41
  42	if (!(process && process->debug_trap_enabled))
  43		return -ENODATA;
  44
  45	mutex_lock(&process->event_mutex);
  46	*event_status = 0;
  47	*queue_id = 0;
  48	*gpu_id = 0;
  49
  50	/* find and report queue events */
  51	pqm = &process->pqm;
  52	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
  53		uint64_t tmp = process->exception_enable_mask;
  54
  55		if (!pqn->q)
  56			continue;
  57
  58		tmp &= pqn->q->properties.exception_status;
  59
  60		if (!tmp)
  61			continue;
  62
  63		*event_status = pqn->q->properties.exception_status;
  64		*queue_id = pqn->q->properties.queue_id;
  65		*gpu_id = pqn->q->device->id;
  66		pqn->q->properties.exception_status &= ~exception_clear_mask;
  67		goto out;
  68	}
  69
  70	/* find and report device events */
  71	for (i = 0; i < process->n_pdds; i++) {
  72		struct kfd_process_device *pdd = process->pdds[i];
  73		uint64_t tmp = process->exception_enable_mask
  74						& pdd->exception_status;
  75
  76		if (!tmp)
  77			continue;
  78
  79		*event_status = pdd->exception_status;
  80		*gpu_id = pdd->dev->id;
  81		pdd->exception_status &= ~exception_clear_mask;
  82		goto out;
  83	}
  84
  85	/* report process events */
  86	if (process->exception_enable_mask & process->exception_status) {
  87		*event_status = process->exception_status;
  88		process->exception_status &= ~exception_clear_mask;
  89	}
  90
  91out:
  92	mutex_unlock(&process->event_mutex);
  93	return *event_status ? 0 : -EAGAIN;
  94}
  95
  96void debug_event_write_work_handler(struct work_struct *work)
  97{
  98	struct kfd_process *process;
  99
 100	static const char write_data = '.';
 101	loff_t pos = 0;
 102
 103	process = container_of(work,
 104			struct kfd_process,
 105			debug_event_workarea);
 106
 107	if (process->debug_trap_enabled && process->dbg_ev_file)
 108		kernel_write(process->dbg_ev_file, &write_data, 1, &pos);
 109}
 110
 111/* update process/device/queue exception status, write to descriptor
 112 * only if exception_status is enabled.
 113 */
 114bool kfd_dbg_ev_raise(uint64_t event_mask,
 115			struct kfd_process *process, struct kfd_node *dev,
 116			unsigned int source_id, bool use_worker,
 117			void *exception_data, size_t exception_data_size)
 118{
 119	struct process_queue_manager *pqm;
 120	struct process_queue_node *pqn;
 121	int i;
 122	static const char write_data = '.';
 123	loff_t pos = 0;
 124	bool is_subscribed = true;
 125
 126	if (!(process && process->debug_trap_enabled))
 127		return false;
 128
 129	mutex_lock(&process->event_mutex);
 130
 131	if (event_mask & KFD_EC_MASK_DEVICE) {
 132		for (i = 0; i < process->n_pdds; i++) {
 133			struct kfd_process_device *pdd = process->pdds[i];
 134
 135			if (pdd->dev != dev)
 136				continue;
 137
 138			pdd->exception_status |= event_mask & KFD_EC_MASK_DEVICE;
 139
 140			if (event_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 141				if (!pdd->vm_fault_exc_data) {
 142					pdd->vm_fault_exc_data = kmemdup(
 143							exception_data,
 144							exception_data_size,
 145							GFP_KERNEL);
 146					if (!pdd->vm_fault_exc_data)
 147						pr_debug("Failed to allocate exception data memory");
 148				} else {
 149					pr_debug("Debugger exception data not saved\n");
 150					print_hex_dump_bytes("exception data: ",
 151							DUMP_PREFIX_OFFSET,
 152							exception_data,
 153							exception_data_size);
 154				}
 155			}
 156			break;
 157		}
 158	} else if (event_mask & KFD_EC_MASK_PROCESS) {
 159		process->exception_status |= event_mask & KFD_EC_MASK_PROCESS;
 160	} else {
 161		pqm = &process->pqm;
 162		list_for_each_entry(pqn, &pqm->queues,
 163				process_queue_list) {
 164			int target_id;
 165
 166			if (!pqn->q)
 167				continue;
 168
 169			target_id = event_mask & KFD_EC_MASK(EC_QUEUE_NEW) ?
 170					pqn->q->properties.queue_id :
 171							pqn->q->doorbell_id;
 172
 173			if (pqn->q->device != dev || target_id != source_id)
 174				continue;
 175
 176			pqn->q->properties.exception_status |= event_mask;
 177			break;
 178		}
 179	}
 180
 181	if (process->exception_enable_mask & event_mask) {
 182		if (use_worker)
 183			schedule_work(&process->debug_event_workarea);
 184		else
 185			kernel_write(process->dbg_ev_file,
 186					&write_data,
 187					1,
 188					&pos);
 189	} else {
 190		is_subscribed = false;
 191	}
 192
 193	mutex_unlock(&process->event_mutex);
 194
 195	return is_subscribed;
 196}
 197
 198/* set pending event queue entry from ring entry  */
 199bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev,
 200				   unsigned int pasid,
 201				   uint32_t doorbell_id,
 202				   uint64_t trap_mask,
 203				   void *exception_data,
 204				   size_t exception_data_size)
 205{
 206	struct kfd_process *p;
 207	struct kfd_process_device *pdd = NULL;
 208	bool signaled_to_debugger_or_runtime = false;
 209
 210	p = kfd_lookup_process_by_pasid(pasid, &pdd);
 211
 212	if (!pdd)
 213		return false;
 214
 215	if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true,
 216			      exception_data, exception_data_size)) {
 217		struct process_queue_manager *pqm;
 218		struct process_queue_node *pqn;
 219
 220		if (!!(trap_mask & KFD_EC_MASK_QUEUE) &&
 221		       p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) {
 222			mutex_lock(&p->mutex);
 223
 224			pqm = &p->pqm;
 225			list_for_each_entry(pqn, &pqm->queues,
 226							process_queue_list) {
 227
 228				if (!(pqn->q && pqn->q->device == dev &&
 229				      pqn->q->doorbell_id == doorbell_id))
 230					continue;
 231
 232				kfd_send_exception_to_runtime(p, pqn->q->properties.queue_id,
 233							      trap_mask);
 234
 235				signaled_to_debugger_or_runtime = true;
 236
 237				break;
 238			}
 239
 240			mutex_unlock(&p->mutex);
 241		} else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 242			kfd_evict_process_device(pdd);
 243			kfd_signal_vm_fault_event(pdd, NULL, exception_data);
 244
 245			signaled_to_debugger_or_runtime = true;
 246		}
 247	} else {
 248		signaled_to_debugger_or_runtime = true;
 249	}
 250
 251	kfd_unref_process(p);
 252
 253	return signaled_to_debugger_or_runtime;
 254}
 255
 256int kfd_dbg_send_exception_to_runtime(struct kfd_process *p,
 257					unsigned int dev_id,
 258					unsigned int queue_id,
 259					uint64_t error_reason)
 260{
 261	if (error_reason & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) {
 262		struct kfd_process_device *pdd = NULL;
 263		struct kfd_hsa_memory_exception_data *data;
 264		int i;
 265
 266		for (i = 0; i < p->n_pdds; i++) {
 267			if (p->pdds[i]->dev->id == dev_id) {
 268				pdd = p->pdds[i];
 269				break;
 270			}
 271		}
 272
 273		if (!pdd)
 274			return -ENODEV;
 275
 276		data = (struct kfd_hsa_memory_exception_data *)
 277						pdd->vm_fault_exc_data;
 278
 279		kfd_evict_process_device(pdd);
 280		kfd_signal_vm_fault_event(pdd, NULL, data);
 281		error_reason &= ~KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION);
 282	}
 283
 284	if (error_reason & (KFD_EC_MASK(EC_PROCESS_RUNTIME))) {
 285		/*
 286		 * block should only happen after the debugger receives runtime
 287		 * enable notice.
 288		 */
 289		up(&p->runtime_enable_sema);
 290		error_reason &= ~KFD_EC_MASK(EC_PROCESS_RUNTIME);
 291	}
 292
 293	if (error_reason)
 294		return kfd_send_exception_to_runtime(p, queue_id, error_reason);
 295
 296	return 0;
 297}
 298
 299static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable)
 300{
 301	struct mqd_update_info minfo = {0};
 302	int err;
 303
 304	if (!q)
 305		return 0;
 306
 307	if (!kfd_dbg_has_cwsr_workaround(q->device))
 308		return 0;
 309
 310	if (enable && q->properties.is_user_cu_masked)
 311		return -EBUSY;
 312
 313	minfo.update_flag = enable ? UPDATE_FLAG_DBG_WA_ENABLE : UPDATE_FLAG_DBG_WA_DISABLE;
 314
 315	q->properties.is_dbg_wa = enable;
 316	err = q->device->dqm->ops.update_queue(q->device->dqm, q, &minfo);
 317	if (err)
 318		q->properties.is_dbg_wa = false;
 319
 320	return err;
 321}
 322
 323static int kfd_dbg_set_workaround(struct kfd_process *target, bool enable)
 324{
 325	struct process_queue_manager *pqm = &target->pqm;
 326	struct process_queue_node *pqn;
 327	int r = 0;
 328
 329	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
 330		r = kfd_dbg_set_queue_workaround(pqn->q, enable);
 331		if (enable && r)
 332			goto unwind;
 333	}
 334
 335	return 0;
 336
 337unwind:
 338	list_for_each_entry(pqn, &pqm->queues, process_queue_list)
 339		kfd_dbg_set_queue_workaround(pqn->q, false);
 340
 341	if (enable)
 342		target->runtime_info.runtime_state = r == -EBUSY ?
 343				DEBUG_RUNTIME_STATE_ENABLED_BUSY :
 344				DEBUG_RUNTIME_STATE_ENABLED_ERROR;
 345
 346	return r;
 347}
 348
 349int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en)
 350{
 351	uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode;
 352	uint32_t flags = pdd->process->dbg_flags;
 353	struct amdgpu_device *adev = pdd->dev->adev;
 354	int r;
 355
 356	if (!kfd_dbg_is_per_vmid_supported(pdd->dev))
 357		return 0;
 358
 359	if (!pdd->proc_ctx_cpu_ptr) {
 360		r = amdgpu_amdkfd_alloc_gtt_mem(adev,
 361			AMDGPU_MES_PROC_CTX_SIZE,
 362			&pdd->proc_ctx_bo,
 363			&pdd->proc_ctx_gpu_addr,
 364			&pdd->proc_ctx_cpu_ptr,
 365			false);
 366		if (r) {
 367			dev_err(adev->dev,
 368			"failed to allocate process context bo\n");
 369			return r;
 370		}
 371		memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
 372	}
 373
 374	return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl,
 375						pdd->watch_points, flags, sq_trap_en);
 376}
 377
 378#define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1
 379static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_id)
 380{
 381	int i;
 382
 383	*watch_id = KFD_DEBUGGER_INVALID_WATCH_POINT_ID;
 384
 385	spin_lock(&pdd->dev->watch_points_lock);
 386
 387	for (i = 0; i < MAX_WATCH_ADDRESSES; i++) {
 388		/* device watchpoint in use so skip */
 389		if ((pdd->dev->alloc_watch_ids >> i) & 0x1)
 390			continue;
 391
 392		pdd->alloc_watch_ids |= 0x1 << i;
 393		pdd->dev->alloc_watch_ids |= 0x1 << i;
 394		*watch_id = i;
 395		spin_unlock(&pdd->dev->watch_points_lock);
 396		return 0;
 397	}
 398
 399	spin_unlock(&pdd->dev->watch_points_lock);
 400
 401	return -ENOMEM;
 402}
 403
 404static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id)
 405{
 406	spin_lock(&pdd->dev->watch_points_lock);
 407
 408	/* process owns device watch point so safe to clear */
 409	if (pdd->alloc_watch_ids & BIT(watch_id)) {
 410		pdd->alloc_watch_ids &= ~BIT(watch_id);
 411		pdd->dev->alloc_watch_ids &= ~BIT(watch_id);
 412	}
 413
 414	spin_unlock(&pdd->dev->watch_points_lock);
 415}
 416
 417static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id)
 418{
 419	bool owns_watch_id = false;
 420
 421	spin_lock(&pdd->dev->watch_points_lock);
 422	owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id);
 423	spin_unlock(&pdd->dev->watch_points_lock);
 424
 425	return owns_watch_id;
 426}
 427
 428int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd,
 429					uint32_t watch_id)
 430{
 431	int r;
 432
 433	if (watch_id >= MAX_WATCH_ADDRESSES)
 434		return -EINVAL;
 435
 436	if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id))
 437		return -EINVAL;
 438
 439	if (!pdd->dev->kfd->shared_resources.enable_mes) {
 440		r = debug_lock_and_unmap(pdd->dev->dqm);
 441		if (r)
 442			return r;
 443	}
 444
 445	amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 446	pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch(
 447							pdd->dev->adev,
 448							watch_id);
 449	amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 450
 451	if (!pdd->dev->kfd->shared_resources.enable_mes)
 452		r = debug_map_and_unlock(pdd->dev->dqm);
 453	else
 454		r = kfd_dbg_set_mes_debug_mode(pdd, true);
 455
 456	kfd_dbg_clear_dev_watch_id(pdd, watch_id);
 457
 458	return r;
 459}
 460
 461int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd,
 462					uint64_t watch_address,
 463					uint32_t watch_address_mask,
 464					uint32_t *watch_id,
 465					uint32_t watch_mode)
 466{
 467	int xcc_id, r = kfd_dbg_get_dev_watch_id(pdd, watch_id);
 468	uint32_t xcc_mask = pdd->dev->xcc_mask;
 469
 470	if (r)
 471		return r;
 472
 473	if (*watch_id >= MAX_WATCH_ADDRESSES)
 474		return -EINVAL;
 475
 476	if (!pdd->dev->kfd->shared_resources.enable_mes) {
 477		r = debug_lock_and_unmap(pdd->dev->dqm);
 478		if (r) {
 479			kfd_dbg_clear_dev_watch_id(pdd, *watch_id);
 480			return r;
 481		}
 482	}
 483
 484	amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 485	for_each_inst(xcc_id, xcc_mask)
 486		pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch(
 487				pdd->dev->adev,
 488				watch_address,
 489				watch_address_mask,
 490				*watch_id,
 491				watch_mode,
 492				pdd->dev->vm_info.last_vmid_kfd,
 493				xcc_id);
 494	amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 495
 496	if (!pdd->dev->kfd->shared_resources.enable_mes)
 497		r = debug_map_and_unlock(pdd->dev->dqm);
 498	else
 499		r = kfd_dbg_set_mes_debug_mode(pdd, true);
 500
 501	/* HWS is broken so no point in HW rollback but release the watchpoint anyways */
 502	if (r)
 503		kfd_dbg_clear_dev_watch_id(pdd, *watch_id);
 504
 505	return 0;
 506}
 507
 508static void kfd_dbg_clear_process_address_watch(struct kfd_process *target)
 509{
 510	int i, j;
 511
 512	for (i = 0; i < target->n_pdds; i++)
 513		for (j = 0; j < MAX_WATCH_ADDRESSES; j++)
 514			kfd_dbg_trap_clear_dev_address_watch(target->pdds[i], j);
 515}
 516
 517int kfd_dbg_trap_set_flags(struct kfd_process *target, uint32_t *flags)
 518{
 519	uint32_t prev_flags = target->dbg_flags;
 520	int i, r = 0, rewind_count = 0;
 521
 522	for (i = 0; i < target->n_pdds; i++) {
 523		struct kfd_topology_device *topo_dev =
 524				kfd_topology_device_by_id(target->pdds[i]->dev->id);
 525		uint32_t caps = topo_dev->node_props.capability;
 526
 527		if (!(caps & HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED) &&
 528			(*flags & KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP)) {
 529			*flags = prev_flags;
 530			return -EACCES;
 531		}
 532
 533		if (!(caps & HSA_CAP_TRAP_DEBUG_PRECISE_ALU_OPERATIONS_SUPPORTED) &&
 534		    (*flags & KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP)) {
 535			*flags = prev_flags;
 536			return -EACCES;
 537		}
 538	}
 539
 540	target->dbg_flags = *flags;
 541	*flags = prev_flags;
 542	for (i = 0; i < target->n_pdds; i++) {
 543		struct kfd_process_device *pdd = target->pdds[i];
 544
 545		if (!kfd_dbg_is_per_vmid_supported(pdd->dev))
 546			continue;
 547
 548		if (!pdd->dev->kfd->shared_resources.enable_mes)
 549			r = debug_refresh_runlist(pdd->dev->dqm);
 550		else
 551			r = kfd_dbg_set_mes_debug_mode(pdd, true);
 552
 553		if (r) {
 554			target->dbg_flags = prev_flags;
 555			break;
 556		}
 557
 558		rewind_count++;
 559	}
 560
 561	/* Rewind flags */
 562	if (r) {
 563		target->dbg_flags = prev_flags;
 564
 565		for (i = 0; i < rewind_count; i++) {
 566			struct kfd_process_device *pdd = target->pdds[i];
 567
 568			if (!kfd_dbg_is_per_vmid_supported(pdd->dev))
 569				continue;
 570
 571			if (!pdd->dev->kfd->shared_resources.enable_mes)
 572				debug_refresh_runlist(pdd->dev->dqm);
 573			else
 574				kfd_dbg_set_mes_debug_mode(pdd, true);
 575		}
 576	}
 577
 578	return r;
 579}
 580
 581/* kfd_dbg_trap_deactivate:
 582 *	target: target process
 583 *	unwind: If this is unwinding a failed kfd_dbg_trap_enable()
 584 *	unwind_count:
 585 *		If unwind == true, how far down the pdd list we need
 586 *				to unwind
 587 *		else: ignored
 588 */
 589void kfd_dbg_trap_deactivate(struct kfd_process *target, bool unwind, int unwind_count)
 590{
 591	int i;
 592
 593	if (!unwind) {
 594		uint32_t flags = 0;
 595		int resume_count = resume_queues(target, 0, NULL);
 596
 597		if (resume_count)
 598			pr_debug("Resumed %d queues\n", resume_count);
 599
 600		cancel_work_sync(&target->debug_event_workarea);
 601		kfd_dbg_clear_process_address_watch(target);
 602		kfd_dbg_trap_set_wave_launch_mode(target, 0);
 603
 604		kfd_dbg_trap_set_flags(target, &flags);
 605	}
 606
 607	for (i = 0; i < target->n_pdds; i++) {
 608		struct kfd_process_device *pdd = target->pdds[i];
 609
 610		/* If this is an unwind, and we have unwound the required
 611		 * enable calls on the pdd list, we need to stop now
 612		 * otherwise we may mess up another debugger session.
 613		 */
 614		if (unwind && i == unwind_count)
 615			break;
 616
 617		kfd_process_set_trap_debug_flag(&pdd->qpd, false);
 618
 619		/* GFX off is already disabled by debug activate if not RLC restore supported. */
 620		if (kfd_dbg_is_rlc_restore_supported(pdd->dev))
 621			amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 622		pdd->spi_dbg_override =
 623				pdd->dev->kfd2kgd->disable_debug_trap(
 624				pdd->dev->adev,
 625				target->runtime_info.ttmp_setup,
 626				pdd->dev->vm_info.last_vmid_kfd);
 627		amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 628
 629		if (!kfd_dbg_is_per_vmid_supported(pdd->dev) &&
 630				release_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd))
 631			pr_err("Failed to release debug vmid on [%i]\n", pdd->dev->id);
 632
 633		if (!pdd->dev->kfd->shared_resources.enable_mes)
 634			debug_refresh_runlist(pdd->dev->dqm);
 635		else
 636			kfd_dbg_set_mes_debug_mode(pdd, !kfd_dbg_has_cwsr_workaround(pdd->dev));
 637	}
 638
 639	kfd_dbg_set_workaround(target, false);
 640}
 641
 642static void kfd_dbg_clean_exception_status(struct kfd_process *target)
 643{
 644	struct process_queue_manager *pqm;
 645	struct process_queue_node *pqn;
 646	int i;
 647
 648	for (i = 0; i < target->n_pdds; i++) {
 649		struct kfd_process_device *pdd = target->pdds[i];
 650
 651		kfd_process_drain_interrupts(pdd);
 652
 653		pdd->exception_status = 0;
 654	}
 655
 656	pqm = &target->pqm;
 657	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
 658		if (!pqn->q)
 659			continue;
 660
 661		pqn->q->properties.exception_status = 0;
 662	}
 663
 664	target->exception_status = 0;
 665}
 666
 667int kfd_dbg_trap_disable(struct kfd_process *target)
 668{
 669	if (!target->debug_trap_enabled)
 670		return 0;
 671
 672	/*
 673	 * Defer deactivation to runtime if runtime not enabled otherwise reset
 674	 * attached running target runtime state to enable for re-attach.
 675	 */
 676	if (target->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED)
 677		kfd_dbg_trap_deactivate(target, false, 0);
 678	else if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
 679		target->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
 680
 681	cancel_work_sync(&target->debug_event_workarea);
 682	fput(target->dbg_ev_file);
 683	target->dbg_ev_file = NULL;
 684
 685	if (target->debugger_process) {
 686		atomic_dec(&target->debugger_process->debugged_process_count);
 687		target->debugger_process = NULL;
 688	}
 689
 690	target->debug_trap_enabled = false;
 691	kfd_dbg_clean_exception_status(target);
 692	kfd_unref_process(target);
 693
 694	return 0;
 695}
 696
 697int kfd_dbg_trap_activate(struct kfd_process *target)
 698{
 699	int i, r = 0;
 700
 701	r = kfd_dbg_set_workaround(target, true);
 702	if (r)
 703		return r;
 704
 705	for (i = 0; i < target->n_pdds; i++) {
 706		struct kfd_process_device *pdd = target->pdds[i];
 707
 708		if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) {
 709			r = reserve_debug_trap_vmid(pdd->dev->dqm, &pdd->qpd);
 710
 711			if (r) {
 712				target->runtime_info.runtime_state = (r == -EBUSY) ?
 713							DEBUG_RUNTIME_STATE_ENABLED_BUSY :
 714							DEBUG_RUNTIME_STATE_ENABLED_ERROR;
 715
 716				goto unwind_err;
 717			}
 718		}
 719
 720		/* Disable GFX OFF to prevent garbage read/writes to debug registers.
 721		 * If RLC restore of debug registers is not supported and runtime enable
 722		 * hasn't done so already on ttmp setup request, restore the trap config registers.
 723		 *
 724		 * If RLC restore of debug registers is not supported, keep gfx off disabled for
 725		 * the debug session.
 726		 */
 727		amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 728		if (!(kfd_dbg_is_rlc_restore_supported(pdd->dev) ||
 729						target->runtime_info.ttmp_setup))
 730			pdd->dev->kfd2kgd->enable_debug_trap(pdd->dev->adev, true,
 731								pdd->dev->vm_info.last_vmid_kfd);
 732
 733		pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
 734					pdd->dev->adev,
 735					false,
 736					pdd->dev->vm_info.last_vmid_kfd);
 737
 738		if (kfd_dbg_is_rlc_restore_supported(pdd->dev))
 739			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 740
 741		/*
 742		 * Setting the debug flag in the trap handler requires that the TMA has been
 743		 * allocated, which occurs during CWSR initialization.
 744		 * In the event that CWSR has not been initialized at this point, setting the
 745		 * flag will be called again during CWSR initialization if the target process
 746		 * is still debug enabled.
 747		 */
 748		kfd_process_set_trap_debug_flag(&pdd->qpd, true);
 749
 750		if (!pdd->dev->kfd->shared_resources.enable_mes)
 751			r = debug_refresh_runlist(pdd->dev->dqm);
 752		else
 753			r = kfd_dbg_set_mes_debug_mode(pdd, true);
 754
 755		if (r) {
 756			target->runtime_info.runtime_state =
 757					DEBUG_RUNTIME_STATE_ENABLED_ERROR;
 758			goto unwind_err;
 759		}
 760	}
 761
 762	return 0;
 763
 764unwind_err:
 765	/* Enabling debug failed, we need to disable on
 766	 * all GPUs so the enable is all or nothing.
 767	 */
 768	kfd_dbg_trap_deactivate(target, true, i);
 769	return r;
 770}
 771
 772int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd,
 773			void __user *runtime_info, uint32_t *runtime_size)
 774{
 775	struct file *f;
 776	uint32_t copy_size;
 777	int i, r = 0;
 778
 779	if (target->debug_trap_enabled)
 780		return -EALREADY;
 781
 782	/* Enable pre-checks */
 783	for (i = 0; i < target->n_pdds; i++) {
 784		struct kfd_process_device *pdd = target->pdds[i];
 785
 786		if (!KFD_IS_SOC15(pdd->dev))
 787			return -ENODEV;
 788
 789		if (pdd->qpd.num_gws && (!kfd_dbg_has_gws_support(pdd->dev) ||
 790					 kfd_dbg_has_cwsr_workaround(pdd->dev)))
 791			return -EBUSY;
 792	}
 793
 794	copy_size = min((size_t)(*runtime_size), sizeof(target->runtime_info));
 795
 796	f = fget(fd);
 797	if (!f) {
 798		pr_err("Failed to get file for (%i)\n", fd);
 799		return -EBADF;
 800	}
 801
 802	target->dbg_ev_file = f;
 803
 804	/* defer activation to runtime if not runtime enabled */
 805	if (target->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED)
 806		kfd_dbg_trap_activate(target);
 807
 808	/* We already hold the process reference but hold another one for the
 809	 * debug session.
 810	 */
 811	kref_get(&target->ref);
 812	target->debug_trap_enabled = true;
 813
 814	if (target->debugger_process)
 815		atomic_inc(&target->debugger_process->debugged_process_count);
 816
 817	if (copy_to_user(runtime_info, (void *)&target->runtime_info, copy_size)) {
 818		kfd_dbg_trap_deactivate(target, false, 0);
 819		r = -EFAULT;
 820	}
 821
 822	*runtime_size = sizeof(target->runtime_info);
 823
 824	return r;
 825}
 826
 827static int kfd_dbg_validate_trap_override_request(struct kfd_process *p,
 828						uint32_t trap_override,
 829						uint32_t trap_mask_request,
 830						uint32_t *trap_mask_supported)
 831{
 832	int i = 0;
 833
 834	*trap_mask_supported = 0xffffffff;
 835
 836	for (i = 0; i < p->n_pdds; i++) {
 837		struct kfd_process_device *pdd = p->pdds[i];
 838		int err = pdd->dev->kfd2kgd->validate_trap_override_request(
 839								pdd->dev->adev,
 840								trap_override,
 841								trap_mask_supported);
 842
 843		if (err)
 844			return err;
 845	}
 846
 847	if (trap_mask_request & ~*trap_mask_supported)
 848		return -EACCES;
 849
 850	return 0;
 851}
 852
 853int kfd_dbg_trap_set_wave_launch_override(struct kfd_process *target,
 854					uint32_t trap_override,
 855					uint32_t trap_mask_bits,
 856					uint32_t trap_mask_request,
 857					uint32_t *trap_mask_prev,
 858					uint32_t *trap_mask_supported)
 859{
 860	int r = 0, i;
 861
 862	r = kfd_dbg_validate_trap_override_request(target,
 863						trap_override,
 864						trap_mask_request,
 865						trap_mask_supported);
 866
 867	if (r)
 868		return r;
 869
 870	for (i = 0; i < target->n_pdds; i++) {
 871		struct kfd_process_device *pdd = target->pdds[i];
 872
 873		amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 874		pdd->spi_dbg_override = pdd->dev->kfd2kgd->set_wave_launch_trap_override(
 875				pdd->dev->adev,
 876				pdd->dev->vm_info.last_vmid_kfd,
 877				trap_override,
 878				trap_mask_bits,
 879				trap_mask_request,
 880				trap_mask_prev,
 881				pdd->spi_dbg_override);
 882		amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 883
 884		if (!pdd->dev->kfd->shared_resources.enable_mes)
 885			r = debug_refresh_runlist(pdd->dev->dqm);
 886		else
 887			r = kfd_dbg_set_mes_debug_mode(pdd, true);
 888
 889		if (r)
 890			break;
 891	}
 892
 893	return r;
 894}
 895
 896int kfd_dbg_trap_set_wave_launch_mode(struct kfd_process *target,
 897					uint8_t wave_launch_mode)
 898{
 899	int r = 0, i;
 900
 901	if (wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_NORMAL &&
 902			wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_HALT &&
 903			wave_launch_mode != KFD_DBG_TRAP_WAVE_LAUNCH_MODE_DEBUG)
 904		return -EINVAL;
 905
 906	for (i = 0; i < target->n_pdds; i++) {
 907		struct kfd_process_device *pdd = target->pdds[i];
 908
 909		amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
 910		pdd->spi_dbg_launch_mode = pdd->dev->kfd2kgd->set_wave_launch_mode(
 911				pdd->dev->adev,
 912				wave_launch_mode,
 913				pdd->dev->vm_info.last_vmid_kfd);
 914		amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
 915
 916		if (!pdd->dev->kfd->shared_resources.enable_mes)
 917			r = debug_refresh_runlist(pdd->dev->dqm);
 918		else
 919			r = kfd_dbg_set_mes_debug_mode(pdd, true);
 920
 921		if (r)
 922			break;
 923	}
 924
 925	return r;
 926}
 927
 928int kfd_dbg_trap_query_exception_info(struct kfd_process *target,
 929		uint32_t source_id,
 930		uint32_t exception_code,
 931		bool clear_exception,
 932		void __user *info,
 933		uint32_t *info_size)
 934{
 935	bool found = false;
 936	int r = 0;
 937	uint32_t copy_size, actual_info_size = 0;
 938	uint64_t *exception_status_ptr = NULL;
 939
 940	if (!target)
 941		return -EINVAL;
 942
 943	if (!info || !info_size)
 944		return -EINVAL;
 945
 946	mutex_lock(&target->event_mutex);
 947
 948	if (KFD_DBG_EC_TYPE_IS_QUEUE(exception_code)) {
 949		/* Per queue exceptions */
 950		struct queue *queue = NULL;
 951		int i;
 952
 953		for (i = 0; i < target->n_pdds; i++) {
 954			struct kfd_process_device *pdd = target->pdds[i];
 955			struct qcm_process_device *qpd = &pdd->qpd;
 956
 957			list_for_each_entry(queue, &qpd->queues_list, list) {
 958				if (!found && queue->properties.queue_id == source_id) {
 959					found = true;
 960					break;
 961				}
 962			}
 963			if (found)
 964				break;
 965		}
 966
 967		if (!found) {
 968			r = -EINVAL;
 969			goto out;
 970		}
 971
 972		if (!(queue->properties.exception_status & KFD_EC_MASK(exception_code))) {
 973			r = -ENODATA;
 974			goto out;
 975		}
 976		exception_status_ptr = &queue->properties.exception_status;
 977	} else if (KFD_DBG_EC_TYPE_IS_DEVICE(exception_code)) {
 978		/* Per device exceptions */
 979		struct kfd_process_device *pdd = NULL;
 980		int i;
 981
 982		for (i = 0; i < target->n_pdds; i++) {
 983			pdd = target->pdds[i];
 984			if (pdd->dev->id == source_id) {
 985				found = true;
 986				break;
 987			}
 988		}
 989
 990		if (!found) {
 991			r = -EINVAL;
 992			goto out;
 993		}
 994
 995		if (!(pdd->exception_status & KFD_EC_MASK(exception_code))) {
 996			r = -ENODATA;
 997			goto out;
 998		}
 999
1000		if (exception_code == EC_DEVICE_MEMORY_VIOLATION) {
1001			copy_size = min((size_t)(*info_size), pdd->vm_fault_exc_data_size);
1002
1003			if (copy_to_user(info, pdd->vm_fault_exc_data, copy_size)) {
1004				r = -EFAULT;
1005				goto out;
1006			}
1007			actual_info_size = pdd->vm_fault_exc_data_size;
1008			if (clear_exception) {
1009				kfree(pdd->vm_fault_exc_data);
1010				pdd->vm_fault_exc_data = NULL;
1011				pdd->vm_fault_exc_data_size = 0;
1012			}
1013		}
1014		exception_status_ptr = &pdd->exception_status;
1015	} else if (KFD_DBG_EC_TYPE_IS_PROCESS(exception_code)) {
1016		/* Per process exceptions */
1017		if (!(target->exception_status & KFD_EC_MASK(exception_code))) {
1018			r = -ENODATA;
1019			goto out;
1020		}
1021
1022		if (exception_code == EC_PROCESS_RUNTIME) {
1023			copy_size = min((size_t)(*info_size), sizeof(target->runtime_info));
1024
1025			if (copy_to_user(info, (void *)&target->runtime_info, copy_size)) {
1026				r = -EFAULT;
1027				goto out;
1028			}
1029
1030			actual_info_size = sizeof(target->runtime_info);
1031		}
1032
1033		exception_status_ptr = &target->exception_status;
1034	} else {
1035		pr_debug("Bad exception type [%i]\n", exception_code);
1036		r = -EINVAL;
1037		goto out;
1038	}
1039
1040	*info_size = actual_info_size;
1041	if (clear_exception)
1042		*exception_status_ptr &= ~KFD_EC_MASK(exception_code);
1043out:
1044	mutex_unlock(&target->event_mutex);
1045	return r;
1046}
1047
1048int kfd_dbg_trap_device_snapshot(struct kfd_process *target,
1049		uint64_t exception_clear_mask,
1050		void __user *user_info,
1051		uint32_t *number_of_device_infos,
1052		uint32_t *entry_size)
1053{
1054	struct kfd_dbg_device_info_entry device_info;
1055	uint32_t tmp_entry_size, tmp_num_devices;
1056	int i, r = 0;
1057
1058	if (!(target && user_info && number_of_device_infos && entry_size))
1059		return -EINVAL;
1060
1061	tmp_entry_size = *entry_size;
1062
1063	tmp_num_devices = min_t(size_t, *number_of_device_infos, target->n_pdds);
1064	*number_of_device_infos = target->n_pdds;
1065	*entry_size = min_t(size_t, *entry_size, sizeof(device_info));
1066
1067	if (!tmp_num_devices)
1068		return 0;
1069
1070	memset(&device_info, 0, sizeof(device_info));
1071
1072	mutex_lock(&target->event_mutex);
1073
1074	/* Run over all pdd of the process */
1075	for (i = 0; i < tmp_num_devices; i++) {
1076		struct kfd_process_device *pdd = target->pdds[i];
1077		struct kfd_topology_device *topo_dev = kfd_topology_device_by_id(pdd->dev->id);
1078
1079		device_info.gpu_id = pdd->dev->id;
1080		device_info.exception_status = pdd->exception_status;
1081		device_info.lds_base = pdd->lds_base;
1082		device_info.lds_limit = pdd->lds_limit;
1083		device_info.scratch_base = pdd->scratch_base;
1084		device_info.scratch_limit = pdd->scratch_limit;
1085		device_info.gpuvm_base = pdd->gpuvm_base;
1086		device_info.gpuvm_limit = pdd->gpuvm_limit;
1087		device_info.location_id = topo_dev->node_props.location_id;
1088		device_info.vendor_id = topo_dev->node_props.vendor_id;
1089		device_info.device_id = topo_dev->node_props.device_id;
1090		device_info.revision_id = pdd->dev->adev->pdev->revision;
1091		device_info.subsystem_vendor_id = pdd->dev->adev->pdev->subsystem_vendor;
1092		device_info.subsystem_device_id = pdd->dev->adev->pdev->subsystem_device;
1093		device_info.fw_version = pdd->dev->kfd->mec_fw_version;
1094		device_info.gfx_target_version =
1095			topo_dev->node_props.gfx_target_version;
1096		device_info.simd_count = topo_dev->node_props.simd_count;
1097		device_info.max_waves_per_simd =
1098			topo_dev->node_props.max_waves_per_simd;
1099		device_info.array_count = topo_dev->node_props.array_count;
1100		device_info.simd_arrays_per_engine =
1101			topo_dev->node_props.simd_arrays_per_engine;
1102		device_info.num_xcc = NUM_XCC(pdd->dev->xcc_mask);
1103		device_info.capability = topo_dev->node_props.capability;
1104		device_info.debug_prop = topo_dev->node_props.debug_prop;
1105
1106		if (exception_clear_mask)
1107			pdd->exception_status &= ~exception_clear_mask;
1108
1109		if (copy_to_user(user_info, &device_info, *entry_size)) {
1110			r = -EFAULT;
1111			break;
1112		}
1113
1114		user_info += tmp_entry_size;
1115	}
1116
1117	mutex_unlock(&target->event_mutex);
1118
1119	return r;
1120}
1121
1122void kfd_dbg_set_enabled_debug_exception_mask(struct kfd_process *target,
1123					uint64_t exception_set_mask)
1124{
1125	uint64_t found_mask = 0;
1126	struct process_queue_manager *pqm;
1127	struct process_queue_node *pqn;
1128	static const char write_data = '.';
1129	loff_t pos = 0;
1130	int i;
1131
1132	mutex_lock(&target->event_mutex);
1133
1134	found_mask |= target->exception_status;
1135
1136	pqm = &target->pqm;
1137	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
1138		if (!pqn->q)
1139			continue;
1140
1141		found_mask |= pqn->q->properties.exception_status;
1142	}
1143
1144	for (i = 0; i < target->n_pdds; i++) {
1145		struct kfd_process_device *pdd = target->pdds[i];
1146
1147		found_mask |= pdd->exception_status;
1148	}
1149
1150	if (exception_set_mask & found_mask)
1151		kernel_write(target->dbg_ev_file, &write_data, 1, &pos);
1152
1153	target->exception_enable_mask = exception_set_mask;
1154
1155	mutex_unlock(&target->event_mutex);
1156}