kfd_chardev.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 */
  23
  24#include <linux/device.h>
  25#include <linux/err.h>
  26#include <linux/fs.h>
  27#include <linux/file.h>
  28#include <linux/sched.h>
  29#include <linux/slab.h>
  30#include <linux/uaccess.h>
  31#include <linux/compat.h>
  32#include <uapi/linux/kfd_ioctl.h>
  33#include <linux/time.h>
  34#include <linux/mm.h>
  35#include <linux/mman.h>
  36#include <linux/ptrace.h>
  37#include <linux/dma-buf.h>
  38#include <linux/processor.h>
  39#include "kfd_priv.h"
  40#include "kfd_device_queue_manager.h"
  41#include "kfd_svm.h"
  42#include "amdgpu_amdkfd.h"
  43#include "kfd_smi_events.h"
  44#include "amdgpu_dma_buf.h"
  45#include "kfd_debug.h"
  46
  47static long kfd_ioctl(struct file *, unsigned int, unsigned long);
  48static int kfd_open(struct inode *, struct file *);
  49static int kfd_release(struct inode *, struct file *);
  50static int kfd_mmap(struct file *, struct vm_area_struct *);
  51
  52static const char kfd_dev_name[] = "kfd";
  53
  54static const struct file_operations kfd_fops = {
  55	.owner = THIS_MODULE,
  56	.unlocked_ioctl = kfd_ioctl,
  57	.compat_ioctl = compat_ptr_ioctl,
  58	.open = kfd_open,
  59	.release = kfd_release,
  60	.mmap = kfd_mmap,
  61};
  62
  63static int kfd_char_dev_major = -1;
  64struct device *kfd_device;
  65static const struct class kfd_class = {
  66	.name = kfd_dev_name,
  67};
  68
  69static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
  70{
  71	struct kfd_process_device *pdd;
  72
  73	mutex_lock(&p->mutex);
  74	pdd = kfd_process_device_data_by_id(p, gpu_id);
  75
  76	if (pdd)
  77		return pdd;
  78
  79	mutex_unlock(&p->mutex);
  80	return NULL;
  81}
  82
  83static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
  84{
  85	mutex_unlock(&pdd->process->mutex);
  86}
  87
  88int kfd_chardev_init(void)
  89{
  90	int err = 0;
  91
  92	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
  93	err = kfd_char_dev_major;
  94	if (err < 0)
  95		goto err_register_chrdev;
  96
  97	err = class_register(&kfd_class);
  98	if (err)
  99		goto err_class_create;
 100
 101	kfd_device = device_create(&kfd_class, NULL,
 102				   MKDEV(kfd_char_dev_major, 0),
 103				   NULL, kfd_dev_name);
 104	err = PTR_ERR(kfd_device);
 105	if (IS_ERR(kfd_device))
 106		goto err_device_create;
 107
 108	return 0;
 109
 110err_device_create:
 111	class_unregister(&kfd_class);
 112err_class_create:
 113	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 114err_register_chrdev:
 115	return err;
 116}
 117
 118void kfd_chardev_exit(void)
 119{
 120	device_destroy(&kfd_class, MKDEV(kfd_char_dev_major, 0));
 121	class_unregister(&kfd_class);
 122	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 123	kfd_device = NULL;
 124}
 125
 126
 127static int kfd_open(struct inode *inode, struct file *filep)
 128{
 129	struct kfd_process *process;
 130	bool is_32bit_user_mode;
 131
 132	if (iminor(inode) != 0)
 133		return -ENODEV;
 134
 135	is_32bit_user_mode = in_compat_syscall();
 136
 137	if (is_32bit_user_mode) {
 138		dev_warn(kfd_device,
 139			"Process %d (32-bit) failed to open /dev/kfd\n"
 140			"32-bit processes are not supported by amdkfd\n",
 141			current->pid);
 142		return -EPERM;
 143	}
 144
 145	process = kfd_create_process(current);
 146	if (IS_ERR(process))
 147		return PTR_ERR(process);
 148
 149	if (kfd_process_init_cwsr_apu(process, filep)) {
 150		kfd_unref_process(process);
 151		return -EFAULT;
 152	}
 153
 154	/* filep now owns the reference returned by kfd_create_process */
 155	filep->private_data = process;
 156
 157	dev_dbg(kfd_device, "process pid %d opened kfd node, compat mode (32 bit) - %d\n",
 158		process->lead_thread->pid, process->is_32bit_user_mode);
 159
 160	return 0;
 161}
 162
 163static int kfd_release(struct inode *inode, struct file *filep)
 164{
 165	struct kfd_process *process = filep->private_data;
 166
 167	if (process)
 168		kfd_unref_process(process);
 169
 170	return 0;
 171}
 172
 173static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
 174					void *data)
 175{
 176	struct kfd_ioctl_get_version_args *args = data;
 177
 178	args->major_version = KFD_IOCTL_MAJOR_VERSION;
 179	args->minor_version = KFD_IOCTL_MINOR_VERSION;
 180
 181	return 0;
 182}
 183
 184static int set_queue_properties_from_user(struct queue_properties *q_properties,
 185				struct kfd_ioctl_create_queue_args *args)
 186{
 187	/*
 188	 * Repurpose queue percentage to accommodate new features:
 189	 * bit 0-7: queue percentage
 190	 * bit 8-15: pm4_target_xcc
 191	 */
 192	if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
 193		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 194		return -EINVAL;
 195	}
 196
 197	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 198		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 199		return -EINVAL;
 200	}
 201
 202	if ((args->ring_base_address) &&
 203		(!access_ok((const void __user *) args->ring_base_address,
 204			sizeof(uint64_t)))) {
 205		pr_err("Can't access ring base address\n");
 206		return -EFAULT;
 207	}
 208
 209	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 210		pr_err("Ring size must be a power of 2 or 0\n");
 211		return -EINVAL;
 212	}
 213
 214	if (args->ring_size < KFD_MIN_QUEUE_RING_SIZE) {
 215		args->ring_size = KFD_MIN_QUEUE_RING_SIZE;
 216		pr_debug("Size lower. clamped to KFD_MIN_QUEUE_RING_SIZE");
 217	}
 218
 219	if (!access_ok((const void __user *) args->read_pointer_address,
 220			sizeof(uint32_t))) {
 221		pr_err("Can't access read pointer\n");
 222		return -EFAULT;
 223	}
 224
 225	if (!access_ok((const void __user *) args->write_pointer_address,
 226			sizeof(uint32_t))) {
 227		pr_err("Can't access write pointer\n");
 228		return -EFAULT;
 229	}
 230
 231	if (args->eop_buffer_address &&
 232		!access_ok((const void __user *) args->eop_buffer_address,
 233			sizeof(uint32_t))) {
 234		pr_debug("Can't access eop buffer");
 235		return -EFAULT;
 236	}
 237
 238	if (args->ctx_save_restore_address &&
 239		!access_ok((const void __user *) args->ctx_save_restore_address,
 240			sizeof(uint32_t))) {
 241		pr_debug("Can't access ctx save restore buffer");
 242		return -EFAULT;
 243	}
 244
 245	q_properties->is_interop = false;
 246	q_properties->is_gws = false;
 247	q_properties->queue_percent = args->queue_percentage & 0xFF;
 248	/* bit 8-15 are repurposed to be PM4 target XCC */
 249	q_properties->pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
 250	q_properties->priority = args->queue_priority;
 251	q_properties->queue_address = args->ring_base_address;
 252	q_properties->queue_size = args->ring_size;
 253	q_properties->read_ptr = (void __user *)args->read_pointer_address;
 254	q_properties->write_ptr = (void __user *)args->write_pointer_address;
 255	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
 256	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
 257	q_properties->ctx_save_restore_area_address =
 258			args->ctx_save_restore_address;
 259	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
 260	q_properties->ctl_stack_size = args->ctl_stack_size;
 261	q_properties->sdma_engine_id = args->sdma_engine_id;
 262	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
 263		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 264		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
 265	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
 266		q_properties->type = KFD_QUEUE_TYPE_SDMA;
 267	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
 268		q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
 269	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID)
 270		q_properties->type = KFD_QUEUE_TYPE_SDMA_BY_ENG_ID;
 271	else
 272		return -ENOTSUPP;
 273
 274	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 275		q_properties->format = KFD_QUEUE_FORMAT_AQL;
 276	else
 277		q_properties->format = KFD_QUEUE_FORMAT_PM4;
 278
 279	pr_debug("Queue Percentage: %d, %d\n",
 280			q_properties->queue_percent, args->queue_percentage);
 281
 282	pr_debug("Queue Priority: %d, %d\n",
 283			q_properties->priority, args->queue_priority);
 284
 285	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
 286			q_properties->queue_address, args->ring_base_address);
 287
 288	pr_debug("Queue Size: 0x%llX, %u\n",
 289			q_properties->queue_size, args->ring_size);
 290
 291	pr_debug("Queue r/w Pointers: %px, %px\n",
 292			q_properties->read_ptr,
 293			q_properties->write_ptr);
 294
 295	pr_debug("Queue Format: %d\n", q_properties->format);
 296
 297	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
 298
 299	pr_debug("Queue CTX save area: 0x%llX\n",
 300			q_properties->ctx_save_restore_area_address);
 301
 302	return 0;
 303}
 304
 305static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 306					void *data)
 307{
 308	struct kfd_ioctl_create_queue_args *args = data;
 309	struct kfd_node *dev;
 310	int err = 0;
 311	unsigned int queue_id;
 312	struct kfd_process_device *pdd;
 313	struct queue_properties q_properties;
 314	uint32_t doorbell_offset_in_process = 0;
 315
 316	memset(&q_properties, 0, sizeof(struct queue_properties));
 317
 318	pr_debug("Creating queue ioctl\n");
 319
 320	err = set_queue_properties_from_user(&q_properties, args);
 321	if (err)
 322		return err;
 323
 324	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
 325
 326	mutex_lock(&p->mutex);
 327
 328	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 329	if (!pdd) {
 330		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 331		err = -EINVAL;
 332		goto err_pdd;
 333	}
 334	dev = pdd->dev;
 335
 336	pdd = kfd_bind_process_to_device(dev, p);
 337	if (IS_ERR(pdd)) {
 338		err = -ESRCH;
 339		goto err_bind_process;
 340	}
 341
 342	if (q_properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
 343		int max_sdma_eng_id = kfd_get_num_sdma_engines(dev) +
 344				      kfd_get_num_xgmi_sdma_engines(dev) - 1;
 345
 346		if (q_properties.sdma_engine_id > max_sdma_eng_id) {
 347			err = -EINVAL;
 348			pr_err("sdma_engine_id %i exceeds maximum id of %i\n",
 349			       q_properties.sdma_engine_id, max_sdma_eng_id);
 350			goto err_sdma_engine_id;
 351		}
 352	}
 353
 354	if (!pdd->qpd.proc_doorbells) {
 355		err = kfd_alloc_process_doorbells(dev->kfd, pdd);
 356		if (err) {
 357			pr_debug("failed to allocate process doorbells\n");
 358			goto err_bind_process;
 359		}
 360	}
 361
 362	err = kfd_queue_acquire_buffers(pdd, &q_properties);
 363	if (err) {
 364		pr_debug("failed to acquire user queue buffers\n");
 365		goto err_acquire_queue_buf;
 366	}
 367
 368	pr_debug("Creating queue for process pid %d on gpu 0x%x\n",
 369			p->lead_thread->pid,
 370			dev->id);
 371
 372	err = pqm_create_queue(&p->pqm, dev, &q_properties, &queue_id,
 373			NULL, NULL, NULL, &doorbell_offset_in_process);
 374	if (err != 0)
 375		goto err_create_queue;
 376
 377	args->queue_id = queue_id;
 378
 379
 380	/* Return gpu_id as doorbell offset for mmap usage */
 381	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
 382	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
 383	if (KFD_IS_SOC15(dev))
 384		/* On SOC15 ASICs, include the doorbell offset within the
 385		 * process doorbell frame, which is 2 pages.
 386		 */
 387		args->doorbell_offset |= doorbell_offset_in_process;
 388
 389	mutex_unlock(&p->mutex);
 390
 391	pr_debug("Queue id %d was created successfully\n", args->queue_id);
 392
 393	pr_debug("Ring buffer address == 0x%016llX\n",
 394			args->ring_base_address);
 395
 396	pr_debug("Read ptr address    == 0x%016llX\n",
 397			args->read_pointer_address);
 398
 399	pr_debug("Write ptr address   == 0x%016llX\n",
 400			args->write_pointer_address);
 401
 402	kfd_dbg_ev_raise(KFD_EC_MASK(EC_QUEUE_NEW), p, dev, queue_id, false, NULL, 0);
 403	return 0;
 404
 405err_create_queue:
 406	kfd_queue_unref_bo_vas(pdd, &q_properties);
 407	kfd_queue_release_buffers(pdd, &q_properties);
 408err_acquire_queue_buf:
 409err_sdma_engine_id:
 410err_bind_process:
 411err_pdd:
 412	mutex_unlock(&p->mutex);
 413	return err;
 414}
 415
 416static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
 417					void *data)
 418{
 419	int retval;
 420	struct kfd_ioctl_destroy_queue_args *args = data;
 421
 422	pr_debug("Destroying queue id %d for process pid %d\n",
 423				args->queue_id,
 424				p->lead_thread->pid);
 425
 426	mutex_lock(&p->mutex);
 427
 428	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
 429
 430	mutex_unlock(&p->mutex);
 431	return retval;
 432}
 433
 434static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 435					void *data)
 436{
 437	int retval;
 438	struct kfd_ioctl_update_queue_args *args = data;
 439	struct queue_properties properties;
 440
 441	/*
 442	 * Repurpose queue percentage to accommodate new features:
 443	 * bit 0-7: queue percentage
 444	 * bit 8-15: pm4_target_xcc
 445	 */
 446	if ((args->queue_percentage & 0xFF) > KFD_MAX_QUEUE_PERCENTAGE) {
 447		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 448		return -EINVAL;
 449	}
 450
 451	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
 452		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 453		return -EINVAL;
 454	}
 455
 456	if ((args->ring_base_address) &&
 457		(!access_ok((const void __user *) args->ring_base_address,
 458			sizeof(uint64_t)))) {
 459		pr_err("Can't access ring base address\n");
 460		return -EFAULT;
 461	}
 462
 463	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
 464		pr_err("Ring size must be a power of 2 or 0\n");
 465		return -EINVAL;
 466	}
 467
 468	if (args->ring_size < KFD_MIN_QUEUE_RING_SIZE) {
 469		args->ring_size = KFD_MIN_QUEUE_RING_SIZE;
 470		pr_debug("Size lower. clamped to KFD_MIN_QUEUE_RING_SIZE");
 471	}
 472
 473	properties.queue_address = args->ring_base_address;
 474	properties.queue_size = args->ring_size;
 475	properties.queue_percent = args->queue_percentage & 0xFF;
 476	/* bit 8-15 are repurposed to be PM4 target XCC */
 477	properties.pm4_target_xcc = (args->queue_percentage >> 8) & 0xFF;
 478	properties.priority = args->queue_priority;
 479
 480	pr_debug("Updating queue id %d for process pid %d\n",
 481			args->queue_id, p->lead_thread->pid);
 482
 483	mutex_lock(&p->mutex);
 484
 485	retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
 486
 487	mutex_unlock(&p->mutex);
 488
 489	return retval;
 490}
 491
 492static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
 493					void *data)
 494{
 495	int retval;
 496	const int max_num_cus = 1024;
 497	struct kfd_ioctl_set_cu_mask_args *args = data;
 498	struct mqd_update_info minfo = {0};
 499	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
 500	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
 501
 502	if ((args->num_cu_mask % 32) != 0) {
 503		pr_debug("num_cu_mask 0x%x must be a multiple of 32",
 504				args->num_cu_mask);
 505		return -EINVAL;
 506	}
 507
 508	minfo.cu_mask.count = args->num_cu_mask;
 509	if (minfo.cu_mask.count == 0) {
 510		pr_debug("CU mask cannot be 0");
 511		return -EINVAL;
 512	}
 513
 514	/* To prevent an unreasonably large CU mask size, set an arbitrary
 515	 * limit of max_num_cus bits.  We can then just drop any CU mask bits
 516	 * past max_num_cus bits and just use the first max_num_cus bits.
 517	 */
 518	if (minfo.cu_mask.count > max_num_cus) {
 519		pr_debug("CU mask cannot be greater than 1024 bits");
 520		minfo.cu_mask.count = max_num_cus;
 521		cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
 522	}
 523
 524	minfo.cu_mask.ptr = memdup_user(cu_mask_ptr, cu_mask_size);
 525	if (IS_ERR(minfo.cu_mask.ptr)) {
 526		pr_debug("Could not copy CU mask from userspace");
 527		return PTR_ERR(minfo.cu_mask.ptr);
 528	}
 529
 530	mutex_lock(&p->mutex);
 531
 532	retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
 533
 534	mutex_unlock(&p->mutex);
 535
 536	kfree(minfo.cu_mask.ptr);
 537	return retval;
 538}
 539
 540static int kfd_ioctl_get_queue_wave_state(struct file *filep,
 541					  struct kfd_process *p, void *data)
 542{
 543	struct kfd_ioctl_get_queue_wave_state_args *args = data;
 544	int r;
 545
 546	mutex_lock(&p->mutex);
 547
 548	r = pqm_get_wave_state(&p->pqm, args->queue_id,
 549			       (void __user *)args->ctl_stack_address,
 550			       &args->ctl_stack_used_size,
 551			       &args->save_area_used_size);
 552
 553	mutex_unlock(&p->mutex);
 554
 555	return r;
 556}
 557
 558static int kfd_ioctl_set_memory_policy(struct file *filep,
 559					struct kfd_process *p, void *data)
 560{
 561	struct kfd_ioctl_set_memory_policy_args *args = data;
 562	int err = 0;
 563	struct kfd_process_device *pdd;
 564	enum cache_policy default_policy, alternate_policy;
 565
 566	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
 567	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 568		return -EINVAL;
 569	}
 570
 571	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
 572	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
 573		return -EINVAL;
 574	}
 575
 576	mutex_lock(&p->mutex);
 577	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 578	if (!pdd) {
 579		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 580		err = -EINVAL;
 581		goto err_pdd;
 582	}
 583
 584	pdd = kfd_bind_process_to_device(pdd->dev, p);
 585	if (IS_ERR(pdd)) {
 586		err = -ESRCH;
 587		goto out;
 588	}
 589
 590	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 591			 ? cache_policy_coherent : cache_policy_noncoherent;
 592
 593	alternate_policy =
 594		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
 595		   ? cache_policy_coherent : cache_policy_noncoherent;
 596
 597	if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
 598				&pdd->qpd,
 599				default_policy,
 600				alternate_policy,
 601				(void __user *)args->alternate_aperture_base,
 602				args->alternate_aperture_size,
 603				args->misc_process_flag))
 604		err = -EINVAL;
 605
 606out:
 607err_pdd:
 608	mutex_unlock(&p->mutex);
 609
 610	return err;
 611}
 612
 613static int kfd_ioctl_set_trap_handler(struct file *filep,
 614					struct kfd_process *p, void *data)
 615{
 616	struct kfd_ioctl_set_trap_handler_args *args = data;
 617	int err = 0;
 618	struct kfd_process_device *pdd;
 619
 620	mutex_lock(&p->mutex);
 621
 622	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 623	if (!pdd) {
 624		err = -EINVAL;
 625		goto err_pdd;
 626	}
 627
 628	pdd = kfd_bind_process_to_device(pdd->dev, p);
 629	if (IS_ERR(pdd)) {
 630		err = -ESRCH;
 631		goto out;
 632	}
 633
 634	kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
 635
 636out:
 637err_pdd:
 638	mutex_unlock(&p->mutex);
 639
 640	return err;
 641}
 642
 643static int kfd_ioctl_dbg_register(struct file *filep,
 644				struct kfd_process *p, void *data)
 645{
 646	return -EPERM;
 647}
 648
 649static int kfd_ioctl_dbg_unregister(struct file *filep,
 650				struct kfd_process *p, void *data)
 651{
 652	return -EPERM;
 653}
 654
 655static int kfd_ioctl_dbg_address_watch(struct file *filep,
 656					struct kfd_process *p, void *data)
 657{
 658	return -EPERM;
 659}
 660
 661/* Parse and generate fixed size data structure for wave control */
 662static int kfd_ioctl_dbg_wave_control(struct file *filep,
 663					struct kfd_process *p, void *data)
 664{
 665	return -EPERM;
 666}
 667
 668static int kfd_ioctl_get_clock_counters(struct file *filep,
 669				struct kfd_process *p, void *data)
 670{
 671	struct kfd_ioctl_get_clock_counters_args *args = data;
 672	struct kfd_process_device *pdd;
 673
 674	mutex_lock(&p->mutex);
 675	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 676	mutex_unlock(&p->mutex);
 677	if (pdd)
 678		/* Reading GPU clock counter from KGD */
 679		args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(pdd->dev->adev);
 680	else
 681		/* Node without GPU resource */
 682		args->gpu_clock_counter = 0;
 683
 684	/* No access to rdtsc. Using raw monotonic time */
 685	args->cpu_clock_counter = ktime_get_raw_ns();
 686	args->system_clock_counter = ktime_get_boottime_ns();
 687
 688	/* Since the counter is in nano-seconds we use 1GHz frequency */
 689	args->system_clock_freq = 1000000000;
 690
 691	return 0;
 692}
 693
 694
 695static int kfd_ioctl_get_process_apertures(struct file *filp,
 696				struct kfd_process *p, void *data)
 697{
 698	struct kfd_ioctl_get_process_apertures_args *args = data;
 699	struct kfd_process_device_apertures *pAperture;
 700	int i;
 701
 702	dev_dbg(kfd_device, "get apertures for process pid %d", p->lead_thread->pid);
 703
 704	args->num_of_nodes = 0;
 705
 706	mutex_lock(&p->mutex);
 707	/* Run over all pdd of the process */
 708	for (i = 0; i < p->n_pdds; i++) {
 709		struct kfd_process_device *pdd = p->pdds[i];
 710
 711		pAperture =
 712			&args->process_apertures[args->num_of_nodes];
 713		pAperture->gpu_id = pdd->dev->id;
 714		pAperture->lds_base = pdd->lds_base;
 715		pAperture->lds_limit = pdd->lds_limit;
 716		pAperture->gpuvm_base = pdd->gpuvm_base;
 717		pAperture->gpuvm_limit = pdd->gpuvm_limit;
 718		pAperture->scratch_base = pdd->scratch_base;
 719		pAperture->scratch_limit = pdd->scratch_limit;
 720
 721		dev_dbg(kfd_device,
 722			"node id %u\n", args->num_of_nodes);
 723		dev_dbg(kfd_device,
 724			"gpu id %u\n", pdd->dev->id);
 725		dev_dbg(kfd_device,
 726			"lds_base %llX\n", pdd->lds_base);
 727		dev_dbg(kfd_device,
 728			"lds_limit %llX\n", pdd->lds_limit);
 729		dev_dbg(kfd_device,
 730			"gpuvm_base %llX\n", pdd->gpuvm_base);
 731		dev_dbg(kfd_device,
 732			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
 733		dev_dbg(kfd_device,
 734			"scratch_base %llX\n", pdd->scratch_base);
 735		dev_dbg(kfd_device,
 736			"scratch_limit %llX\n", pdd->scratch_limit);
 737
 738		if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
 739			break;
 740	}
 741	mutex_unlock(&p->mutex);
 742
 743	return 0;
 744}
 745
 746static int kfd_ioctl_get_process_apertures_new(struct file *filp,
 747				struct kfd_process *p, void *data)
 748{
 749	struct kfd_ioctl_get_process_apertures_new_args *args = data;
 750	struct kfd_process_device_apertures *pa;
 751	int ret;
 752	int i;
 753
 754	dev_dbg(kfd_device, "get apertures for process pid %d",
 755			p->lead_thread->pid);
 756
 757	if (args->num_of_nodes == 0) {
 758		/* Return number of nodes, so that user space can alloacate
 759		 * sufficient memory
 760		 */
 761		mutex_lock(&p->mutex);
 762		args->num_of_nodes = p->n_pdds;
 763		goto out_unlock;
 764	}
 765
 766	/* Fill in process-aperture information for all available
 767	 * nodes, but not more than args->num_of_nodes as that is
 768	 * the amount of memory allocated by user
 769	 */
 770	pa = kcalloc(args->num_of_nodes, sizeof(struct kfd_process_device_apertures),
 771		     GFP_KERNEL);
 772	if (!pa)
 773		return -ENOMEM;
 774
 775	mutex_lock(&p->mutex);
 776
 777	if (!p->n_pdds) {
 778		args->num_of_nodes = 0;
 779		kfree(pa);
 780		goto out_unlock;
 781	}
 782
 783	/* Run over all pdd of the process */
 784	for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
 785		struct kfd_process_device *pdd = p->pdds[i];
 786
 787		pa[i].gpu_id = pdd->dev->id;
 788		pa[i].lds_base = pdd->lds_base;
 789		pa[i].lds_limit = pdd->lds_limit;
 790		pa[i].gpuvm_base = pdd->gpuvm_base;
 791		pa[i].gpuvm_limit = pdd->gpuvm_limit;
 792		pa[i].scratch_base = pdd->scratch_base;
 793		pa[i].scratch_limit = pdd->scratch_limit;
 794
 795		dev_dbg(kfd_device,
 796			"gpu id %u\n", pdd->dev->id);
 797		dev_dbg(kfd_device,
 798			"lds_base %llX\n", pdd->lds_base);
 799		dev_dbg(kfd_device,
 800			"lds_limit %llX\n", pdd->lds_limit);
 801		dev_dbg(kfd_device,
 802			"gpuvm_base %llX\n", pdd->gpuvm_base);
 803		dev_dbg(kfd_device,
 804			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
 805		dev_dbg(kfd_device,
 806			"scratch_base %llX\n", pdd->scratch_base);
 807		dev_dbg(kfd_device,
 808			"scratch_limit %llX\n", pdd->scratch_limit);
 809	}
 810	mutex_unlock(&p->mutex);
 811
 812	args->num_of_nodes = i;
 813	ret = copy_to_user(
 814			(void __user *)args->kfd_process_device_apertures_ptr,
 815			pa,
 816			(i * sizeof(struct kfd_process_device_apertures)));
 817	kfree(pa);
 818	return ret ? -EFAULT : 0;
 819
 820out_unlock:
 821	mutex_unlock(&p->mutex);
 822	return 0;
 823}
 824
 825static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
 826					void *data)
 827{
 828	struct kfd_ioctl_create_event_args *args = data;
 829	int err;
 830
 831	/* For dGPUs the event page is allocated in user mode. The
 832	 * handle is passed to KFD with the first call to this IOCTL
 833	 * through the event_page_offset field.
 834	 */
 835	if (args->event_page_offset) {
 836		mutex_lock(&p->mutex);
 837		err = kfd_kmap_event_page(p, args->event_page_offset);
 838		mutex_unlock(&p->mutex);
 839		if (err)
 840			return err;
 841	}
 842
 843	err = kfd_event_create(filp, p, args->event_type,
 844				args->auto_reset != 0, args->node_id,
 845				&args->event_id, &args->event_trigger_data,
 846				&args->event_page_offset,
 847				&args->event_slot_index);
 848
 849	pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
 850	return err;
 851}
 852
 853static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
 854					void *data)
 855{
 856	struct kfd_ioctl_destroy_event_args *args = data;
 857
 858	return kfd_event_destroy(p, args->event_id);
 859}
 860
 861static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
 862				void *data)
 863{
 864	struct kfd_ioctl_set_event_args *args = data;
 865
 866	return kfd_set_event(p, args->event_id);
 867}
 868
 869static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
 870				void *data)
 871{
 872	struct kfd_ioctl_reset_event_args *args = data;
 873
 874	return kfd_reset_event(p, args->event_id);
 875}
 876
 877static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 878				void *data)
 879{
 880	struct kfd_ioctl_wait_events_args *args = data;
 881
 882	return kfd_wait_on_events(p, args->num_events,
 883			(void __user *)args->events_ptr,
 884			(args->wait_for_all != 0),
 885			&args->timeout, &args->wait_result);
 886}
 887static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
 888					struct kfd_process *p, void *data)
 889{
 890	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
 891	struct kfd_process_device *pdd;
 892	struct kfd_node *dev;
 893	long err;
 894
 895	mutex_lock(&p->mutex);
 896	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 897	if (!pdd) {
 898		err = -EINVAL;
 899		goto err_pdd;
 900	}
 901	dev = pdd->dev;
 902
 903	pdd = kfd_bind_process_to_device(dev, p);
 904	if (IS_ERR(pdd)) {
 905		err = PTR_ERR(pdd);
 906		goto bind_process_to_device_fail;
 907	}
 908
 909	pdd->qpd.sh_hidden_private_base = args->va_addr;
 910
 911	mutex_unlock(&p->mutex);
 912
 913	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
 914	    pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
 915		dev->kfd2kgd->set_scratch_backing_va(
 916			dev->adev, args->va_addr, pdd->qpd.vmid);
 917
 918	return 0;
 919
 920bind_process_to_device_fail:
 921err_pdd:
 922	mutex_unlock(&p->mutex);
 923	return err;
 924}
 925
 926static int kfd_ioctl_get_tile_config(struct file *filep,
 927		struct kfd_process *p, void *data)
 928{
 929	struct kfd_ioctl_get_tile_config_args *args = data;
 930	struct kfd_process_device *pdd;
 931	struct tile_config config;
 932	int err = 0;
 933
 934	mutex_lock(&p->mutex);
 935	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 936	mutex_unlock(&p->mutex);
 937	if (!pdd)
 938		return -EINVAL;
 939
 940	amdgpu_amdkfd_get_tile_config(pdd->dev->adev, &config);
 941
 942	args->gb_addr_config = config.gb_addr_config;
 943	args->num_banks = config.num_banks;
 944	args->num_ranks = config.num_ranks;
 945
 946	if (args->num_tile_configs > config.num_tile_configs)
 947		args->num_tile_configs = config.num_tile_configs;
 948	err = copy_to_user((void __user *)args->tile_config_ptr,
 949			config.tile_config_ptr,
 950			args->num_tile_configs * sizeof(uint32_t));
 951	if (err) {
 952		args->num_tile_configs = 0;
 953		return -EFAULT;
 954	}
 955
 956	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
 957		args->num_macro_tile_configs =
 958				config.num_macro_tile_configs;
 959	err = copy_to_user((void __user *)args->macro_tile_config_ptr,
 960			config.macro_tile_config_ptr,
 961			args->num_macro_tile_configs * sizeof(uint32_t));
 962	if (err) {
 963		args->num_macro_tile_configs = 0;
 964		return -EFAULT;
 965	}
 966
 967	return 0;
 968}
 969
 970static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
 971				void *data)
 972{
 973	struct kfd_ioctl_acquire_vm_args *args = data;
 974	struct kfd_process_device *pdd;
 975	struct file *drm_file;
 976	int ret;
 977
 978	drm_file = fget(args->drm_fd);
 979	if (!drm_file)
 980		return -EINVAL;
 981
 982	mutex_lock(&p->mutex);
 983	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
 984	if (!pdd) {
 985		ret = -EINVAL;
 986		goto err_pdd;
 987	}
 988
 989	if (pdd->drm_file) {
 990		ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
 991		goto err_drm_file;
 992	}
 993
 994	ret = kfd_process_device_init_vm(pdd, drm_file);
 995	if (ret)
 996		goto err_unlock;
 997
 998	/* On success, the PDD keeps the drm_file reference */
 999	mutex_unlock(&p->mutex);
1000
1001	return 0;
1002
1003err_unlock:
1004err_pdd:
1005err_drm_file:
1006	mutex_unlock(&p->mutex);
1007	fput(drm_file);
1008	return ret;
1009}
1010
1011bool kfd_dev_is_large_bar(struct kfd_node *dev)
1012{
1013	if (dev->kfd->adev->debug_largebar) {
1014		pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1015		return true;
1016	}
1017
1018	if (dev->local_mem_info.local_mem_size_private == 0 &&
1019	    dev->local_mem_info.local_mem_size_public > 0)
1020		return true;
1021
1022	if (dev->local_mem_info.local_mem_size_public == 0 &&
1023	    dev->kfd->adev->gmc.is_app_apu) {
1024		pr_debug("APP APU, Consider like a large bar system\n");
1025		return true;
1026	}
1027
1028	return false;
1029}
1030
1031static int kfd_ioctl_get_available_memory(struct file *filep,
1032					  struct kfd_process *p, void *data)
1033{
1034	struct kfd_ioctl_get_available_memory_args *args = data;
1035	struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
1036
1037	if (!pdd)
1038		return -EINVAL;
1039	args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev,
1040							pdd->dev->node_id);
1041	kfd_unlock_pdd(pdd);
1042	return 0;
1043}
1044
1045static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1046					struct kfd_process *p, void *data)
1047{
1048	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1049	struct kfd_process_device *pdd;
1050	void *mem;
1051	struct kfd_node *dev;
1052	int idr_handle;
1053	long err;
1054	uint64_t offset = args->mmap_offset;
1055	uint32_t flags = args->flags;
1056
1057	if (args->size == 0)
1058		return -EINVAL;
1059
1060#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1061	/* Flush pending deferred work to avoid racing with deferred actions
1062	 * from previous memory map changes (e.g. munmap).
1063	 */
1064	svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1065	mutex_lock(&p->svms.lock);
1066	mmap_write_unlock(current->mm);
1067
1068	/* Skip a special case that allocates VRAM without VA,
1069	 * VA will be invalid of 0.
1070	 */
1071	if (!(!args->va_addr && (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) &&
1072	    interval_tree_iter_first(&p->svms.objects,
1073				     args->va_addr >> PAGE_SHIFT,
1074				     (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1075		pr_err("Address: 0x%llx already allocated by SVM\n",
1076			args->va_addr);
1077		mutex_unlock(&p->svms.lock);
1078		return -EADDRINUSE;
1079	}
1080
1081	/* When register user buffer check if it has been registered by svm by
1082	 * buffer cpu virtual address.
1083	 */
1084	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) &&
1085	    interval_tree_iter_first(&p->svms.objects,
1086				     args->mmap_offset >> PAGE_SHIFT,
1087				     (args->mmap_offset  + args->size - 1) >> PAGE_SHIFT)) {
1088		pr_err("User Buffer Address: 0x%llx already allocated by SVM\n",
1089			args->mmap_offset);
1090		mutex_unlock(&p->svms.lock);
1091		return -EADDRINUSE;
1092	}
1093
1094	mutex_unlock(&p->svms.lock);
1095#endif
1096	mutex_lock(&p->mutex);
1097	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1098	if (!pdd) {
1099		err = -EINVAL;
1100		goto err_pdd;
1101	}
1102
1103	dev = pdd->dev;
1104
1105	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1106		(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1107		!kfd_dev_is_large_bar(dev)) {
1108		pr_err("Alloc host visible vram on small bar is not allowed\n");
1109		err = -EINVAL;
1110		goto err_large_bar;
1111	}
1112
1113	pdd = kfd_bind_process_to_device(dev, p);
1114	if (IS_ERR(pdd)) {
1115		err = PTR_ERR(pdd);
1116		goto err_unlock;
1117	}
1118
1119	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1120		if (args->size != kfd_doorbell_process_slice(dev->kfd)) {
1121			err = -EINVAL;
1122			goto err_unlock;
1123		}
1124		offset = kfd_get_process_doorbells(pdd);
1125		if (!offset) {
1126			err = -ENOMEM;
1127			goto err_unlock;
1128		}
1129	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1130		if (args->size != PAGE_SIZE) {
1131			err = -EINVAL;
1132			goto err_unlock;
1133		}
1134		offset = dev->adev->rmmio_remap.bus_addr;
1135		if (!offset || (PAGE_SIZE > 4096)) {
1136			err = -ENOMEM;
1137			goto err_unlock;
1138		}
1139	}
1140
1141	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1142		dev->adev, args->va_addr, args->size,
1143		pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1144		flags, false);
1145
1146	if (err)
1147		goto err_unlock;
1148
1149	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1150	if (idr_handle < 0) {
1151		err = -EFAULT;
1152		goto err_free;
1153	}
1154
1155	/* Update the VRAM usage count */
1156	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
1157		uint64_t size = args->size;
1158
1159		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM)
1160			size >>= 1;
1161		atomic64_add(PAGE_ALIGN(size), &pdd->vram_usage);
1162	}
1163
1164	mutex_unlock(&p->mutex);
1165
1166	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1167	args->mmap_offset = offset;
1168
1169	/* MMIO is mapped through kfd device
1170	 * Generate a kfd mmap offset
1171	 */
1172	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1173		args->mmap_offset = KFD_MMAP_TYPE_MMIO
1174					| KFD_MMAP_GPU_ID(args->gpu_id);
1175
1176	return 0;
1177
1178err_free:
1179	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
1180					       pdd->drm_priv, NULL);
1181err_unlock:
1182err_pdd:
1183err_large_bar:
1184	mutex_unlock(&p->mutex);
1185	return err;
1186}
1187
1188static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1189					struct kfd_process *p, void *data)
1190{
1191	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1192	struct kfd_process_device *pdd;
1193	void *mem;
1194	int ret;
1195	uint64_t size = 0;
1196
1197	mutex_lock(&p->mutex);
1198	/*
1199	 * Safeguard to prevent user space from freeing signal BO.
1200	 * It will be freed at process termination.
1201	 */
1202	if (p->signal_handle && (p->signal_handle == args->handle)) {
1203		pr_err("Free signal BO is not allowed\n");
1204		ret = -EPERM;
1205		goto err_unlock;
1206	}
1207
1208	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1209	if (!pdd) {
1210		pr_err("Process device data doesn't exist\n");
1211		ret = -EINVAL;
1212		goto err_pdd;
1213	}
1214
1215	mem = kfd_process_device_translate_handle(
1216		pdd, GET_IDR_HANDLE(args->handle));
1217	if (!mem) {
1218		ret = -EINVAL;
1219		goto err_unlock;
1220	}
1221
1222	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev,
1223				(struct kgd_mem *)mem, pdd->drm_priv, &size);
1224
1225	/* If freeing the buffer failed, leave the handle in place for
1226	 * clean-up during process tear-down.
1227	 */
1228	if (!ret)
1229		kfd_process_device_remove_obj_handle(
1230			pdd, GET_IDR_HANDLE(args->handle));
1231
1232	atomic64_sub(size, &pdd->vram_usage);
1233
1234err_unlock:
1235err_pdd:
1236	mutex_unlock(&p->mutex);
1237	return ret;
1238}
1239
1240static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1241					struct kfd_process *p, void *data)
1242{
1243	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1244	struct kfd_process_device *pdd, *peer_pdd;
1245	void *mem;
1246	struct kfd_node *dev;
1247	long err = 0;
1248	int i;
1249	uint32_t *devices_arr = NULL;
1250
1251	if (!args->n_devices) {
1252		pr_debug("Device IDs array empty\n");
1253		return -EINVAL;
1254	}
1255	if (args->n_success > args->n_devices) {
1256		pr_debug("n_success exceeds n_devices\n");
1257		return -EINVAL;
1258	}
1259
1260	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1261				    GFP_KERNEL);
1262	if (!devices_arr)
1263		return -ENOMEM;
1264
1265	err = copy_from_user(devices_arr,
1266			     (void __user *)args->device_ids_array_ptr,
1267			     args->n_devices * sizeof(*devices_arr));
1268	if (err != 0) {
1269		err = -EFAULT;
1270		goto copy_from_user_failed;
1271	}
1272
1273	mutex_lock(&p->mutex);
1274	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1275	if (!pdd) {
1276		err = -EINVAL;
1277		goto get_process_device_data_failed;
1278	}
1279	dev = pdd->dev;
1280
1281	pdd = kfd_bind_process_to_device(dev, p);
1282	if (IS_ERR(pdd)) {
1283		err = PTR_ERR(pdd);
1284		goto bind_process_to_device_failed;
1285	}
1286
1287	mem = kfd_process_device_translate_handle(pdd,
1288						GET_IDR_HANDLE(args->handle));
1289	if (!mem) {
1290		err = -ENOMEM;
1291		goto get_mem_obj_from_handle_failed;
1292	}
1293
1294	for (i = args->n_success; i < args->n_devices; i++) {
1295		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1296		if (!peer_pdd) {
1297			pr_debug("Getting device by id failed for 0x%x\n",
1298				 devices_arr[i]);
1299			err = -EINVAL;
1300			goto get_mem_obj_from_handle_failed;
1301		}
1302
1303		peer_pdd = kfd_bind_process_to_device(peer_pdd->dev, p);
1304		if (IS_ERR(peer_pdd)) {
1305			err = PTR_ERR(peer_pdd);
1306			goto get_mem_obj_from_handle_failed;
1307		}
1308
1309		err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1310			peer_pdd->dev->adev, (struct kgd_mem *)mem,
1311			peer_pdd->drm_priv);
1312		if (err) {
1313			struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1314
1315			dev_err(dev->adev->dev,
1316			       "Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1317			       pci_domain_nr(pdev->bus),
1318			       pdev->bus->number,
1319			       PCI_SLOT(pdev->devfn),
1320			       PCI_FUNC(pdev->devfn),
1321			       ((struct kgd_mem *)mem)->domain);
1322			goto map_memory_to_gpu_failed;
1323		}
1324		args->n_success = i+1;
1325	}
1326
1327	err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev, (struct kgd_mem *) mem, true);
1328	if (err) {
1329		pr_debug("Sync memory failed, wait interrupted by user signal\n");
1330		goto sync_memory_failed;
1331	}
1332
1333	mutex_unlock(&p->mutex);
1334
1335	/* Flush TLBs after waiting for the page table updates to complete */
1336	for (i = 0; i < args->n_devices; i++) {
1337		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1338		if (WARN_ON_ONCE(!peer_pdd))
1339			continue;
1340		kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1341	}
1342	kfree(devices_arr);
1343
1344	return err;
1345
1346get_process_device_data_failed:
1347bind_process_to_device_failed:
1348get_mem_obj_from_handle_failed:
1349map_memory_to_gpu_failed:
1350sync_memory_failed:
1351	mutex_unlock(&p->mutex);
1352copy_from_user_failed:
1353	kfree(devices_arr);
1354
1355	return err;
1356}
1357
1358static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1359					struct kfd_process *p, void *data)
1360{
1361	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1362	struct kfd_process_device *pdd, *peer_pdd;
1363	void *mem;
1364	long err = 0;
1365	uint32_t *devices_arr = NULL, i;
1366	bool flush_tlb;
1367
1368	if (!args->n_devices) {
1369		pr_debug("Device IDs array empty\n");
1370		return -EINVAL;
1371	}
1372	if (args->n_success > args->n_devices) {
1373		pr_debug("n_success exceeds n_devices\n");
1374		return -EINVAL;
1375	}
1376
1377	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1378				    GFP_KERNEL);
1379	if (!devices_arr)
1380		return -ENOMEM;
1381
1382	err = copy_from_user(devices_arr,
1383			     (void __user *)args->device_ids_array_ptr,
1384			     args->n_devices * sizeof(*devices_arr));
1385	if (err != 0) {
1386		err = -EFAULT;
1387		goto copy_from_user_failed;
1388	}
1389
1390	mutex_lock(&p->mutex);
1391	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1392	if (!pdd) {
1393		err = -EINVAL;
1394		goto bind_process_to_device_failed;
1395	}
1396
1397	mem = kfd_process_device_translate_handle(pdd,
1398						GET_IDR_HANDLE(args->handle));
1399	if (!mem) {
1400		err = -ENOMEM;
1401		goto get_mem_obj_from_handle_failed;
1402	}
1403
1404	for (i = args->n_success; i < args->n_devices; i++) {
1405		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1406		if (!peer_pdd) {
1407			err = -EINVAL;
1408			goto get_mem_obj_from_handle_failed;
1409		}
1410		err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1411			peer_pdd->dev->adev, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1412		if (err) {
1413			pr_debug("Failed to unmap from gpu %d/%d\n", i, args->n_devices);
1414			goto unmap_memory_from_gpu_failed;
1415		}
1416		args->n_success = i+1;
1417	}
1418
1419	flush_tlb = kfd_flush_tlb_after_unmap(pdd->dev->kfd);
1420	if (flush_tlb) {
1421		err = amdgpu_amdkfd_gpuvm_sync_memory(pdd->dev->adev,
1422				(struct kgd_mem *) mem, true);
1423		if (err) {
1424			pr_debug("Sync memory failed, wait interrupted by user signal\n");
1425			goto sync_memory_failed;
1426		}
1427	}
1428
1429	/* Flush TLBs after waiting for the page table updates to complete */
1430	for (i = 0; i < args->n_devices; i++) {
1431		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1432		if (WARN_ON_ONCE(!peer_pdd))
1433			continue;
1434		if (flush_tlb)
1435			kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1436
1437		/* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */
1438		err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, peer_pdd->drm_priv);
1439		if (err)
1440			goto sync_memory_failed;
1441	}
1442
1443	mutex_unlock(&p->mutex);
1444
1445	kfree(devices_arr);
1446
1447	return 0;
1448
1449bind_process_to_device_failed:
1450get_mem_obj_from_handle_failed:
1451unmap_memory_from_gpu_failed:
1452sync_memory_failed:
1453	mutex_unlock(&p->mutex);
1454copy_from_user_failed:
1455	kfree(devices_arr);
1456	return err;
1457}
1458
1459static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1460		struct kfd_process *p, void *data)
1461{
1462	int retval;
1463	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1464	struct queue *q;
1465	struct kfd_node *dev;
1466
1467	mutex_lock(&p->mutex);
1468	q = pqm_get_user_queue(&p->pqm, args->queue_id);
1469
1470	if (q) {
1471		dev = q->device;
1472	} else {
1473		retval = -EINVAL;
1474		goto out_unlock;
1475	}
1476
1477	if (!dev->gws) {
1478		retval = -ENODEV;
1479		goto out_unlock;
1480	}
1481
1482	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1483		retval = -ENODEV;
1484		goto out_unlock;
1485	}
1486
1487	if (p->debug_trap_enabled && (!kfd_dbg_has_gws_support(dev) ||
1488				      kfd_dbg_has_cwsr_workaround(dev))) {
1489		retval = -EBUSY;
1490		goto out_unlock;
1491	}
1492
1493	retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1494	mutex_unlock(&p->mutex);
1495
1496	args->first_gws = 0;
1497	return retval;
1498
1499out_unlock:
1500	mutex_unlock(&p->mutex);
1501	return retval;
1502}
1503
1504static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1505		struct kfd_process *p, void *data)
1506{
1507	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1508	struct kfd_node *dev = NULL;
1509	struct amdgpu_device *dmabuf_adev;
1510	void *metadata_buffer = NULL;
1511	uint32_t flags;
1512	int8_t xcp_id;
1513	unsigned int i;
1514	int r;
1515
1516	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1517	for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1518		if (dev && !kfd_devcgroup_check_permission(dev))
1519			break;
1520	if (!dev)
1521		return -EINVAL;
1522
1523	if (args->metadata_ptr) {
1524		metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1525		if (!metadata_buffer)
1526			return -ENOMEM;
1527	}
1528
1529	/* Get dmabuf info from KGD */
1530	r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
1531					  &dmabuf_adev, &args->size,
1532					  metadata_buffer, args->metadata_size,
1533					  &args->metadata_size, &flags, &xcp_id);
1534	if (r)
1535		goto exit;
1536
1537	if (xcp_id >= 0)
1538		args->gpu_id = dmabuf_adev->kfd.dev->nodes[xcp_id]->id;
1539	else
1540		args->gpu_id = dev->id;
1541	args->flags = flags;
1542
1543	/* Copy metadata buffer to user mode */
1544	if (metadata_buffer) {
1545		r = copy_to_user((void __user *)args->metadata_ptr,
1546				 metadata_buffer, args->metadata_size);
1547		if (r != 0)
1548			r = -EFAULT;
1549	}
1550
1551exit:
1552	kfree(metadata_buffer);
1553
1554	return r;
1555}
1556
1557static int kfd_ioctl_import_dmabuf(struct file *filep,
1558				   struct kfd_process *p, void *data)
1559{
1560	struct kfd_ioctl_import_dmabuf_args *args = data;
1561	struct kfd_process_device *pdd;
1562	int idr_handle;
1563	uint64_t size;
1564	void *mem;
1565	int r;
1566
1567	mutex_lock(&p->mutex);
1568	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1569	if (!pdd) {
1570		r = -EINVAL;
1571		goto err_unlock;
1572	}
1573
1574	pdd = kfd_bind_process_to_device(pdd->dev, p);
1575	if (IS_ERR(pdd)) {
1576		r = PTR_ERR(pdd);
1577		goto err_unlock;
1578	}
1579
1580	r = amdgpu_amdkfd_gpuvm_import_dmabuf_fd(pdd->dev->adev, args->dmabuf_fd,
1581						 args->va_addr, pdd->drm_priv,
1582						 (struct kgd_mem **)&mem, &size,
1583						 NULL);
1584	if (r)
1585		goto err_unlock;
1586
1587	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1588	if (idr_handle < 0) {
1589		r = -EFAULT;
1590		goto err_free;
1591	}
1592
1593	mutex_unlock(&p->mutex);
1594
1595	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1596
1597	return 0;
1598
1599err_free:
1600	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, (struct kgd_mem *)mem,
1601					       pdd->drm_priv, NULL);
1602err_unlock:
1603	mutex_unlock(&p->mutex);
1604	return r;
1605}
1606
1607static int kfd_ioctl_export_dmabuf(struct file *filep,
1608				   struct kfd_process *p, void *data)
1609{
1610	struct kfd_ioctl_export_dmabuf_args *args = data;
1611	struct kfd_process_device *pdd;
1612	struct dma_buf *dmabuf;
1613	struct kfd_node *dev;
1614	void *mem;
1615	int ret = 0;
1616
1617	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1618	if (!dev)
1619		return -EINVAL;
1620
1621	mutex_lock(&p->mutex);
1622
1623	pdd = kfd_get_process_device_data(dev, p);
1624	if (!pdd) {
1625		ret = -EINVAL;
1626		goto err_unlock;
1627	}
1628
1629	mem = kfd_process_device_translate_handle(pdd,
1630						GET_IDR_HANDLE(args->handle));
1631	if (!mem) {
1632		ret = -EINVAL;
1633		goto err_unlock;
1634	}
1635
1636	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
1637	mutex_unlock(&p->mutex);
1638	if (ret)
1639		goto err_out;
1640
1641	ret = dma_buf_fd(dmabuf, args->flags);
1642	if (ret < 0) {
1643		dma_buf_put(dmabuf);
1644		goto err_out;
1645	}
1646	/* dma_buf_fd assigns the reference count to the fd, no need to
1647	 * put the reference here.
1648	 */
1649	args->dmabuf_fd = ret;
1650
1651	return 0;
1652
1653err_unlock:
1654	mutex_unlock(&p->mutex);
1655err_out:
1656	return ret;
1657}
1658
1659/* Handle requests for watching SMI events */
1660static int kfd_ioctl_smi_events(struct file *filep,
1661				struct kfd_process *p, void *data)
1662{
1663	struct kfd_ioctl_smi_events_args *args = data;
1664	struct kfd_process_device *pdd;
1665
1666	mutex_lock(&p->mutex);
1667
1668	pdd = kfd_process_device_data_by_id(p, args->gpuid);
1669	mutex_unlock(&p->mutex);
1670	if (!pdd)
1671		return -EINVAL;
1672
1673	return kfd_smi_event_open(pdd->dev, &args->anon_fd);
1674}
1675
1676#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1677
1678static int kfd_ioctl_set_xnack_mode(struct file *filep,
1679				    struct kfd_process *p, void *data)
1680{
1681	struct kfd_ioctl_set_xnack_mode_args *args = data;
1682	int r = 0;
1683
1684	mutex_lock(&p->mutex);
1685	if (args->xnack_enabled >= 0) {
1686		if (!list_empty(&p->pqm.queues)) {
1687			pr_debug("Process has user queues running\n");
1688			r = -EBUSY;
1689			goto out_unlock;
1690		}
1691
1692		if (p->xnack_enabled == args->xnack_enabled)
1693			goto out_unlock;
1694
1695		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) {
1696			r = -EPERM;
1697			goto out_unlock;
1698		}
1699
1700		r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled);
1701	} else {
1702		args->xnack_enabled = p->xnack_enabled;
1703	}
1704
1705out_unlock:
1706	mutex_unlock(&p->mutex);
1707
1708	return r;
1709}
1710
1711static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1712{
1713	struct kfd_ioctl_svm_args *args = data;
1714	int r = 0;
1715
1716	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1717		 args->start_addr, args->size, args->op, args->nattr);
1718
1719	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1720		return -EINVAL;
1721	if (!args->start_addr || !args->size)
1722		return -EINVAL;
1723
1724	r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1725		      args->attrs);
1726
1727	return r;
1728}
1729#else
1730static int kfd_ioctl_set_xnack_mode(struct file *filep,
1731				    struct kfd_process *p, void *data)
1732{
1733	return -EPERM;
1734}
1735static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1736{
1737	return -EPERM;
1738}
1739#endif
1740
1741static int criu_checkpoint_process(struct kfd_process *p,
1742			     uint8_t __user *user_priv_data,
1743			     uint64_t *priv_offset)
1744{
1745	struct kfd_criu_process_priv_data process_priv;
1746	int ret;
1747
1748	memset(&process_priv, 0, sizeof(process_priv));
1749
1750	process_priv.version = KFD_CRIU_PRIV_VERSION;
1751	/* For CR, we don't consider negative xnack mode which is used for
1752	 * querying without changing it, here 0 simply means disabled and 1
1753	 * means enabled so retry for finding a valid PTE.
1754	 */
1755	process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1756
1757	ret = copy_to_user(user_priv_data + *priv_offset,
1758				&process_priv, sizeof(process_priv));
1759
1760	if (ret) {
1761		pr_err("Failed to copy process information to user\n");
1762		ret = -EFAULT;
1763	}
1764
1765	*priv_offset += sizeof(process_priv);
1766	return ret;
1767}
1768
1769static int criu_checkpoint_devices(struct kfd_process *p,
1770			     uint32_t num_devices,
1771			     uint8_t __user *user_addr,
1772			     uint8_t __user *user_priv_data,
1773			     uint64_t *priv_offset)
1774{
1775	struct kfd_criu_device_priv_data *device_priv = NULL;
1776	struct kfd_criu_device_bucket *device_buckets = NULL;
1777	int ret = 0, i;
1778
1779	device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
1780	if (!device_buckets) {
1781		ret = -ENOMEM;
1782		goto exit;
1783	}
1784
1785	device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
1786	if (!device_priv) {
1787		ret = -ENOMEM;
1788		goto exit;
1789	}
1790
1791	for (i = 0; i < num_devices; i++) {
1792		struct kfd_process_device *pdd = p->pdds[i];
1793
1794		device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1795		device_buckets[i].actual_gpu_id = pdd->dev->id;
1796
1797		/*
1798		 * priv_data does not contain useful information for now and is reserved for
1799		 * future use, so we do not set its contents.
1800		 */
1801	}
1802
1803	ret = copy_to_user(user_addr, device_buckets, num_devices * sizeof(*device_buckets));
1804	if (ret) {
1805		pr_err("Failed to copy device information to user\n");
1806		ret = -EFAULT;
1807		goto exit;
1808	}
1809
1810	ret = copy_to_user(user_priv_data + *priv_offset,
1811			   device_priv,
1812			   num_devices * sizeof(*device_priv));
1813	if (ret) {
1814		pr_err("Failed to copy device information to user\n");
1815		ret = -EFAULT;
1816	}
1817	*priv_offset += num_devices * sizeof(*device_priv);
1818
1819exit:
1820	kvfree(device_buckets);
1821	kvfree(device_priv);
1822	return ret;
1823}
1824
1825static uint32_t get_process_num_bos(struct kfd_process *p)
1826{
1827	uint32_t num_of_bos = 0;
1828	int i;
1829
1830	/* Run over all PDDs of the process */
1831	for (i = 0; i < p->n_pdds; i++) {
1832		struct kfd_process_device *pdd = p->pdds[i];
1833		void *mem;
1834		int id;
1835
1836		idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1837			struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1838
1839			if (!kgd_mem->va || kgd_mem->va > pdd->gpuvm_base)
1840				num_of_bos++;
1841		}
1842	}
1843	return num_of_bos;
1844}
1845
1846static int criu_get_prime_handle(struct kgd_mem *mem,
1847				 int flags, u32 *shared_fd,
1848				 struct file **file)
1849{
1850	struct dma_buf *dmabuf;
1851	int ret;
1852
1853	ret = amdgpu_amdkfd_gpuvm_export_dmabuf(mem, &dmabuf);
1854	if (ret) {
1855		pr_err("dmabuf export failed for the BO\n");
1856		return ret;
1857	}
1858
1859	ret = get_unused_fd_flags(flags);
1860	if (ret < 0) {
1861		pr_err("dmabuf create fd failed, ret:%d\n", ret);
1862		goto out_free_dmabuf;
1863	}
1864
1865	*shared_fd = ret;
1866	*file = dmabuf->file;
1867	return 0;
1868
1869out_free_dmabuf:
1870	dma_buf_put(dmabuf);
1871	return ret;
1872}
1873
1874static void commit_files(struct file **files,
1875			 struct kfd_criu_bo_bucket *bo_buckets,
1876			 unsigned int count,
1877			 int err)
1878{
1879	while (count--) {
1880		struct file *file = files[count];
1881
1882		if (!file)
1883			continue;
1884		if (err) {
1885			fput(file);
1886			put_unused_fd(bo_buckets[count].dmabuf_fd);
1887		} else {
1888			fd_install(bo_buckets[count].dmabuf_fd, file);
1889		}
1890	}
1891}
1892
1893static int criu_checkpoint_bos(struct kfd_process *p,
1894			       uint32_t num_bos,
1895			       uint8_t __user *user_bos,
1896			       uint8_t __user *user_priv_data,
1897			       uint64_t *priv_offset)
1898{
1899	struct kfd_criu_bo_bucket *bo_buckets;
1900	struct kfd_criu_bo_priv_data *bo_privs;
1901	struct file **files = NULL;
1902	int ret = 0, pdd_index, bo_index = 0, id;
1903	void *mem;
1904
1905	bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1906	if (!bo_buckets)
1907		return -ENOMEM;
1908
1909	bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
1910	if (!bo_privs) {
1911		ret = -ENOMEM;
1912		goto exit;
1913	}
1914
1915	files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL);
1916	if (!files) {
1917		ret = -ENOMEM;
1918		goto exit;
1919	}
1920
1921	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1922		struct kfd_process_device *pdd = p->pdds[pdd_index];
1923		struct amdgpu_bo *dumper_bo;
1924		struct kgd_mem *kgd_mem;
1925
1926		idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1927			struct kfd_criu_bo_bucket *bo_bucket;
1928			struct kfd_criu_bo_priv_data *bo_priv;
1929			int i, dev_idx = 0;
1930
1931			kgd_mem = (struct kgd_mem *)mem;
1932			dumper_bo = kgd_mem->bo;
1933
1934			/* Skip checkpointing BOs that are used for Trap handler
1935			 * code and state. Currently, these BOs have a VA that
1936			 * is less GPUVM Base
1937			 */
1938			if (kgd_mem->va && kgd_mem->va <= pdd->gpuvm_base)
1939				continue;
1940
1941			bo_bucket = &bo_buckets[bo_index];
1942			bo_priv = &bo_privs[bo_index];
1943
1944			bo_bucket->gpu_id = pdd->user_gpu_id;
1945			bo_bucket->addr = (uint64_t)kgd_mem->va;
1946			bo_bucket->size = amdgpu_bo_size(dumper_bo);
1947			bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1948			bo_priv->idr_handle = id;
1949
1950			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1951				ret = amdgpu_ttm_tt_get_userptr(&dumper_bo->tbo,
1952								&bo_priv->user_addr);
1953				if (ret) {
1954					pr_err("Failed to obtain user address for user-pointer bo\n");
1955					goto exit;
1956				}
1957			}
1958			if (bo_bucket->alloc_flags
1959			    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1960				ret = criu_get_prime_handle(kgd_mem,
1961						bo_bucket->alloc_flags &
1962						KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1963						&bo_bucket->dmabuf_fd, &files[bo_index]);
1964				if (ret)
1965					goto exit;
1966			} else {
1967				bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1968			}
1969
1970			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1971				bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1972					KFD_MMAP_GPU_ID(pdd->dev->id);
1973			else if (bo_bucket->alloc_flags &
1974				KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1975				bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1976					KFD_MMAP_GPU_ID(pdd->dev->id);
1977			else
1978				bo_bucket->offset = amdgpu_bo_mmap_offset(dumper_bo);
1979
1980			for (i = 0; i < p->n_pdds; i++) {
1981				if (amdgpu_amdkfd_bo_mapped_to_dev(p->pdds[i]->drm_priv, kgd_mem))
1982					bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1983			}
1984
1985			pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1986					"gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1987					bo_bucket->size,
1988					bo_bucket->addr,
1989					bo_bucket->offset,
1990					bo_bucket->gpu_id,
1991					bo_bucket->alloc_flags,
1992					bo_priv->idr_handle);
1993			bo_index++;
1994		}
1995	}
1996
1997	ret = copy_to_user(user_bos, bo_buckets, num_bos * sizeof(*bo_buckets));
1998	if (ret) {
1999		pr_err("Failed to copy BO information to user\n");
2000		ret = -EFAULT;
2001		goto exit;
2002	}
2003
2004	ret = copy_to_user(user_priv_data + *priv_offset, bo_privs, num_bos * sizeof(*bo_privs));
2005	if (ret) {
2006		pr_err("Failed to copy BO priv information to user\n");
2007		ret = -EFAULT;
2008		goto exit;
2009	}
2010
2011	*priv_offset += num_bos * sizeof(*bo_privs);
2012
2013exit:
2014	commit_files(files, bo_buckets, bo_index, ret);
2015	kvfree(files);
2016	kvfree(bo_buckets);
2017	kvfree(bo_privs);
2018	return ret;
2019}
2020
2021static int criu_get_process_object_info(struct kfd_process *p,
2022					uint32_t *num_devices,
2023					uint32_t *num_bos,
2024					uint32_t *num_objects,
2025					uint64_t *objs_priv_size)
2026{
2027	uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
2028	uint32_t num_queues, num_events, num_svm_ranges;
2029	int ret;
2030
2031	*num_devices = p->n_pdds;
2032	*num_bos = get_process_num_bos(p);
2033
2034	ret = kfd_process_get_queue_info(p, &num_queues, &queues_priv_data_size);
2035	if (ret)
2036		return ret;
2037
2038	num_events = kfd_get_num_events(p);
2039
2040	svm_range_get_info(p, &num_svm_ranges, &svm_priv_data_size);
2041
2042	*num_objects = num_queues + num_events + num_svm_ranges;
2043
2044	if (objs_priv_size) {
2045		priv_size = sizeof(struct kfd_criu_process_priv_data);
2046		priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
2047		priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
2048		priv_size += queues_priv_data_size;
2049		priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
2050		priv_size += svm_priv_data_size;
2051		*objs_priv_size = priv_size;
2052	}
2053	return 0;
2054}
2055
2056static int criu_checkpoint(struct file *filep,
2057			   struct kfd_process *p,
2058			   struct kfd_ioctl_criu_args *args)
2059{
2060	int ret;
2061	uint32_t num_devices, num_bos, num_objects;
2062	uint64_t priv_size, priv_offset = 0, bo_priv_offset;
2063
2064	if (!args->devices || !args->bos || !args->priv_data)
2065		return -EINVAL;
2066
2067	mutex_lock(&p->mutex);
2068
2069	if (!p->n_pdds) {
2070		pr_err("No pdd for given process\n");
2071		ret = -ENODEV;
2072		goto exit_unlock;
2073	}
2074
2075	/* Confirm all process queues are evicted */
2076	if (!p->queues_paused) {
2077		pr_err("Cannot dump process when queues are not in evicted state\n");
2078		/* CRIU plugin did not call op PROCESS_INFO before checkpointing */
2079		ret = -EINVAL;
2080		goto exit_unlock;
2081	}
2082
2083	ret = criu_get_process_object_info(p, &num_devices, &num_bos, &num_objects, &priv_size);
2084	if (ret)
2085		goto exit_unlock;
2086
2087	if (num_devices != args->num_devices ||
2088	    num_bos != args->num_bos ||
2089	    num_objects != args->num_objects ||
2090	    priv_size != args->priv_data_size) {
2091
2092		ret = -EINVAL;
2093		goto exit_unlock;
2094	}
2095
2096	/* each function will store private data inside priv_data and adjust priv_offset */
2097	ret = criu_checkpoint_process(p, (uint8_t __user *)args->priv_data, &priv_offset);
2098	if (ret)
2099		goto exit_unlock;
2100
2101	ret = criu_checkpoint_devices(p, num_devices, (uint8_t __user *)args->devices,
2102				(uint8_t __user *)args->priv_data, &priv_offset);
2103	if (ret)
2104		goto exit_unlock;
2105
2106	/* Leave room for BOs in the private data. They need to be restored
2107	 * before events, but we checkpoint them last to simplify the error
2108	 * handling.
2109	 */
2110	bo_priv_offset = priv_offset;
2111	priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2112
2113	if (num_objects) {
2114		ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data,
2115						 &priv_offset);
2116		if (ret)
2117			goto exit_unlock;
2118
2119		ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data,
2120						 &priv_offset);
2121		if (ret)
2122			goto exit_unlock;
2123
2124		ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset);
2125		if (ret)
2126			goto exit_unlock;
2127	}
2128
2129	/* This must be the last thing in this function that can fail.
2130	 * Otherwise we leak dmabuf file descriptors.
2131	 */
2132	ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos,
2133			   (uint8_t __user *)args->priv_data, &bo_priv_offset);
2134
2135exit_unlock:
2136	mutex_unlock(&p->mutex);
2137	if (ret)
2138		pr_err("Failed to dump CRIU ret:%d\n", ret);
2139	else
2140		pr_debug("CRIU dump ret:%d\n", ret);
2141
2142	return ret;
2143}
2144
2145static int criu_restore_process(struct kfd_process *p,
2146				struct kfd_ioctl_criu_args *args,
2147				uint64_t *priv_offset,
2148				uint64_t max_priv_data_size)
2149{
2150	int ret = 0;
2151	struct kfd_criu_process_priv_data process_priv;
2152
2153	if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2154		return -EINVAL;
2155
2156	ret = copy_from_user(&process_priv,
2157				(void __user *)(args->priv_data + *priv_offset),
2158				sizeof(process_priv));
2159	if (ret) {
2160		pr_err("Failed to copy process private information from user\n");
2161		ret = -EFAULT;
2162		goto exit;
2163	}
2164	*priv_offset += sizeof(process_priv);
2165
2166	if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2167		pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2168			process_priv.version, KFD_CRIU_PRIV_VERSION);
2169		return -EINVAL;
2170	}
2171
2172	pr_debug("Setting XNACK mode\n");
2173	if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, true)) {
2174		pr_err("xnack mode cannot be set\n");
2175		ret = -EPERM;
2176		goto exit;
2177	} else {
2178		pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2179		p->xnack_enabled = process_priv.xnack_mode;
2180	}
2181
2182exit:
2183	return ret;
2184}
2185
2186static int criu_restore_devices(struct kfd_process *p,
2187				struct kfd_ioctl_criu_args *args,
2188				uint64_t *priv_offset,
2189				uint64_t max_priv_data_size)
2190{
2191	struct kfd_criu_device_bucket *device_buckets;
2192	struct kfd_criu_device_priv_data *device_privs;
2193	int ret = 0;
2194	uint32_t i;
2195
2196	if (args->num_devices != p->n_pdds)
2197		return -EINVAL;
2198
2199	if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2200		return -EINVAL;
2201
2202	device_buckets = kmalloc_array(args->num_devices, sizeof(*device_buckets), GFP_KERNEL);
2203	if (!device_buckets)
2204		return -ENOMEM;
2205
2206	ret = copy_from_user(device_buckets, (void __user *)args->devices,
2207				args->num_devices * sizeof(*device_buckets));
2208	if (ret) {
2209		pr_err("Failed to copy devices buckets from user\n");
2210		ret = -EFAULT;
2211		goto exit;
2212	}
2213
2214	for (i = 0; i < args->num_devices; i++) {
2215		struct kfd_node *dev;
2216		struct kfd_process_device *pdd;
2217		struct file *drm_file;
2218
2219		/* device private data is not currently used */
2220
2221		if (!device_buckets[i].user_gpu_id) {
2222			pr_err("Invalid user gpu_id\n");
2223			ret = -EINVAL;
2224			goto exit;
2225		}
2226
2227		dev = kfd_device_by_id(device_buckets[i].actual_gpu_id);
2228		if (!dev) {
2229			pr_err("Failed to find device with gpu_id = %x\n",
2230				device_buckets[i].actual_gpu_id);
2231			ret = -EINVAL;
2232			goto exit;
2233		}
2234
2235		pdd = kfd_get_process_device_data(dev, p);
2236		if (!pdd) {
2237			pr_err("Failed to get pdd for gpu_id = %x\n",
2238					device_buckets[i].actual_gpu_id);
2239			ret = -EINVAL;
2240			goto exit;
2241		}
2242		pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2243
2244		drm_file = fget(device_buckets[i].drm_fd);
2245		if (!drm_file) {
2246			pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2247				device_buckets[i].drm_fd);
2248			ret = -EINVAL;
2249			goto exit;
2250		}
2251
2252		if (pdd->drm_file) {
2253			ret = -EINVAL;
2254			goto exit;
2255		}
2256
2257		/* create the vm using render nodes for kfd pdd */
2258		if (kfd_process_device_init_vm(pdd, drm_file)) {
2259			pr_err("could not init vm for given pdd\n");
2260			/* On success, the PDD keeps the drm_file reference */
2261			fput(drm_file);
2262			ret = -EINVAL;
2263			goto exit;
2264		}
2265		/*
2266		 * pdd now already has the vm bound to render node so below api won't create a new
2267		 * exclusive kfd mapping but use existing one with renderDXXX but is still needed
2268		 * for iommu v2 binding  and runtime pm.
2269		 */
2270		pdd = kfd_bind_process_to_device(dev, p);
2271		if (IS_ERR(pdd)) {
2272			ret = PTR_ERR(pdd);
2273			goto exit;
2274		}
2275
2276		if (!pdd->qpd.proc_doorbells) {
2277			ret = kfd_alloc_process_doorbells(dev->kfd, pdd);
2278			if (ret)
2279				goto exit;
2280		}
2281	}
2282
2283	/*
2284	 * We are not copying device private data from user as we are not using the data for now,
2285	 * but we still adjust for its private data.
2286	 */
2287	*priv_offset += args->num_devices * sizeof(*device_privs);
2288
2289exit:
2290	kfree(device_buckets);
2291	return ret;
2292}
2293
2294static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2295				      struct kfd_criu_bo_bucket *bo_bucket,
2296				      struct kfd_criu_bo_priv_data *bo_priv,
2297				      struct kgd_mem **kgd_mem)
2298{
2299	int idr_handle;
2300	int ret;
2301	const bool criu_resume = true;
2302	u64 offset;
2303
2304	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2305		if (bo_bucket->size !=
2306				kfd_doorbell_process_slice(pdd->dev->kfd))
2307			return -EINVAL;
2308
2309		offset = kfd_get_process_doorbells(pdd);
2310		if (!offset)
2311			return -ENOMEM;
2312	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2313		/* MMIO BOs need remapped bus address */
2314		if (bo_bucket->size != PAGE_SIZE) {
2315			pr_err("Invalid page size\n");
2316			return -EINVAL;
2317		}
2318		offset = pdd->dev->adev->rmmio_remap.bus_addr;
2319		if (!offset || (PAGE_SIZE > 4096)) {
2320			pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2321			return -ENOMEM;
2322		}
2323	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2324		offset = bo_priv->user_addr;
2325	}
2326	/* Create the BO */
2327	ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(pdd->dev->adev, bo_bucket->addr,
2328						      bo_bucket->size, pdd->drm_priv, kgd_mem,
2329						      &offset, bo_bucket->alloc_flags, criu_resume);
2330	if (ret) {
2331		pr_err("Could not create the BO\n");
2332		return ret;
2333	}
2334	pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2335		 bo_bucket->size, bo_bucket->addr, offset);
2336
2337	/* Restore previous IDR handle */
2338	pr_debug("Restoring old IDR handle for the BO");
2339	idr_handle = idr_alloc(&pdd->alloc_idr, *kgd_mem, bo_priv->idr_handle,
2340			       bo_priv->idr_handle + 1, GFP_KERNEL);
2341
2342	if (idr_handle < 0) {
2343		pr_err("Could not allocate idr\n");
2344		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, *kgd_mem, pdd->drm_priv,
2345						       NULL);
2346		return -ENOMEM;
2347	}
2348
2349	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2350		bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2351	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2352		bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2353	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2354		bo_bucket->restored_offset = offset;
2355	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2356		bo_bucket->restored_offset = offset;
2357		/* Update the VRAM usage count */
2358		atomic64_add(bo_bucket->size, &pdd->vram_usage);
2359	}
2360	return 0;
2361}
2362
2363static int criu_restore_bo(struct kfd_process *p,
2364			   struct kfd_criu_bo_bucket *bo_bucket,
2365			   struct kfd_criu_bo_priv_data *bo_priv,
2366			   struct file **file)
2367{
2368	struct kfd_process_device *pdd;
2369	struct kgd_mem *kgd_mem;
2370	int ret;
2371	int j;
2372
2373	pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2374		 bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2375		 bo_priv->idr_handle);
2376
2377	pdd = kfd_process_device_data_by_id(p, bo_bucket->gpu_id);
2378	if (!pdd) {
2379		pr_err("Failed to get pdd\n");
2380		return -ENODEV;
2381	}
2382
2383	ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, &kgd_mem);
2384	if (ret)
2385		return ret;
2386
2387	/* now map these BOs to GPU/s */
2388	for (j = 0; j < p->n_pdds; j++) {
2389		struct kfd_node *peer;
2390		struct kfd_process_device *peer_pdd;
2391
2392		if (!bo_priv->mapped_gpuids[j])
2393			break;
2394
2395		peer_pdd = kfd_process_device_data_by_id(p, bo_priv->mapped_gpuids[j]);
2396		if (!peer_pdd)
2397			return -EINVAL;
2398
2399		peer = peer_pdd->dev;
2400
2401		peer_pdd = kfd_bind_process_to_device(peer, p);
2402		if (IS_ERR(peer_pdd))
2403			return PTR_ERR(peer_pdd);
2404
2405		ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem,
2406							    peer_pdd->drm_priv);
2407		if (ret) {
2408			pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2409			return ret;
2410		}
2411	}
2412
2413	pr_debug("map memory was successful for the BO\n");
2414	/* create the dmabuf object and export the bo */
2415	if (bo_bucket->alloc_flags
2416	    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2417		ret = criu_get_prime_handle(kgd_mem, DRM_RDWR,
2418					    &bo_bucket->dmabuf_fd, file);
2419		if (ret)
2420			return ret;
2421	} else {
2422		bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2423	}
2424
2425	return 0;
2426}
2427
2428static int criu_restore_bos(struct kfd_process *p,
2429			    struct kfd_ioctl_criu_args *args,
2430			    uint64_t *priv_offset,
2431			    uint64_t max_priv_data_size)
2432{
2433	struct kfd_criu_bo_bucket *bo_buckets = NULL;
2434	struct kfd_criu_bo_priv_data *bo_privs = NULL;
2435	struct file **files = NULL;
2436	int ret = 0;
2437	uint32_t i = 0;
2438
2439	if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2440		return -EINVAL;
2441
2442	/* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2443	amdgpu_amdkfd_block_mmu_notifications(p->kgd_process_info);
2444
2445	bo_buckets = kvmalloc_array(args->num_bos, sizeof(*bo_buckets), GFP_KERNEL);
2446	if (!bo_buckets)
2447		return -ENOMEM;
2448
2449	files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL);
2450	if (!files) {
2451		ret = -ENOMEM;
2452		goto exit;
2453	}
2454
2455	ret = copy_from_user(bo_buckets, (void __user *)args->bos,
2456			     args->num_bos * sizeof(*bo_buckets));
2457	if (ret) {
2458		pr_err("Failed to copy BOs information from user\n");
2459		ret = -EFAULT;
2460		goto exit;
2461	}
2462
2463	bo_privs = kvmalloc_array(args->num_bos, sizeof(*bo_privs), GFP_KERNEL);
2464	if (!bo_privs) {
2465		ret = -ENOMEM;
2466		goto exit;
2467	}
2468
2469	ret = copy_from_user(bo_privs, (void __user *)args->priv_data + *priv_offset,
2470			     args->num_bos * sizeof(*bo_privs));
2471	if (ret) {
2472		pr_err("Failed to copy BOs information from user\n");
2473		ret = -EFAULT;
2474		goto exit;
2475	}
2476	*priv_offset += args->num_bos * sizeof(*bo_privs);
2477
2478	/* Create and map new BOs */
2479	for (; i < args->num_bos; i++) {
2480		ret = criu_restore_bo(p, &bo_buckets[i], &bo_privs[i], &files[i]);
2481		if (ret) {
2482			pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2483			goto exit;
2484		}
2485	} /* done */
2486
2487	/* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2488	ret = copy_to_user((void __user *)args->bos,
2489				bo_buckets,
2490				(args->num_bos * sizeof(*bo_buckets)));
2491	if (ret)
2492		ret = -EFAULT;
2493
2494exit:
2495	commit_files(files, bo_buckets, i, ret);
2496	kvfree(files);
2497	kvfree(bo_buckets);
2498	kvfree(bo_privs);
2499	return ret;
2500}
2501
2502static int criu_restore_objects(struct file *filep,
2503				struct kfd_process *p,
2504				struct kfd_ioctl_criu_args *args,
2505				uint64_t *priv_offset,
2506				uint64_t max_priv_data_size)
2507{
2508	int ret = 0;
2509	uint32_t i;
2510
2511	BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2512	BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2513	BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2514
2515	for (i = 0; i < args->num_objects; i++) {
2516		uint32_t object_type;
2517
2518		if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2519			pr_err("Invalid private data size\n");
2520			return -EINVAL;
2521		}
2522
2523		ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2524		if (ret) {
2525			pr_err("Failed to copy private information from user\n");
2526			goto exit;
2527		}
2528
2529		switch (object_type) {
2530		case KFD_CRIU_OBJECT_TYPE_QUEUE:
2531			ret = kfd_criu_restore_queue(p, (uint8_t __user *)args->priv_data,
2532						     priv_offset, max_priv_data_size);
2533			if (ret)
2534				goto exit;
2535			break;
2536		case KFD_CRIU_OBJECT_TYPE_EVENT:
2537			ret = kfd_criu_restore_event(filep, p, (uint8_t __user *)args->priv_data,
2538						     priv_offset, max_priv_data_size);
2539			if (ret)
2540				goto exit;
2541			break;
2542		case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2543			ret = kfd_criu_restore_svm(p, (uint8_t __user *)args->priv_data,
2544						     priv_offset, max_priv_data_size);
2545			if (ret)
2546				goto exit;
2547			break;
2548		default:
2549			pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2550			ret = -EINVAL;
2551			goto exit;
2552		}
2553	}
2554exit:
2555	return ret;
2556}
2557
2558static int criu_restore(struct file *filep,
2559			struct kfd_process *p,
2560			struct kfd_ioctl_criu_args *args)
2561{
2562	uint64_t priv_offset = 0;
2563	int ret = 0;
2564
2565	pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2566		 args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2567
2568	if ((args->num_bos > 0 && !args->bos) || !args->devices || !args->priv_data ||
2569	    !args->priv_data_size || !args->num_devices)
2570		return -EINVAL;
2571
2572	mutex_lock(&p->mutex);
2573
2574	/*
2575	 * Set the process to evicted state to avoid running any new queues before all the memory
2576	 * mappings are ready.
2577	 */
2578	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_RESTORE);
2579	if (ret)
2580		goto exit_unlock;
2581
2582	/* Each function will adjust priv_offset based on how many bytes they consumed */
2583	ret = criu_restore_process(p, args, &priv_offset, args->priv_data_size);
2584	if (ret)
2585		goto exit_unlock;
2586
2587	ret = criu_restore_devices(p, args, &priv_offset, args->priv_data_size);
2588	if (ret)
2589		goto exit_unlock;
2590
2591	ret = criu_restore_bos(p, args, &priv_offset, args->priv_data_size);
2592	if (ret)
2593		goto exit_unlock;
2594
2595	ret = criu_restore_objects(filep, p, args, &priv_offset, args->priv_data_size);
2596	if (ret)
2597		goto exit_unlock;
2598
2599	if (priv_offset != args->priv_data_size) {
2600		pr_err("Invalid private data size\n");
2601		ret = -EINVAL;
2602	}
2603
2604exit_unlock:
2605	mutex_unlock(&p->mutex);
2606	if (ret)
2607		pr_err("Failed to restore CRIU ret:%d\n", ret);
2608	else
2609		pr_debug("CRIU restore successful\n");
2610
2611	return ret;
2612}
2613
2614static int criu_unpause(struct file *filep,
2615			struct kfd_process *p,
2616			struct kfd_ioctl_criu_args *args)
2617{
2618	int ret;
2619
2620	mutex_lock(&p->mutex);
2621
2622	if (!p->queues_paused) {
2623		mutex_unlock(&p->mutex);
2624		return -EINVAL;
2625	}
2626
2627	ret = kfd_process_restore_queues(p);
2628	if (ret)
2629		pr_err("Failed to unpause queues ret:%d\n", ret);
2630	else
2631		p->queues_paused = false;
2632
2633	mutex_unlock(&p->mutex);
2634
2635	return ret;
2636}
2637
2638static int criu_resume(struct file *filep,
2639			struct kfd_process *p,
2640			struct kfd_ioctl_criu_args *args)
2641{
2642	struct kfd_process *target = NULL;
2643	struct pid *pid = NULL;
2644	int ret = 0;
2645
2646	pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2647		 args->pid);
2648
2649	pid = find_get_pid(args->pid);
2650	if (!pid) {
2651		pr_err("Cannot find pid info for %i\n", args->pid);
2652		return -ESRCH;
2653	}
2654
2655	pr_debug("calling kfd_lookup_process_by_pid\n");
2656	target = kfd_lookup_process_by_pid(pid);
2657
2658	put_pid(pid);
2659
2660	if (!target) {
2661		pr_debug("Cannot find process info for %i\n", args->pid);
2662		return -ESRCH;
2663	}
2664
2665	mutex_lock(&target->mutex);
2666	ret = kfd_criu_resume_svm(target);
2667	if (ret) {
2668		pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2669		goto exit;
2670	}
2671
2672	ret =  amdgpu_amdkfd_criu_resume(target->kgd_process_info);
2673	if (ret)
2674		pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2675
2676exit:
2677	mutex_unlock(&target->mutex);
2678
2679	kfd_unref_process(target);
2680	return ret;
2681}
2682
2683static int criu_process_info(struct file *filep,
2684				struct kfd_process *p,
2685				struct kfd_ioctl_criu_args *args)
2686{
2687	int ret = 0;
2688
2689	mutex_lock(&p->mutex);
2690
2691	if (!p->n_pdds) {
2692		pr_err("No pdd for given process\n");
2693		ret = -ENODEV;
2694		goto err_unlock;
2695	}
2696
2697	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2698	if (ret)
2699		goto err_unlock;
2700
2701	p->queues_paused = true;
2702
2703	args->pid = task_pid_nr_ns(p->lead_thread,
2704					task_active_pid_ns(p->lead_thread));
2705
2706	ret = criu_get_process_object_info(p, &args->num_devices, &args->num_bos,
2707					   &args->num_objects, &args->priv_data_size);
2708	if (ret)
2709		goto err_unlock;
2710
2711	dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2712				args->num_devices, args->num_bos, args->num_objects,
2713				args->priv_data_size);
2714
2715err_unlock:
2716	if (ret) {
2717		kfd_process_restore_queues(p);
2718		p->queues_paused = false;
2719	}
2720	mutex_unlock(&p->mutex);
2721	return ret;
2722}
2723
2724static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2725{
2726	struct kfd_ioctl_criu_args *args = data;
2727	int ret;
2728
2729	dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2730	switch (args->op) {
2731	case KFD_CRIU_OP_PROCESS_INFO:
2732		ret = criu_process_info(filep, p, args);
2733		break;
2734	case KFD_CRIU_OP_CHECKPOINT:
2735		ret = criu_checkpoint(filep, p, args);
2736		break;
2737	case KFD_CRIU_OP_UNPAUSE:
2738		ret = criu_unpause(filep, p, args);
2739		break;
2740	case KFD_CRIU_OP_RESTORE:
2741		ret = criu_restore(filep, p, args);
2742		break;
2743	case KFD_CRIU_OP_RESUME:
2744		ret = criu_resume(filep, p, args);
2745		break;
2746	default:
2747		dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2748		ret = -EINVAL;
2749		break;
2750	}
2751
2752	if (ret)
2753		dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2754
2755	return ret;
2756}
2757
2758static int runtime_enable(struct kfd_process *p, uint64_t r_debug,
2759			bool enable_ttmp_setup)
2760{
2761	int i = 0, ret = 0;
2762
2763	if (p->is_runtime_retry)
2764		goto retry;
2765
2766	if (p->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_DISABLED)
2767		return -EBUSY;
2768
2769	for (i = 0; i < p->n_pdds; i++) {
2770		struct kfd_process_device *pdd = p->pdds[i];
2771
2772		if (pdd->qpd.queue_count)
2773			return -EEXIST;
2774
2775		/*
2776		 * Setup TTMPs by default.
2777		 * Note that this call must remain here for MES ADD QUEUE to
2778		 * skip_process_ctx_clear unconditionally as the first call to
2779		 * SET_SHADER_DEBUGGER clears any stale process context data
2780		 * saved in MES.
2781		 */
2782		if (pdd->dev->kfd->shared_resources.enable_mes)
2783			kfd_dbg_set_mes_debug_mode(pdd, !kfd_dbg_has_cwsr_workaround(pdd->dev));
2784	}
2785
2786	p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_ENABLED;
2787	p->runtime_info.r_debug = r_debug;
2788	p->runtime_info.ttmp_setup = enable_ttmp_setup;
2789
2790	if (p->runtime_info.ttmp_setup) {
2791		for (i = 0; i < p->n_pdds; i++) {
2792			struct kfd_process_device *pdd = p->pdds[i];
2793
2794			if (!kfd_dbg_is_rlc_restore_supported(pdd->dev)) {
2795				amdgpu_gfx_off_ctrl(pdd->dev->adev, false);
2796				pdd->dev->kfd2kgd->enable_debug_trap(
2797						pdd->dev->adev,
2798						true,
2799						pdd->dev->vm_info.last_vmid_kfd);
2800			} else if (kfd_dbg_is_per_vmid_supported(pdd->dev)) {
2801				pdd->spi_dbg_override = pdd->dev->kfd2kgd->enable_debug_trap(
2802						pdd->dev->adev,
2803						false,
2804						0);
2805			}
2806		}
2807	}
2808
2809retry:
2810	if (p->debug_trap_enabled) {
2811		if (!p->is_runtime_retry) {
2812			kfd_dbg_trap_activate(p);
2813			kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2814					p, NULL, 0, false, NULL, 0);
2815		}
2816
2817		mutex_unlock(&p->mutex);
2818		ret = down_interruptible(&p->runtime_enable_sema);
2819		mutex_lock(&p->mutex);
2820
2821		p->is_runtime_retry = !!ret;
2822	}
2823
2824	return ret;
2825}
2826
2827static int runtime_disable(struct kfd_process *p)
2828{
2829	int i = 0, ret = 0;
2830	bool was_enabled = p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED;
2831
2832	p->runtime_info.runtime_state = DEBUG_RUNTIME_STATE_DISABLED;
2833	p->runtime_info.r_debug = 0;
2834
2835	if (p->debug_trap_enabled) {
2836		if (was_enabled)
2837			kfd_dbg_trap_deactivate(p, false, 0);
2838
2839		if (!p->is_runtime_retry)
2840			kfd_dbg_ev_raise(KFD_EC_MASK(EC_PROCESS_RUNTIME),
2841					p, NULL, 0, false, NULL, 0);
2842
2843		mutex_unlock(&p->mutex);
2844		ret = down_interruptible(&p->runtime_enable_sema);
2845		mutex_lock(&p->mutex);
2846
2847		p->is_runtime_retry = !!ret;
2848		if (ret)
2849			return ret;
2850	}
2851
2852	if (was_enabled && p->runtime_info.ttmp_setup) {
2853		for (i = 0; i < p->n_pdds; i++) {
2854			struct kfd_process_device *pdd = p->pdds[i];
2855
2856			if (!kfd_dbg_is_rlc_restore_supported(pdd->dev))
2857				amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
2858		}
2859	}
2860
2861	p->runtime_info.ttmp_setup = false;
2862
2863	/* disable ttmp setup */
2864	for (i = 0; i < p->n_pdds; i++) {
2865		struct kfd_process_device *pdd = p->pdds[i];
2866		int last_err = 0;
2867
2868		if (kfd_dbg_is_per_vmid_supported(pdd->dev)) {
2869			pdd->spi_dbg_override =
2870					pdd->dev->kfd2kgd->disable_debug_trap(
2871					pdd->dev->adev,
2872					false,
2873					pdd->dev->vm_info.last_vmid_kfd);
2874
2875			if (!pdd->dev->kfd->shared_resources.enable_mes)
2876				last_err = debug_refresh_runlist(pdd->dev->dqm);
2877			else
2878				last_err = kfd_dbg_set_mes_debug_mode(pdd,
2879							   !kfd_dbg_has_cwsr_workaround(pdd->dev));
2880
2881			if (last_err)
2882				ret = last_err;
2883		}
2884	}
2885
2886	return ret;
2887}
2888
2889static int kfd_ioctl_runtime_enable(struct file *filep, struct kfd_process *p, void *data)
2890{
2891	struct kfd_ioctl_runtime_enable_args *args = data;
2892	int r;
2893
2894	mutex_lock(&p->mutex);
2895
2896	if (args->mode_mask & KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK)
2897		r = runtime_enable(p, args->r_debug,
2898				!!(args->mode_mask & KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK));
2899	else
2900		r = runtime_disable(p);
2901
2902	mutex_unlock(&p->mutex);
2903
2904	return r;
2905}
2906
2907static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, void *data)
2908{
2909	struct kfd_ioctl_dbg_trap_args *args = data;
2910	struct task_struct *thread = NULL;
2911	struct mm_struct *mm = NULL;
2912	struct pid *pid = NULL;
2913	struct kfd_process *target = NULL;
2914	struct kfd_process_device *pdd = NULL;
2915	int r = 0;
2916
2917	if (sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2918		pr_err("Debugging does not support sched_policy %i", sched_policy);
2919		return -EINVAL;
2920	}
2921
2922	pid = find_get_pid(args->pid);
2923	if (!pid) {
2924		pr_debug("Cannot find pid info for %i\n", args->pid);
2925		r = -ESRCH;
2926		goto out;
2927	}
2928
2929	thread = get_pid_task(pid, PIDTYPE_PID);
2930	if (!thread) {
2931		r = -ESRCH;
2932		goto out;
2933	}
2934
2935	mm = get_task_mm(thread);
2936	if (!mm) {
2937		r = -ESRCH;
2938		goto out;
2939	}
2940
2941	if (args->op == KFD_IOC_DBG_TRAP_ENABLE) {
2942		bool create_process;
2943
2944		rcu_read_lock();
2945		create_process = thread && thread != current && ptrace_parent(thread) == current;
2946		rcu_read_unlock();
2947
2948		target = create_process ? kfd_create_process(thread) :
2949					kfd_lookup_process_by_pid(pid);
2950	} else {
2951		target = kfd_lookup_process_by_pid(pid);
2952	}
2953
2954	if (IS_ERR_OR_NULL(target)) {
2955		pr_debug("Cannot find process PID %i to debug\n", args->pid);
2956		r = target ? PTR_ERR(target) : -ESRCH;
2957		target = NULL;
2958		goto out;
2959	}
2960
2961	/* Check if target is still PTRACED. */
2962	rcu_read_lock();
2963	if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE
2964				&& ptrace_parent(target->lead_thread) != current) {
2965		pr_err("PID %i is not PTRACED and cannot be debugged\n", args->pid);
2966		r = -EPERM;
2967	}
2968	rcu_read_unlock();
2969
2970	if (r)
2971		goto out;
2972
2973	mutex_lock(&target->mutex);
2974
2975	if (args->op != KFD_IOC_DBG_TRAP_ENABLE && !target->debug_trap_enabled) {
2976		pr_err("PID %i not debug enabled for op %i\n", args->pid, args->op);
2977		r = -EINVAL;
2978		goto unlock_out;
2979	}
2980
2981	if (target->runtime_info.runtime_state != DEBUG_RUNTIME_STATE_ENABLED &&
2982			(args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE ||
2983			 args->op == KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE ||
2984			 args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ||
2985			 args->op == KFD_IOC_DBG_TRAP_RESUME_QUEUES ||
2986			 args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2987			 args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH ||
2988			 args->op == KFD_IOC_DBG_TRAP_SET_FLAGS)) {
2989		r = -EPERM;
2990		goto unlock_out;
2991	}
2992
2993	if (args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ||
2994	    args->op == KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH) {
2995		int user_gpu_id = kfd_process_get_user_gpu_id(target,
2996				args->op == KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH ?
2997					args->set_node_address_watch.gpu_id :
2998					args->clear_node_address_watch.gpu_id);
2999
3000		pdd = kfd_process_device_data_by_id(target, user_gpu_id);
3001		if (user_gpu_id == -EINVAL || !pdd) {
3002			r = -ENODEV;
3003			goto unlock_out;
3004		}
3005	}
3006
3007	switch (args->op) {
3008	case KFD_IOC_DBG_TRAP_ENABLE:
3009		if (target != p)
3010			target->debugger_process = p;
3011
3012		r = kfd_dbg_trap_enable(target,
3013					args->enable.dbg_fd,
3014					(void __user *)args->enable.rinfo_ptr,
3015					&args->enable.rinfo_size);
3016		if (!r)
3017			target->exception_enable_mask = args->enable.exception_mask;
3018
3019		break;
3020	case KFD_IOC_DBG_TRAP_DISABLE:
3021		r = kfd_dbg_trap_disable(target);
3022		break;
3023	case KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT:
3024		r = kfd_dbg_send_exception_to_runtime(target,
3025				args->send_runtime_event.gpu_id,
3026				args->send_runtime_event.queue_id,
3027				args->send_runtime_event.exception_mask);
3028		break;
3029	case KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED:
3030		kfd_dbg_set_enabled_debug_exception_mask(target,
3031				args->set_exceptions_enabled.exception_mask);
3032		break;
3033	case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE:
3034		r = kfd_dbg_trap_set_wave_launch_override(target,
3035				args->launch_override.override_mode,
3036				args->launch_override.enable_mask,
3037				args->launch_override.support_request_mask,
3038				&args->launch_override.enable_mask,
3039				&args->launch_override.support_request_mask);
3040		break;
3041	case KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE:
3042		r = kfd_dbg_trap_set_wave_launch_mode(target,
3043				args->launch_mode.launch_mode);
3044		break;
3045	case KFD_IOC_DBG_TRAP_SUSPEND_QUEUES:
3046		r = suspend_queues(target,
3047				args->suspend_queues.num_queues,
3048				args->suspend_queues.grace_period,
3049				args->suspend_queues.exception_mask,
3050				(uint32_t *)args->suspend_queues.queue_array_ptr);
3051
3052		break;
3053	case KFD_IOC_DBG_TRAP_RESUME_QUEUES:
3054		r = resume_queues(target, args->resume_queues.num_queues,
3055				(uint32_t *)args->resume_queues.queue_array_ptr);
3056		break;
3057	case KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH:
3058		r = kfd_dbg_trap_set_dev_address_watch(pdd,
3059				args->set_node_address_watch.address,
3060				args->set_node_address_watch.mask,
3061				&args->set_node_address_watch.id,
3062				args->set_node_address_watch.mode);
3063		break;
3064	case KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH:
3065		r = kfd_dbg_trap_clear_dev_address_watch(pdd,
3066				args->clear_node_address_watch.id);
3067		break;
3068	case KFD_IOC_DBG_TRAP_SET_FLAGS:
3069		r = kfd_dbg_trap_set_flags(target, &args->set_flags.flags);
3070		break;
3071	case KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT:
3072		r = kfd_dbg_ev_query_debug_event(target,
3073				&args->query_debug_event.queue_id,
3074				&args->query_debug_event.gpu_id,
3075				args->query_debug_event.exception_mask,
3076				&args->query_debug_event.exception_mask);
3077		break;
3078	case KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO:
3079		r = kfd_dbg_trap_query_exception_info(target,
3080				args->query_exception_info.source_id,
3081				args->query_exception_info.exception_code,
3082				args->query_exception_info.clear_exception,
3083				(void __user *)args->query_exception_info.info_ptr,
3084				&args->query_exception_info.info_size);
3085		break;
3086	case KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT:
3087		r = pqm_get_queue_snapshot(&target->pqm,
3088				args->queue_snapshot.exception_mask,
3089				(void __user *)args->queue_snapshot.snapshot_buf_ptr,
3090				&args->queue_snapshot.num_queues,
3091				&args->queue_snapshot.entry_size);
3092		break;
3093	case KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT:
3094		r = kfd_dbg_trap_device_snapshot(target,
3095				args->device_snapshot.exception_mask,
3096				(void __user *)args->device_snapshot.snapshot_buf_ptr,
3097				&args->device_snapshot.num_devices,
3098				&args->device_snapshot.entry_size);
3099		break;
3100	default:
3101		pr_err("Invalid option: %i\n", args->op);
3102		r = -EINVAL;
3103	}
3104
3105unlock_out:
3106	mutex_unlock(&target->mutex);
3107
3108out:
3109	if (thread)
3110		put_task_struct(thread);
3111
3112	if (mm)
3113		mmput(mm);
3114
3115	if (pid)
3116		put_pid(pid);
3117
3118	if (target)
3119		kfd_unref_process(target);
3120
3121	return r;
3122}
3123
3124#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
3125	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
3126			    .cmd_drv = 0, .name = #ioctl}
3127
3128/** Ioctl table */
3129static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
3130	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
3131			kfd_ioctl_get_version, 0),
3132
3133	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
3134			kfd_ioctl_create_queue, 0),
3135
3136	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
3137			kfd_ioctl_destroy_queue, 0),
3138
3139	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
3140			kfd_ioctl_set_memory_policy, 0),
3141
3142	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
3143			kfd_ioctl_get_clock_counters, 0),
3144
3145	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
3146			kfd_ioctl_get_process_apertures, 0),
3147
3148	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
3149			kfd_ioctl_update_queue, 0),
3150
3151	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
3152			kfd_ioctl_create_event, 0),
3153
3154	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
3155			kfd_ioctl_destroy_event, 0),
3156
3157	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
3158			kfd_ioctl_set_event, 0),
3159
3160	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
3161			kfd_ioctl_reset_event, 0),
3162
3163	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
3164			kfd_ioctl_wait_events, 0),
3165
3166	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
3167			kfd_ioctl_dbg_register, 0),
3168
3169	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
3170			kfd_ioctl_dbg_unregister, 0),
3171
3172	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
3173			kfd_ioctl_dbg_address_watch, 0),
3174
3175	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
3176			kfd_ioctl_dbg_wave_control, 0),
3177
3178	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
3179			kfd_ioctl_set_scratch_backing_va, 0),
3180
3181	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
3182			kfd_ioctl_get_tile_config, 0),
3183
3184	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
3185			kfd_ioctl_set_trap_handler, 0),
3186
3187	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
3188			kfd_ioctl_get_process_apertures_new, 0),
3189
3190	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
3191			kfd_ioctl_acquire_vm, 0),
3192
3193	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
3194			kfd_ioctl_alloc_memory_of_gpu, 0),
3195
3196	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
3197			kfd_ioctl_free_memory_of_gpu, 0),
3198
3199	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
3200			kfd_ioctl_map_memory_to_gpu, 0),
3201
3202	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
3203			kfd_ioctl_unmap_memory_from_gpu, 0),
3204
3205	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
3206			kfd_ioctl_set_cu_mask, 0),
3207
3208	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
3209			kfd_ioctl_get_queue_wave_state, 0),
3210
3211	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
3212				kfd_ioctl_get_dmabuf_info, 0),
3213
3214	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
3215				kfd_ioctl_import_dmabuf, 0),
3216
3217	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
3218			kfd_ioctl_alloc_queue_gws, 0),
3219
3220	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
3221			kfd_ioctl_smi_events, 0),
3222
3223	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
3224
3225	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
3226			kfd_ioctl_set_xnack_mode, 0),
3227
3228	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
3229			kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
3230
3231	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
3232			kfd_ioctl_get_available_memory, 0),
3233
3234	AMDKFD_IOCTL_DEF(AMDKFD_IOC_EXPORT_DMABUF,
3235				kfd_ioctl_export_dmabuf, 0),
3236
3237	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RUNTIME_ENABLE,
3238			kfd_ioctl_runtime_enable, 0),
3239
3240	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_TRAP,
3241			kfd_ioctl_set_debug_trap, 0),
3242};
3243
3244#define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
3245
3246static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
3247{
3248	struct kfd_process *process;
3249	amdkfd_ioctl_t *func;
3250	const struct amdkfd_ioctl_desc *ioctl = NULL;
3251	unsigned int nr = _IOC_NR(cmd);
3252	char stack_kdata[128];
3253	char *kdata = NULL;
3254	unsigned int usize, asize;
3255	int retcode = -EINVAL;
3256	bool ptrace_attached = false;
3257
3258	if (nr >= AMDKFD_CORE_IOCTL_COUNT) {
3259		retcode = -ENOTTY;
3260		goto err_i1;
3261	}
3262
3263	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
3264		u32 amdkfd_size;
3265
3266		ioctl = &amdkfd_ioctls[nr];
3267
3268		amdkfd_size = _IOC_SIZE(ioctl->cmd);
3269		usize = asize = _IOC_SIZE(cmd);
3270		if (amdkfd_size > asize)
3271			asize = amdkfd_size;
3272
3273		cmd = ioctl->cmd;
3274	} else {
3275		retcode = -ENOTTY;
3276		goto err_i1;
3277	}
3278
3279	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
3280
3281	/* Get the process struct from the filep. Only the process
3282	 * that opened /dev/kfd can use the file descriptor. Child
3283	 * processes need to create their own KFD device context.
3284	 */
3285	process = filep->private_data;
3286
3287	rcu_read_lock();
3288	if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
3289	    ptrace_parent(process->lead_thread) == current)
3290		ptrace_attached = true;
3291	rcu_read_unlock();
3292
3293	if (process->lead_thread != current->group_leader
3294	    && !ptrace_attached) {
3295		dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
3296		retcode = -EBADF;
3297		goto err_i1;
3298	}
3299
3300	/* Do not trust userspace, use our own definition */
3301	func = ioctl->func;
3302
3303	if (unlikely(!func)) {
3304		dev_dbg(kfd_device, "no function\n");
3305		retcode = -EINVAL;
3306		goto err_i1;
3307	}
3308
3309	/*
3310	 * Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
3311	 * CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
3312	 * more priviledged access.
3313	 */
3314	if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
3315		if (!capable(CAP_CHECKPOINT_RESTORE) &&
3316						!capable(CAP_SYS_ADMIN)) {
3317			retcode = -EACCES;
3318			goto err_i1;
3319		}
3320	}
3321
3322	if (cmd & (IOC_IN | IOC_OUT)) {
3323		if (asize <= sizeof(stack_kdata)) {
3324			kdata = stack_kdata;
3325		} else {
3326			kdata = kmalloc(asize, GFP_KERNEL);
3327			if (!kdata) {
3328				retcode = -ENOMEM;
3329				goto err_i1;
3330			}
3331		}
3332		if (asize > usize)
3333			memset(kdata + usize, 0, asize - usize);
3334	}
3335
3336	if (cmd & IOC_IN) {
3337		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
3338			retcode = -EFAULT;
3339			goto err_i1;
3340		}
3341	} else if (cmd & IOC_OUT) {
3342		memset(kdata, 0, usize);
3343	}
3344
3345	retcode = func(filep, process, kdata);
3346
3347	if (cmd & IOC_OUT)
3348		if (copy_to_user((void __user *)arg, kdata, usize) != 0)
3349			retcode = -EFAULT;
3350
3351err_i1:
3352	if (!ioctl)
3353		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
3354			  task_pid_nr(current), cmd, nr);
3355
3356	if (kdata != stack_kdata)
3357		kfree(kdata);
3358
3359	if (retcode)
3360		dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
3361				nr, arg, retcode);
3362
3363	return retcode;
3364}
3365
3366static int kfd_mmio_mmap(struct kfd_node *dev, struct kfd_process *process,
3367		      struct vm_area_struct *vma)
3368{
3369	phys_addr_t address;
3370
3371	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3372		return -EINVAL;
3373
3374	if (PAGE_SIZE > 4096)
3375		return -EINVAL;
3376
3377	address = dev->adev->rmmio_remap.bus_addr;
3378
3379	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
3380				VM_DONTDUMP | VM_PFNMAP);
3381
3382	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3383
3384	pr_debug("process pid %d mapping mmio page\n"
3385		 "     target user address == 0x%08llX\n"
3386		 "     physical address    == 0x%08llX\n"
3387		 "     vm_flags            == 0x%04lX\n"
3388		 "     size                == 0x%04lX\n",
3389		 process->lead_thread->pid, (unsigned long long) vma->vm_start,
3390		 address, vma->vm_flags, PAGE_SIZE);
3391
3392	return io_remap_pfn_range(vma,
3393				vma->vm_start,
3394				address >> PAGE_SHIFT,
3395				PAGE_SIZE,
3396				vma->vm_page_prot);
3397}
3398
3399
3400static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
3401{
3402	struct kfd_process *process;
3403	struct kfd_node *dev = NULL;
3404	unsigned long mmap_offset;
3405	unsigned int gpu_id;
3406
3407	process = kfd_get_process(current);
3408	if (IS_ERR(process))
3409		return PTR_ERR(process);
3410
3411	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
3412	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
3413	if (gpu_id)
3414		dev = kfd_device_by_id(gpu_id);
3415
3416	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
3417	case KFD_MMAP_TYPE_DOORBELL:
3418		if (!dev)
3419			return -ENODEV;
3420		return kfd_doorbell_mmap(dev, process, vma);
3421
3422	case KFD_MMAP_TYPE_EVENTS:
3423		return kfd_event_mmap(process, vma);
3424
3425	case KFD_MMAP_TYPE_RESERVED_MEM:
3426		if (!dev)
3427			return -ENODEV;
3428		return kfd_reserved_mem_mmap(dev, process, vma);
3429	case KFD_MMAP_TYPE_MMIO:
3430		if (!dev)
3431			return -ENODEV;
3432		return kfd_mmio_mmap(dev, process, vma);
3433	}
3434
3435	return -EFAULT;
3436}