vfio_main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * VFIO core
   4 *
   5 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6 *     Author: Alex Williamson <alex.williamson@redhat.com>
   7 *
   8 * Derived from original vfio:
   9 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10 * Author: Tom Lyon, pugs@cisco.com
  11 */
  12
  13#include <linux/cdev.h>
  14#include <linux/compat.h>
  15#include <linux/device.h>
  16#include <linux/fs.h>
  17#include <linux/idr.h>
  18#include <linux/iommu.h>
  19#if IS_ENABLED(CONFIG_KVM)
  20#include <linux/kvm_host.h>
  21#endif
  22#include <linux/list.h>
  23#include <linux/miscdevice.h>
  24#include <linux/module.h>
  25#include <linux/mount.h>
  26#include <linux/mutex.h>
  27#include <linux/pci.h>
  28#include <linux/pseudo_fs.h>
  29#include <linux/rwsem.h>
  30#include <linux/sched.h>
  31#include <linux/seq_file.h>
  32#include <linux/slab.h>
  33#include <linux/stat.h>
  34#include <linux/string.h>
  35#include <linux/uaccess.h>
  36#include <linux/vfio.h>
  37#include <linux/wait.h>
  38#include <linux/sched/signal.h>
  39#include <linux/pm_runtime.h>
  40#include <linux/interval_tree.h>
  41#include <linux/iova_bitmap.h>
  42#include <linux/iommufd.h>
  43#include "vfio.h"
  44
  45#define DRIVER_VERSION	"0.3"
  46#define DRIVER_AUTHOR	"Alex Williamson <alex.williamson@redhat.com>"
  47#define DRIVER_DESC	"VFIO - User Level meta-driver"
  48
  49#define VFIO_MAGIC 0x5646494f /* "VFIO" */
  50
  51static struct vfio {
  52	struct class			*device_class;
  53	struct ida			device_ida;
  54	struct vfsmount			*vfs_mount;
  55	int				fs_count;
  56} vfio;
  57
  58#ifdef CONFIG_VFIO_NOIOMMU
  59bool vfio_noiommu __read_mostly;
  60module_param_named(enable_unsafe_noiommu_mode,
  61		   vfio_noiommu, bool, S_IRUGO | S_IWUSR);
  62MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
  63#endif
  64
  65static DEFINE_XARRAY(vfio_device_set_xa);
  66
  67int vfio_assign_device_set(struct vfio_device *device, void *set_id)
  68{
  69	unsigned long idx = (unsigned long)set_id;
  70	struct vfio_device_set *new_dev_set;
  71	struct vfio_device_set *dev_set;
  72
  73	if (WARN_ON(!set_id))
  74		return -EINVAL;
  75
  76	/*
  77	 * Atomically acquire a singleton object in the xarray for this set_id
  78	 */
  79	xa_lock(&vfio_device_set_xa);
  80	dev_set = xa_load(&vfio_device_set_xa, idx);
  81	if (dev_set)
  82		goto found_get_ref;
  83	xa_unlock(&vfio_device_set_xa);
  84
  85	new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL);
  86	if (!new_dev_set)
  87		return -ENOMEM;
  88	mutex_init(&new_dev_set->lock);
  89	INIT_LIST_HEAD(&new_dev_set->device_list);
  90	new_dev_set->set_id = set_id;
  91
  92	xa_lock(&vfio_device_set_xa);
  93	dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
  94			       GFP_KERNEL);
  95	if (!dev_set) {
  96		dev_set = new_dev_set;
  97		goto found_get_ref;
  98	}
  99
 100	kfree(new_dev_set);
 101	if (xa_is_err(dev_set)) {
 102		xa_unlock(&vfio_device_set_xa);
 103		return xa_err(dev_set);
 104	}
 105
 106found_get_ref:
 107	dev_set->device_count++;
 108	xa_unlock(&vfio_device_set_xa);
 109	mutex_lock(&dev_set->lock);
 110	device->dev_set = dev_set;
 111	list_add_tail(&device->dev_set_list, &dev_set->device_list);
 112	mutex_unlock(&dev_set->lock);
 113	return 0;
 114}
 115EXPORT_SYMBOL_GPL(vfio_assign_device_set);
 116
 117static void vfio_release_device_set(struct vfio_device *device)
 118{
 119	struct vfio_device_set *dev_set = device->dev_set;
 120
 121	if (!dev_set)
 122		return;
 123
 124	mutex_lock(&dev_set->lock);
 125	list_del(&device->dev_set_list);
 126	mutex_unlock(&dev_set->lock);
 127
 128	xa_lock(&vfio_device_set_xa);
 129	if (!--dev_set->device_count) {
 130		__xa_erase(&vfio_device_set_xa,
 131			   (unsigned long)dev_set->set_id);
 132		mutex_destroy(&dev_set->lock);
 133		kfree(dev_set);
 134	}
 135	xa_unlock(&vfio_device_set_xa);
 136}
 137
 138unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set)
 139{
 140	struct vfio_device *cur;
 141	unsigned int open_count = 0;
 142
 143	lockdep_assert_held(&dev_set->lock);
 144
 145	list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 146		open_count += cur->open_count;
 147	return open_count;
 148}
 149EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
 150
 151struct vfio_device *
 152vfio_find_device_in_devset(struct vfio_device_set *dev_set,
 153			   struct device *dev)
 154{
 155	struct vfio_device *cur;
 156
 157	lockdep_assert_held(&dev_set->lock);
 158
 159	list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 160		if (cur->dev == dev)
 161			return cur;
 162	return NULL;
 163}
 164EXPORT_SYMBOL_GPL(vfio_find_device_in_devset);
 165
 166/*
 167 * Device objects - create, release, get, put, search
 168 */
 169/* Device reference always implies a group reference */
 170void vfio_device_put_registration(struct vfio_device *device)
 171{
 172	if (refcount_dec_and_test(&device->refcount))
 173		complete(&device->comp);
 174}
 175EXPORT_SYMBOL_GPL(vfio_device_put_registration);
 176
 177bool vfio_device_try_get_registration(struct vfio_device *device)
 178{
 179	return refcount_inc_not_zero(&device->refcount);
 180}
 181EXPORT_SYMBOL_GPL(vfio_device_try_get_registration);
 182
 183/*
 184 * VFIO driver API
 185 */
 186/* Release helper called by vfio_put_device() */
 187static void vfio_device_release(struct device *dev)
 188{
 189	struct vfio_device *device =
 190			container_of(dev, struct vfio_device, device);
 191
 192	vfio_release_device_set(device);
 193	ida_free(&vfio.device_ida, device->index);
 194
 195	if (device->ops->release)
 196		device->ops->release(device);
 197
 198	iput(device->inode);
 199	simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 200	kvfree(device);
 201}
 202
 203static int vfio_init_device(struct vfio_device *device, struct device *dev,
 204			    const struct vfio_device_ops *ops);
 205
 206/*
 207 * Allocate and initialize vfio_device so it can be registered to vfio
 208 * core.
 209 *
 210 * Drivers should use the wrapper vfio_alloc_device() for allocation.
 211 * @size is the size of the structure to be allocated, including any
 212 * private data used by the driver.
 213 *
 214 * Driver may provide an @init callback to cover device private data.
 215 *
 216 * Use vfio_put_device() to release the structure after success return.
 217 */
 218struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
 219				       const struct vfio_device_ops *ops)
 220{
 221	struct vfio_device *device;
 222	int ret;
 223
 224	if (WARN_ON(size < sizeof(struct vfio_device)))
 225		return ERR_PTR(-EINVAL);
 226
 227	device = kvzalloc(size, GFP_KERNEL);
 228	if (!device)
 229		return ERR_PTR(-ENOMEM);
 230
 231	ret = vfio_init_device(device, dev, ops);
 232	if (ret)
 233		goto out_free;
 234	return device;
 235
 236out_free:
 237	kvfree(device);
 238	return ERR_PTR(ret);
 239}
 240EXPORT_SYMBOL_GPL(_vfio_alloc_device);
 241
 242static int vfio_fs_init_fs_context(struct fs_context *fc)
 243{
 244	return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM;
 245}
 246
 247static struct file_system_type vfio_fs_type = {
 248	.name = "vfio",
 249	.owner = THIS_MODULE,
 250	.init_fs_context = vfio_fs_init_fs_context,
 251	.kill_sb = kill_anon_super,
 252};
 253
 254static struct inode *vfio_fs_inode_new(void)
 255{
 256	struct inode *inode;
 257	int ret;
 258
 259	ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count);
 260	if (ret)
 261		return ERR_PTR(ret);
 262
 263	inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb);
 264	if (IS_ERR(inode))
 265		simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 266
 267	return inode;
 268}
 269
 270/*
 271 * Initialize a vfio_device so it can be registered to vfio core.
 272 */
 273static int vfio_init_device(struct vfio_device *device, struct device *dev,
 274			    const struct vfio_device_ops *ops)
 275{
 276	int ret;
 277
 278	ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
 279	if (ret < 0) {
 280		dev_dbg(dev, "Error to alloc index\n");
 281		return ret;
 282	}
 283
 284	device->index = ret;
 285	init_completion(&device->comp);
 286	device->dev = dev;
 287	device->ops = ops;
 288	device->inode = vfio_fs_inode_new();
 289	if (IS_ERR(device->inode)) {
 290		ret = PTR_ERR(device->inode);
 291		goto out_inode;
 292	}
 293
 294	if (ops->init) {
 295		ret = ops->init(device);
 296		if (ret)
 297			goto out_uninit;
 298	}
 299
 300	device_initialize(&device->device);
 301	device->device.release = vfio_device_release;
 302	device->device.class = vfio.device_class;
 303	device->device.parent = device->dev;
 304	return 0;
 305
 306out_uninit:
 307	iput(device->inode);
 308	simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 309out_inode:
 310	vfio_release_device_set(device);
 311	ida_free(&vfio.device_ida, device->index);
 312	return ret;
 313}
 314
 315static int __vfio_register_dev(struct vfio_device *device,
 316			       enum vfio_group_type type)
 317{
 318	int ret;
 319
 320	if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
 321		    (!device->ops->bind_iommufd ||
 322		     !device->ops->unbind_iommufd ||
 323		     !device->ops->attach_ioas ||
 324		     !device->ops->detach_ioas)))
 325		return -EINVAL;
 326
 327	/*
 328	 * If the driver doesn't specify a set then the device is added to a
 329	 * singleton set just for itself.
 330	 */
 331	if (!device->dev_set)
 332		vfio_assign_device_set(device, device);
 333
 334	ret = dev_set_name(&device->device, "vfio%d", device->index);
 335	if (ret)
 336		return ret;
 337
 338	ret = vfio_device_set_group(device, type);
 339	if (ret)
 340		return ret;
 341
 342	/*
 343	 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
 344	 * restore cache coherency. It has to be checked here because it is only
 345	 * valid for cases where we are using iommu groups.
 346	 */
 347	if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) &&
 348	    !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) {
 349		ret = -EINVAL;
 350		goto err_out;
 351	}
 352
 353	ret = vfio_device_add(device);
 354	if (ret)
 355		goto err_out;
 356
 357	/* Refcounting can't start until the driver calls register */
 358	refcount_set(&device->refcount, 1);
 359
 360	vfio_device_group_register(device);
 361	vfio_device_debugfs_init(device);
 362
 363	return 0;
 364err_out:
 365	vfio_device_remove_group(device);
 366	return ret;
 367}
 368
 369int vfio_register_group_dev(struct vfio_device *device)
 370{
 371	return __vfio_register_dev(device, VFIO_IOMMU);
 372}
 373EXPORT_SYMBOL_GPL(vfio_register_group_dev);
 374
 375/*
 376 * Register a virtual device without IOMMU backing.  The user of this
 377 * device must not be able to directly trigger unmediated DMA.
 378 */
 379int vfio_register_emulated_iommu_dev(struct vfio_device *device)
 380{
 381	return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
 382}
 383EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
 384
 385/*
 386 * Decrement the device reference count and wait for the device to be
 387 * removed.  Open file descriptors for the device... */
 388void vfio_unregister_group_dev(struct vfio_device *device)
 389{
 390	unsigned int i = 0;
 391	bool interrupted = false;
 392	long rc;
 393
 394	/*
 395	 * Prevent new device opened by userspace via the
 396	 * VFIO_GROUP_GET_DEVICE_FD in the group path.
 397	 */
 398	vfio_device_group_unregister(device);
 399
 400	/*
 401	 * Balances vfio_device_add() in register path, also prevents
 402	 * new device opened by userspace in the cdev path.
 403	 */
 404	vfio_device_del(device);
 405
 406	vfio_device_put_registration(device);
 407	rc = try_wait_for_completion(&device->comp);
 408	while (rc <= 0) {
 409		if (device->ops->request)
 410			device->ops->request(device, i++);
 411
 412		if (interrupted) {
 413			rc = wait_for_completion_timeout(&device->comp,
 414							 HZ * 10);
 415		} else {
 416			rc = wait_for_completion_interruptible_timeout(
 417				&device->comp, HZ * 10);
 418			if (rc < 0) {
 419				interrupted = true;
 420				dev_warn(device->dev,
 421					 "Device is currently in use, task"
 422					 " \"%s\" (%d) "
 423					 "blocked until device is released",
 424					 current->comm, task_pid_nr(current));
 425			}
 426		}
 427	}
 428
 429	vfio_device_debugfs_exit(device);
 430	/* Balances vfio_device_set_group in register path */
 431	vfio_device_remove_group(device);
 432}
 433EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
 434
 435#if IS_ENABLED(CONFIG_KVM)
 436void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
 437{
 438	void (*pfn)(struct kvm *kvm);
 439	bool (*fn)(struct kvm *kvm);
 440	bool ret;
 441
 442	lockdep_assert_held(&device->dev_set->lock);
 443
 444	if (!kvm)
 445		return;
 446
 447	pfn = symbol_get(kvm_put_kvm);
 448	if (WARN_ON(!pfn))
 449		return;
 450
 451	fn = symbol_get(kvm_get_kvm_safe);
 452	if (WARN_ON(!fn)) {
 453		symbol_put(kvm_put_kvm);
 454		return;
 455	}
 456
 457	ret = fn(kvm);
 458	symbol_put(kvm_get_kvm_safe);
 459	if (!ret) {
 460		symbol_put(kvm_put_kvm);
 461		return;
 462	}
 463
 464	device->put_kvm = pfn;
 465	device->kvm = kvm;
 466}
 467
 468void vfio_device_put_kvm(struct vfio_device *device)
 469{
 470	lockdep_assert_held(&device->dev_set->lock);
 471
 472	if (!device->kvm)
 473		return;
 474
 475	if (WARN_ON(!device->put_kvm))
 476		goto clear;
 477
 478	device->put_kvm(device->kvm);
 479	device->put_kvm = NULL;
 480	symbol_put(kvm_put_kvm);
 481
 482clear:
 483	device->kvm = NULL;
 484}
 485#endif
 486
 487/* true if the vfio_device has open_device() called but not close_device() */
 488static bool vfio_assert_device_open(struct vfio_device *device)
 489{
 490	return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
 491}
 492
 493struct vfio_device_file *
 494vfio_allocate_device_file(struct vfio_device *device)
 495{
 496	struct vfio_device_file *df;
 497
 498	df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT);
 499	if (!df)
 500		return ERR_PTR(-ENOMEM);
 501
 502	df->device = device;
 503	spin_lock_init(&df->kvm_ref_lock);
 504
 505	return df;
 506}
 507
 508static int vfio_df_device_first_open(struct vfio_device_file *df)
 509{
 510	struct vfio_device *device = df->device;
 511	struct iommufd_ctx *iommufd = df->iommufd;
 512	int ret;
 513
 514	lockdep_assert_held(&device->dev_set->lock);
 515
 516	if (!try_module_get(device->dev->driver->owner))
 517		return -ENODEV;
 518
 519	if (iommufd)
 520		ret = vfio_df_iommufd_bind(df);
 521	else
 522		ret = vfio_device_group_use_iommu(device);
 523	if (ret)
 524		goto err_module_put;
 525
 526	if (device->ops->open_device) {
 527		ret = device->ops->open_device(device);
 528		if (ret)
 529			goto err_unuse_iommu;
 530	}
 531	return 0;
 532
 533err_unuse_iommu:
 534	if (iommufd)
 535		vfio_df_iommufd_unbind(df);
 536	else
 537		vfio_device_group_unuse_iommu(device);
 538err_module_put:
 539	module_put(device->dev->driver->owner);
 540	return ret;
 541}
 542
 543static void vfio_df_device_last_close(struct vfio_device_file *df)
 544{
 545	struct vfio_device *device = df->device;
 546	struct iommufd_ctx *iommufd = df->iommufd;
 547
 548	lockdep_assert_held(&device->dev_set->lock);
 549
 550	if (device->ops->close_device)
 551		device->ops->close_device(device);
 552	if (iommufd)
 553		vfio_df_iommufd_unbind(df);
 554	else
 555		vfio_device_group_unuse_iommu(device);
 556	module_put(device->dev->driver->owner);
 557}
 558
 559int vfio_df_open(struct vfio_device_file *df)
 560{
 561	struct vfio_device *device = df->device;
 562	int ret = 0;
 563
 564	lockdep_assert_held(&device->dev_set->lock);
 565
 566	/*
 567	 * Only the group path allows the device to be opened multiple
 568	 * times.  The device cdev path doesn't have a secure way for it.
 569	 */
 570	if (device->open_count != 0 && !df->group)
 571		return -EINVAL;
 572
 573	device->open_count++;
 574	if (device->open_count == 1) {
 575		ret = vfio_df_device_first_open(df);
 576		if (ret)
 577			device->open_count--;
 578	}
 579
 580	return ret;
 581}
 582
 583void vfio_df_close(struct vfio_device_file *df)
 584{
 585	struct vfio_device *device = df->device;
 586
 587	lockdep_assert_held(&device->dev_set->lock);
 588
 589	if (!vfio_assert_device_open(device))
 590		return;
 591	if (device->open_count == 1)
 592		vfio_df_device_last_close(df);
 593	device->open_count--;
 594}
 595
 596/*
 597 * Wrapper around pm_runtime_resume_and_get().
 598 * Return error code on failure or 0 on success.
 599 */
 600static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
 601{
 602	struct device *dev = device->dev;
 603
 604	if (dev->driver && dev->driver->pm) {
 605		int ret;
 606
 607		ret = pm_runtime_resume_and_get(dev);
 608		if (ret) {
 609			dev_info_ratelimited(dev,
 610				"vfio: runtime resume failed %d\n", ret);
 611			return -EIO;
 612		}
 613	}
 614
 615	return 0;
 616}
 617
 618/*
 619 * Wrapper around pm_runtime_put().
 620 */
 621static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
 622{
 623	struct device *dev = device->dev;
 624
 625	if (dev->driver && dev->driver->pm)
 626		pm_runtime_put(dev);
 627}
 628
 629/*
 630 * VFIO Device fd
 631 */
 632static int vfio_device_fops_release(struct inode *inode, struct file *filep)
 633{
 634	struct vfio_device_file *df = filep->private_data;
 635	struct vfio_device *device = df->device;
 636
 637	if (df->group)
 638		vfio_df_group_close(df);
 639	else
 640		vfio_df_unbind_iommufd(df);
 641
 642	vfio_device_put_registration(device);
 643
 644	kfree(df);
 645
 646	return 0;
 647}
 648
 649/*
 650 * vfio_mig_get_next_state - Compute the next step in the FSM
 651 * @cur_fsm - The current state the device is in
 652 * @new_fsm - The target state to reach
 653 * @next_fsm - Pointer to the next step to get to new_fsm
 654 *
 655 * Return 0 upon success, otherwise -errno
 656 * Upon success the next step in the state progression between cur_fsm and
 657 * new_fsm will be set in next_fsm.
 658 *
 659 * This breaks down requests for combination transitions into smaller steps and
 660 * returns the next step to get to new_fsm. The function may need to be called
 661 * multiple times before reaching new_fsm.
 662 *
 663 */
 664int vfio_mig_get_next_state(struct vfio_device *device,
 665			    enum vfio_device_mig_state cur_fsm,
 666			    enum vfio_device_mig_state new_fsm,
 667			    enum vfio_device_mig_state *next_fsm)
 668{
 669	enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 };
 670	/*
 671	 * The coding in this table requires the driver to implement the
 672	 * following FSM arcs:
 673	 *         RESUMING -> STOP
 674	 *         STOP -> RESUMING
 675	 *         STOP -> STOP_COPY
 676	 *         STOP_COPY -> STOP
 677	 *
 678	 * If P2P is supported then the driver must also implement these FSM
 679	 * arcs:
 680	 *         RUNNING -> RUNNING_P2P
 681	 *         RUNNING_P2P -> RUNNING
 682	 *         RUNNING_P2P -> STOP
 683	 *         STOP -> RUNNING_P2P
 684	 *
 685	 * If precopy is supported then the driver must support these additional
 686	 * FSM arcs:
 687	 *         RUNNING -> PRE_COPY
 688	 *         PRE_COPY -> RUNNING
 689	 *         PRE_COPY -> STOP_COPY
 690	 * However, if precopy and P2P are supported together then the driver
 691	 * must support these additional arcs beyond the P2P arcs above:
 692	 *         PRE_COPY -> RUNNING
 693	 *         PRE_COPY -> PRE_COPY_P2P
 694	 *         PRE_COPY_P2P -> PRE_COPY
 695	 *         PRE_COPY_P2P -> RUNNING_P2P
 696	 *         PRE_COPY_P2P -> STOP_COPY
 697	 *         RUNNING -> PRE_COPY
 698	 *         RUNNING_P2P -> PRE_COPY_P2P
 699	 *
 700	 * Without P2P and precopy the driver must implement:
 701	 *         RUNNING -> STOP
 702	 *         STOP -> RUNNING
 703	 *
 704	 * The coding will step through multiple states for some combination
 705	 * transitions; if all optional features are supported, this means the
 706	 * following ones:
 707	 *         PRE_COPY -> PRE_COPY_P2P -> STOP_COPY
 708	 *         PRE_COPY -> RUNNING -> RUNNING_P2P
 709	 *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP
 710	 *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 711	 *         PRE_COPY_P2P -> RUNNING_P2P -> RUNNING
 712	 *         PRE_COPY_P2P -> RUNNING_P2P -> STOP
 713	 *         PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING
 714	 *         RESUMING -> STOP -> RUNNING_P2P
 715	 *         RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P
 716	 *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING
 717	 *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 718	 *         RESUMING -> STOP -> STOP_COPY
 719	 *         RUNNING -> RUNNING_P2P -> PRE_COPY_P2P
 720	 *         RUNNING -> RUNNING_P2P -> STOP
 721	 *         RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 722	 *         RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
 723	 *         RUNNING_P2P -> RUNNING -> PRE_COPY
 724	 *         RUNNING_P2P -> STOP -> RESUMING
 725	 *         RUNNING_P2P -> STOP -> STOP_COPY
 726	 *         STOP -> RUNNING_P2P -> PRE_COPY_P2P
 727	 *         STOP -> RUNNING_P2P -> RUNNING
 728	 *         STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 729	 *         STOP_COPY -> STOP -> RESUMING
 730	 *         STOP_COPY -> STOP -> RUNNING_P2P
 731	 *         STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
 732	 *
 733	 *  The following transitions are blocked:
 734	 *         STOP_COPY -> PRE_COPY
 735	 *         STOP_COPY -> PRE_COPY_P2P
 736	 */
 737	static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
 738		[VFIO_DEVICE_STATE_STOP] = {
 739			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 740			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 741			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 742			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 743			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 744			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 745			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 746			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 747		},
 748		[VFIO_DEVICE_STATE_RUNNING] = {
 749			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 750			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 751			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 752			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 753			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 754			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 755			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 756			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 757		},
 758		[VFIO_DEVICE_STATE_PRE_COPY] = {
 759			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING,
 760			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 761			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 762			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 763			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 764			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING,
 765			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING,
 766			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 767		},
 768		[VFIO_DEVICE_STATE_PRE_COPY_P2P] = {
 769			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 770			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 771			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 772			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 773			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 774			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 775			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 776			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 777		},
 778		[VFIO_DEVICE_STATE_STOP_COPY] = {
 779			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 780			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 781			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 782			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 783			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 784			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 785			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 786			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 787		},
 788		[VFIO_DEVICE_STATE_RESUMING] = {
 789			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 790			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 791			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP,
 792			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP,
 793			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 794			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 795			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 796			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 797		},
 798		[VFIO_DEVICE_STATE_RUNNING_P2P] = {
 799			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 800			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 801			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING,
 802			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 803			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 804			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 805			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 806			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 807		},
 808		[VFIO_DEVICE_STATE_ERROR] = {
 809			[VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
 810			[VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
 811			[VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 812			[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 813			[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
 814			[VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
 815			[VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
 816			[VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 817		},
 818	};
 819
 820	static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
 821		[VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
 822		[VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
 823		[VFIO_DEVICE_STATE_PRE_COPY] =
 824			VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY,
 825		[VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY |
 826						   VFIO_MIGRATION_P2P |
 827						   VFIO_MIGRATION_PRE_COPY,
 828		[VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
 829		[VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
 830		[VFIO_DEVICE_STATE_RUNNING_P2P] =
 831			VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
 832		[VFIO_DEVICE_STATE_ERROR] = ~0U,
 833	};
 834
 835	if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 836		    (state_flags_table[cur_fsm] & device->migration_flags) !=
 837			state_flags_table[cur_fsm]))
 838		return -EINVAL;
 839
 840	if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 841	   (state_flags_table[new_fsm] & device->migration_flags) !=
 842			state_flags_table[new_fsm])
 843		return -EINVAL;
 844
 845	/*
 846	 * Arcs touching optional and unsupported states are skipped over. The
 847	 * driver will instead see an arc from the original state to the next
 848	 * logical state, as per the above comment.
 849	 */
 850	*next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
 851	while ((state_flags_table[*next_fsm] & device->migration_flags) !=
 852			state_flags_table[*next_fsm])
 853		*next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
 854
 855	return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
 856}
 857EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
 858
 859/*
 860 * Convert the drivers's struct file into a FD number and return it to userspace
 861 */
 862static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
 863				   struct vfio_device_feature_mig_state *mig)
 864{
 865	int ret;
 866	int fd;
 867
 868	fd = get_unused_fd_flags(O_CLOEXEC);
 869	if (fd < 0) {
 870		ret = fd;
 871		goto out_fput;
 872	}
 873
 874	mig->data_fd = fd;
 875	if (copy_to_user(arg, mig, sizeof(*mig))) {
 876		ret = -EFAULT;
 877		goto out_put_unused;
 878	}
 879	fd_install(fd, filp);
 880	return 0;
 881
 882out_put_unused:
 883	put_unused_fd(fd);
 884out_fput:
 885	fput(filp);
 886	return ret;
 887}
 888
 889static int
 890vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
 891					   u32 flags, void __user *arg,
 892					   size_t argsz)
 893{
 894	size_t minsz =
 895		offsetofend(struct vfio_device_feature_mig_state, data_fd);
 896	struct vfio_device_feature_mig_state mig;
 897	struct file *filp = NULL;
 898	int ret;
 899
 900	if (!device->mig_ops)
 901		return -ENOTTY;
 902
 903	ret = vfio_check_feature(flags, argsz,
 904				 VFIO_DEVICE_FEATURE_SET |
 905				 VFIO_DEVICE_FEATURE_GET,
 906				 sizeof(mig));
 907	if (ret != 1)
 908		return ret;
 909
 910	if (copy_from_user(&mig, arg, minsz))
 911		return -EFAULT;
 912
 913	if (flags & VFIO_DEVICE_FEATURE_GET) {
 914		enum vfio_device_mig_state curr_state;
 915
 916		ret = device->mig_ops->migration_get_state(device,
 917							   &curr_state);
 918		if (ret)
 919			return ret;
 920		mig.device_state = curr_state;
 921		goto out_copy;
 922	}
 923
 924	/* Handle the VFIO_DEVICE_FEATURE_SET */
 925	filp = device->mig_ops->migration_set_state(device, mig.device_state);
 926	if (IS_ERR(filp) || !filp)
 927		goto out_copy;
 928
 929	return vfio_ioct_mig_return_fd(filp, arg, &mig);
 930out_copy:
 931	mig.data_fd = -1;
 932	if (copy_to_user(arg, &mig, sizeof(mig)))
 933		return -EFAULT;
 934	if (IS_ERR(filp))
 935		return PTR_ERR(filp);
 936	return 0;
 937}
 938
 939static int
 940vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device,
 941					      u32 flags, void __user *arg,
 942					      size_t argsz)
 943{
 944	struct vfio_device_feature_mig_data_size data_size = {};
 945	unsigned long stop_copy_length;
 946	int ret;
 947
 948	if (!device->mig_ops)
 949		return -ENOTTY;
 950
 951	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 952				 sizeof(data_size));
 953	if (ret != 1)
 954		return ret;
 955
 956	ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length);
 957	if (ret)
 958		return ret;
 959
 960	data_size.stop_copy_length = stop_copy_length;
 961	if (copy_to_user(arg, &data_size, sizeof(data_size)))
 962		return -EFAULT;
 963
 964	return 0;
 965}
 966
 967static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
 968					       u32 flags, void __user *arg,
 969					       size_t argsz)
 970{
 971	struct vfio_device_feature_migration mig = {
 972		.flags = device->migration_flags,
 973	};
 974	int ret;
 975
 976	if (!device->mig_ops)
 977		return -ENOTTY;
 978
 979	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 980				 sizeof(mig));
 981	if (ret != 1)
 982		return ret;
 983	if (copy_to_user(arg, &mig, sizeof(mig)))
 984		return -EFAULT;
 985	return 0;
 986}
 987
 988void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes,
 989			      u32 req_nodes)
 990{
 991	struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
 992	unsigned long min_gap, curr_gap;
 993
 994	/* Special shortcut when a single range is required */
 995	if (req_nodes == 1) {
 996		unsigned long last;
 997
 998		comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
 999
1000		/* Empty list */
1001		if (WARN_ON_ONCE(!comb_start))
1002			return;
1003
1004		curr = comb_start;
1005		while (curr) {
1006			last = curr->last;
1007			prev = curr;
1008			curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1009			if (prev != comb_start)
1010				interval_tree_remove(prev, root);
1011		}
1012		comb_start->last = last;
1013		return;
1014	}
1015
1016	/* Combine ranges which have the smallest gap */
1017	while (cur_nodes > req_nodes) {
1018		prev = NULL;
1019		min_gap = ULONG_MAX;
1020		curr = interval_tree_iter_first(root, 0, ULONG_MAX);
1021		while (curr) {
1022			if (prev) {
1023				curr_gap = curr->start - prev->last;
1024				if (curr_gap < min_gap) {
1025					min_gap = curr_gap;
1026					comb_start = prev;
1027					comb_end = curr;
1028				}
1029			}
1030			prev = curr;
1031			curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1032		}
1033
1034		/* Empty list or no nodes to combine */
1035		if (WARN_ON_ONCE(min_gap == ULONG_MAX))
1036			break;
1037
1038		comb_start->last = comb_end->last;
1039		interval_tree_remove(comb_end, root);
1040		cur_nodes--;
1041	}
1042}
1043EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges);
1044
1045/* Ranges should fit into a single kernel page */
1046#define LOG_MAX_RANGES \
1047	(PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
1048
1049static int
1050vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
1051					u32 flags, void __user *arg,
1052					size_t argsz)
1053{
1054	size_t minsz =
1055		offsetofend(struct vfio_device_feature_dma_logging_control,
1056			    ranges);
1057	struct vfio_device_feature_dma_logging_range __user *ranges;
1058	struct vfio_device_feature_dma_logging_control control;
1059	struct vfio_device_feature_dma_logging_range range;
1060	struct rb_root_cached root = RB_ROOT_CACHED;
1061	struct interval_tree_node *nodes;
1062	u64 iova_end;
1063	u32 nnodes;
1064	int i, ret;
1065
1066	if (!device->log_ops)
1067		return -ENOTTY;
1068
1069	ret = vfio_check_feature(flags, argsz,
1070				 VFIO_DEVICE_FEATURE_SET,
1071				 sizeof(control));
1072	if (ret != 1)
1073		return ret;
1074
1075	if (copy_from_user(&control, arg, minsz))
1076		return -EFAULT;
1077
1078	nnodes = control.num_ranges;
1079	if (!nnodes)
1080		return -EINVAL;
1081
1082	if (nnodes > LOG_MAX_RANGES)
1083		return -E2BIG;
1084
1085	ranges = u64_to_user_ptr(control.ranges);
1086	nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
1087			      GFP_KERNEL);
1088	if (!nodes)
1089		return -ENOMEM;
1090
1091	for (i = 0; i < nnodes; i++) {
1092		if (copy_from_user(&range, &ranges[i], sizeof(range))) {
1093			ret = -EFAULT;
1094			goto end;
1095		}
1096		if (!IS_ALIGNED(range.iova, control.page_size) ||
1097		    !IS_ALIGNED(range.length, control.page_size)) {
1098			ret = -EINVAL;
1099			goto end;
1100		}
1101
1102		if (check_add_overflow(range.iova, range.length, &iova_end) ||
1103		    iova_end > ULONG_MAX) {
1104			ret = -EOVERFLOW;
1105			goto end;
1106		}
1107
1108		nodes[i].start = range.iova;
1109		nodes[i].last = range.iova + range.length - 1;
1110		if (interval_tree_iter_first(&root, nodes[i].start,
1111					     nodes[i].last)) {
1112			/* Range overlapping */
1113			ret = -EINVAL;
1114			goto end;
1115		}
1116		interval_tree_insert(nodes + i, &root);
1117	}
1118
1119	ret = device->log_ops->log_start(device, &root, nnodes,
1120					 &control.page_size);
1121	if (ret)
1122		goto end;
1123
1124	if (copy_to_user(arg, &control, sizeof(control))) {
1125		ret = -EFAULT;
1126		device->log_ops->log_stop(device);
1127	}
1128
1129end:
1130	kfree(nodes);
1131	return ret;
1132}
1133
1134static int
1135vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
1136				       u32 flags, void __user *arg,
1137				       size_t argsz)
1138{
1139	int ret;
1140
1141	if (!device->log_ops)
1142		return -ENOTTY;
1143
1144	ret = vfio_check_feature(flags, argsz,
1145				 VFIO_DEVICE_FEATURE_SET, 0);
1146	if (ret != 1)
1147		return ret;
1148
1149	return device->log_ops->log_stop(device);
1150}
1151
1152static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
1153					  unsigned long iova, size_t length,
1154					  void *opaque)
1155{
1156	struct vfio_device *device = opaque;
1157
1158	return device->log_ops->log_read_and_clear(device, iova, length, iter);
1159}
1160
1161static int
1162vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
1163					 u32 flags, void __user *arg,
1164					 size_t argsz)
1165{
1166	size_t minsz =
1167		offsetofend(struct vfio_device_feature_dma_logging_report,
1168			    bitmap);
1169	struct vfio_device_feature_dma_logging_report report;
1170	struct iova_bitmap *iter;
1171	u64 iova_end;
1172	int ret;
1173
1174	if (!device->log_ops)
1175		return -ENOTTY;
1176
1177	ret = vfio_check_feature(flags, argsz,
1178				 VFIO_DEVICE_FEATURE_GET,
1179				 sizeof(report));
1180	if (ret != 1)
1181		return ret;
1182
1183	if (copy_from_user(&report, arg, minsz))
1184		return -EFAULT;
1185
1186	if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
1187		return -EINVAL;
1188
1189	if (check_add_overflow(report.iova, report.length, &iova_end) ||
1190	    iova_end > ULONG_MAX)
1191		return -EOVERFLOW;
1192
1193	iter = iova_bitmap_alloc(report.iova, report.length,
1194				 report.page_size,
1195				 u64_to_user_ptr(report.bitmap));
1196	if (IS_ERR(iter))
1197		return PTR_ERR(iter);
1198
1199	ret = iova_bitmap_for_each(iter, device,
1200				   vfio_device_log_read_and_clear);
1201
1202	iova_bitmap_free(iter);
1203	return ret;
1204}
1205
1206static int vfio_ioctl_device_feature(struct vfio_device *device,
1207				     struct vfio_device_feature __user *arg)
1208{
1209	size_t minsz = offsetofend(struct vfio_device_feature, flags);
1210	struct vfio_device_feature feature;
1211
1212	if (copy_from_user(&feature, arg, minsz))
1213		return -EFAULT;
1214
1215	if (feature.argsz < minsz)
1216		return -EINVAL;
1217
1218	/* Check unknown flags */
1219	if (feature.flags &
1220	    ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
1221	      VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
1222		return -EINVAL;
1223
1224	/* GET & SET are mutually exclusive except with PROBE */
1225	if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1226	    (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1227	    (feature.flags & VFIO_DEVICE_FEATURE_GET))
1228		return -EINVAL;
1229
1230	switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1231	case VFIO_DEVICE_FEATURE_MIGRATION:
1232		return vfio_ioctl_device_feature_migration(
1233			device, feature.flags, arg->data,
1234			feature.argsz - minsz);
1235	case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
1236		return vfio_ioctl_device_feature_mig_device_state(
1237			device, feature.flags, arg->data,
1238			feature.argsz - minsz);
1239	case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
1240		return vfio_ioctl_device_feature_logging_start(
1241			device, feature.flags, arg->data,
1242			feature.argsz - minsz);
1243	case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
1244		return vfio_ioctl_device_feature_logging_stop(
1245			device, feature.flags, arg->data,
1246			feature.argsz - minsz);
1247	case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
1248		return vfio_ioctl_device_feature_logging_report(
1249			device, feature.flags, arg->data,
1250			feature.argsz - minsz);
1251	case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE:
1252		return vfio_ioctl_device_feature_migration_data_size(
1253			device, feature.flags, arg->data,
1254			feature.argsz - minsz);
1255	default:
1256		if (unlikely(!device->ops->device_feature))
1257			return -ENOTTY;
1258		return device->ops->device_feature(device, feature.flags,
1259						   arg->data,
1260						   feature.argsz - minsz);
1261	}
1262}
1263
1264static long vfio_get_region_info(struct vfio_device *device,
1265				 struct vfio_region_info __user *arg)
1266{
1267	unsigned long minsz = offsetofend(struct vfio_region_info, offset);
1268	struct vfio_region_info info = {};
1269	struct vfio_info_cap caps = {};
1270	int ret;
1271
1272	if (unlikely(!device->ops->get_region_info_caps))
1273		return -EINVAL;
1274
1275	if (copy_from_user(&info, arg, minsz))
1276		return -EFAULT;
1277	if (info.argsz < minsz)
1278		return -EINVAL;
1279
1280	ret = device->ops->get_region_info_caps(device, &info, &caps);
1281	if (ret)
1282		goto out_free;
1283
1284	if (caps.size) {
1285		info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1286		if (info.argsz < sizeof(info) + caps.size) {
1287			info.argsz = sizeof(info) + caps.size;
1288			info.cap_offset = 0;
1289		} else {
1290			vfio_info_cap_shift(&caps, sizeof(info));
1291			if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1292				ret = -EFAULT;
1293				goto out_free;
1294			}
1295			info.cap_offset = sizeof(info);
1296		}
1297	}
1298
1299	if (copy_to_user(arg, &info, minsz)){
1300		ret = -EFAULT;
1301		goto out_free;
1302	}
1303
1304out_free:
1305	kfree(caps.buf);
1306	return ret;
1307}
1308
1309static long vfio_device_fops_unl_ioctl(struct file *filep,
1310				       unsigned int cmd, unsigned long arg)
1311{
1312	struct vfio_device_file *df = filep->private_data;
1313	struct vfio_device *device = df->device;
1314	void __user *uptr = (void __user *)arg;
1315	int ret;
1316
1317	if (cmd == VFIO_DEVICE_BIND_IOMMUFD)
1318		return vfio_df_ioctl_bind_iommufd(df, uptr);
1319
1320	/* Paired with smp_store_release() following vfio_df_open() */
1321	if (!smp_load_acquire(&df->access_granted))
1322		return -EINVAL;
1323
1324	ret = vfio_device_pm_runtime_get(device);
1325	if (ret)
1326		return ret;
1327
1328	/* cdev only ioctls */
1329	if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) {
1330		switch (cmd) {
1331		case VFIO_DEVICE_ATTACH_IOMMUFD_PT:
1332			ret = vfio_df_ioctl_attach_pt(df, uptr);
1333			goto out;
1334
1335		case VFIO_DEVICE_DETACH_IOMMUFD_PT:
1336			ret = vfio_df_ioctl_detach_pt(df, uptr);
1337			goto out;
1338		}
1339	}
1340
1341	switch (cmd) {
1342	case VFIO_DEVICE_FEATURE:
1343		ret = vfio_ioctl_device_feature(device, uptr);
1344		break;
1345
1346	case VFIO_DEVICE_GET_REGION_INFO:
1347		ret = vfio_get_region_info(device, uptr);
1348		break;
1349
1350	default:
1351		if (unlikely(!device->ops->ioctl))
1352			ret = -EINVAL;
1353		else
1354			ret = device->ops->ioctl(device, cmd, arg);
1355		break;
1356	}
1357out:
1358	vfio_device_pm_runtime_put(device);
1359	return ret;
1360}
1361
1362static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1363				     size_t count, loff_t *ppos)
1364{
1365	struct vfio_device_file *df = filep->private_data;
1366	struct vfio_device *device = df->device;
1367
1368	/* Paired with smp_store_release() following vfio_df_open() */
1369	if (!smp_load_acquire(&df->access_granted))
1370		return -EINVAL;
1371
1372	if (unlikely(!device->ops->read))
1373		return -EINVAL;
1374
1375	return device->ops->read(device, buf, count, ppos);
1376}
1377
1378static ssize_t vfio_device_fops_write(struct file *filep,
1379				      const char __user *buf,
1380				      size_t count, loff_t *ppos)
1381{
1382	struct vfio_device_file *df = filep->private_data;
1383	struct vfio_device *device = df->device;
1384
1385	/* Paired with smp_store_release() following vfio_df_open() */
1386	if (!smp_load_acquire(&df->access_granted))
1387		return -EINVAL;
1388
1389	if (unlikely(!device->ops->write))
1390		return -EINVAL;
1391
1392	return device->ops->write(device, buf, count, ppos);
1393}
1394
1395static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1396{
1397	struct vfio_device_file *df = filep->private_data;
1398	struct vfio_device *device = df->device;
1399
1400	/* Paired with smp_store_release() following vfio_df_open() */
1401	if (!smp_load_acquire(&df->access_granted))
1402		return -EINVAL;
1403
1404	if (unlikely(!device->ops->mmap))
1405		return -EINVAL;
1406
1407	return device->ops->mmap(device, vma);
1408}
1409
1410#ifdef CONFIG_PROC_FS
1411static void vfio_device_show_fdinfo(struct seq_file *m, struct file *filep)
1412{
1413	char *path;
1414	struct vfio_device_file *df = filep->private_data;
1415	struct vfio_device *device = df->device;
1416
1417	path = kobject_get_path(&device->dev->kobj, GFP_KERNEL);
1418	if (!path)
1419		return;
1420
1421	seq_printf(m, "vfio-device-syspath: /sys%s\n", path);
1422	kfree(path);
1423}
1424#endif
1425
1426const struct file_operations vfio_device_fops = {
1427	.owner		= THIS_MODULE,
1428	.open		= vfio_device_fops_cdev_open,
1429	.release	= vfio_device_fops_release,
1430	.read		= vfio_device_fops_read,
1431	.write		= vfio_device_fops_write,
1432	.unlocked_ioctl	= vfio_device_fops_unl_ioctl,
1433	.compat_ioctl	= compat_ptr_ioctl,
1434	.mmap		= vfio_device_fops_mmap,
1435#ifdef CONFIG_PROC_FS
1436	.show_fdinfo	= vfio_device_show_fdinfo,
1437#endif
1438};
1439
1440static struct vfio_device *vfio_device_from_file(struct file *file)
1441{
1442	struct vfio_device_file *df = file->private_data;
1443
1444	if (file->f_op != &vfio_device_fops)
1445		return NULL;
1446	return df->device;
1447}
1448
1449/**
1450 * vfio_file_is_valid - True if the file is valid vfio file
1451 * @file: VFIO group file or VFIO device file
1452 */
1453bool vfio_file_is_valid(struct file *file)
1454{
1455	return vfio_group_from_file(file) ||
1456	       vfio_device_from_file(file);
1457}
1458EXPORT_SYMBOL_GPL(vfio_file_is_valid);
1459
1460/**
1461 * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1462 *        is always CPU cache coherent
1463 * @file: VFIO group file or VFIO device file
1464 *
1465 * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1466 * bit in DMA transactions. A return of false indicates that the user has
1467 * rights to access additional instructions such as wbinvd on x86.
1468 */
1469bool vfio_file_enforced_coherent(struct file *file)
1470{
1471	struct vfio_device *device;
1472	struct vfio_group *group;
1473
1474	group = vfio_group_from_file(file);
1475	if (group)
1476		return vfio_group_enforced_coherent(group);
1477
1478	device = vfio_device_from_file(file);
1479	if (device)
1480		return device_iommu_capable(device->dev,
1481					    IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
1482
1483	return true;
1484}
1485EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1486
1487static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
1488{
1489	struct vfio_device_file *df = file->private_data;
1490
1491	/*
1492	 * The kvm is first recorded in the vfio_device_file, and will
1493	 * be propagated to vfio_device::kvm when the file is bound to
1494	 * iommufd successfully in the vfio device cdev path.
1495	 */
1496	spin_lock(&df->kvm_ref_lock);
1497	df->kvm = kvm;
1498	spin_unlock(&df->kvm_ref_lock);
1499}
1500
1501/**
1502 * vfio_file_set_kvm - Link a kvm with VFIO drivers
1503 * @file: VFIO group file or VFIO device file
1504 * @kvm: KVM to link
1505 *
1506 * When a VFIO device is first opened the KVM will be available in
1507 * device->kvm if one was associated with the file.
1508 */
1509void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1510{
1511	struct vfio_group *group;
1512
1513	group = vfio_group_from_file(file);
1514	if (group)
1515		vfio_group_set_kvm(group, kvm);
1516
1517	if (vfio_device_from_file(file))
1518		vfio_device_file_set_kvm(file, kvm);
1519}
1520EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1521
1522/*
1523 * Sub-module support
1524 */
1525/*
1526 * Helper for managing a buffer of info chain capabilities, allocate or
1527 * reallocate a buffer with additional @size, filling in @id and @version
1528 * of the capability.  A pointer to the new capability is returned.
1529 *
1530 * NB. The chain is based at the head of the buffer, so new entries are
1531 * added to the tail, vfio_info_cap_shift() should be called to fixup the
1532 * next offsets prior to copying to the user buffer.
1533 */
1534struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1535					       size_t size, u16 id, u16 version)
1536{
1537	void *buf;
1538	struct vfio_info_cap_header *header, *tmp;
1539
1540	/* Ensure that the next capability struct will be aligned */
1541	size = ALIGN(size, sizeof(u64));
1542
1543	buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1544	if (!buf) {
1545		kfree(caps->buf);
1546		caps->buf = NULL;
1547		caps->size = 0;
1548		return ERR_PTR(-ENOMEM);
1549	}
1550
1551	caps->buf = buf;
1552	header = buf + caps->size;
1553
1554	/* Eventually copied to user buffer, zero */
1555	memset(header, 0, size);
1556
1557	header->id = id;
1558	header->version = version;
1559
1560	/* Add to the end of the capability chain */
1561	for (tmp = buf; tmp->next; tmp = buf + tmp->next)
1562		; /* nothing */
1563
1564	tmp->next = caps->size;
1565	caps->size += size;
1566
1567	return header;
1568}
1569EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1570
1571void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1572{
1573	struct vfio_info_cap_header *tmp;
1574	void *buf = (void *)caps->buf;
1575
1576	/* Capability structs should start with proper alignment */
1577	WARN_ON(!IS_ALIGNED(offset, sizeof(u64)));
1578
1579	for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
1580		tmp->next += offset;
1581}
1582EXPORT_SYMBOL(vfio_info_cap_shift);
1583
1584int vfio_info_add_capability(struct vfio_info_cap *caps,
1585			     struct vfio_info_cap_header *cap, size_t size)
1586{
1587	struct vfio_info_cap_header *header;
1588
1589	header = vfio_info_cap_add(caps, size, cap->id, cap->version);
1590	if (IS_ERR(header))
1591		return PTR_ERR(header);
1592
1593	memcpy(header + 1, cap + 1, size - sizeof(*header));
1594
1595	return 0;
1596}
1597EXPORT_SYMBOL(vfio_info_add_capability);
1598
1599int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
1600				       int max_irq_type, size_t *data_size)
1601{
1602	unsigned long minsz;
1603	size_t size;
1604
1605	minsz = offsetofend(struct vfio_irq_set, count);
1606
1607	if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
1608	    (hdr->count >= (U32_MAX - hdr->start)) ||
1609	    (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
1610				VFIO_IRQ_SET_ACTION_TYPE_MASK)))
1611		return -EINVAL;
1612
1613	if (data_size)
1614		*data_size = 0;
1615
1616	if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
1617		return -EINVAL;
1618
1619	switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1620	case VFIO_IRQ_SET_DATA_NONE:
1621		size = 0;
1622		break;
1623	case VFIO_IRQ_SET_DATA_BOOL:
1624		size = sizeof(uint8_t);
1625		break;
1626	case VFIO_IRQ_SET_DATA_EVENTFD:
1627		size = sizeof(int32_t);
1628		break;
1629	default:
1630		return -EINVAL;
1631	}
1632
1633	if (size) {
1634		if (hdr->argsz - minsz < hdr->count * size)
1635			return -EINVAL;
1636
1637		if (!data_size)
1638			return -EINVAL;
1639
1640		*data_size = hdr->count * size;
1641	}
1642
1643	return 0;
1644}
1645EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1646
1647/*
1648 * Pin contiguous user pages and return their associated host pages for local
1649 * domain only.
1650 * @device [in]  : device
1651 * @iova [in]    : starting IOVA of user pages to be pinned.
1652 * @npage [in]   : count of pages to be pinned.  This count should not
1653 *		   be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1654 * @prot [in]    : protection flags
1655 * @pages[out]   : array of host pages
1656 * Return error or number of pages pinned.
1657 *
1658 * A driver may only call this function if the vfio_device was created
1659 * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1660 */
1661int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1662		   int npage, int prot, struct page **pages)
1663{
1664	/* group->container cannot change while a vfio device is open */
1665	if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1666		return -EINVAL;
1667	if (!device->ops->dma_unmap)
1668		return -EINVAL;
1669	if (vfio_device_has_container(device))
1670		return vfio_device_container_pin_pages(device, iova,
1671						       npage, prot, pages);
1672	if (device->iommufd_access) {
1673		int ret;
1674
1675		if (iova > ULONG_MAX)
1676			return -EINVAL;
1677		/*
1678		 * VFIO ignores the sub page offset, npages is from the start of
1679		 * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1680		 * the sub page offset by doing:
1681		 *     pages[0] + (iova % PAGE_SIZE)
1682		 */
1683		ret = iommufd_access_pin_pages(
1684			device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1685			npage * PAGE_SIZE, pages,
1686			(prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1687		if (ret)
1688			return ret;
1689		return npage;
1690	}
1691	return -EINVAL;
1692}
1693EXPORT_SYMBOL(vfio_pin_pages);
1694
1695/*
1696 * Unpin contiguous host pages for local domain only.
1697 * @device [in]  : device
1698 * @iova [in]    : starting address of user pages to be unpinned.
1699 * @npage [in]   : count of pages to be unpinned.  This count should not
1700 *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1701 */
1702void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1703{
1704	if (WARN_ON(!vfio_assert_device_open(device)))
1705		return;
1706	if (WARN_ON(!device->ops->dma_unmap))
1707		return;
1708
1709	if (vfio_device_has_container(device)) {
1710		vfio_device_container_unpin_pages(device, iova, npage);
1711		return;
1712	}
1713	if (device->iommufd_access) {
1714		if (WARN_ON(iova > ULONG_MAX))
1715			return;
1716		iommufd_access_unpin_pages(device->iommufd_access,
1717					   ALIGN_DOWN(iova, PAGE_SIZE),
1718					   npage * PAGE_SIZE);
1719		return;
1720	}
1721}
1722EXPORT_SYMBOL(vfio_unpin_pages);
1723
1724/*
1725 * This interface allows the CPUs to perform some sort of virtual DMA on
1726 * behalf of the device.
1727 *
1728 * CPUs read/write from/into a range of IOVAs pointing to user space memory
1729 * into/from a kernel buffer.
1730 *
1731 * As the read/write of user space memory is conducted via the CPUs and is
1732 * not a real device DMA, it is not necessary to pin the user space memory.
1733 *
1734 * @device [in]		: VFIO device
1735 * @iova [in]		: base IOVA of a user space buffer
1736 * @data [in]		: pointer to kernel buffer
1737 * @len [in]		: kernel buffer length
1738 * @write		: indicate read or write
1739 * Return error code on failure or 0 on success.
1740 */
1741int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1742		size_t len, bool write)
1743{
1744	if (!data || len <= 0 || !vfio_assert_device_open(device))
1745		return -EINVAL;
1746
1747	if (vfio_device_has_container(device))
1748		return vfio_device_container_dma_rw(device, iova,
1749						    data, len, write);
1750
1751	if (device->iommufd_access) {
1752		unsigned int flags = 0;
1753
1754		if (iova > ULONG_MAX)
1755			return -EINVAL;
1756
1757		/* VFIO historically tries to auto-detect a kthread */
1758		if (!current->mm)
1759			flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1760		if (write)
1761			flags |= IOMMUFD_ACCESS_RW_WRITE;
1762		return iommufd_access_rw(device->iommufd_access, iova, data,
1763					 len, flags);
1764	}
1765	return -EINVAL;
1766}
1767EXPORT_SYMBOL(vfio_dma_rw);
1768
1769/*
1770 * Module/class support
1771 */
1772static int __init vfio_init(void)
1773{
1774	int ret;
1775
1776	ida_init(&vfio.device_ida);
1777
1778	ret = vfio_group_init();
1779	if (ret)
1780		return ret;
1781
1782	ret = vfio_virqfd_init();
1783	if (ret)
1784		goto err_virqfd;
1785
1786	/* /sys/class/vfio-dev/vfioX */
1787	vfio.device_class = class_create("vfio-dev");
1788	if (IS_ERR(vfio.device_class)) {
1789		ret = PTR_ERR(vfio.device_class);
1790		goto err_dev_class;
1791	}
1792
1793	ret = vfio_cdev_init(vfio.device_class);
1794	if (ret)
1795		goto err_alloc_dev_chrdev;
1796
1797	vfio_debugfs_create_root();
1798	pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1799	return 0;
1800
1801err_alloc_dev_chrdev:
1802	class_destroy(vfio.device_class);
1803	vfio.device_class = NULL;
1804err_dev_class:
1805	vfio_virqfd_exit();
1806err_virqfd:
1807	vfio_group_cleanup();
1808	return ret;
1809}
1810
1811static void __exit vfio_cleanup(void)
1812{
1813	vfio_debugfs_remove_root();
1814	ida_destroy(&vfio.device_ida);
1815	vfio_cdev_cleanup();
1816	class_destroy(vfio.device_class);
1817	vfio.device_class = NULL;
1818	vfio_virqfd_exit();
1819	vfio_group_cleanup();
1820	xa_destroy(&vfio_device_set_xa);
1821}
1822
1823module_init(vfio_init);
1824module_exit(vfio_cleanup);
1825
1826MODULE_IMPORT_NS("IOMMUFD");
1827MODULE_VERSION(DRIVER_VERSION);
1828MODULE_LICENSE("GPL v2");
1829MODULE_AUTHOR(DRIVER_AUTHOR);
1830MODULE_DESCRIPTION(DRIVER_DESC);
1831MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");