vfio_pci_core.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   4 *     Author: Alex Williamson <alex.williamson@redhat.com>
   5 *
   6 * Derived from original vfio:
   7 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
   8 * Author: Tom Lyon, pugs@cisco.com
   9 */
  10
  11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13#include <linux/aperture.h>
  14#include <linux/device.h>
  15#include <linux/eventfd.h>
  16#include <linux/file.h>
  17#include <linux/interrupt.h>
  18#include <linux/iommu.h>
  19#include <linux/module.h>
  20#include <linux/mutex.h>
  21#include <linux/notifier.h>
  22#include <linux/pci.h>
  23#include <linux/pm_runtime.h>
  24#include <linux/slab.h>
  25#include <linux/types.h>
  26#include <linux/uaccess.h>
  27#include <linux/vgaarb.h>
  28#include <linux/nospec.h>
  29#include <linux/sched/mm.h>
  30#include <linux/iommufd.h>
  31#include <linux/pci-p2pdma.h>
  32#if IS_ENABLED(CONFIG_EEH)
  33#include <asm/eeh.h>
  34#endif
  35
  36#include "vfio_pci_priv.h"
  37
  38#define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  39#define DRIVER_DESC "core driver for VFIO based PCI devices"
  40
  41static bool nointxmask;
  42static bool disable_vga;
  43static bool disable_idle_d3;
  44
  45static void vfio_pci_eventfd_rcu_free(struct rcu_head *rcu)
  46{
  47	struct vfio_pci_eventfd *eventfd =
  48		container_of(rcu, struct vfio_pci_eventfd, rcu);
  49
  50	eventfd_ctx_put(eventfd->ctx);
  51	kfree(eventfd);
  52}
  53
  54int vfio_pci_eventfd_replace_locked(struct vfio_pci_core_device *vdev,
  55				    struct vfio_pci_eventfd __rcu **peventfd,
  56				    struct eventfd_ctx *ctx)
  57{
  58	struct vfio_pci_eventfd *new = NULL;
  59	struct vfio_pci_eventfd *old;
  60
  61	lockdep_assert_held(&vdev->igate);
  62
  63	if (ctx) {
  64		new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT);
  65		if (!new)
  66			return -ENOMEM;
  67
  68		new->ctx = ctx;
  69	}
  70
  71	old = rcu_replace_pointer(*peventfd, new,
  72				  lockdep_is_held(&vdev->igate));
  73	if (old)
  74		call_rcu(&old->rcu, vfio_pci_eventfd_rcu_free);
  75
  76	return 0;
  77}
  78
  79/* List of PF's that vfio_pci_core_sriov_configure() has been called on */
  80static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex);
  81static LIST_HEAD(vfio_pci_sriov_pfs);
  82
  83struct vfio_pci_dummy_resource {
  84	struct resource		resource;
  85	int			index;
  86	struct list_head	res_next;
  87};
  88
  89struct vfio_pci_vf_token {
  90	struct mutex		lock;
  91	uuid_t			uuid;
  92	int			users;
  93};
  94
  95static inline bool vfio_vga_disabled(void)
  96{
  97#ifdef CONFIG_VFIO_PCI_VGA
  98	return disable_vga;
  99#else
 100	return true;
 101#endif
 102}
 103
 104/*
 105 * Our VGA arbiter participation is limited since we don't know anything
 106 * about the device itself.  However, if the device is the only VGA device
 107 * downstream of a bridge and VFIO VGA support is disabled, then we can
 108 * safely return legacy VGA IO and memory as not decoded since the user
 109 * has no way to get to it and routing can be disabled externally at the
 110 * bridge.
 111 */
 112static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga)
 113{
 114	struct pci_dev *tmp = NULL;
 115	unsigned char max_busnr;
 116	unsigned int decodes;
 117
 118	if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))
 119		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
 120		       VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
 121
 122	max_busnr = pci_bus_max_busnr(pdev->bus);
 123	decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 124
 125	while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
 126		if (tmp == pdev ||
 127		    pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||
 128		    pci_is_root_bus(tmp->bus))
 129			continue;
 130
 131		if (tmp->bus->number >= pdev->bus->number &&
 132		    tmp->bus->number <= max_busnr) {
 133			pci_dev_put(tmp);
 134			decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
 135			break;
 136		}
 137	}
 138
 139	return decodes;
 140}
 141
 142static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev)
 143{
 144	struct resource *res;
 145	int i;
 146	struct vfio_pci_dummy_resource *dummy_res;
 147
 148	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 149		int bar = i + PCI_STD_RESOURCES;
 150
 151		res = &vdev->pdev->resource[bar];
 152
 153		if (vdev->pdev->non_mappable_bars)
 154			goto no_mmap;
 155
 156		if (!(res->flags & IORESOURCE_MEM))
 157			goto no_mmap;
 158
 159		/*
 160		 * The PCI core shouldn't set up a resource with a
 161		 * type but zero size. But there may be bugs that
 162		 * cause us to do that.
 163		 */
 164		if (!resource_size(res))
 165			goto no_mmap;
 166
 167		if (resource_size(res) >= PAGE_SIZE) {
 168			vdev->bar_mmap_supported[bar] = true;
 169			continue;
 170		}
 171
 172		if (!(res->start & ~PAGE_MASK)) {
 173			/*
 174			 * Add a dummy resource to reserve the remainder
 175			 * of the exclusive page in case that hot-add
 176			 * device's bar is assigned into it.
 177			 */
 178			dummy_res =
 179				kzalloc(sizeof(*dummy_res), GFP_KERNEL_ACCOUNT);
 180			if (dummy_res == NULL)
 181				goto no_mmap;
 182
 183			dummy_res->resource.name = "vfio sub-page reserved";
 184			dummy_res->resource.start = res->end + 1;
 185			dummy_res->resource.end = res->start + PAGE_SIZE - 1;
 186			dummy_res->resource.flags = res->flags;
 187			if (request_resource(res->parent,
 188						&dummy_res->resource)) {
 189				kfree(dummy_res);
 190				goto no_mmap;
 191			}
 192			dummy_res->index = bar;
 193			list_add(&dummy_res->res_next,
 194					&vdev->dummy_resources_list);
 195			vdev->bar_mmap_supported[bar] = true;
 196			continue;
 197		}
 198		/*
 199		 * Here we don't handle the case when the BAR is not page
 200		 * aligned because we can't expect the BAR will be
 201		 * assigned into the same location in a page in guest
 202		 * when we passthrough the BAR. And it's hard to access
 203		 * this BAR in userspace because we have no way to get
 204		 * the BAR's location in a page.
 205		 */
 206no_mmap:
 207		vdev->bar_mmap_supported[bar] = false;
 208	}
 209}
 210
 211struct vfio_pci_group_info;
 212static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
 213static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
 214				      struct vfio_pci_group_info *groups,
 215				      struct iommufd_ctx *iommufd_ctx);
 216
 217/*
 218 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
 219 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
 220 * If a device implements the former but not the latter we would typically
 221 * expect broken_intx_masking be set and require an exclusive interrupt.
 222 * However since we do have control of the device's ability to assert INTx,
 223 * we can instead pretend that the device does not implement INTx, virtualizing
 224 * the pin register to report zero and maintaining DisINTx set on the host.
 225 */
 226static bool vfio_pci_nointx(struct pci_dev *pdev)
 227{
 228	switch (pdev->vendor) {
 229	case PCI_VENDOR_ID_INTEL:
 230		switch (pdev->device) {
 231		/* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
 232		case 0x1572:
 233		case 0x1574:
 234		case 0x1580 ... 0x1581:
 235		case 0x1583 ... 0x158b:
 236		case 0x37d0 ... 0x37d2:
 237		/* X550 */
 238		case 0x1563:
 239			return true;
 240		default:
 241			return false;
 242		}
 243	}
 244
 245	return false;
 246}
 247
 248static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
 249{
 250	struct pci_dev *pdev = vdev->pdev;
 251	u16 pmcsr;
 252
 253	if (!pdev->pm_cap)
 254		return;
 255
 256	pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
 257
 258	vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
 259}
 260
 261/*
 262 * pci_set_power_state() wrapper handling devices which perform a soft reset on
 263 * D3->D0 transition.  Save state prior to D0/1/2->D3, stash it on the vdev,
 264 * restore when returned to D0.  Saved separately from pci_saved_state for use
 265 * by PM capability emulation and separately from pci_dev internal saved state
 266 * to avoid it being overwritten and consumed around other resets.
 267 */
 268int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)
 269{
 270	struct pci_dev *pdev = vdev->pdev;
 271	bool needs_restore = false, needs_save = false;
 272	int ret;
 273
 274	/* Prevent changing power state for PFs with VFs enabled */
 275	if (pci_num_vf(pdev) && state > PCI_D0)
 276		return -EBUSY;
 277
 278	if (vdev->needs_pm_restore) {
 279		if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
 280			pci_save_state(pdev);
 281			needs_save = true;
 282		}
 283
 284		if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
 285			needs_restore = true;
 286	}
 287
 288	ret = pci_set_power_state(pdev, state);
 289
 290	if (!ret) {
 291		/* D3 might be unsupported via quirk, skip unless in D3 */
 292		if (needs_save && pdev->current_state >= PCI_D3hot) {
 293			/*
 294			 * The current PCI state will be saved locally in
 295			 * 'pm_save' during the D3hot transition. When the
 296			 * device state is changed to D0 again with the current
 297			 * function, then pci_store_saved_state() will restore
 298			 * the state and will free the memory pointed by
 299			 * 'pm_save'. There are few cases where the PCI power
 300			 * state can be changed to D0 without the involvement
 301			 * of the driver. For these cases, free the earlier
 302			 * allocated memory first before overwriting 'pm_save'
 303			 * to prevent the memory leak.
 304			 */
 305			kfree(vdev->pm_save);
 306			vdev->pm_save = pci_store_saved_state(pdev);
 307		} else if (needs_restore) {
 308			pci_load_and_free_saved_state(pdev, &vdev->pm_save);
 309			pci_restore_state(pdev);
 310		}
 311	}
 312
 313	return ret;
 314}
 315
 316static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,
 317				     struct eventfd_ctx *efdctx)
 318{
 319	/*
 320	 * The vdev power related flags are protected with 'memory_lock'
 321	 * semaphore.
 322	 */
 323	vfio_pci_zap_and_down_write_memory_lock(vdev);
 324	vfio_pci_dma_buf_move(vdev, true);
 325
 326	if (vdev->pm_runtime_engaged) {
 327		up_write(&vdev->memory_lock);
 328		return -EINVAL;
 329	}
 330
 331	vdev->pm_runtime_engaged = true;
 332	vdev->pm_wake_eventfd_ctx = efdctx;
 333	pm_runtime_put_noidle(&vdev->pdev->dev);
 334	up_write(&vdev->memory_lock);
 335
 336	return 0;
 337}
 338
 339static int vfio_pci_core_pm_entry(struct vfio_pci_core_device *vdev, u32 flags,
 340				  void __user *arg, size_t argsz)
 341{
 342	int ret;
 343
 344	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
 345	if (ret != 1)
 346		return ret;
 347
 348	/*
 349	 * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count
 350	 * will be decremented. The pm_runtime_put() will be invoked again
 351	 * while returning from the ioctl and then the device can go into
 352	 * runtime suspended state.
 353	 */
 354	return vfio_pci_runtime_pm_entry(vdev, NULL);
 355}
 356
 357static int vfio_pci_core_pm_entry_with_wakeup(
 358	struct vfio_pci_core_device *vdev, u32 flags,
 359	struct vfio_device_low_power_entry_with_wakeup __user *arg,
 360	size_t argsz)
 361{
 362	struct vfio_device_low_power_entry_with_wakeup entry;
 363	struct eventfd_ctx *efdctx;
 364	int ret;
 365
 366	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
 367				 sizeof(entry));
 368	if (ret != 1)
 369		return ret;
 370
 371	if (copy_from_user(&entry, arg, sizeof(entry)))
 372		return -EFAULT;
 373
 374	if (entry.wakeup_eventfd < 0)
 375		return -EINVAL;
 376
 377	efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd);
 378	if (IS_ERR(efdctx))
 379		return PTR_ERR(efdctx);
 380
 381	ret = vfio_pci_runtime_pm_entry(vdev, efdctx);
 382	if (ret)
 383		eventfd_ctx_put(efdctx);
 384
 385	return ret;
 386}
 387
 388static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
 389{
 390	if (vdev->pm_runtime_engaged) {
 391		vdev->pm_runtime_engaged = false;
 392		pm_runtime_get_noresume(&vdev->pdev->dev);
 393
 394		if (vdev->pm_wake_eventfd_ctx) {
 395			eventfd_ctx_put(vdev->pm_wake_eventfd_ctx);
 396			vdev->pm_wake_eventfd_ctx = NULL;
 397		}
 398	}
 399}
 400
 401static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
 402{
 403	/*
 404	 * The vdev power related flags are protected with 'memory_lock'
 405	 * semaphore.
 406	 */
 407	down_write(&vdev->memory_lock);
 408	__vfio_pci_runtime_pm_exit(vdev);
 409	if (__vfio_pci_memory_enabled(vdev))
 410		vfio_pci_dma_buf_move(vdev, false);
 411	up_write(&vdev->memory_lock);
 412}
 413
 414static int vfio_pci_core_pm_exit(struct vfio_pci_core_device *vdev, u32 flags,
 415				 void __user *arg, size_t argsz)
 416{
 417	int ret;
 418
 419	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
 420	if (ret != 1)
 421		return ret;
 422
 423	/*
 424	 * The device is always in the active state here due to pm wrappers
 425	 * around ioctls. If the device had entered a low power state and
 426	 * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has
 427	 * already signaled the eventfd and exited low power mode itself.
 428	 * pm_runtime_engaged protects the redundant call here.
 429	 */
 430	vfio_pci_runtime_pm_exit(vdev);
 431	return 0;
 432}
 433
 434#ifdef CONFIG_PM
 435static int vfio_pci_core_runtime_suspend(struct device *dev)
 436{
 437	struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
 438
 439	down_write(&vdev->memory_lock);
 440	/*
 441	 * The user can move the device into D3hot state before invoking
 442	 * power management IOCTL. Move the device into D0 state here and then
 443	 * the pci-driver core runtime PM suspend function will move the device
 444	 * into the low power state. Also, for the devices which have
 445	 * NoSoftRst-, it will help in restoring the original state
 446	 * (saved locally in 'vdev->pm_save').
 447	 */
 448	vfio_pci_set_power_state(vdev, PCI_D0);
 449	up_write(&vdev->memory_lock);
 450
 451	/*
 452	 * If INTx is enabled, then mask INTx before going into the runtime
 453	 * suspended state and unmask the same in the runtime resume.
 454	 * If INTx has already been masked by the user, then
 455	 * vfio_pci_intx_mask() will return false and in that case, INTx
 456	 * should not be unmasked in the runtime resume.
 457	 */
 458	vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) &&
 459				vfio_pci_intx_mask(vdev));
 460
 461	return 0;
 462}
 463
 464static int vfio_pci_core_runtime_resume(struct device *dev)
 465{
 466	struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
 467
 468	/*
 469	 * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit
 470	 * low power mode.
 471	 */
 472	down_write(&vdev->memory_lock);
 473	if (vdev->pm_wake_eventfd_ctx) {
 474		eventfd_signal(vdev->pm_wake_eventfd_ctx);
 475		__vfio_pci_runtime_pm_exit(vdev);
 476	}
 477	up_write(&vdev->memory_lock);
 478
 479	if (vdev->pm_intx_masked)
 480		vfio_pci_intx_unmask(vdev);
 481
 482	return 0;
 483}
 484#endif /* CONFIG_PM */
 485
 486/*
 487 * The pci-driver core runtime PM routines always save the device state
 488 * before going into suspended state. If the device is going into low power
 489 * state with only with runtime PM ops, then no explicit handling is needed
 490 * for the devices which have NoSoftRst-.
 491 */
 492static const struct dev_pm_ops vfio_pci_core_pm_ops = {
 493	SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend,
 494			   vfio_pci_core_runtime_resume,
 495			   NULL)
 496};
 497
 498int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
 499{
 500	struct pci_dev *pdev = vdev->pdev;
 501	int ret;
 502	u16 cmd;
 503	u8 msix_pos;
 504
 505	if (!disable_idle_d3) {
 506		ret = pm_runtime_resume_and_get(&pdev->dev);
 507		if (ret < 0)
 508			return ret;
 509	}
 510
 511	/* Don't allow our initial saved state to include busmaster */
 512	pci_clear_master(pdev);
 513
 514	ret = pci_enable_device(pdev);
 515	if (ret)
 516		goto out_power;
 517
 518	/* If reset fails because of the device lock, fail this path entirely */
 519	ret = pci_try_reset_function(pdev);
 520	if (ret == -EAGAIN)
 521		goto out_disable_device;
 522
 523	vdev->reset_works = !ret;
 524	pci_save_state(pdev);
 525	vdev->pci_saved_state = pci_store_saved_state(pdev);
 526	if (!vdev->pci_saved_state)
 527		pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
 528
 529	if (likely(!nointxmask)) {
 530		if (vfio_pci_nointx(pdev)) {
 531			pci_info(pdev, "Masking broken INTx support\n");
 532			vdev->nointx = true;
 533			pci_intx(pdev, 0);
 534		} else
 535			vdev->pci_2_3 = pci_intx_mask_supported(pdev);
 536	}
 537
 538	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
 539	if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
 540		cmd &= ~PCI_COMMAND_INTX_DISABLE;
 541		pci_write_config_word(pdev, PCI_COMMAND, cmd);
 542	}
 543
 544	ret = vfio_pci_zdev_open_device(vdev);
 545	if (ret)
 546		goto out_free_state;
 547
 548	ret = vfio_config_init(vdev);
 549	if (ret)
 550		goto out_free_zdev;
 551
 552	msix_pos = pdev->msix_cap;
 553	if (msix_pos) {
 554		u16 flags;
 555		u32 table;
 556
 557		pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
 558		pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
 559
 560		vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
 561		vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
 562		vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
 563		vdev->has_dyn_msix = pci_msix_can_alloc_dyn(pdev);
 564	} else {
 565		vdev->msix_bar = 0xFF;
 566		vdev->has_dyn_msix = false;
 567	}
 568
 569	if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
 570		vdev->has_vga = true;
 571
 572
 573	return 0;
 574
 575out_free_zdev:
 576	vfio_pci_zdev_close_device(vdev);
 577out_free_state:
 578	kfree(vdev->pci_saved_state);
 579	vdev->pci_saved_state = NULL;
 580out_disable_device:
 581	pci_disable_device(pdev);
 582out_power:
 583	if (!disable_idle_d3)
 584		pm_runtime_put(&pdev->dev);
 585	return ret;
 586}
 587EXPORT_SYMBOL_GPL(vfio_pci_core_enable);
 588
 589void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
 590{
 591	struct pci_dev *bridge;
 592	struct pci_dev *pdev = vdev->pdev;
 593	struct vfio_pci_dummy_resource *dummy_res, *tmp;
 594	struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp;
 595	int i, bar;
 596
 597	/* For needs_reset */
 598	lockdep_assert_held(&vdev->vdev.dev_set->lock);
 599
 600	/*
 601	 * This function can be invoked while the power state is non-D0.
 602	 * This non-D0 power state can be with or without runtime PM.
 603	 * vfio_pci_runtime_pm_exit() will internally increment the usage
 604	 * count corresponding to pm_runtime_put() called during low power
 605	 * feature entry and then pm_runtime_resume() will wake up the device,
 606	 * if the device has already gone into the suspended state. Otherwise,
 607	 * the vfio_pci_set_power_state() will change the device power state
 608	 * to D0.
 609	 */
 610	vfio_pci_runtime_pm_exit(vdev);
 611	pm_runtime_resume(&pdev->dev);
 612
 613	/*
 614	 * This function calls __pci_reset_function_locked() which internally
 615	 * can use pci_pm_reset() for the function reset. pci_pm_reset() will
 616	 * fail if the power state is non-D0. Also, for the devices which
 617	 * have NoSoftRst-, the reset function can cause the PCI config space
 618	 * reset without restoring the original state (saved locally in
 619	 * 'vdev->pm_save').
 620	 */
 621	vfio_pci_set_power_state(vdev, PCI_D0);
 622
 623	/* Stop the device from further DMA */
 624	pci_clear_master(pdev);
 625
 626	vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
 627				VFIO_IRQ_SET_ACTION_TRIGGER,
 628				vdev->irq_type, 0, 0, NULL);
 629
 630	/* Device closed, don't need mutex here */
 631	list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
 632				 &vdev->ioeventfds_list, next) {
 633		vfio_virqfd_disable(&ioeventfd->virqfd);
 634		list_del(&ioeventfd->next);
 635		kfree(ioeventfd);
 636	}
 637	vdev->ioeventfds_nr = 0;
 638
 639	vdev->virq_disabled = false;
 640
 641	for (i = 0; i < vdev->num_regions; i++)
 642		vdev->region[i].ops->release(vdev, &vdev->region[i]);
 643
 644	vdev->num_regions = 0;
 645	kfree(vdev->region);
 646	vdev->region = NULL; /* don't krealloc a freed pointer */
 647
 648	vfio_config_free(vdev);
 649
 650	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
 651		bar = i + PCI_STD_RESOURCES;
 652		if (!vdev->barmap[bar])
 653			continue;
 654		pci_iounmap(pdev, vdev->barmap[bar]);
 655		pci_release_selected_regions(pdev, 1 << bar);
 656		vdev->barmap[bar] = NULL;
 657	}
 658
 659	list_for_each_entry_safe(dummy_res, tmp,
 660				 &vdev->dummy_resources_list, res_next) {
 661		list_del(&dummy_res->res_next);
 662		release_resource(&dummy_res->resource);
 663		kfree(dummy_res);
 664	}
 665
 666	vdev->needs_reset = true;
 667
 668	vfio_pci_zdev_close_device(vdev);
 669
 670	/*
 671	 * If we have saved state, restore it.  If we can reset the device,
 672	 * even better.  Resetting with current state seems better than
 673	 * nothing, but saving and restoring current state without reset
 674	 * is just busy work.
 675	 */
 676	if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
 677		pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
 678
 679		if (!vdev->reset_works)
 680			goto out;
 681
 682		pci_save_state(pdev);
 683	}
 684
 685	/*
 686	 * Disable INTx and MSI, presumably to avoid spurious interrupts
 687	 * during reset.  Stolen from pci_reset_function()
 688	 */
 689	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
 690
 691	/*
 692	 * Try to get the locks ourselves to prevent a deadlock. The
 693	 * success of this is dependent on being able to lock the device,
 694	 * which is not always possible.
 695	 * We can not use the "try" reset interface here, which will
 696	 * overwrite the previously restored configuration information.
 697	 */
 698	if (vdev->reset_works) {
 699		bridge = pci_upstream_bridge(pdev);
 700		if (bridge && !pci_dev_trylock(bridge))
 701			goto out_restore_state;
 702		if (pci_dev_trylock(pdev)) {
 703			if (!__pci_reset_function_locked(pdev))
 704				vdev->needs_reset = false;
 705			pci_dev_unlock(pdev);
 706		}
 707		if (bridge)
 708			pci_dev_unlock(bridge);
 709	}
 710
 711out_restore_state:
 712	pci_restore_state(pdev);
 713out:
 714	pci_disable_device(pdev);
 715
 716	vfio_pci_dev_set_try_reset(vdev->vdev.dev_set);
 717
 718	/* Put the pm-runtime usage counter acquired during enable */
 719	if (!disable_idle_d3)
 720		pm_runtime_put(&pdev->dev);
 721}
 722EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
 723
 724void vfio_pci_core_close_device(struct vfio_device *core_vdev)
 725{
 726	struct vfio_pci_core_device *vdev =
 727		container_of(core_vdev, struct vfio_pci_core_device, vdev);
 728
 729	if (vdev->sriov_pf_core_dev) {
 730		mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
 731		WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users);
 732		vdev->sriov_pf_core_dev->vf_token->users--;
 733		mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
 734	}
 735#if IS_ENABLED(CONFIG_EEH)
 736	eeh_dev_release(vdev->pdev);
 737#endif
 738	vfio_pci_core_disable(vdev);
 739
 740	vfio_pci_dma_buf_cleanup(vdev);
 741
 742	mutex_lock(&vdev->igate);
 743	vfio_pci_eventfd_replace_locked(vdev, &vdev->err_trigger, NULL);
 744	vfio_pci_eventfd_replace_locked(vdev, &vdev->req_trigger, NULL);
 745	mutex_unlock(&vdev->igate);
 746}
 747EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
 748
 749void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
 750{
 751	vfio_pci_probe_mmaps(vdev);
 752#if IS_ENABLED(CONFIG_EEH)
 753	eeh_dev_open(vdev->pdev);
 754#endif
 755
 756	if (vdev->sriov_pf_core_dev) {
 757		mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
 758		vdev->sriov_pf_core_dev->vf_token->users++;
 759		mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
 760	}
 761}
 762EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
 763
 764static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type)
 765{
 766	if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
 767		return vdev->vconfig[PCI_INTERRUPT_PIN] ? 1 : 0;
 768	} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
 769		u8 pos;
 770		u16 flags;
 771
 772		pos = vdev->pdev->msi_cap;
 773		if (pos) {
 774			pci_read_config_word(vdev->pdev,
 775					     pos + PCI_MSI_FLAGS, &flags);
 776			return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
 777		}
 778	} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
 779		u8 pos;
 780		u16 flags;
 781
 782		pos = vdev->pdev->msix_cap;
 783		if (pos) {
 784			pci_read_config_word(vdev->pdev,
 785					     pos + PCI_MSIX_FLAGS, &flags);
 786
 787			return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
 788		}
 789	} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
 790		if (pci_is_pcie(vdev->pdev))
 791			return 1;
 792	} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
 793		return 1;
 794	}
 795
 796	return 0;
 797}
 798
 799static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
 800{
 801	(*(int *)data)++;
 802	return 0;
 803}
 804
 805struct vfio_pci_fill_info {
 806	struct vfio_device *vdev;
 807	struct vfio_pci_dependent_device *devices;
 808	int nr_devices;
 809	u32 count;
 810	u32 flags;
 811};
 812
 813static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
 814{
 815	struct vfio_pci_dependent_device *info;
 816	struct vfio_pci_fill_info *fill = data;
 817
 818	/* The topology changed since we counted devices */
 819	if (fill->count >= fill->nr_devices)
 820		return -EAGAIN;
 821
 822	info = &fill->devices[fill->count++];
 823	info->segment = pci_domain_nr(pdev->bus);
 824	info->bus = pdev->bus->number;
 825	info->devfn = pdev->devfn;
 826
 827	if (fill->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID) {
 828		struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev);
 829		struct vfio_device_set *dev_set = fill->vdev->dev_set;
 830		struct vfio_device *vdev;
 831
 832		/*
 833		 * hot-reset requires all affected devices be represented in
 834		 * the dev_set.
 835		 */
 836		vdev = vfio_find_device_in_devset(dev_set, &pdev->dev);
 837		if (!vdev) {
 838			info->devid = VFIO_PCI_DEVID_NOT_OWNED;
 839		} else {
 840			int id = vfio_iommufd_get_dev_id(vdev, iommufd);
 841
 842			if (id > 0)
 843				info->devid = id;
 844			else if (id == -ENOENT)
 845				info->devid = VFIO_PCI_DEVID_OWNED;
 846			else
 847				info->devid = VFIO_PCI_DEVID_NOT_OWNED;
 848		}
 849		/* If devid is VFIO_PCI_DEVID_NOT_OWNED, clear owned flag. */
 850		if (info->devid == VFIO_PCI_DEVID_NOT_OWNED)
 851			fill->flags &= ~VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;
 852	} else {
 853		struct iommu_group *iommu_group;
 854
 855		iommu_group = iommu_group_get(&pdev->dev);
 856		if (!iommu_group)
 857			return -EPERM; /* Cannot reset non-isolated devices */
 858
 859		info->group_id = iommu_group_id(iommu_group);
 860		iommu_group_put(iommu_group);
 861	}
 862
 863	return 0;
 864}
 865
 866struct vfio_pci_group_info {
 867	int count;
 868	struct file **files;
 869};
 870
 871static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
 872{
 873	for (; pdev; pdev = pdev->bus->self)
 874		if (pdev->bus == slot->bus)
 875			return (pdev->slot == slot);
 876	return false;
 877}
 878
 879struct vfio_pci_walk_info {
 880	int (*fn)(struct pci_dev *pdev, void *data);
 881	void *data;
 882	struct pci_dev *pdev;
 883	bool slot;
 884	int ret;
 885};
 886
 887static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
 888{
 889	struct vfio_pci_walk_info *walk = data;
 890
 891	if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
 892		walk->ret = walk->fn(pdev, walk->data);
 893
 894	return walk->ret;
 895}
 896
 897static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
 898					 int (*fn)(struct pci_dev *,
 899						   void *data), void *data,
 900					 bool slot)
 901{
 902	struct vfio_pci_walk_info walk = {
 903		.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
 904	};
 905
 906	pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
 907
 908	return walk.ret;
 909}
 910
 911static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,
 912			      struct vfio_info_cap *caps)
 913{
 914	struct vfio_info_cap_header header = {
 915		.id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
 916		.version = 1
 917	};
 918
 919	return vfio_info_add_capability(caps, &header, sizeof(header));
 920}
 921
 922int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
 923				      unsigned int type, unsigned int subtype,
 924				      const struct vfio_pci_regops *ops,
 925				      size_t size, u32 flags, void *data)
 926{
 927	struct vfio_pci_region *region;
 928
 929	region = krealloc(vdev->region,
 930			  (vdev->num_regions + 1) * sizeof(*region),
 931			  GFP_KERNEL_ACCOUNT);
 932	if (!region)
 933		return -ENOMEM;
 934
 935	vdev->region = region;
 936	vdev->region[vdev->num_regions].type = type;
 937	vdev->region[vdev->num_regions].subtype = subtype;
 938	vdev->region[vdev->num_regions].ops = ops;
 939	vdev->region[vdev->num_regions].size = size;
 940	vdev->region[vdev->num_regions].flags = flags;
 941	vdev->region[vdev->num_regions].data = data;
 942
 943	vdev->num_regions++;
 944
 945	return 0;
 946}
 947EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region);
 948
 949static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev,
 950				    struct vfio_info_cap *caps)
 951{
 952	struct vfio_device_info_cap_pci_atomic_comp cap = {
 953		.header.id = VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP,
 954		.header.version = 1
 955	};
 956	struct pci_dev *pdev = pci_physfn(vdev->pdev);
 957	u32 devcap2;
 958
 959	pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &devcap2);
 960
 961	if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
 962	    !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32))
 963		cap.flags |= VFIO_PCI_ATOMIC_COMP32;
 964
 965	if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
 966	    !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64))
 967		cap.flags |= VFIO_PCI_ATOMIC_COMP64;
 968
 969	if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP128) &&
 970	    !pci_enable_atomic_ops_to_root(pdev,
 971					   PCI_EXP_DEVCAP2_ATOMIC_COMP128))
 972		cap.flags |= VFIO_PCI_ATOMIC_COMP128;
 973
 974	if (!cap.flags)
 975		return -ENODEV;
 976
 977	return vfio_info_add_capability(caps, &cap.header, sizeof(cap));
 978}
 979
 980static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
 981				   struct vfio_device_info __user *arg)
 982{
 983	unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs);
 984	struct vfio_device_info info = {};
 985	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
 986	int ret;
 987
 988	if (copy_from_user(&info, arg, minsz))
 989		return -EFAULT;
 990
 991	if (info.argsz < minsz)
 992		return -EINVAL;
 993
 994	minsz = min_t(size_t, info.argsz, sizeof(info));
 995
 996	info.flags = VFIO_DEVICE_FLAGS_PCI;
 997
 998	if (vdev->reset_works)
 999		info.flags |= VFIO_DEVICE_FLAGS_RESET;
1000
1001	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
1002	info.num_irqs = VFIO_PCI_NUM_IRQS;
1003
1004	ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
1005	if (ret && ret != -ENODEV) {
1006		pci_warn(vdev->pdev,
1007			 "Failed to setup zPCI info capabilities\n");
1008		return ret;
1009	}
1010
1011	ret = vfio_pci_info_atomic_cap(vdev, &caps);
1012	if (ret && ret != -ENODEV) {
1013		pci_warn(vdev->pdev,
1014			 "Failed to setup AtomicOps info capability\n");
1015		return ret;
1016	}
1017
1018	if (caps.size) {
1019		info.flags |= VFIO_DEVICE_FLAGS_CAPS;
1020		if (info.argsz < sizeof(info) + caps.size) {
1021			info.argsz = sizeof(info) + caps.size;
1022		} else {
1023			vfio_info_cap_shift(&caps, sizeof(info));
1024			if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1025				kfree(caps.buf);
1026				return -EFAULT;
1027			}
1028			info.cap_offset = sizeof(*arg);
1029		}
1030
1031		kfree(caps.buf);
1032	}
1033
1034	return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1035}
1036
1037int vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
1038				   struct vfio_region_info *info,
1039				   struct vfio_info_cap *caps)
1040{
1041	struct vfio_pci_core_device *vdev =
1042		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1043	struct pci_dev *pdev = vdev->pdev;
1044	int i, ret;
1045
1046	switch (info->index) {
1047	case VFIO_PCI_CONFIG_REGION_INDEX:
1048		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1049		info->size = pdev->cfg_size;
1050		info->flags = VFIO_REGION_INFO_FLAG_READ |
1051			      VFIO_REGION_INFO_FLAG_WRITE;
1052		break;
1053	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1054		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1055		info->size = pci_resource_len(pdev, info->index);
1056		if (!info->size) {
1057			info->flags = 0;
1058			break;
1059		}
1060
1061		info->flags = VFIO_REGION_INFO_FLAG_READ |
1062			      VFIO_REGION_INFO_FLAG_WRITE;
1063		if (vdev->bar_mmap_supported[info->index]) {
1064			info->flags |= VFIO_REGION_INFO_FLAG_MMAP;
1065			if (info->index == vdev->msix_bar) {
1066				ret = msix_mmappable_cap(vdev, caps);
1067				if (ret)
1068					return ret;
1069			}
1070		}
1071
1072		break;
1073	case VFIO_PCI_ROM_REGION_INDEX: {
1074		void __iomem *io;
1075		size_t size;
1076		u16 cmd;
1077
1078		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1079		info->flags = 0;
1080		info->size = 0;
1081
1082		if (pci_resource_start(pdev, PCI_ROM_RESOURCE)) {
1083			/*
1084			 * Check ROM content is valid. Need to enable memory
1085			 * decode for ROM access in pci_map_rom().
1086			 */
1087			cmd = vfio_pci_memory_lock_and_enable(vdev);
1088			io = pci_map_rom(pdev, &size);
1089			if (io) {
1090				info->flags = VFIO_REGION_INFO_FLAG_READ;
1091				/* Report the BAR size, not the ROM size. */
1092				info->size = pci_resource_len(pdev,
1093							      PCI_ROM_RESOURCE);
1094				pci_unmap_rom(pdev, io);
1095			}
1096			vfio_pci_memory_unlock_and_restore(vdev, cmd);
1097		} else if (pdev->rom && pdev->romlen) {
1098			info->flags = VFIO_REGION_INFO_FLAG_READ;
1099			/* Report BAR size as power of two. */
1100			info->size = roundup_pow_of_two(pdev->romlen);
1101		}
1102
1103		break;
1104	}
1105	case VFIO_PCI_VGA_REGION_INDEX:
1106		if (!vdev->has_vga)
1107			return -EINVAL;
1108
1109		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1110		info->size = 0xc0000;
1111		info->flags = VFIO_REGION_INFO_FLAG_READ |
1112			      VFIO_REGION_INFO_FLAG_WRITE;
1113
1114		break;
1115	default: {
1116		struct vfio_region_info_cap_type cap_type = {
1117			.header.id = VFIO_REGION_INFO_CAP_TYPE,
1118			.header.version = 1
1119		};
1120
1121		if (info->index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1122			return -EINVAL;
1123		info->index = array_index_nospec(
1124			info->index, VFIO_PCI_NUM_REGIONS + vdev->num_regions);
1125
1126		i = info->index - VFIO_PCI_NUM_REGIONS;
1127
1128		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1129		info->size = vdev->region[i].size;
1130		info->flags = vdev->region[i].flags;
1131
1132		cap_type.type = vdev->region[i].type;
1133		cap_type.subtype = vdev->region[i].subtype;
1134
1135		ret = vfio_info_add_capability(caps, &cap_type.header,
1136					       sizeof(cap_type));
1137		if (ret)
1138			return ret;
1139
1140		if (vdev->region[i].ops->add_capability) {
1141			ret = vdev->region[i].ops->add_capability(
1142				vdev, &vdev->region[i], caps);
1143			if (ret)
1144				return ret;
1145		}
1146	}
1147	}
1148	return 0;
1149}
1150EXPORT_SYMBOL_GPL(vfio_pci_ioctl_get_region_info);
1151
1152static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev,
1153				       struct vfio_irq_info __user *arg)
1154{
1155	unsigned long minsz = offsetofend(struct vfio_irq_info, count);
1156	struct vfio_irq_info info;
1157
1158	if (copy_from_user(&info, arg, minsz))
1159		return -EFAULT;
1160
1161	if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1162		return -EINVAL;
1163
1164	switch (info.index) {
1165	case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
1166	case VFIO_PCI_REQ_IRQ_INDEX:
1167		break;
1168	case VFIO_PCI_ERR_IRQ_INDEX:
1169		if (pci_is_pcie(vdev->pdev))
1170			break;
1171		fallthrough;
1172	default:
1173		return -EINVAL;
1174	}
1175
1176	info.flags = VFIO_IRQ_INFO_EVENTFD;
1177
1178	info.count = vfio_pci_get_irq_count(vdev, info.index);
1179
1180	if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1181		info.flags |=
1182			(VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED);
1183	else if (info.index != VFIO_PCI_MSIX_IRQ_INDEX || !vdev->has_dyn_msix)
1184		info.flags |= VFIO_IRQ_INFO_NORESIZE;
1185
1186	return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1187}
1188
1189static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,
1190				   struct vfio_irq_set __user *arg)
1191{
1192	unsigned long minsz = offsetofend(struct vfio_irq_set, count);
1193	struct vfio_irq_set hdr;
1194	u8 *data = NULL;
1195	int max, ret = 0;
1196	size_t data_size = 0;
1197
1198	if (copy_from_user(&hdr, arg, minsz))
1199		return -EFAULT;
1200
1201	max = vfio_pci_get_irq_count(vdev, hdr.index);
1202
1203	ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS,
1204						 &data_size);
1205	if (ret)
1206		return ret;
1207
1208	if (data_size) {
1209		data = memdup_user(&arg->data, data_size);
1210		if (IS_ERR(data))
1211			return PTR_ERR(data);
1212	}
1213
1214	mutex_lock(&vdev->igate);
1215
1216	ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start,
1217				      hdr.count, data);
1218
1219	mutex_unlock(&vdev->igate);
1220	kfree(data);
1221
1222	return ret;
1223}
1224
1225static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
1226				void __user *arg)
1227{
1228	int ret;
1229
1230	if (!vdev->reset_works)
1231		return -EINVAL;
1232
1233	vfio_pci_zap_and_down_write_memory_lock(vdev);
1234
1235	/*
1236	 * This function can be invoked while the power state is non-D0. If
1237	 * pci_try_reset_function() has been called while the power state is
1238	 * non-D0, then pci_try_reset_function() will internally set the power
1239	 * state to D0 without vfio driver involvement. For the devices which
1240	 * have NoSoftRst-, the reset function can cause the PCI config space
1241	 * reset without restoring the original state (saved locally in
1242	 * 'vdev->pm_save').
1243	 */
1244	vfio_pci_set_power_state(vdev, PCI_D0);
1245
1246	vfio_pci_dma_buf_move(vdev, true);
1247	ret = pci_try_reset_function(vdev->pdev);
1248	if (__vfio_pci_memory_enabled(vdev))
1249		vfio_pci_dma_buf_move(vdev, false);
1250	up_write(&vdev->memory_lock);
1251
1252	return ret;
1253}
1254
1255static int vfio_pci_ioctl_get_pci_hot_reset_info(
1256	struct vfio_pci_core_device *vdev,
1257	struct vfio_pci_hot_reset_info __user *arg)
1258{
1259	unsigned long minsz =
1260		offsetofend(struct vfio_pci_hot_reset_info, count);
1261	struct vfio_pci_dependent_device *devices = NULL;
1262	struct vfio_pci_hot_reset_info hdr;
1263	struct vfio_pci_fill_info fill = {};
1264	bool slot = false;
1265	int ret, count = 0;
1266
1267	if (copy_from_user(&hdr, arg, minsz))
1268		return -EFAULT;
1269
1270	if (hdr.argsz < minsz)
1271		return -EINVAL;
1272
1273	hdr.flags = 0;
1274
1275	/* Can we do a slot or bus reset or neither? */
1276	if (!pci_probe_reset_slot(vdev->pdev->slot))
1277		slot = true;
1278	else if (pci_probe_reset_bus(vdev->pdev->bus))
1279		return -ENODEV;
1280
1281	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1282					    &count, slot);
1283	if (ret)
1284		return ret;
1285
1286	if (WARN_ON(!count)) /* Should always be at least one */
1287		return -ERANGE;
1288
1289	if (count > (hdr.argsz - sizeof(hdr)) / sizeof(*devices)) {
1290		hdr.count = count;
1291		ret = -ENOSPC;
1292		goto header;
1293	}
1294
1295	devices = kcalloc(count, sizeof(*devices), GFP_KERNEL);
1296	if (!devices)
1297		return -ENOMEM;
1298
1299	fill.devices = devices;
1300	fill.nr_devices = count;
1301	fill.vdev = &vdev->vdev;
1302
1303	if (vfio_device_cdev_opened(&vdev->vdev))
1304		fill.flags |= VFIO_PCI_HOT_RESET_FLAG_DEV_ID |
1305			     VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;
1306
1307	mutex_lock(&vdev->vdev.dev_set->lock);
1308	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs,
1309					    &fill, slot);
1310	mutex_unlock(&vdev->vdev.dev_set->lock);
1311	if (ret)
1312		goto out;
1313
1314	if (copy_to_user(arg->devices, devices,
1315			 sizeof(*devices) * fill.count)) {
1316		ret = -EFAULT;
1317		goto out;
1318	}
1319
1320	hdr.count = fill.count;
1321	hdr.flags = fill.flags;
1322
1323header:
1324	if (copy_to_user(arg, &hdr, minsz))
1325		ret = -EFAULT;
1326out:
1327	kfree(devices);
1328	return ret;
1329}
1330
1331static int
1332vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device *vdev,
1333				    u32 array_count, bool slot,
1334				    struct vfio_pci_hot_reset __user *arg)
1335{
1336	int32_t *group_fds;
1337	struct file **files;
1338	struct vfio_pci_group_info info;
1339	int file_idx, count = 0, ret = 0;
1340
1341	/*
1342	 * We can't let userspace give us an arbitrarily large buffer to copy,
1343	 * so verify how many we think there could be.  Note groups can have
1344	 * multiple devices so one group per device is the max.
1345	 */
1346	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1347					    &count, slot);
1348	if (ret)
1349		return ret;
1350
1351	if (array_count > count)
1352		return -EINVAL;
1353
1354	group_fds = kcalloc(array_count, sizeof(*group_fds), GFP_KERNEL);
1355	files = kcalloc(array_count, sizeof(*files), GFP_KERNEL);
1356	if (!group_fds || !files) {
1357		kfree(group_fds);
1358		kfree(files);
1359		return -ENOMEM;
1360	}
1361
1362	if (copy_from_user(group_fds, arg->group_fds,
1363			   array_count * sizeof(*group_fds))) {
1364		kfree(group_fds);
1365		kfree(files);
1366		return -EFAULT;
1367	}
1368
1369	/*
1370	 * Get the group file for each fd to ensure the group is held across
1371	 * the reset
1372	 */
1373	for (file_idx = 0; file_idx < array_count; file_idx++) {
1374		struct file *file = fget(group_fds[file_idx]);
1375
1376		if (!file) {
1377			ret = -EBADF;
1378			break;
1379		}
1380
1381		/* Ensure the FD is a vfio group FD.*/
1382		if (!vfio_file_is_group(file)) {
1383			fput(file);
1384			ret = -EINVAL;
1385			break;
1386		}
1387
1388		files[file_idx] = file;
1389	}
1390
1391	kfree(group_fds);
1392
1393	/* release reference to groups on error */
1394	if (ret)
1395		goto hot_reset_release;
1396
1397	info.count = array_count;
1398	info.files = files;
1399
1400	ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info, NULL);
1401
1402hot_reset_release:
1403	for (file_idx--; file_idx >= 0; file_idx--)
1404		fput(files[file_idx]);
1405
1406	kfree(files);
1407	return ret;
1408}
1409
1410static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev,
1411					struct vfio_pci_hot_reset __user *arg)
1412{
1413	unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count);
1414	struct vfio_pci_hot_reset hdr;
1415	bool slot = false;
1416
1417	if (copy_from_user(&hdr, arg, minsz))
1418		return -EFAULT;
1419
1420	if (hdr.argsz < minsz || hdr.flags)
1421		return -EINVAL;
1422
1423	/* zero-length array is only for cdev opened devices */
1424	if (!!hdr.count == vfio_device_cdev_opened(&vdev->vdev))
1425		return -EINVAL;
1426
1427	/* Can we do a slot or bus reset or neither? */
1428	if (!pci_probe_reset_slot(vdev->pdev->slot))
1429		slot = true;
1430	else if (pci_probe_reset_bus(vdev->pdev->bus))
1431		return -ENODEV;
1432
1433	if (hdr.count)
1434		return vfio_pci_ioctl_pci_hot_reset_groups(vdev, hdr.count, slot, arg);
1435
1436	return vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, NULL,
1437					  vfio_iommufd_device_ictx(&vdev->vdev));
1438}
1439
1440static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev,
1441				    struct vfio_device_ioeventfd __user *arg)
1442{
1443	unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd);
1444	struct vfio_device_ioeventfd ioeventfd;
1445	int count;
1446
1447	if (copy_from_user(&ioeventfd, arg, minsz))
1448		return -EFAULT;
1449
1450	if (ioeventfd.argsz < minsz)
1451		return -EINVAL;
1452
1453	if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
1454		return -EINVAL;
1455
1456	count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
1457
1458	if (hweight8(count) != 1 || ioeventfd.fd < -1)
1459		return -EINVAL;
1460
1461	return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count,
1462				  ioeventfd.fd);
1463}
1464
1465long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
1466			 unsigned long arg)
1467{
1468	struct vfio_pci_core_device *vdev =
1469		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1470	void __user *uarg = (void __user *)arg;
1471
1472	switch (cmd) {
1473	case VFIO_DEVICE_GET_INFO:
1474		return vfio_pci_ioctl_get_info(vdev, uarg);
1475	case VFIO_DEVICE_GET_IRQ_INFO:
1476		return vfio_pci_ioctl_get_irq_info(vdev, uarg);
1477	case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO:
1478		return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg);
1479	case VFIO_DEVICE_IOEVENTFD:
1480		return vfio_pci_ioctl_ioeventfd(vdev, uarg);
1481	case VFIO_DEVICE_PCI_HOT_RESET:
1482		return vfio_pci_ioctl_pci_hot_reset(vdev, uarg);
1483	case VFIO_DEVICE_RESET:
1484		return vfio_pci_ioctl_reset(vdev, uarg);
1485	case VFIO_DEVICE_SET_IRQS:
1486		return vfio_pci_ioctl_set_irqs(vdev, uarg);
1487	default:
1488		return -ENOTTY;
1489	}
1490}
1491EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);
1492
1493static int vfio_pci_core_feature_token(struct vfio_pci_core_device *vdev,
1494				       u32 flags, uuid_t __user *arg,
1495				       size_t argsz)
1496{
1497	uuid_t uuid;
1498	int ret;
1499
1500	if (!vdev->vf_token)
1501		return -ENOTTY;
1502	/*
1503	 * We do not support GET of the VF Token UUID as this could
1504	 * expose the token of the previous device user.
1505	 */
1506	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
1507				 sizeof(uuid));
1508	if (ret != 1)
1509		return ret;
1510
1511	if (copy_from_user(&uuid, arg, sizeof(uuid)))
1512		return -EFAULT;
1513
1514	mutex_lock(&vdev->vf_token->lock);
1515	uuid_copy(&vdev->vf_token->uuid, &uuid);
1516	mutex_unlock(&vdev->vf_token->lock);
1517	return 0;
1518}
1519
1520int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
1521				void __user *arg, size_t argsz)
1522{
1523	struct vfio_pci_core_device *vdev =
1524		container_of(device, struct vfio_pci_core_device, vdev);
1525
1526	switch (flags & VFIO_DEVICE_FEATURE_MASK) {
1527	case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY:
1528		return vfio_pci_core_pm_entry(vdev, flags, arg, argsz);
1529	case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP:
1530		return vfio_pci_core_pm_entry_with_wakeup(vdev, flags,
1531							  arg, argsz);
1532	case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT:
1533		return vfio_pci_core_pm_exit(vdev, flags, arg, argsz);
1534	case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
1535		return vfio_pci_core_feature_token(vdev, flags, arg, argsz);
1536	case VFIO_DEVICE_FEATURE_DMA_BUF:
1537		return vfio_pci_core_feature_dma_buf(vdev, flags, arg, argsz);
1538	default:
1539		return -ENOTTY;
1540	}
1541}
1542EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl_feature);
1543
1544static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf,
1545			   size_t count, loff_t *ppos, bool iswrite)
1546{
1547	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1548	int ret;
1549
1550	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1551		return -EINVAL;
1552
1553	ret = pm_runtime_resume_and_get(&vdev->pdev->dev);
1554	if (ret) {
1555		pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n",
1556				     ret);
1557		return -EIO;
1558	}
1559
1560	switch (index) {
1561	case VFIO_PCI_CONFIG_REGION_INDEX:
1562		ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1563		break;
1564
1565	case VFIO_PCI_ROM_REGION_INDEX:
1566		if (iswrite)
1567			ret = -EINVAL;
1568		else
1569			ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1570		break;
1571
1572	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1573		ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1574		break;
1575
1576	case VFIO_PCI_VGA_REGION_INDEX:
1577		ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1578		break;
1579
1580	default:
1581		index -= VFIO_PCI_NUM_REGIONS;
1582		ret = vdev->region[index].ops->rw(vdev, buf,
1583						   count, ppos, iswrite);
1584		break;
1585	}
1586
1587	pm_runtime_put(&vdev->pdev->dev);
1588	return ret;
1589}
1590
1591ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
1592		size_t count, loff_t *ppos)
1593{
1594	struct vfio_pci_core_device *vdev =
1595		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1596
1597	if (!count)
1598		return 0;
1599
1600	return vfio_pci_rw(vdev, buf, count, ppos, false);
1601}
1602EXPORT_SYMBOL_GPL(vfio_pci_core_read);
1603
1604ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
1605		size_t count, loff_t *ppos)
1606{
1607	struct vfio_pci_core_device *vdev =
1608		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1609
1610	if (!count)
1611		return 0;
1612
1613	return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
1614}
1615EXPORT_SYMBOL_GPL(vfio_pci_core_write);
1616
1617static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev)
1618{
1619	struct vfio_device *core_vdev = &vdev->vdev;
1620	loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);
1621	loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);
1622	loff_t len = end - start;
1623
1624	unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true);
1625}
1626
1627void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
1628{
1629	down_write(&vdev->memory_lock);
1630	vfio_pci_zap_bars(vdev);
1631}
1632
1633u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)
1634{
1635	u16 cmd;
1636
1637	down_write(&vdev->memory_lock);
1638	pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
1639	if (!(cmd & PCI_COMMAND_MEMORY))
1640		pci_write_config_word(vdev->pdev, PCI_COMMAND,
1641				      cmd | PCI_COMMAND_MEMORY);
1642
1643	return cmd;
1644}
1645
1646void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd)
1647{
1648	pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
1649	up_write(&vdev->memory_lock);
1650}
1651
1652static unsigned long vma_to_pfn(struct vm_area_struct *vma)
1653{
1654	struct vfio_pci_core_device *vdev = vma->vm_private_data;
1655	int index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1656	u64 pgoff;
1657
1658	pgoff = vma->vm_pgoff &
1659		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1660
1661	return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff;
1662}
1663
1664vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
1665				   struct vm_fault *vmf,
1666				   unsigned long pfn,
1667				   unsigned int order)
1668{
1669	lockdep_assert_held_read(&vdev->memory_lock);
1670
1671	if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev))
1672		return VM_FAULT_SIGBUS;
1673
1674	switch (order) {
1675	case 0:
1676		return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
1677#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
1678	case PMD_ORDER:
1679		return vmf_insert_pfn_pmd(vmf, pfn, false);
1680#endif
1681#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
1682	case PUD_ORDER:
1683		return vmf_insert_pfn_pud(vmf, pfn, false);
1684		break;
1685#endif
1686	default:
1687		return VM_FAULT_FALLBACK;
1688	}
1689}
1690EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn);
1691
1692static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,
1693					   unsigned int order)
1694{
1695	struct vm_area_struct *vma = vmf->vma;
1696	struct vfio_pci_core_device *vdev = vma->vm_private_data;
1697	unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1);
1698	unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
1699	unsigned long pfn = vma_to_pfn(vma) + pgoff;
1700	vm_fault_t ret = VM_FAULT_FALLBACK;
1701
1702	if (is_aligned_for_order(vma, addr, pfn, order)) {
1703		scoped_guard(rwsem_read, &vdev->memory_lock)
1704			ret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order);
1705	}
1706
1707	dev_dbg_ratelimited(&vdev->pdev->dev,
1708			   "%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\n",
1709			    __func__, order,
1710			    vma->vm_pgoff >>
1711				(VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT),
1712			    pgoff, (unsigned int)ret);
1713
1714	return ret;
1715}
1716
1717static vm_fault_t vfio_pci_mmap_page_fault(struct vm_fault *vmf)
1718{
1719	return vfio_pci_mmap_huge_fault(vmf, 0);
1720}
1721
1722static const struct vm_operations_struct vfio_pci_mmap_ops = {
1723	.fault = vfio_pci_mmap_page_fault,
1724#ifdef CONFIG_ARCH_SUPPORTS_HUGE_PFNMAP
1725	.huge_fault = vfio_pci_mmap_huge_fault,
1726#endif
1727};
1728
1729int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
1730{
1731	struct vfio_pci_core_device *vdev =
1732		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1733	struct pci_dev *pdev = vdev->pdev;
1734	unsigned int index;
1735	u64 phys_len, req_len, pgoff, req_start;
1736	int ret;
1737
1738	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1739
1740	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1741		return -EINVAL;
1742	if (vma->vm_end < vma->vm_start)
1743		return -EINVAL;
1744	if ((vma->vm_flags & VM_SHARED) == 0)
1745		return -EINVAL;
1746	if (index >= VFIO_PCI_NUM_REGIONS) {
1747		int regnum = index - VFIO_PCI_NUM_REGIONS;
1748		struct vfio_pci_region *region = vdev->region + regnum;
1749
1750		if (region->ops && region->ops->mmap &&
1751		    (region->flags & VFIO_REGION_INFO_FLAG_MMAP))
1752			return region->ops->mmap(vdev, region, vma);
1753		return -EINVAL;
1754	}
1755	if (index >= VFIO_PCI_ROM_REGION_INDEX)
1756		return -EINVAL;
1757	if (!vdev->bar_mmap_supported[index])
1758		return -EINVAL;
1759
1760	phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1761	req_len = vma->vm_end - vma->vm_start;
1762	pgoff = vma->vm_pgoff &
1763		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1764	req_start = pgoff << PAGE_SHIFT;
1765
1766	if (req_start + req_len > phys_len)
1767		return -EINVAL;
1768
1769	/*
1770	 * Even though we don't make use of the barmap for the mmap,
1771	 * we need to request the region and the barmap tracks that.
1772	 */
1773	ret = vfio_pci_core_setup_barmap(vdev, index);
1774	if (ret)
1775		return ret;
1776
1777	vma->vm_private_data = vdev;
1778	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1779	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1780
1781	/*
1782	 * Set vm_flags now, they should not be changed in the fault handler.
1783	 * We want the same flags and page protection (decrypted above) as
1784	 * io_remap_pfn_range() would set.
1785	 *
1786	 * VM_ALLOW_ANY_UNCACHED: The VMA flag is implemented for ARM64,
1787	 * allowing KVM stage 2 device mapping attributes to use Normal-NC
1788	 * rather than DEVICE_nGnRE, which allows guest mappings
1789	 * supporting write-combining attributes (WC). ARM does not
1790	 * architecturally guarantee this is safe, and indeed some MMIO
1791	 * regions like the GICv2 VCPU interface can trigger uncontained
1792	 * faults if Normal-NC is used.
1793	 *
1794	 * To safely use VFIO in KVM the platform must guarantee full
1795	 * safety in the guest where no action taken against a MMIO
1796	 * mapping can trigger an uncontained failure. The assumption is
1797	 * that most VFIO PCI platforms support this for both mapping types,
1798	 * at least in common flows, based on some expectations of how
1799	 * PCI IP is integrated. Hence VM_ALLOW_ANY_UNCACHED is set in
1800	 * the VMA flags.
1801	 */
1802	vm_flags_set(vma, VM_ALLOW_ANY_UNCACHED | VM_IO | VM_PFNMAP |
1803			VM_DONTEXPAND | VM_DONTDUMP);
1804	vma->vm_ops = &vfio_pci_mmap_ops;
1805
1806	return 0;
1807}
1808EXPORT_SYMBOL_GPL(vfio_pci_core_mmap);
1809
1810void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count)
1811{
1812	struct vfio_pci_core_device *vdev =
1813		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1814	struct pci_dev *pdev = vdev->pdev;
1815	struct vfio_pci_eventfd *eventfd;
1816
1817	rcu_read_lock();
1818	eventfd = rcu_dereference(vdev->req_trigger);
1819	if (eventfd) {
1820		if (!(count % 10))
1821			pci_notice_ratelimited(pdev,
1822				"Relaying device request to user (#%u)\n",
1823				count);
1824		eventfd_signal(eventfd->ctx);
1825	} else if (count == 0) {
1826		pci_warn(pdev,
1827			"No device request channel registered, blocked until released by user\n");
1828	}
1829	rcu_read_unlock();
1830}
1831EXPORT_SYMBOL_GPL(vfio_pci_core_request);
1832
1833int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev,
1834				   const uuid_t *uuid)
1835
1836{
1837	struct vfio_pci_core_device *vdev =
1838		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1839
1840	/*
1841	 * There's always some degree of trust or collaboration between SR-IOV
1842	 * PF and VFs, even if just that the PF hosts the SR-IOV capability and
1843	 * can disrupt VFs with a reset, but often the PF has more explicit
1844	 * access to deny service to the VF or access data passed through the
1845	 * VF.  We therefore require an opt-in via a shared VF token (UUID) to
1846	 * represent this trust.  This both prevents that a VF driver might
1847	 * assume the PF driver is a trusted, in-kernel driver, and also that
1848	 * a PF driver might be replaced with a rogue driver, unknown to in-use
1849	 * VF drivers.
1850	 *
1851	 * Therefore when presented with a VF, if the PF is a vfio device and
1852	 * it is bound to the vfio-pci driver, the user needs to provide a VF
1853	 * token to access the device, in the form of appending a vf_token to
1854	 * the device name, for example:
1855	 *
1856	 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
1857	 *
1858	 * When presented with a PF which has VFs in use, the user must also
1859	 * provide the current VF token to prove collaboration with existing
1860	 * VF users.  If VFs are not in use, the VF token provided for the PF
1861	 * device will act to set the VF token.
1862	 *
1863	 * If the VF token is provided but unused, an error is generated.
1864	 */
1865	if (vdev->pdev->is_virtfn) {
1866		struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev;
1867		bool match;
1868
1869		if (!pf_vdev) {
1870			if (!uuid)
1871				return 0; /* PF is not vfio-pci, no VF token */
1872
1873			pci_info_ratelimited(vdev->pdev,
1874				"VF token incorrectly provided, PF not bound to vfio-pci\n");
1875			return -EINVAL;
1876		}
1877
1878		if (!uuid) {
1879			pci_info_ratelimited(vdev->pdev,
1880				"VF token required to access device\n");
1881			return -EACCES;
1882		}
1883
1884		mutex_lock(&pf_vdev->vf_token->lock);
1885		match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
1886		mutex_unlock(&pf_vdev->vf_token->lock);
1887
1888		if (!match) {
1889			pci_info_ratelimited(vdev->pdev,
1890				"Incorrect VF token provided for device\n");
1891			return -EACCES;
1892		}
1893	} else if (vdev->vf_token) {
1894		mutex_lock(&vdev->vf_token->lock);
1895		if (vdev->vf_token->users) {
1896			if (!uuid) {
1897				mutex_unlock(&vdev->vf_token->lock);
1898				pci_info_ratelimited(vdev->pdev,
1899					"VF token required to access device\n");
1900				return -EACCES;
1901			}
1902
1903			if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
1904				mutex_unlock(&vdev->vf_token->lock);
1905				pci_info_ratelimited(vdev->pdev,
1906					"Incorrect VF token provided for device\n");
1907				return -EACCES;
1908			}
1909		} else if (uuid) {
1910			uuid_copy(&vdev->vf_token->uuid, uuid);
1911		}
1912
1913		mutex_unlock(&vdev->vf_token->lock);
1914	} else if (uuid) {
1915		pci_info_ratelimited(vdev->pdev,
1916			"VF token incorrectly provided, not a PF or VF\n");
1917		return -EINVAL;
1918	}
1919
1920	return 0;
1921}
1922EXPORT_SYMBOL_GPL(vfio_pci_core_match_token_uuid);
1923
1924#define VF_TOKEN_ARG "vf_token="
1925
1926int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf)
1927{
1928	struct vfio_pci_core_device *vdev =
1929		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1930	bool vf_token = false;
1931	uuid_t uuid;
1932	int ret;
1933
1934	if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
1935		return 0; /* No match */
1936
1937	if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
1938		buf += strlen(pci_name(vdev->pdev));
1939
1940		if (*buf != ' ')
1941			return 0; /* No match: non-whitespace after name */
1942
1943		while (*buf) {
1944			if (*buf == ' ') {
1945				buf++;
1946				continue;
1947			}
1948
1949			if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
1950						  strlen(VF_TOKEN_ARG))) {
1951				buf += strlen(VF_TOKEN_ARG);
1952
1953				if (strlen(buf) < UUID_STRING_LEN)
1954					return -EINVAL;
1955
1956				ret = uuid_parse(buf, &uuid);
1957				if (ret)
1958					return ret;
1959
1960				vf_token = true;
1961				buf += UUID_STRING_LEN;
1962			} else {
1963				/* Unknown/duplicate option */
1964				return -EINVAL;
1965			}
1966		}
1967	}
1968
1969	ret = core_vdev->ops->match_token_uuid(core_vdev,
1970					       vf_token ? &uuid : NULL);
1971	if (ret)
1972		return ret;
1973
1974	return 1; /* Match */
1975}
1976EXPORT_SYMBOL_GPL(vfio_pci_core_match);
1977
1978static int vfio_pci_bus_notifier(struct notifier_block *nb,
1979				 unsigned long action, void *data)
1980{
1981	struct vfio_pci_core_device *vdev = container_of(nb,
1982						    struct vfio_pci_core_device, nb);
1983	struct device *dev = data;
1984	struct pci_dev *pdev = to_pci_dev(dev);
1985	struct pci_dev *physfn = pci_physfn(pdev);
1986
1987	if (action == BUS_NOTIFY_ADD_DEVICE &&
1988	    pdev->is_virtfn && physfn == vdev->pdev) {
1989		pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
1990			 pci_name(pdev));
1991		pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
1992						  vdev->vdev.ops->name);
1993		WARN_ON(!pdev->driver_override);
1994	} else if (action == BUS_NOTIFY_BOUND_DRIVER &&
1995		   pdev->is_virtfn && physfn == vdev->pdev) {
1996		struct pci_driver *drv = pci_dev_driver(pdev);
1997
1998		if (drv && drv != pci_dev_driver(vdev->pdev))
1999			pci_warn(vdev->pdev,
2000				 "VF %s bound to driver %s while PF bound to driver %s\n",
2001				 pci_name(pdev), drv->name,
2002				 pci_dev_driver(vdev->pdev)->name);
2003	}
2004
2005	return 0;
2006}
2007
2008static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)
2009{
2010	struct pci_dev *pdev = vdev->pdev;
2011	struct vfio_pci_core_device *cur;
2012	struct pci_dev *physfn;
2013	int ret;
2014
2015	if (pdev->is_virtfn) {
2016		/*
2017		 * If this VF was created by our vfio_pci_core_sriov_configure()
2018		 * then we can find the PF vfio_pci_core_device now, and due to
2019		 * the locking in pci_disable_sriov() it cannot change until
2020		 * this VF device driver is removed.
2021		 */
2022		physfn = pci_physfn(vdev->pdev);
2023		mutex_lock(&vfio_pci_sriov_pfs_mutex);
2024		list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) {
2025			if (cur->pdev == physfn) {
2026				vdev->sriov_pf_core_dev = cur;
2027				break;
2028			}
2029		}
2030		mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2031		return 0;
2032	}
2033
2034	/* Not a SRIOV PF */
2035	if (!pdev->is_physfn)
2036		return 0;
2037
2038	vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
2039	if (!vdev->vf_token)
2040		return -ENOMEM;
2041
2042	mutex_init(&vdev->vf_token->lock);
2043	uuid_gen(&vdev->vf_token->uuid);
2044
2045	vdev->nb.notifier_call = vfio_pci_bus_notifier;
2046	ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
2047	if (ret) {
2048		kfree(vdev->vf_token);
2049		return ret;
2050	}
2051	return 0;
2052}
2053
2054static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev)
2055{
2056	if (!vdev->vf_token)
2057		return;
2058
2059	bus_unregister_notifier(&pci_bus_type, &vdev->nb);
2060	WARN_ON(vdev->vf_token->users);
2061	mutex_destroy(&vdev->vf_token->lock);
2062	kfree(vdev->vf_token);
2063}
2064
2065static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev)
2066{
2067	struct pci_dev *pdev = vdev->pdev;
2068	int ret;
2069
2070	if (!vfio_pci_is_vga(pdev))
2071		return 0;
2072
2073	ret = aperture_remove_conflicting_pci_devices(pdev, vdev->vdev.ops->name);
2074	if (ret)
2075		return ret;
2076
2077	ret = vga_client_register(pdev, vfio_pci_set_decode);
2078	if (ret)
2079		return ret;
2080	vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false));
2081	return 0;
2082}
2083
2084static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev)
2085{
2086	struct pci_dev *pdev = vdev->pdev;
2087
2088	if (!vfio_pci_is_vga(pdev))
2089		return;
2090	vga_client_unregister(pdev);
2091	vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
2092					      VGA_RSRC_LEGACY_IO |
2093					      VGA_RSRC_LEGACY_MEM);
2094}
2095
2096int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
2097{
2098	struct vfio_pci_core_device *vdev =
2099		container_of(core_vdev, struct vfio_pci_core_device, vdev);
2100	int ret;
2101
2102	vdev->pdev = to_pci_dev(core_vdev->dev);
2103	vdev->irq_type = VFIO_PCI_NUM_IRQS;
2104	mutex_init(&vdev->igate);
2105	spin_lock_init(&vdev->irqlock);
2106	mutex_init(&vdev->ioeventfds_lock);
2107	INIT_LIST_HEAD(&vdev->dummy_resources_list);
2108	INIT_LIST_HEAD(&vdev->ioeventfds_list);
2109	INIT_LIST_HEAD(&vdev->sriov_pfs_item);
2110	ret = pcim_p2pdma_init(vdev->pdev);
2111	if (ret && ret != -EOPNOTSUPP)
2112		return ret;
2113	INIT_LIST_HEAD(&vdev->dmabufs);
2114	init_rwsem(&vdev->memory_lock);
2115	xa_init(&vdev->ctx);
2116
2117	return 0;
2118}
2119EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev);
2120
2121void vfio_pci_core_release_dev(struct vfio_device *core_vdev)
2122{
2123	struct vfio_pci_core_device *vdev =
2124		container_of(core_vdev, struct vfio_pci_core_device, vdev);
2125
2126	mutex_destroy(&vdev->igate);
2127	mutex_destroy(&vdev->ioeventfds_lock);
2128	kfree(vdev->region);
2129	kfree(vdev->pm_save);
2130}
2131EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev);
2132
2133int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
2134{
2135	struct pci_dev *pdev = vdev->pdev;
2136	struct device *dev = &pdev->dev;
2137	int ret;
2138
2139	/* Drivers must set the vfio_pci_core_device to their drvdata */
2140	if (WARN_ON(vdev != dev_get_drvdata(dev)))
2141		return -EINVAL;
2142
2143	if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
2144		return -EINVAL;
2145
2146	if (vdev->vdev.mig_ops) {
2147		if (!(vdev->vdev.mig_ops->migration_get_state &&
2148		      vdev->vdev.mig_ops->migration_set_state &&
2149		      vdev->vdev.mig_ops->migration_get_data_size) ||
2150		    !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY))
2151			return -EINVAL;
2152	}
2153
2154	if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start &&
2155	    vdev->vdev.log_ops->log_stop &&
2156	    vdev->vdev.log_ops->log_read_and_clear))
2157		return -EINVAL;
2158
2159	/*
2160	 * Prevent binding to PFs with VFs enabled, the VFs might be in use
2161	 * by the host or other users.  We cannot capture the VFs if they
2162	 * already exist, nor can we track VF users.  Disabling SR-IOV here
2163	 * would initiate removing the VFs, which would unbind the driver,
2164	 * which is prone to blocking if that VF is also in use by vfio-pci.
2165	 * Just reject these PFs and let the user sort it out.
2166	 */
2167	if (pci_num_vf(pdev)) {
2168		pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
2169		return -EBUSY;
2170	}
2171
2172	if (pci_is_root_bus(pdev->bus) || pdev->is_virtfn) {
2173		ret = vfio_assign_device_set(&vdev->vdev, vdev);
2174	} else if (!pci_probe_reset_slot(pdev->slot)) {
2175		ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
2176	} else {
2177		/*
2178		 * If there is no slot reset support for this device, the whole
2179		 * bus needs to be grouped together to support bus-wide resets.
2180		 */
2181		ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
2182	}
2183
2184	if (ret)
2185		return ret;
2186	ret = vfio_pci_vf_init(vdev);
2187	if (ret)
2188		return ret;
2189	ret = vfio_pci_vga_init(vdev);
2190	if (ret)
2191		goto out_vf;
2192
2193	vfio_pci_probe_power_state(vdev);
2194
2195	/*
2196	 * pci-core sets the device power state to an unknown value at
2197	 * bootup and after being removed from a driver.  The only
2198	 * transition it allows from this unknown state is to D0, which
2199	 * typically happens when a driver calls pci_enable_device().
2200	 * We're not ready to enable the device yet, but we do want to
2201	 * be able to get to D3.  Therefore first do a D0 transition
2202	 * before enabling runtime PM.
2203	 */
2204	vfio_pci_set_power_state(vdev, PCI_D0);
2205
2206	dev->driver->pm = &vfio_pci_core_pm_ops;
2207	pm_runtime_allow(dev);
2208	if (!disable_idle_d3)
2209		pm_runtime_put(dev);
2210
2211	ret = vfio_register_group_dev(&vdev->vdev);
2212	if (ret)
2213		goto out_power;
2214	return 0;
2215
2216out_power:
2217	if (!disable_idle_d3)
2218		pm_runtime_get_noresume(dev);
2219
2220	pm_runtime_forbid(dev);
2221out_vf:
2222	vfio_pci_vf_uninit(vdev);
2223	return ret;
2224}
2225EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);
2226
2227void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
2228{
2229	vfio_pci_core_sriov_configure(vdev, 0);
2230
2231	vfio_unregister_group_dev(&vdev->vdev);
2232
2233	vfio_pci_vf_uninit(vdev);
2234	vfio_pci_vga_uninit(vdev);
2235
2236	if (!disable_idle_d3)
2237		pm_runtime_get_noresume(&vdev->pdev->dev);
2238
2239	pm_runtime_forbid(&vdev->pdev->dev);
2240}
2241EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
2242
2243pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
2244						pci_channel_state_t state)
2245{
2246	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
2247	struct vfio_pci_eventfd *eventfd;
2248
2249	rcu_read_lock();
2250	eventfd = rcu_dereference(vdev->err_trigger);
2251	if (eventfd)
2252		eventfd_signal(eventfd->ctx);
2253	rcu_read_unlock();
2254
2255	return PCI_ERS_RESULT_CAN_RECOVER;
2256}
2257EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected);
2258
2259int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
2260				  int nr_virtfn)
2261{
2262	struct pci_dev *pdev = vdev->pdev;
2263	int ret = 0;
2264
2265	device_lock_assert(&pdev->dev);
2266
2267	if (nr_virtfn) {
2268		mutex_lock(&vfio_pci_sriov_pfs_mutex);
2269		/*
2270		 * The thread that adds the vdev to the list is the only thread
2271		 * that gets to call pci_enable_sriov() and we will only allow
2272		 * it to be called once without going through
2273		 * pci_disable_sriov()
2274		 */
2275		if (!list_empty(&vdev->sriov_pfs_item)) {
2276			ret = -EINVAL;
2277			goto out_unlock;
2278		}
2279		list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs);
2280		mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2281
2282		/*
2283		 * The PF power state should always be higher than the VF power
2284		 * state. The PF can be in low power state either with runtime
2285		 * power management (when there is no user) or PCI_PM_CTRL
2286		 * register write by the user. If PF is in the low power state,
2287		 * then change the power state to D0 first before enabling
2288		 * SR-IOV. Also, this function can be called at any time, and
2289		 * userspace PCI_PM_CTRL write can race against this code path,
2290		 * so protect the same with 'memory_lock'.
2291		 */
2292		ret = pm_runtime_resume_and_get(&pdev->dev);
2293		if (ret)
2294			goto out_del;
2295
2296		down_write(&vdev->memory_lock);
2297		vfio_pci_set_power_state(vdev, PCI_D0);
2298		ret = pci_enable_sriov(pdev, nr_virtfn);
2299		up_write(&vdev->memory_lock);
2300		if (ret) {
2301			pm_runtime_put(&pdev->dev);
2302			goto out_del;
2303		}
2304		return nr_virtfn;
2305	}
2306
2307	if (pci_num_vf(pdev)) {
2308		pci_disable_sriov(pdev);
2309		pm_runtime_put(&pdev->dev);
2310	}
2311
2312out_del:
2313	mutex_lock(&vfio_pci_sriov_pfs_mutex);
2314	list_del_init(&vdev->sriov_pfs_item);
2315out_unlock:
2316	mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2317	return ret;
2318}
2319EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
2320
2321const struct pci_error_handlers vfio_pci_core_err_handlers = {
2322	.error_detected = vfio_pci_core_aer_err_detected,
2323};
2324EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);
2325
2326static bool vfio_dev_in_groups(struct vfio_device *vdev,
2327			       struct vfio_pci_group_info *groups)
2328{
2329	unsigned int i;
2330
2331	if (!groups)
2332		return false;
2333
2334	for (i = 0; i < groups->count; i++)
2335		if (vfio_file_has_dev(groups->files[i], vdev))
2336			return true;
2337	return false;
2338}
2339
2340static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data)
2341{
2342	struct vfio_device_set *dev_set = data;
2343
2344	return vfio_find_device_in_devset(dev_set, &pdev->dev) ? 0 : -ENODEV;
2345}
2346
2347/*
2348 * vfio-core considers a group to be viable and will create a vfio_device even
2349 * if some devices are bound to drivers like pci-stub or pcieport. Here we
2350 * require all PCI devices to be inside our dev_set since that ensures they stay
2351 * put and that every driver controlling the device can co-ordinate with the
2352 * device reset.
2353 *
2354 * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
2355 * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
2356 */
2357static struct pci_dev *
2358vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
2359{
2360	struct pci_dev *pdev;
2361
2362	lockdep_assert_held(&dev_set->lock);
2363
2364	/*
2365	 * By definition all PCI devices in the dev_set share the same PCI
2366	 * reset, so any pci_dev will have the same outcomes for
2367	 * pci_probe_reset_*() and pci_reset_bus().
2368	 */
2369	pdev = list_first_entry(&dev_set->device_list,
2370				struct vfio_pci_core_device,
2371				vdev.dev_set_list)->pdev;
2372
2373	/* pci_reset_bus() is supported */
2374	if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
2375		return NULL;
2376
2377	if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
2378					  dev_set,
2379					  !pci_probe_reset_slot(pdev->slot)))
2380		return NULL;
2381	return pdev;
2382}
2383
2384static int vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set *dev_set)
2385{
2386	struct vfio_pci_core_device *cur;
2387	int ret;
2388
2389	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2390		ret = pm_runtime_resume_and_get(&cur->pdev->dev);
2391		if (ret)
2392			goto unwind;
2393	}
2394
2395	return 0;
2396
2397unwind:
2398	list_for_each_entry_continue_reverse(cur, &dev_set->device_list,
2399					     vdev.dev_set_list)
2400		pm_runtime_put(&cur->pdev->dev);
2401
2402	return ret;
2403}
2404
2405static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
2406				      struct vfio_pci_group_info *groups,
2407				      struct iommufd_ctx *iommufd_ctx)
2408{
2409	struct vfio_pci_core_device *vdev;
2410	struct pci_dev *pdev;
2411	int ret;
2412
2413	mutex_lock(&dev_set->lock);
2414
2415	pdev = vfio_pci_dev_set_resettable(dev_set);
2416	if (!pdev) {
2417		ret = -EINVAL;
2418		goto err_unlock;
2419	}
2420
2421	/*
2422	 * Some of the devices in the dev_set can be in the runtime suspended
2423	 * state. Increment the usage count for all the devices in the dev_set
2424	 * before reset and decrement the same after reset.
2425	 */
2426	ret = vfio_pci_dev_set_pm_runtime_get(dev_set);
2427	if (ret)
2428		goto err_unlock;
2429
2430	list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) {
2431		bool owned;
2432
2433		/*
2434		 * Test whether all the affected devices can be reset by the
2435		 * user.
2436		 *
2437		 * If called from a group opened device and the user provides
2438		 * a set of groups, all the devices in the dev_set should be
2439		 * contained by the set of groups provided by the user.
2440		 *
2441		 * If called from a cdev opened device and the user provides
2442		 * a zero-length array, all the devices in the dev_set must
2443		 * be bound to the same iommufd_ctx as the input iommufd_ctx.
2444		 * If there is any device that has not been bound to any
2445		 * iommufd_ctx yet, check if its iommu_group has any device
2446		 * bound to the input iommufd_ctx.  Such devices can be
2447		 * considered owned by the input iommufd_ctx as the device
2448		 * cannot be owned by another iommufd_ctx when its iommu_group
2449		 * is owned.
2450		 *
2451		 * Otherwise, reset is not allowed.
2452		 */
2453		if (iommufd_ctx) {
2454			int devid = vfio_iommufd_get_dev_id(&vdev->vdev,
2455							    iommufd_ctx);
2456
2457			owned = (devid > 0 || devid == -ENOENT);
2458		} else {
2459			owned = vfio_dev_in_groups(&vdev->vdev, groups);
2460		}
2461
2462		if (!owned) {
2463			ret = -EINVAL;
2464			break;
2465		}
2466
2467		/*
2468		 * Take the memory write lock for each device and zap BAR
2469		 * mappings to prevent the user accessing the device while in
2470		 * reset.  Locking multiple devices is prone to deadlock,
2471		 * runaway and unwind if we hit contention.
2472		 */
2473		if (!down_write_trylock(&vdev->memory_lock)) {
2474			ret = -EBUSY;
2475			break;
2476		}
2477
2478		vfio_pci_dma_buf_move(vdev, true);
2479		vfio_pci_zap_bars(vdev);
2480	}
2481
2482	if (!list_entry_is_head(vdev,
2483				&dev_set->device_list, vdev.dev_set_list)) {
2484		vdev = list_prev_entry(vdev, vdev.dev_set_list);
2485		goto err_undo;
2486	}
2487
2488	/*
2489	 * The pci_reset_bus() will reset all the devices in the bus.
2490	 * The power state can be non-D0 for some of the devices in the bus.
2491	 * For these devices, the pci_reset_bus() will internally set
2492	 * the power state to D0 without vfio driver involvement.
2493	 * For the devices which have NoSoftRst-, the reset function can
2494	 * cause the PCI config space reset without restoring the original
2495	 * state (saved locally in 'vdev->pm_save').
2496	 */
2497	list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list)
2498		vfio_pci_set_power_state(vdev, PCI_D0);
2499
2500	ret = pci_reset_bus(pdev);
2501
2502	vdev = list_last_entry(&dev_set->device_list,
2503			       struct vfio_pci_core_device, vdev.dev_set_list);
2504
2505err_undo:
2506	list_for_each_entry_from_reverse(vdev, &dev_set->device_list,
2507					 vdev.dev_set_list) {
2508		if (vdev->vdev.open_count && __vfio_pci_memory_enabled(vdev))
2509			vfio_pci_dma_buf_move(vdev, false);
2510		up_write(&vdev->memory_lock);
2511	}
2512
2513	list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list)
2514		pm_runtime_put(&vdev->pdev->dev);
2515
2516err_unlock:
2517	mutex_unlock(&dev_set->lock);
2518	return ret;
2519}
2520
2521static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
2522{
2523	struct vfio_pci_core_device *cur;
2524	bool needs_reset = false;
2525
2526	/* No other VFIO device in the set can be open. */
2527	if (vfio_device_set_open_count(dev_set) > 1)
2528		return false;
2529
2530	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2531		needs_reset |= cur->needs_reset;
2532	return needs_reset;
2533}
2534
2535/*
2536 * If a bus or slot reset is available for the provided dev_set and:
2537 *  - All of the devices affected by that bus or slot reset are unused
2538 *  - At least one of the affected devices is marked dirty via
2539 *    needs_reset (such as by lack of FLR support)
2540 * Then attempt to perform that bus or slot reset.
2541 */
2542static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
2543{
2544	struct vfio_pci_core_device *cur;
2545	struct pci_dev *pdev;
2546	bool reset_done = false;
2547
2548	if (!vfio_pci_dev_set_needs_reset(dev_set))
2549		return;
2550
2551	pdev = vfio_pci_dev_set_resettable(dev_set);
2552	if (!pdev)
2553		return;
2554
2555	/*
2556	 * Some of the devices in the bus can be in the runtime suspended
2557	 * state. Increment the usage count for all the devices in the dev_set
2558	 * before reset and decrement the same after reset.
2559	 */
2560	if (!disable_idle_d3 && vfio_pci_dev_set_pm_runtime_get(dev_set))
2561		return;
2562
2563	if (!pci_reset_bus(pdev))
2564		reset_done = true;
2565
2566	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2567		if (reset_done)
2568			cur->needs_reset = false;
2569
2570		if (!disable_idle_d3)
2571			pm_runtime_put(&cur->pdev->dev);
2572	}
2573}
2574
2575void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga,
2576			      bool is_disable_idle_d3)
2577{
2578	nointxmask = is_nointxmask;
2579	disable_vga = is_disable_vga;
2580	disable_idle_d3 = is_disable_idle_d3;
2581}
2582EXPORT_SYMBOL_GPL(vfio_pci_core_set_params);
2583
2584static void vfio_pci_core_cleanup(void)
2585{
2586	vfio_pci_uninit_perm_bits();
2587}
2588
2589static int __init vfio_pci_core_init(void)
2590{
2591	/* Allocate shared config space permission data used by all devices */
2592	return vfio_pci_init_perm_bits();
2593}
2594
2595module_init(vfio_pci_core_init);
2596module_exit(vfio_pci_core_cleanup);
2597
2598MODULE_LICENSE("GPL v2");
2599MODULE_AUTHOR(DRIVER_AUTHOR);
2600MODULE_DESCRIPTION(DRIVER_DESC);