kvmgt.c

   1/*
   2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM
   3 *
   4 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
   5 *
   6 * Permission is hereby granted, free of charge, to any person obtaining a
   7 * copy of this software and associated documentation files (the "Software"),
   8 * to deal in the Software without restriction, including without limitation
   9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 * and/or sell copies of the Software, and to permit persons to whom the
  11 * Software is furnished to do so, subject to the following conditions:
  12 *
  13 * The above copyright notice and this permission notice (including the next
  14 * paragraph) shall be included in all copies or substantial portions of the
  15 * Software.
  16 *
  17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23 * SOFTWARE.
  24 *
  25 * Authors:
  26 *    Kevin Tian <kevin.tian@intel.com>
  27 *    Jike Song <jike.song@intel.com>
  28 *    Xiaoguang Chen <xiaoguang.chen@intel.com>
  29 *    Eddie Dong <eddie.dong@intel.com>
  30 *
  31 * Contributors:
  32 *    Niu Bing <bing.niu@intel.com>
  33 *    Zhi Wang <zhi.a.wang@intel.com>
  34 */
  35
  36#include <linux/init.h>
  37#include <linux/mm.h>
  38#include <linux/kthread.h>
  39#include <linux/sched/mm.h>
  40#include <linux/types.h>
  41#include <linux/list.h>
  42#include <linux/rbtree.h>
  43#include <linux/spinlock.h>
  44#include <linux/eventfd.h>
  45#include <linux/mdev.h>
  46#include <linux/debugfs.h>
  47
  48#include <linux/nospec.h>
  49
  50#include <drm/drm_edid.h>
  51#include <drm/drm_print.h>
  52
  53#include "i915_drv.h"
  54#include "intel_gvt.h"
  55#include "gvt.h"
  56
  57MODULE_IMPORT_NS("DMA_BUF");
  58MODULE_IMPORT_NS("I915_GVT");
  59
  60/* helper macros copied from vfio-pci */
  61#define VFIO_PCI_OFFSET_SHIFT   40
  62#define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
  63#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
  64#define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
  65
  66#define EDID_BLOB_OFFSET (PAGE_SIZE/2)
  67
  68#define OPREGION_SIGNATURE "IntelGraphicsMem"
  69
  70struct vfio_region;
  71struct intel_vgpu_regops {
  72	size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
  73			size_t count, loff_t *ppos, bool iswrite);
  74	void (*release)(struct intel_vgpu *vgpu,
  75			struct vfio_region *region);
  76};
  77
  78struct vfio_region {
  79	u32				type;
  80	u32				subtype;
  81	size_t				size;
  82	u32				flags;
  83	const struct intel_vgpu_regops	*ops;
  84	void				*data;
  85};
  86
  87struct vfio_edid_region {
  88	struct vfio_region_gfx_edid vfio_edid_regs;
  89	void *edid_blob;
  90};
  91
  92struct kvmgt_pgfn {
  93	gfn_t gfn;
  94	struct hlist_node hnode;
  95};
  96
  97struct gvt_dma {
  98	struct intel_vgpu *vgpu;
  99	struct rb_node gfn_node;
 100	struct rb_node dma_addr_node;
 101	gfn_t gfn;
 102	dma_addr_t dma_addr;
 103	unsigned long size;
 104	struct kref ref;
 105};
 106
 107#define vfio_dev_to_vgpu(vfio_dev) \
 108	container_of((vfio_dev), struct intel_vgpu, vfio_device)
 109
 110static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,
 111				   struct kvm_page_track_notifier_node *node);
 112static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,
 113					   struct kvm_page_track_notifier_node *node);
 114
 115static ssize_t intel_vgpu_show_description(struct mdev_type *mtype, char *buf)
 116{
 117	struct intel_vgpu_type *type =
 118		container_of(mtype, struct intel_vgpu_type, type);
 119
 120	return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
 121		       "fence: %d\nresolution: %s\n"
 122		       "weight: %d\n",
 123		       BYTES_TO_MB(type->conf->low_mm),
 124		       BYTES_TO_MB(type->conf->high_mm),
 125		       type->conf->fence, vgpu_edid_str(type->conf->edid),
 126		       type->conf->weight);
 127}
 128
 129static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 130		unsigned long size)
 131{
 132	vfio_unpin_pages(&vgpu->vfio_device, gfn << PAGE_SHIFT,
 133			 DIV_ROUND_UP(size, PAGE_SIZE));
 134}
 135
 136/* Pin a normal or compound guest page for dma. */
 137static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 138		unsigned long size, struct page **page)
 139{
 140	int total_pages = DIV_ROUND_UP(size, PAGE_SIZE);
 141	struct page *base_page = NULL;
 142	int npage;
 143	int ret;
 144
 145	/*
 146	 * We pin the pages one-by-one to avoid allocating a big array
 147	 * on stack to hold pfns.
 148	 */
 149	for (npage = 0; npage < total_pages; npage++) {
 150		dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT;
 151		struct page *cur_page;
 152
 153		ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1,
 154				     IOMMU_READ | IOMMU_WRITE, &cur_page);
 155		if (ret != 1) {
 156			gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n",
 157				     &cur_iova, ret);
 158			goto err;
 159		}
 160
 161		if (npage == 0)
 162			base_page = cur_page;
 163		else if (page_to_pfn(base_page) + npage != page_to_pfn(cur_page)) {
 164			ret = -EINVAL;
 165			npage++;
 166			goto err;
 167		}
 168	}
 169
 170	*page = base_page;
 171	return 0;
 172err:
 173	if (npage)
 174		gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
 175	return ret;
 176}
 177
 178static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
 179		dma_addr_t *dma_addr, unsigned long size)
 180{
 181	struct device *dev = vgpu->gvt->gt->i915->drm.dev;
 182	struct page *page = NULL;
 183	int ret;
 184
 185	ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
 186	if (ret)
 187		return ret;
 188
 189	/* Setup DMA mapping. */
 190	*dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL);
 191	if (dma_mapping_error(dev, *dma_addr)) {
 192		gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
 193			     page_to_pfn(page), ret);
 194		gvt_unpin_guest_page(vgpu, gfn, size);
 195		return -ENOMEM;
 196	}
 197
 198	return 0;
 199}
 200
 201static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
 202		dma_addr_t dma_addr, unsigned long size)
 203{
 204	struct device *dev = vgpu->gvt->gt->i915->drm.dev;
 205
 206	dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL);
 207	gvt_unpin_guest_page(vgpu, gfn, size);
 208}
 209
 210static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
 211		dma_addr_t dma_addr)
 212{
 213	struct rb_node *node = vgpu->dma_addr_cache.rb_node;
 214	struct gvt_dma *itr;
 215
 216	while (node) {
 217		itr = rb_entry(node, struct gvt_dma, dma_addr_node);
 218
 219		if (dma_addr < itr->dma_addr)
 220			node = node->rb_left;
 221		else if (dma_addr > itr->dma_addr)
 222			node = node->rb_right;
 223		else
 224			return itr;
 225	}
 226	return NULL;
 227}
 228
 229static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
 230{
 231	struct rb_node *node = vgpu->gfn_cache.rb_node;
 232	struct gvt_dma *itr;
 233
 234	while (node) {
 235		itr = rb_entry(node, struct gvt_dma, gfn_node);
 236
 237		if (gfn < itr->gfn)
 238			node = node->rb_left;
 239		else if (gfn > itr->gfn)
 240			node = node->rb_right;
 241		else
 242			return itr;
 243	}
 244	return NULL;
 245}
 246
 247static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
 248		dma_addr_t dma_addr, unsigned long size)
 249{
 250	struct gvt_dma *new, *itr;
 251	struct rb_node **link, *parent = NULL;
 252
 253	new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
 254	if (!new)
 255		return -ENOMEM;
 256
 257	new->vgpu = vgpu;
 258	new->gfn = gfn;
 259	new->dma_addr = dma_addr;
 260	new->size = size;
 261	kref_init(&new->ref);
 262
 263	/* gfn_cache maps gfn to struct gvt_dma. */
 264	link = &vgpu->gfn_cache.rb_node;
 265	while (*link) {
 266		parent = *link;
 267		itr = rb_entry(parent, struct gvt_dma, gfn_node);
 268
 269		if (gfn < itr->gfn)
 270			link = &parent->rb_left;
 271		else
 272			link = &parent->rb_right;
 273	}
 274	rb_link_node(&new->gfn_node, parent, link);
 275	rb_insert_color(&new->gfn_node, &vgpu->gfn_cache);
 276
 277	/* dma_addr_cache maps dma addr to struct gvt_dma. */
 278	parent = NULL;
 279	link = &vgpu->dma_addr_cache.rb_node;
 280	while (*link) {
 281		parent = *link;
 282		itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
 283
 284		if (dma_addr < itr->dma_addr)
 285			link = &parent->rb_left;
 286		else
 287			link = &parent->rb_right;
 288	}
 289	rb_link_node(&new->dma_addr_node, parent, link);
 290	rb_insert_color(&new->dma_addr_node, &vgpu->dma_addr_cache);
 291
 292	vgpu->nr_cache_entries++;
 293	return 0;
 294}
 295
 296static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
 297				struct gvt_dma *entry)
 298{
 299	rb_erase(&entry->gfn_node, &vgpu->gfn_cache);
 300	rb_erase(&entry->dma_addr_node, &vgpu->dma_addr_cache);
 301	kfree(entry);
 302	vgpu->nr_cache_entries--;
 303}
 304
 305static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 306{
 307	struct gvt_dma *dma;
 308	struct rb_node *node = NULL;
 309
 310	for (;;) {
 311		mutex_lock(&vgpu->cache_lock);
 312		node = rb_first(&vgpu->gfn_cache);
 313		if (!node) {
 314			mutex_unlock(&vgpu->cache_lock);
 315			break;
 316		}
 317		dma = rb_entry(node, struct gvt_dma, gfn_node);
 318		gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
 319		__gvt_cache_remove_entry(vgpu, dma);
 320		mutex_unlock(&vgpu->cache_lock);
 321	}
 322}
 323
 324static void gvt_cache_init(struct intel_vgpu *vgpu)
 325{
 326	vgpu->gfn_cache = RB_ROOT;
 327	vgpu->dma_addr_cache = RB_ROOT;
 328	vgpu->nr_cache_entries = 0;
 329	mutex_init(&vgpu->cache_lock);
 330}
 331
 332static void kvmgt_protect_table_init(struct intel_vgpu *info)
 333{
 334	hash_init(info->ptable);
 335}
 336
 337static void kvmgt_protect_table_destroy(struct intel_vgpu *info)
 338{
 339	struct kvmgt_pgfn *p;
 340	struct hlist_node *tmp;
 341	int i;
 342
 343	hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
 344		hash_del(&p->hnode);
 345		kfree(p);
 346	}
 347}
 348
 349static struct kvmgt_pgfn *
 350__kvmgt_protect_table_find(struct intel_vgpu *info, gfn_t gfn)
 351{
 352	struct kvmgt_pgfn *p, *res = NULL;
 353
 354	lockdep_assert_held(&info->vgpu_lock);
 355
 356	hash_for_each_possible(info->ptable, p, hnode, gfn) {
 357		if (gfn == p->gfn) {
 358			res = p;
 359			break;
 360		}
 361	}
 362
 363	return res;
 364}
 365
 366static bool kvmgt_gfn_is_write_protected(struct intel_vgpu *info, gfn_t gfn)
 367{
 368	struct kvmgt_pgfn *p;
 369
 370	p = __kvmgt_protect_table_find(info, gfn);
 371	return !!p;
 372}
 373
 374static void kvmgt_protect_table_add(struct intel_vgpu *info, gfn_t gfn)
 375{
 376	struct kvmgt_pgfn *p;
 377
 378	if (kvmgt_gfn_is_write_protected(info, gfn))
 379		return;
 380
 381	p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
 382	if (WARN(!p, "gfn: 0x%llx\n", gfn))
 383		return;
 384
 385	p->gfn = gfn;
 386	hash_add(info->ptable, &p->hnode, gfn);
 387}
 388
 389static void kvmgt_protect_table_del(struct intel_vgpu *info, gfn_t gfn)
 390{
 391	struct kvmgt_pgfn *p;
 392
 393	p = __kvmgt_protect_table_find(info, gfn);
 394	if (p) {
 395		hash_del(&p->hnode);
 396		kfree(p);
 397	}
 398}
 399
 400static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
 401		size_t count, loff_t *ppos, bool iswrite)
 402{
 403	unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 404			VFIO_PCI_NUM_REGIONS;
 405	void *base = vgpu->region[i].data;
 406	loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 407
 408
 409	if (pos >= vgpu->region[i].size || iswrite) {
 410		gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
 411		return -EINVAL;
 412	}
 413	count = min(count, (size_t)(vgpu->region[i].size - pos));
 414	memcpy(buf, base + pos, count);
 415
 416	return count;
 417}
 418
 419static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
 420		struct vfio_region *region)
 421{
 422}
 423
 424static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
 425	.rw = intel_vgpu_reg_rw_opregion,
 426	.release = intel_vgpu_reg_release_opregion,
 427};
 428
 429static bool edid_valid(const void *edid, size_t size)
 430{
 431	const struct drm_edid *drm_edid;
 432	bool is_valid;
 433
 434	drm_edid = drm_edid_alloc(edid, size);
 435	is_valid = drm_edid_valid(drm_edid);
 436	drm_edid_free(drm_edid);
 437
 438	return is_valid;
 439}
 440
 441static int handle_edid_regs(struct intel_vgpu *vgpu,
 442			struct vfio_edid_region *region, char *buf,
 443			size_t count, u16 offset, bool is_write)
 444{
 445	struct vfio_region_gfx_edid *regs = &region->vfio_edid_regs;
 446	unsigned int data;
 447
 448	if (offset + count > sizeof(*regs))
 449		return -EINVAL;
 450
 451	if (count != 4)
 452		return -EINVAL;
 453
 454	if (is_write) {
 455		data = *((unsigned int *)buf);
 456		switch (offset) {
 457		case offsetof(struct vfio_region_gfx_edid, link_state):
 458			if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) {
 459				if (!edid_valid(region->edid_blob, EDID_SIZE)) {
 460					gvt_vgpu_err("invalid EDID blob\n");
 461					return -EINVAL;
 462				}
 463				intel_vgpu_emulate_hotplug(vgpu, true);
 464			} else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN)
 465				intel_vgpu_emulate_hotplug(vgpu, false);
 466			else {
 467				gvt_vgpu_err("invalid EDID link state %d\n",
 468					regs->link_state);
 469				return -EINVAL;
 470			}
 471			regs->link_state = data;
 472			break;
 473		case offsetof(struct vfio_region_gfx_edid, edid_size):
 474			if (data > regs->edid_max_size) {
 475				gvt_vgpu_err("EDID size is bigger than %d!\n",
 476					regs->edid_max_size);
 477				return -EINVAL;
 478			}
 479			regs->edid_size = data;
 480			break;
 481		default:
 482			/* read-only regs */
 483			gvt_vgpu_err("write read-only EDID region at offset %d\n",
 484				offset);
 485			return -EPERM;
 486		}
 487	} else {
 488		memcpy(buf, (char *)regs + offset, count);
 489	}
 490
 491	return count;
 492}
 493
 494static int handle_edid_blob(struct vfio_edid_region *region, char *buf,
 495			size_t count, u16 offset, bool is_write)
 496{
 497	if (offset + count > region->vfio_edid_regs.edid_size)
 498		return -EINVAL;
 499
 500	if (is_write)
 501		memcpy(region->edid_blob + offset, buf, count);
 502	else
 503		memcpy(buf, region->edid_blob + offset, count);
 504
 505	return count;
 506}
 507
 508static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf,
 509		size_t count, loff_t *ppos, bool iswrite)
 510{
 511	int ret;
 512	unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 513			VFIO_PCI_NUM_REGIONS;
 514	struct vfio_edid_region *region = vgpu->region[i].data;
 515	loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 516
 517	if (pos < region->vfio_edid_regs.edid_offset) {
 518		ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite);
 519	} else {
 520		pos -= EDID_BLOB_OFFSET;
 521		ret = handle_edid_blob(region, buf, count, pos, iswrite);
 522	}
 523
 524	if (ret < 0)
 525		gvt_vgpu_err("failed to access EDID region\n");
 526
 527	return ret;
 528}
 529
 530static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu,
 531					struct vfio_region *region)
 532{
 533	kfree(region->data);
 534}
 535
 536static const struct intel_vgpu_regops intel_vgpu_regops_edid = {
 537	.rw = intel_vgpu_reg_rw_edid,
 538	.release = intel_vgpu_reg_release_edid,
 539};
 540
 541static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
 542		unsigned int type, unsigned int subtype,
 543		const struct intel_vgpu_regops *ops,
 544		size_t size, u32 flags, void *data)
 545{
 546	struct vfio_region *region;
 547
 548	region = krealloc(vgpu->region,
 549			(vgpu->num_regions + 1) * sizeof(*region),
 550			GFP_KERNEL);
 551	if (!region)
 552		return -ENOMEM;
 553
 554	vgpu->region = region;
 555	vgpu->region[vgpu->num_regions].type = type;
 556	vgpu->region[vgpu->num_regions].subtype = subtype;
 557	vgpu->region[vgpu->num_regions].ops = ops;
 558	vgpu->region[vgpu->num_regions].size = size;
 559	vgpu->region[vgpu->num_regions].flags = flags;
 560	vgpu->region[vgpu->num_regions].data = data;
 561	vgpu->num_regions++;
 562	return 0;
 563}
 564
 565int intel_gvt_set_opregion(struct intel_vgpu *vgpu)
 566{
 567	void *base;
 568	int ret;
 569
 570	/* Each vgpu has its own opregion, although VFIO would create another
 571	 * one later. This one is used to expose opregion to VFIO. And the
 572	 * other one created by VFIO later, is used by guest actually.
 573	 */
 574	base = vgpu_opregion(vgpu)->va;
 575	if (!base)
 576		return -ENOMEM;
 577
 578	if (memcmp(base, OPREGION_SIGNATURE, 16)) {
 579		memunmap(base);
 580		return -EINVAL;
 581	}
 582
 583	ret = intel_vgpu_register_reg(vgpu,
 584			PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
 585			VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
 586			&intel_vgpu_regops_opregion, INTEL_GVT_OPREGION_SIZE,
 587			VFIO_REGION_INFO_FLAG_READ, base);
 588
 589	return ret;
 590}
 591
 592int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num)
 593{
 594	struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
 595	struct vfio_edid_region *base;
 596	int ret;
 597
 598	base = kzalloc(sizeof(*base), GFP_KERNEL);
 599	if (!base)
 600		return -ENOMEM;
 601
 602	/* TODO: Add multi-port and EDID extension block support */
 603	base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET;
 604	base->vfio_edid_regs.edid_max_size = EDID_SIZE;
 605	base->vfio_edid_regs.edid_size = EDID_SIZE;
 606	base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id);
 607	base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id);
 608	base->edid_blob = port->edid->edid_block;
 609
 610	ret = intel_vgpu_register_reg(vgpu,
 611			VFIO_REGION_TYPE_GFX,
 612			VFIO_REGION_SUBTYPE_GFX_EDID,
 613			&intel_vgpu_regops_edid, EDID_SIZE,
 614			VFIO_REGION_INFO_FLAG_READ |
 615			VFIO_REGION_INFO_FLAG_WRITE |
 616			VFIO_REGION_INFO_FLAG_CAPS, base);
 617
 618	return ret;
 619}
 620
 621static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova,
 622				 u64 length)
 623{
 624	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 625	struct gvt_dma *entry;
 626	u64 iov_pfn = iova >> PAGE_SHIFT;
 627	u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE;
 628
 629	mutex_lock(&vgpu->cache_lock);
 630	for (; iov_pfn < end_iov_pfn; iov_pfn++) {
 631		entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
 632		if (!entry)
 633			continue;
 634
 635		gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
 636				   entry->size);
 637		__gvt_cache_remove_entry(vgpu, entry);
 638	}
 639	mutex_unlock(&vgpu->cache_lock);
 640}
 641
 642static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu)
 643{
 644	struct intel_vgpu *itr;
 645	int id;
 646	bool ret = false;
 647
 648	mutex_lock(&vgpu->gvt->lock);
 649	for_each_active_vgpu(vgpu->gvt, itr, id) {
 650		if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, itr->status))
 651			continue;
 652
 653		if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) {
 654			ret = true;
 655			goto out;
 656		}
 657	}
 658out:
 659	mutex_unlock(&vgpu->gvt->lock);
 660	return ret;
 661}
 662
 663static int intel_vgpu_open_device(struct vfio_device *vfio_dev)
 664{
 665	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 666	int ret;
 667
 668	if (__kvmgt_vgpu_exist(vgpu))
 669		return -EEXIST;
 670
 671	vgpu->track_node.track_write = kvmgt_page_track_write;
 672	vgpu->track_node.track_remove_region = kvmgt_page_track_remove_region;
 673	ret = kvm_page_track_register_notifier(vgpu->vfio_device.kvm,
 674					       &vgpu->track_node);
 675	if (ret) {
 676		gvt_vgpu_err("KVM is required to use Intel vGPU\n");
 677		return ret;
 678	}
 679
 680	set_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status);
 681
 682	debugfs_create_ulong(KVMGT_DEBUGFS_FILENAME, 0444, vgpu->debugfs,
 683			     &vgpu->nr_cache_entries);
 684
 685	intel_gvt_activate_vgpu(vgpu);
 686
 687	return 0;
 688}
 689
 690static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
 691{
 692	struct eventfd_ctx *trigger;
 693
 694	trigger = vgpu->msi_trigger;
 695	if (trigger) {
 696		eventfd_ctx_put(trigger);
 697		vgpu->msi_trigger = NULL;
 698	}
 699}
 700
 701static void intel_vgpu_close_device(struct vfio_device *vfio_dev)
 702{
 703	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 704
 705	intel_gvt_release_vgpu(vgpu);
 706
 707	clear_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status);
 708
 709	debugfs_lookup_and_remove(KVMGT_DEBUGFS_FILENAME, vgpu->debugfs);
 710
 711	kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm,
 712					   &vgpu->track_node);
 713
 714	kvmgt_protect_table_destroy(vgpu);
 715	gvt_cache_destroy(vgpu);
 716
 717	WARN_ON(vgpu->nr_cache_entries);
 718
 719	vgpu->gfn_cache = RB_ROOT;
 720	vgpu->dma_addr_cache = RB_ROOT;
 721
 722	intel_vgpu_release_msi_eventfd_ctx(vgpu);
 723}
 724
 725static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
 726{
 727	u32 start_lo, start_hi;
 728	u32 mem_type;
 729
 730	start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 731			PCI_BASE_ADDRESS_MEM_MASK;
 732	mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 733			PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 734
 735	switch (mem_type) {
 736	case PCI_BASE_ADDRESS_MEM_TYPE_64:
 737		start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
 738						+ bar + 4));
 739		break;
 740	case PCI_BASE_ADDRESS_MEM_TYPE_32:
 741	case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 742		/* 1M mem BAR treated as 32-bit BAR */
 743	default:
 744		/* mem unknown type treated as 32-bit BAR */
 745		start_hi = 0;
 746		break;
 747	}
 748
 749	return ((u64)start_hi << 32) | start_lo;
 750}
 751
 752static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off,
 753			     void *buf, unsigned int count, bool is_write)
 754{
 755	u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
 756	int ret;
 757
 758	if (is_write)
 759		ret = intel_vgpu_emulate_mmio_write(vgpu,
 760					bar_start + off, buf, count);
 761	else
 762		ret = intel_vgpu_emulate_mmio_read(vgpu,
 763					bar_start + off, buf, count);
 764	return ret;
 765}
 766
 767static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off)
 768{
 769	return off >= vgpu_aperture_offset(vgpu) &&
 770	       off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
 771}
 772
 773static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off,
 774		void *buf, unsigned long count, bool is_write)
 775{
 776	void __iomem *aperture_va;
 777
 778	if (!intel_vgpu_in_aperture(vgpu, off) ||
 779	    !intel_vgpu_in_aperture(vgpu, off + count)) {
 780		gvt_vgpu_err("Invalid aperture offset %llu\n", off);
 781		return -EINVAL;
 782	}
 783
 784	aperture_va = io_mapping_map_wc(&vgpu->gvt->gt->ggtt->iomap,
 785					ALIGN_DOWN(off, PAGE_SIZE),
 786					count + offset_in_page(off));
 787	if (!aperture_va)
 788		return -EIO;
 789
 790	if (is_write)
 791		memcpy_toio(aperture_va + offset_in_page(off), buf, count);
 792	else
 793		memcpy_fromio(buf, aperture_va + offset_in_page(off), count);
 794
 795	io_mapping_unmap(aperture_va);
 796
 797	return 0;
 798}
 799
 800static ssize_t intel_vgpu_rw(struct intel_vgpu *vgpu, char *buf,
 801			size_t count, loff_t *ppos, bool is_write)
 802{
 803	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 804	u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;
 805	int ret = -EINVAL;
 806
 807
 808	if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) {
 809		gvt_vgpu_err("invalid index: %u\n", index);
 810		return -EINVAL;
 811	}
 812
 813	switch (index) {
 814	case VFIO_PCI_CONFIG_REGION_INDEX:
 815		if (is_write)
 816			ret = intel_vgpu_emulate_cfg_write(vgpu, pos,
 817						buf, count);
 818		else
 819			ret = intel_vgpu_emulate_cfg_read(vgpu, pos,
 820						buf, count);
 821		break;
 822	case VFIO_PCI_BAR0_REGION_INDEX:
 823		ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
 824					buf, count, is_write);
 825		break;
 826	case VFIO_PCI_BAR2_REGION_INDEX:
 827		ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
 828		break;
 829	case VFIO_PCI_BAR1_REGION_INDEX:
 830	case VFIO_PCI_BAR3_REGION_INDEX:
 831	case VFIO_PCI_BAR4_REGION_INDEX:
 832	case VFIO_PCI_BAR5_REGION_INDEX:
 833	case VFIO_PCI_VGA_REGION_INDEX:
 834	case VFIO_PCI_ROM_REGION_INDEX:
 835		break;
 836	default:
 837		if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions)
 838			return -EINVAL;
 839
 840		index -= VFIO_PCI_NUM_REGIONS;
 841		return vgpu->region[index].ops->rw(vgpu, buf, count,
 842				ppos, is_write);
 843	}
 844
 845	return ret == 0 ? count : ret;
 846}
 847
 848static bool gtt_entry(struct intel_vgpu *vgpu, loff_t *ppos)
 849{
 850	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 851	struct intel_gvt *gvt = vgpu->gvt;
 852	int offset;
 853
 854	/* Only allow MMIO GGTT entry access */
 855	if (index != PCI_BASE_ADDRESS_0)
 856		return false;
 857
 858	offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
 859		intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
 860
 861	return (offset >= gvt->device_info.gtt_start_offset &&
 862		offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
 863			true : false;
 864}
 865
 866static ssize_t intel_vgpu_read(struct vfio_device *vfio_dev, char __user *buf,
 867			size_t count, loff_t *ppos)
 868{
 869	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 870	unsigned int done = 0;
 871	int ret;
 872
 873	while (count) {
 874		size_t filled;
 875
 876		/* Only support GGTT entry 8 bytes read */
 877		if (count >= 8 && !(*ppos % 8) &&
 878			gtt_entry(vgpu, ppos)) {
 879			u64 val;
 880
 881			ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 882					ppos, false);
 883			if (ret <= 0)
 884				goto read_err;
 885
 886			if (copy_to_user(buf, &val, sizeof(val)))
 887				goto read_err;
 888
 889			filled = 8;
 890		} else if (count >= 4 && !(*ppos % 4)) {
 891			u32 val;
 892
 893			ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 894					ppos, false);
 895			if (ret <= 0)
 896				goto read_err;
 897
 898			if (copy_to_user(buf, &val, sizeof(val)))
 899				goto read_err;
 900
 901			filled = 4;
 902		} else if (count >= 2 && !(*ppos % 2)) {
 903			u16 val;
 904
 905			ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 906					ppos, false);
 907			if (ret <= 0)
 908				goto read_err;
 909
 910			if (copy_to_user(buf, &val, sizeof(val)))
 911				goto read_err;
 912
 913			filled = 2;
 914		} else {
 915			u8 val;
 916
 917			ret = intel_vgpu_rw(vgpu, &val, sizeof(val), ppos,
 918					false);
 919			if (ret <= 0)
 920				goto read_err;
 921
 922			if (copy_to_user(buf, &val, sizeof(val)))
 923				goto read_err;
 924
 925			filled = 1;
 926		}
 927
 928		count -= filled;
 929		done += filled;
 930		*ppos += filled;
 931		buf += filled;
 932	}
 933
 934	return done;
 935
 936read_err:
 937	return -EFAULT;
 938}
 939
 940static ssize_t intel_vgpu_write(struct vfio_device *vfio_dev,
 941				const char __user *buf,
 942				size_t count, loff_t *ppos)
 943{
 944	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
 945	unsigned int done = 0;
 946	int ret;
 947
 948	while (count) {
 949		size_t filled;
 950
 951		/* Only support GGTT entry 8 bytes write */
 952		if (count >= 8 && !(*ppos % 8) &&
 953			gtt_entry(vgpu, ppos)) {
 954			u64 val;
 955
 956			if (copy_from_user(&val, buf, sizeof(val)))
 957				goto write_err;
 958
 959			ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 960					ppos, true);
 961			if (ret <= 0)
 962				goto write_err;
 963
 964			filled = 8;
 965		} else if (count >= 4 && !(*ppos % 4)) {
 966			u32 val;
 967
 968			if (copy_from_user(&val, buf, sizeof(val)))
 969				goto write_err;
 970
 971			ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),
 972					ppos, true);
 973			if (ret <= 0)
 974				goto write_err;
 975
 976			filled = 4;
 977		} else if (count >= 2 && !(*ppos % 2)) {
 978			u16 val;
 979
 980			if (copy_from_user(&val, buf, sizeof(val)))
 981				goto write_err;
 982
 983			ret = intel_vgpu_rw(vgpu, (char *)&val,
 984					sizeof(val), ppos, true);
 985			if (ret <= 0)
 986				goto write_err;
 987
 988			filled = 2;
 989		} else {
 990			u8 val;
 991
 992			if (copy_from_user(&val, buf, sizeof(val)))
 993				goto write_err;
 994
 995			ret = intel_vgpu_rw(vgpu, &val, sizeof(val),
 996					ppos, true);
 997			if (ret <= 0)
 998				goto write_err;
 999
1000			filled = 1;
1001		}
1002
1003		count -= filled;
1004		done += filled;
1005		*ppos += filled;
1006		buf += filled;
1007	}
1008
1009	return done;
1010write_err:
1011	return -EFAULT;
1012}
1013
1014static int intel_vgpu_mmap(struct vfio_device *vfio_dev,
1015		struct vm_area_struct *vma)
1016{
1017	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1018	unsigned int index;
1019	u64 virtaddr;
1020	unsigned long req_size, pgoff, req_start;
1021	pgprot_t pg_prot;
1022
1023	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1024	if (index >= VFIO_PCI_ROM_REGION_INDEX)
1025		return -EINVAL;
1026
1027	if (vma->vm_end < vma->vm_start)
1028		return -EINVAL;
1029	if ((vma->vm_flags & VM_SHARED) == 0)
1030		return -EINVAL;
1031	if (index != VFIO_PCI_BAR2_REGION_INDEX)
1032		return -EINVAL;
1033
1034	pg_prot = vma->vm_page_prot;
1035	virtaddr = vma->vm_start;
1036	req_size = vma->vm_end - vma->vm_start;
1037	pgoff = vma->vm_pgoff &
1038		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1039	req_start = pgoff << PAGE_SHIFT;
1040
1041	if (!intel_vgpu_in_aperture(vgpu, req_start))
1042		return -EINVAL;
1043	if (req_start + req_size >
1044	    vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu))
1045		return -EINVAL;
1046
1047	pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff;
1048
1049	return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1050}
1051
1052static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1053{
1054	if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1055		return 1;
1056
1057	return 0;
1058}
1059
1060static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1061			unsigned int index, unsigned int start,
1062			unsigned int count, u32 flags,
1063			void *data)
1064{
1065	return 0;
1066}
1067
1068static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1069			unsigned int index, unsigned int start,
1070			unsigned int count, u32 flags, void *data)
1071{
1072	return 0;
1073}
1074
1075static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1076		unsigned int index, unsigned int start, unsigned int count,
1077		u32 flags, void *data)
1078{
1079	return 0;
1080}
1081
1082static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1083		unsigned int index, unsigned int start, unsigned int count,
1084		u32 flags, void *data)
1085{
1086	struct eventfd_ctx *trigger;
1087
1088	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1089		int fd = *(int *)data;
1090
1091		trigger = eventfd_ctx_fdget(fd);
1092		if (IS_ERR(trigger)) {
1093			gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1094			return PTR_ERR(trigger);
1095		}
1096		vgpu->msi_trigger = trigger;
1097	} else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1098		intel_vgpu_release_msi_eventfd_ctx(vgpu);
1099
1100	return 0;
1101}
1102
1103static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags,
1104		unsigned int index, unsigned int start, unsigned int count,
1105		void *data)
1106{
1107	int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1108			unsigned int start, unsigned int count, u32 flags,
1109			void *data) = NULL;
1110
1111	switch (index) {
1112	case VFIO_PCI_INTX_IRQ_INDEX:
1113		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1114		case VFIO_IRQ_SET_ACTION_MASK:
1115			func = intel_vgpu_set_intx_mask;
1116			break;
1117		case VFIO_IRQ_SET_ACTION_UNMASK:
1118			func = intel_vgpu_set_intx_unmask;
1119			break;
1120		case VFIO_IRQ_SET_ACTION_TRIGGER:
1121			func = intel_vgpu_set_intx_trigger;
1122			break;
1123		}
1124		break;
1125	case VFIO_PCI_MSI_IRQ_INDEX:
1126		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1127		case VFIO_IRQ_SET_ACTION_MASK:
1128		case VFIO_IRQ_SET_ACTION_UNMASK:
1129			/* XXX Need masking support exported */
1130			break;
1131		case VFIO_IRQ_SET_ACTION_TRIGGER:
1132			func = intel_vgpu_set_msi_trigger;
1133			break;
1134		}
1135		break;
1136	}
1137
1138	if (!func)
1139		return -ENOTTY;
1140
1141	return func(vgpu, index, start, count, flags, data);
1142}
1143
1144static int intel_vgpu_ioctl_get_region_info(struct vfio_device *vfio_dev,
1145					    struct vfio_region_info *info,
1146					    struct vfio_info_cap *caps)
1147{
1148	struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1149	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1150	int nr_areas = 1;
1151	int cap_type_id;
1152	unsigned int i;
1153	int ret;
1154
1155	switch (info->index) {
1156	case VFIO_PCI_CONFIG_REGION_INDEX:
1157		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1158		info->size = vgpu->gvt->device_info.cfg_space_size;
1159		info->flags = VFIO_REGION_INFO_FLAG_READ |
1160			      VFIO_REGION_INFO_FLAG_WRITE;
1161		break;
1162	case VFIO_PCI_BAR0_REGION_INDEX:
1163		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1164		info->size = vgpu->cfg_space.bar[info->index].size;
1165		if (!info->size) {
1166			info->flags = 0;
1167			break;
1168		}
1169
1170		info->flags = VFIO_REGION_INFO_FLAG_READ |
1171			      VFIO_REGION_INFO_FLAG_WRITE;
1172		break;
1173	case VFIO_PCI_BAR1_REGION_INDEX:
1174		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1175		info->size = 0;
1176		info->flags = 0;
1177		break;
1178	case VFIO_PCI_BAR2_REGION_INDEX:
1179		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1180		info->flags = VFIO_REGION_INFO_FLAG_CAPS |
1181			      VFIO_REGION_INFO_FLAG_MMAP |
1182			      VFIO_REGION_INFO_FLAG_READ |
1183			      VFIO_REGION_INFO_FLAG_WRITE;
1184		info->size = gvt_aperture_sz(vgpu->gvt);
1185
1186		sparse = kzalloc(struct_size(sparse, areas, nr_areas),
1187				 GFP_KERNEL);
1188		if (!sparse)
1189			return -ENOMEM;
1190
1191		sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1192		sparse->header.version = 1;
1193		sparse->nr_areas = nr_areas;
1194		cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1195		sparse->areas[0].offset =
1196			PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1197		sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1198		break;
1199
1200	case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1201		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1202		info->size = 0;
1203		info->flags = 0;
1204
1205		gvt_dbg_core("get region info bar:%d\n", info->index);
1206		break;
1207
1208	case VFIO_PCI_ROM_REGION_INDEX:
1209	case VFIO_PCI_VGA_REGION_INDEX:
1210		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1211		info->size = 0;
1212		info->flags = 0;
1213
1214		gvt_dbg_core("get region info index:%d\n", info->index);
1215		break;
1216	default: {
1217		struct vfio_region_info_cap_type cap_type = {
1218			.header.id = VFIO_REGION_INFO_CAP_TYPE,
1219			.header.version = 1
1220		};
1221
1222		if (info->index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions)
1223			return -EINVAL;
1224		info->index = array_index_nospec(
1225			info->index, VFIO_PCI_NUM_REGIONS + vgpu->num_regions);
1226
1227		i = info->index - VFIO_PCI_NUM_REGIONS;
1228
1229		info->offset = VFIO_PCI_INDEX_TO_OFFSET(info->index);
1230		info->size = vgpu->region[i].size;
1231		info->flags = vgpu->region[i].flags;
1232
1233		cap_type.type = vgpu->region[i].type;
1234		cap_type.subtype = vgpu->region[i].subtype;
1235
1236		ret = vfio_info_add_capability(caps, &cap_type.header,
1237					       sizeof(cap_type));
1238		if (ret)
1239			return ret;
1240	}
1241	}
1242
1243	if ((info->flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1244		ret = -EINVAL;
1245		if (cap_type_id == VFIO_REGION_INFO_CAP_SPARSE_MMAP) {
1246			ret = vfio_info_add_capability(
1247				caps, &sparse->header,
1248				struct_size(sparse, areas, sparse->nr_areas));
1249		}
1250		if (ret) {
1251			kfree(sparse);
1252			return ret;
1253		}
1254	}
1255
1256	kfree(sparse);
1257	return 0;
1258}
1259
1260static long intel_vgpu_ioctl(struct vfio_device *vfio_dev, unsigned int cmd,
1261			     unsigned long arg)
1262{
1263	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1264	unsigned long minsz;
1265
1266	gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1267
1268	if (cmd == VFIO_DEVICE_GET_INFO) {
1269		struct vfio_device_info info;
1270
1271		minsz = offsetofend(struct vfio_device_info, num_irqs);
1272
1273		if (copy_from_user(&info, (void __user *)arg, minsz))
1274			return -EFAULT;
1275
1276		if (info.argsz < minsz)
1277			return -EINVAL;
1278
1279		info.flags = VFIO_DEVICE_FLAGS_PCI;
1280		info.flags |= VFIO_DEVICE_FLAGS_RESET;
1281		info.num_regions = VFIO_PCI_NUM_REGIONS +
1282				vgpu->num_regions;
1283		info.num_irqs = VFIO_PCI_NUM_IRQS;
1284
1285		return copy_to_user((void __user *)arg, &info, minsz) ?
1286			-EFAULT : 0;
1287
1288	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1289		struct vfio_irq_info info;
1290
1291		minsz = offsetofend(struct vfio_irq_info, count);
1292
1293		if (copy_from_user(&info, (void __user *)arg, minsz))
1294			return -EFAULT;
1295
1296		if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1297			return -EINVAL;
1298
1299		switch (info.index) {
1300		case VFIO_PCI_INTX_IRQ_INDEX:
1301		case VFIO_PCI_MSI_IRQ_INDEX:
1302			break;
1303		default:
1304			return -EINVAL;
1305		}
1306
1307		info.flags = VFIO_IRQ_INFO_EVENTFD;
1308
1309		info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1310
1311		if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1312			info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1313				       VFIO_IRQ_INFO_AUTOMASKED);
1314		else
1315			info.flags |= VFIO_IRQ_INFO_NORESIZE;
1316
1317		return copy_to_user((void __user *)arg, &info, minsz) ?
1318			-EFAULT : 0;
1319	} else if (cmd == VFIO_DEVICE_SET_IRQS) {
1320		struct vfio_irq_set hdr;
1321		u8 *data = NULL;
1322		int ret = 0;
1323		size_t data_size = 0;
1324
1325		minsz = offsetofend(struct vfio_irq_set, count);
1326
1327		if (copy_from_user(&hdr, (void __user *)arg, minsz))
1328			return -EFAULT;
1329
1330		if (!is_power_of_2(hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) ||
1331		    !is_power_of_2(hdr.flags & VFIO_IRQ_SET_ACTION_TYPE_MASK))
1332			return -EINVAL;
1333
1334		if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1335			int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1336
1337			if (!hdr.count)
1338				return -EINVAL;
1339
1340			ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1341						VFIO_PCI_NUM_IRQS, &data_size);
1342			if (ret) {
1343				gvt_vgpu_err("vfio_set_irqs_validate_and_prepare failed\n");
1344				return ret;
1345			}
1346
1347			data = memdup_user((void __user *)(arg + minsz),
1348					   data_size);
1349			if (IS_ERR(data))
1350				return PTR_ERR(data);
1351		}
1352
1353		ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1354					hdr.start, hdr.count, data);
1355		kfree(data);
1356
1357		return ret;
1358	} else if (cmd == VFIO_DEVICE_RESET) {
1359		intel_gvt_reset_vgpu(vgpu);
1360		return 0;
1361	} else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1362		struct vfio_device_gfx_plane_info dmabuf = {};
1363		int ret = 0;
1364
1365		minsz = offsetofend(struct vfio_device_gfx_plane_info,
1366				    dmabuf_id);
1367		if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1368			return -EFAULT;
1369		if (dmabuf.argsz < minsz)
1370			return -EINVAL;
1371
1372		ret = intel_vgpu_query_plane(vgpu, &dmabuf);
1373		if (ret != 0)
1374			return ret;
1375
1376		return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1377								-EFAULT : 0;
1378	} else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1379		__u32 dmabuf_id;
1380
1381		if (get_user(dmabuf_id, (__u32 __user *)arg))
1382			return -EFAULT;
1383		return intel_vgpu_get_dmabuf(vgpu, dmabuf_id);
1384	}
1385
1386	return -ENOTTY;
1387}
1388
1389static ssize_t
1390vgpu_id_show(struct device *dev, struct device_attribute *attr,
1391	     char *buf)
1392{
1393	struct intel_vgpu *vgpu = dev_get_drvdata(dev);
1394
1395	return sprintf(buf, "%d\n", vgpu->id);
1396}
1397
1398static DEVICE_ATTR_RO(vgpu_id);
1399
1400static struct attribute *intel_vgpu_attrs[] = {
1401	&dev_attr_vgpu_id.attr,
1402	NULL
1403};
1404
1405static const struct attribute_group intel_vgpu_group = {
1406	.name = "intel_vgpu",
1407	.attrs = intel_vgpu_attrs,
1408};
1409
1410static const struct attribute_group *intel_vgpu_groups[] = {
1411	&intel_vgpu_group,
1412	NULL,
1413};
1414
1415static int intel_vgpu_init_dev(struct vfio_device *vfio_dev)
1416{
1417	struct mdev_device *mdev = to_mdev_device(vfio_dev->dev);
1418	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1419	struct intel_vgpu_type *type =
1420		container_of(mdev->type, struct intel_vgpu_type, type);
1421	int ret;
1422
1423	vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;
1424	ret = intel_gvt_create_vgpu(vgpu, type->conf);
1425	if (ret)
1426		return ret;
1427
1428	kvmgt_protect_table_init(vgpu);
1429	gvt_cache_init(vgpu);
1430
1431	return 0;
1432}
1433
1434static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
1435{
1436	struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
1437
1438	intel_gvt_destroy_vgpu(vgpu);
1439}
1440
1441static const struct vfio_device_ops intel_vgpu_dev_ops = {
1442	.init		= intel_vgpu_init_dev,
1443	.release	= intel_vgpu_release_dev,
1444	.open_device	= intel_vgpu_open_device,
1445	.close_device	= intel_vgpu_close_device,
1446	.read		= intel_vgpu_read,
1447	.write		= intel_vgpu_write,
1448	.mmap		= intel_vgpu_mmap,
1449	.ioctl		= intel_vgpu_ioctl,
1450	.get_region_info_caps = intel_vgpu_ioctl_get_region_info,
1451	.dma_unmap	= intel_vgpu_dma_unmap,
1452	.bind_iommufd	= vfio_iommufd_emulated_bind,
1453	.unbind_iommufd = vfio_iommufd_emulated_unbind,
1454	.attach_ioas	= vfio_iommufd_emulated_attach_ioas,
1455	.detach_ioas	= vfio_iommufd_emulated_detach_ioas,
1456};
1457
1458static int intel_vgpu_probe(struct mdev_device *mdev)
1459{
1460	struct intel_vgpu *vgpu;
1461	int ret;
1462
1463	vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev,
1464				 &intel_vgpu_dev_ops);
1465	if (IS_ERR(vgpu)) {
1466		gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu));
1467		return PTR_ERR(vgpu);
1468	}
1469
1470	dev_set_drvdata(&mdev->dev, vgpu);
1471	ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device);
1472	if (ret)
1473		goto out_put_vdev;
1474
1475	gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
1476		     dev_name(mdev_dev(mdev)));
1477	return 0;
1478
1479out_put_vdev:
1480	vfio_put_device(&vgpu->vfio_device);
1481	return ret;
1482}
1483
1484static void intel_vgpu_remove(struct mdev_device *mdev)
1485{
1486	struct intel_vgpu *vgpu = dev_get_drvdata(&mdev->dev);
1487
1488	vfio_unregister_group_dev(&vgpu->vfio_device);
1489	vfio_put_device(&vgpu->vfio_device);
1490}
1491
1492static unsigned int intel_vgpu_get_available(struct mdev_type *mtype)
1493{
1494	struct intel_vgpu_type *type =
1495		container_of(mtype, struct intel_vgpu_type, type);
1496	struct intel_gvt *gvt = kdev_to_i915(mtype->parent->dev)->gvt;
1497	unsigned int low_gm_avail, high_gm_avail, fence_avail;
1498
1499	mutex_lock(&gvt->lock);
1500	low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE -
1501		gvt->gm.vgpu_allocated_low_gm_size;
1502	high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE -
1503		gvt->gm.vgpu_allocated_high_gm_size;
1504	fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
1505		gvt->fence.vgpu_allocated_fence_num;
1506	mutex_unlock(&gvt->lock);
1507
1508	return min3(low_gm_avail / type->conf->low_mm,
1509		    high_gm_avail / type->conf->high_mm,
1510		    fence_avail / type->conf->fence);
1511}
1512
1513static struct mdev_driver intel_vgpu_mdev_driver = {
1514	.device_api	= VFIO_DEVICE_API_PCI_STRING,
1515	.driver = {
1516		.name		= "intel_vgpu_mdev",
1517		.owner		= THIS_MODULE,
1518		.dev_groups	= intel_vgpu_groups,
1519	},
1520	.probe			= intel_vgpu_probe,
1521	.remove			= intel_vgpu_remove,
1522	.get_available		= intel_vgpu_get_available,
1523	.show_description	= intel_vgpu_show_description,
1524};
1525
1526int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn)
1527{
1528	int r;
1529
1530	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))
1531		return -ESRCH;
1532
1533	if (kvmgt_gfn_is_write_protected(info, gfn))
1534		return 0;
1535
1536	r = kvm_write_track_add_gfn(info->vfio_device.kvm, gfn);
1537	if (r)
1538		return r;
1539
1540	kvmgt_protect_table_add(info, gfn);
1541	return 0;
1542}
1543
1544int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn)
1545{
1546	int r;
1547
1548	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))
1549		return -ESRCH;
1550
1551	if (!kvmgt_gfn_is_write_protected(info, gfn))
1552		return 0;
1553
1554	r = kvm_write_track_remove_gfn(info->vfio_device.kvm, gfn);
1555	if (r)
1556		return r;
1557
1558	kvmgt_protect_table_del(info, gfn);
1559	return 0;
1560}
1561
1562static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,
1563				   struct kvm_page_track_notifier_node *node)
1564{
1565	struct intel_vgpu *info =
1566		container_of(node, struct intel_vgpu, track_node);
1567
1568	mutex_lock(&info->vgpu_lock);
1569
1570	if (kvmgt_gfn_is_write_protected(info, gpa >> PAGE_SHIFT))
1571		intel_vgpu_page_track_handler(info, gpa,
1572						     (void *)val, len);
1573
1574	mutex_unlock(&info->vgpu_lock);
1575}
1576
1577static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,
1578					   struct kvm_page_track_notifier_node *node)
1579{
1580	unsigned long i;
1581	struct intel_vgpu *info =
1582		container_of(node, struct intel_vgpu, track_node);
1583
1584	mutex_lock(&info->vgpu_lock);
1585
1586	for (i = 0; i < nr_pages; i++) {
1587		if (kvmgt_gfn_is_write_protected(info, gfn + i))
1588			kvmgt_protect_table_del(info, gfn + i);
1589	}
1590
1591	mutex_unlock(&info->vgpu_lock);
1592}
1593
1594void intel_vgpu_detach_regions(struct intel_vgpu *vgpu)
1595{
1596	int i;
1597
1598	if (!vgpu->region)
1599		return;
1600
1601	for (i = 0; i < vgpu->num_regions; i++)
1602		if (vgpu->region[i].ops->release)
1603			vgpu->region[i].ops->release(vgpu,
1604					&vgpu->region[i]);
1605	vgpu->num_regions = 0;
1606	kfree(vgpu->region);
1607	vgpu->region = NULL;
1608}
1609
1610int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
1611		unsigned long size, dma_addr_t *dma_addr)
1612{
1613	struct gvt_dma *entry;
1614	int ret;
1615
1616	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1617		return -EINVAL;
1618
1619	mutex_lock(&vgpu->cache_lock);
1620
1621	entry = __gvt_cache_find_gfn(vgpu, gfn);
1622	if (!entry) {
1623		ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1624		if (ret)
1625			goto err_unlock;
1626
1627		ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);
1628		if (ret)
1629			goto err_unmap;
1630	} else if (entry->size != size) {
1631		/* the same gfn with different size: unmap and re-map */
1632		gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size);
1633		__gvt_cache_remove_entry(vgpu, entry);
1634
1635		ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1636		if (ret)
1637			goto err_unlock;
1638
1639		ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);
1640		if (ret)
1641			goto err_unmap;
1642	} else {
1643		kref_get(&entry->ref);
1644		*dma_addr = entry->dma_addr;
1645	}
1646
1647	mutex_unlock(&vgpu->cache_lock);
1648	return 0;
1649
1650err_unmap:
1651	gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1652err_unlock:
1653	mutex_unlock(&vgpu->cache_lock);
1654	return ret;
1655}
1656
1657int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr)
1658{
1659	struct gvt_dma *entry;
1660	int ret = 0;
1661
1662	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1663		return -EINVAL;
1664
1665	mutex_lock(&vgpu->cache_lock);
1666	entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
1667	if (entry)
1668		kref_get(&entry->ref);
1669	else
1670		ret = -ENOMEM;
1671	mutex_unlock(&vgpu->cache_lock);
1672
1673	return ret;
1674}
1675
1676static void __gvt_dma_release(struct kref *ref)
1677{
1678	struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1679
1680	gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
1681			   entry->size);
1682	__gvt_cache_remove_entry(entry->vgpu, entry);
1683}
1684
1685void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu,
1686		dma_addr_t dma_addr)
1687{
1688	struct gvt_dma *entry;
1689
1690	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
1691		return;
1692
1693	mutex_lock(&vgpu->cache_lock);
1694	entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
1695	if (entry)
1696		kref_put(&entry->ref, __gvt_dma_release);
1697	mutex_unlock(&vgpu->cache_lock);
1698}
1699
1700static void init_device_info(struct intel_gvt *gvt)
1701{
1702	struct intel_gvt_device_info *info = &gvt->device_info;
1703	struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev);
1704
1705	info->max_support_vgpus = 8;
1706	info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE;
1707	info->mmio_size = 2 * 1024 * 1024;
1708	info->mmio_bar = 0;
1709	info->gtt_start_offset = 8 * 1024 * 1024;
1710	info->gtt_entry_size = 8;
1711	info->gtt_entry_size_shift = 3;
1712	info->gmadr_bytes_in_cmd = 8;
1713	info->max_surface_size = 36 * 1024 * 1024;
1714	info->msi_cap_offset = pdev->msi_cap;
1715}
1716
1717static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt)
1718{
1719	struct intel_vgpu *vgpu;
1720	int id;
1721
1722	mutex_lock(&gvt->lock);
1723	idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) {
1724		if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK + id,
1725				       (void *)&gvt->service_request)) {
1726			if (test_bit(INTEL_VGPU_STATUS_ACTIVE, vgpu->status))
1727				intel_vgpu_emulate_vblank(vgpu);
1728		}
1729	}
1730	mutex_unlock(&gvt->lock);
1731}
1732
1733static int gvt_service_thread(void *data)
1734{
1735	struct intel_gvt *gvt = (struct intel_gvt *)data;
1736	int ret;
1737
1738	gvt_dbg_core("service thread start\n");
1739
1740	while (!kthread_should_stop()) {
1741		ret = wait_event_interruptible(gvt->service_thread_wq,
1742				kthread_should_stop() || gvt->service_request);
1743
1744		if (kthread_should_stop())
1745			break;
1746
1747		if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))
1748			continue;
1749
1750		intel_gvt_test_and_emulate_vblank(gvt);
1751
1752		if (test_bit(INTEL_GVT_REQUEST_SCHED,
1753				(void *)&gvt->service_request) ||
1754			test_bit(INTEL_GVT_REQUEST_EVENT_SCHED,
1755					(void *)&gvt->service_request)) {
1756			intel_gvt_schedule(gvt);
1757		}
1758	}
1759
1760	return 0;
1761}
1762
1763static void clean_service_thread(struct intel_gvt *gvt)
1764{
1765	kthread_stop(gvt->service_thread);
1766}
1767
1768static int init_service_thread(struct intel_gvt *gvt)
1769{
1770	init_waitqueue_head(&gvt->service_thread_wq);
1771
1772	gvt->service_thread = kthread_run(gvt_service_thread,
1773			gvt, "gvt_service_thread");
1774	if (IS_ERR(gvt->service_thread)) {
1775		gvt_err("fail to start service thread.\n");
1776		return PTR_ERR(gvt->service_thread);
1777	}
1778	return 0;
1779}
1780
1781/**
1782 * intel_gvt_clean_device - clean a GVT device
1783 * @i915: i915 private
1784 *
1785 * This function is called at the driver unloading stage, to free the
1786 * resources owned by a GVT device.
1787 *
1788 */
1789static void intel_gvt_clean_device(struct drm_i915_private *i915)
1790{
1791	struct intel_gvt *gvt = fetch_and_zero(&i915->gvt);
1792
1793	if (drm_WARN_ON(&i915->drm, !gvt))
1794		return;
1795
1796	mdev_unregister_parent(&gvt->parent);
1797	intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
1798	intel_gvt_clean_vgpu_types(gvt);
1799
1800	intel_gvt_debugfs_clean(gvt);
1801	clean_service_thread(gvt);
1802	intel_gvt_clean_cmd_parser(gvt);
1803	intel_gvt_clean_sched_policy(gvt);
1804	intel_gvt_clean_workload_scheduler(gvt);
1805	intel_gvt_clean_gtt(gvt);
1806	intel_gvt_free_firmware(gvt);
1807	intel_gvt_clean_mmio_info(gvt);
1808	idr_destroy(&gvt->vgpu_idr);
1809
1810	kfree(i915->gvt);
1811}
1812
1813/**
1814 * intel_gvt_init_device - initialize a GVT device
1815 * @i915: drm i915 private data
1816 *
1817 * This function is called at the initialization stage, to initialize
1818 * necessary GVT components.
1819 *
1820 * Returns:
1821 * Zero on success, negative error code if failed.
1822 *
1823 */
1824static int intel_gvt_init_device(struct drm_i915_private *i915)
1825{
1826	struct intel_gvt *gvt;
1827	struct intel_vgpu *vgpu;
1828	int ret;
1829
1830	if (drm_WARN_ON(&i915->drm, i915->gvt))
1831		return -EEXIST;
1832
1833	gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL);
1834	if (!gvt)
1835		return -ENOMEM;
1836
1837	gvt_dbg_core("init gvt device\n");
1838
1839	idr_init_base(&gvt->vgpu_idr, 1);
1840	spin_lock_init(&gvt->scheduler.mmio_context_lock);
1841	mutex_init(&gvt->lock);
1842	mutex_init(&gvt->sched_lock);
1843	gvt->gt = to_gt(i915);
1844	i915->gvt = gvt;
1845
1846	init_device_info(gvt);
1847
1848	ret = intel_gvt_setup_mmio_info(gvt);
1849	if (ret)
1850		goto out_clean_idr;
1851
1852	intel_gvt_init_engine_mmio_context(gvt);
1853
1854	ret = intel_gvt_load_firmware(gvt);
1855	if (ret)
1856		goto out_clean_mmio_info;
1857
1858	ret = intel_gvt_init_irq(gvt);
1859	if (ret)
1860		goto out_free_firmware;
1861
1862	ret = intel_gvt_init_gtt(gvt);
1863	if (ret)
1864		goto out_free_firmware;
1865
1866	ret = intel_gvt_init_workload_scheduler(gvt);
1867	if (ret)
1868		goto out_clean_gtt;
1869
1870	ret = intel_gvt_init_sched_policy(gvt);
1871	if (ret)
1872		goto out_clean_workload_scheduler;
1873
1874	ret = intel_gvt_init_cmd_parser(gvt);
1875	if (ret)
1876		goto out_clean_sched_policy;
1877
1878	ret = init_service_thread(gvt);
1879	if (ret)
1880		goto out_clean_cmd_parser;
1881
1882	ret = intel_gvt_init_vgpu_types(gvt);
1883	if (ret)
1884		goto out_clean_thread;
1885
1886	vgpu = intel_gvt_create_idle_vgpu(gvt);
1887	if (IS_ERR(vgpu)) {
1888		ret = PTR_ERR(vgpu);
1889		gvt_err("failed to create idle vgpu\n");
1890		goto out_clean_types;
1891	}
1892	gvt->idle_vgpu = vgpu;
1893
1894	intel_gvt_debugfs_init(gvt);
1895
1896	ret = mdev_register_parent(&gvt->parent, i915->drm.dev,
1897				   &intel_vgpu_mdev_driver,
1898				   gvt->mdev_types, gvt->num_types);
1899	if (ret)
1900		goto out_destroy_idle_vgpu;
1901
1902	gvt_dbg_core("gvt device initialization is done\n");
1903	return 0;
1904
1905out_destroy_idle_vgpu:
1906	intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);
1907	intel_gvt_debugfs_clean(gvt);
1908out_clean_types:
1909	intel_gvt_clean_vgpu_types(gvt);
1910out_clean_thread:
1911	clean_service_thread(gvt);
1912out_clean_cmd_parser:
1913	intel_gvt_clean_cmd_parser(gvt);
1914out_clean_sched_policy:
1915	intel_gvt_clean_sched_policy(gvt);
1916out_clean_workload_scheduler:
1917	intel_gvt_clean_workload_scheduler(gvt);
1918out_clean_gtt:
1919	intel_gvt_clean_gtt(gvt);
1920out_free_firmware:
1921	intel_gvt_free_firmware(gvt);
1922out_clean_mmio_info:
1923	intel_gvt_clean_mmio_info(gvt);
1924out_clean_idr:
1925	idr_destroy(&gvt->vgpu_idr);
1926	kfree(gvt);
1927	i915->gvt = NULL;
1928	return ret;
1929}
1930
1931static void intel_gvt_pm_resume(struct drm_i915_private *i915)
1932{
1933	struct intel_gvt *gvt = i915->gvt;
1934
1935	intel_gvt_restore_fence(gvt);
1936	intel_gvt_restore_mmio(gvt);
1937	intel_gvt_restore_ggtt(gvt);
1938}
1939
1940static const struct intel_vgpu_ops intel_gvt_vgpu_ops = {
1941	.init_device	= intel_gvt_init_device,
1942	.clean_device	= intel_gvt_clean_device,
1943	.pm_resume	= intel_gvt_pm_resume,
1944};
1945
1946static int __init kvmgt_init(void)
1947{
1948	int ret;
1949
1950	ret = intel_gvt_set_ops(&intel_gvt_vgpu_ops);
1951	if (ret)
1952		return ret;
1953
1954	ret = mdev_register_driver(&intel_vgpu_mdev_driver);
1955	if (ret)
1956		intel_gvt_clear_ops(&intel_gvt_vgpu_ops);
1957	return ret;
1958}
1959
1960static void __exit kvmgt_exit(void)
1961{
1962	mdev_unregister_driver(&intel_vgpu_mdev_driver);
1963	intel_gvt_clear_ops(&intel_gvt_vgpu_ops);
1964}
1965
1966module_init(kvmgt_init);
1967module_exit(kvmgt_exit);
1968
1969MODULE_DESCRIPTION("Intel mediated pass-through framework for KVM");
1970MODULE_LICENSE("GPL and additional rights");
1971MODULE_AUTHOR("Intel Corporation");