vfio_iommu_type1.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * VFIO: IOMMU DMA mapping support for Type1 IOMMU
   4 *
   5 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6 *     Author: Alex Williamson <alex.williamson@redhat.com>
   7 *
   8 * Derived from original vfio:
   9 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10 * Author: Tom Lyon, pugs@cisco.com
  11 *
  12 * We arbitrarily define a Type1 IOMMU as one matching the below code.
  13 * It could be called the x86 IOMMU as it's designed for AMD-Vi & Intel
  14 * VT-d, but that makes it harder to re-use as theoretically anyone
  15 * implementing a similar IOMMU could make use of this.  We expect the
  16 * IOMMU to support the IOMMU API and have few to no restrictions around
  17 * the IOVA range that can be mapped.  The Type1 IOMMU is currently
  18 * optimized for relatively static mappings of a userspace process with
  19 * userspace pages pinned into memory.  We also assume devices and IOMMU
  20 * domains are PCI based as the IOMMU API is still centered around a
  21 * device/bus interface rather than a group interface.
  22 */
  23
  24#include <linux/compat.h>
  25#include <linux/device.h>
  26#include <linux/fs.h>
  27#include <linux/highmem.h>
  28#include <linux/iommu.h>
  29#include <linux/module.h>
  30#include <linux/mm.h>
  31#include <linux/kthread.h>
  32#include <linux/rbtree.h>
  33#include <linux/sched/signal.h>
  34#include <linux/sched/mm.h>
  35#include <linux/slab.h>
  36#include <linux/uaccess.h>
  37#include <linux/vfio.h>
  38#include <linux/workqueue.h>
  39#include <linux/notifier.h>
  40#include <linux/mm_inline.h>
  41#include <linux/overflow.h>
  42#include "vfio.h"
  43
  44#define DRIVER_VERSION  "0.2"
  45#define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  46#define DRIVER_DESC     "Type1 IOMMU driver for VFIO"
  47
  48static bool allow_unsafe_interrupts;
  49module_param_named(allow_unsafe_interrupts,
  50		   allow_unsafe_interrupts, bool, S_IRUGO | S_IWUSR);
  51MODULE_PARM_DESC(allow_unsafe_interrupts,
  52		 "Enable VFIO IOMMU support for on platforms without interrupt remapping support.");
  53
  54static bool disable_hugepages;
  55module_param_named(disable_hugepages,
  56		   disable_hugepages, bool, S_IRUGO | S_IWUSR);
  57MODULE_PARM_DESC(disable_hugepages,
  58		 "Disable VFIO IOMMU support for IOMMU hugepages.");
  59
  60static unsigned int dma_entry_limit __read_mostly = U16_MAX;
  61module_param_named(dma_entry_limit, dma_entry_limit, uint, 0644);
  62MODULE_PARM_DESC(dma_entry_limit,
  63		 "Maximum number of user DMA mappings per container (65535).");
  64
  65struct vfio_iommu {
  66	struct list_head	domain_list;
  67	struct list_head	iova_list;
  68	struct mutex		lock;
  69	struct rb_root		dma_list;
  70	struct list_head	device_list;
  71	struct mutex		device_list_lock;
  72	unsigned int		dma_avail;
  73	unsigned int		vaddr_invalid_count;
  74	uint64_t		pgsize_bitmap;
  75	uint64_t		num_non_pinned_groups;
  76	bool			v2;
  77	bool			dirty_page_tracking;
  78	struct list_head	emulated_iommu_groups;
  79};
  80
  81struct vfio_domain {
  82	struct iommu_domain	*domain;
  83	struct list_head	next;
  84	struct list_head	group_list;
  85	bool			enforce_cache_coherency : 1;
  86};
  87
  88struct vfio_dma {
  89	struct rb_node		node;
  90	dma_addr_t		iova;		/* Device address */
  91	unsigned long		vaddr;		/* Process virtual addr */
  92	size_t			size;		/* Map size (bytes) */
  93	int			prot;		/* IOMMU_READ/WRITE */
  94	bool			iommu_mapped;
  95	bool			lock_cap;	/* capable(CAP_IPC_LOCK) */
  96	bool			vaddr_invalid;
  97	bool			has_rsvd;	/* has 1 or more rsvd pfns */
  98	struct task_struct	*task;
  99	struct rb_root		pfn_list;	/* Ex-user pinned pfn list */
 100	unsigned long		*bitmap;
 101	struct mm_struct	*mm;
 102	size_t			locked_vm;
 103};
 104
 105struct vfio_batch {
 106	struct page		**pages;	/* for pin_user_pages_remote */
 107	struct page		*fallback_page; /* if pages alloc fails */
 108	unsigned int		capacity;	/* length of pages array */
 109	unsigned int		size;		/* of batch currently */
 110	unsigned int		offset;		/* of next entry in pages */
 111};
 112
 113struct vfio_iommu_group {
 114	struct iommu_group	*iommu_group;
 115	struct list_head	next;
 116	bool			pinned_page_dirty_scope;
 117};
 118
 119struct vfio_iova {
 120	struct list_head	list;
 121	dma_addr_t		start;
 122	dma_addr_t		end;
 123};
 124
 125/*
 126 * Guest RAM pinning working set or DMA target
 127 */
 128struct vfio_pfn {
 129	struct rb_node		node;
 130	dma_addr_t		iova;		/* Device address */
 131	unsigned long		pfn;		/* Host pfn */
 132	unsigned int		ref_count;
 133};
 134
 135struct vfio_regions {
 136	struct list_head list;
 137	dma_addr_t iova;
 138	phys_addr_t phys;
 139	size_t len;
 140};
 141
 142#define DIRTY_BITMAP_BYTES(n)	(ALIGN(n, BITS_PER_TYPE(u64)) / BITS_PER_BYTE)
 143
 144/*
 145 * Input argument of number of bits to bitmap_set() is unsigned integer, which
 146 * further casts to signed integer for unaligned multi-bit operation,
 147 * __bitmap_set().
 148 * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte,
 149 * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page
 150 * system.
 151 */
 152#define DIRTY_BITMAP_PAGES_MAX	 ((u64)INT_MAX)
 153#define DIRTY_BITMAP_SIZE_MAX	 DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
 154
 155static int put_pfn(unsigned long pfn, int prot);
 156
 157static struct vfio_iommu_group*
 158vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
 159			    struct iommu_group *iommu_group);
 160
 161/*
 162 * This code handles mapping and unmapping of user data buffers
 163 * into DMA'ble space using the IOMMU
 164 */
 165
 166static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu,
 167				      dma_addr_t start, size_t size)
 168{
 169	struct rb_node *node = iommu->dma_list.rb_node;
 170
 171	WARN_ON(!size);
 172
 173	while (node) {
 174		struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
 175
 176		if (start + size - 1 < dma->iova)
 177			node = node->rb_left;
 178		else if (start > dma->iova + dma->size - 1)
 179			node = node->rb_right;
 180		else
 181			return dma;
 182	}
 183
 184	return NULL;
 185}
 186
 187static struct rb_node *vfio_find_dma_first_node(struct vfio_iommu *iommu,
 188						dma_addr_t start,
 189						dma_addr_t end)
 190{
 191	struct rb_node *res = NULL;
 192	struct rb_node *node = iommu->dma_list.rb_node;
 193	struct vfio_dma *dma_res = NULL;
 194
 195	WARN_ON(end < start);
 196
 197	while (node) {
 198		struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node);
 199
 200		if (start <= dma->iova + dma->size - 1) {
 201			res = node;
 202			dma_res = dma;
 203			if (start >= dma->iova)
 204				break;
 205			node = node->rb_left;
 206		} else {
 207			node = node->rb_right;
 208		}
 209	}
 210	if (res && dma_res->iova > end)
 211		res = NULL;
 212	return res;
 213}
 214
 215static void vfio_link_dma(struct vfio_iommu *iommu, struct vfio_dma *new)
 216{
 217	struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL;
 218	struct vfio_dma *dma;
 219
 220	WARN_ON(new->size != 0);
 221
 222	while (*link) {
 223		parent = *link;
 224		dma = rb_entry(parent, struct vfio_dma, node);
 225
 226		if (new->iova <= dma->iova)
 227			link = &(*link)->rb_left;
 228		else
 229			link = &(*link)->rb_right;
 230	}
 231
 232	rb_link_node(&new->node, parent, link);
 233	rb_insert_color(&new->node, &iommu->dma_list);
 234}
 235
 236static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
 237{
 238	rb_erase(&old->node, &iommu->dma_list);
 239}
 240
 241
 242static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize)
 243{
 244	uint64_t npages = dma->size / pgsize;
 245
 246	if (npages > DIRTY_BITMAP_PAGES_MAX)
 247		return -EINVAL;
 248
 249	/*
 250	 * Allocate extra 64 bits that are used to calculate shift required for
 251	 * bitmap_shift_left() to manipulate and club unaligned number of pages
 252	 * in adjacent vfio_dma ranges.
 253	 */
 254	dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages) + sizeof(u64),
 255			       GFP_KERNEL);
 256	if (!dma->bitmap)
 257		return -ENOMEM;
 258
 259	return 0;
 260}
 261
 262static void vfio_dma_bitmap_free(struct vfio_dma *dma)
 263{
 264	kvfree(dma->bitmap);
 265	dma->bitmap = NULL;
 266}
 267
 268static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize)
 269{
 270	struct rb_node *p;
 271	unsigned long pgshift = __ffs(pgsize);
 272
 273	for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) {
 274		struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node);
 275
 276		bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1);
 277	}
 278}
 279
 280static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu)
 281{
 282	struct rb_node *n;
 283	unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
 284
 285	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
 286		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
 287
 288		bitmap_set(dma->bitmap, 0, dma->size >> pgshift);
 289	}
 290}
 291
 292static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)
 293{
 294	struct rb_node *n;
 295
 296	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
 297		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
 298		int ret;
 299
 300		ret = vfio_dma_bitmap_alloc(dma, pgsize);
 301		if (ret) {
 302			struct rb_node *p;
 303
 304			for (p = rb_prev(n); p; p = rb_prev(p)) {
 305				struct vfio_dma *dma = rb_entry(p,
 306							struct vfio_dma, node);
 307
 308				vfio_dma_bitmap_free(dma);
 309			}
 310			return ret;
 311		}
 312		vfio_dma_populate_bitmap(dma, pgsize);
 313	}
 314	return 0;
 315}
 316
 317static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu)
 318{
 319	struct rb_node *n;
 320
 321	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
 322		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
 323
 324		vfio_dma_bitmap_free(dma);
 325	}
 326}
 327
 328/*
 329 * Helper Functions for host iova-pfn list
 330 */
 331
 332/*
 333 * Find the highest vfio_pfn that overlapping the range
 334 * [iova_start, iova_end) in rb tree.
 335 */
 336static struct vfio_pfn *vfio_find_vpfn_range(struct vfio_dma *dma,
 337		dma_addr_t iova_start, dma_addr_t iova_end)
 338{
 339	struct vfio_pfn *vpfn;
 340	struct rb_node *node = dma->pfn_list.rb_node;
 341
 342	while (node) {
 343		vpfn = rb_entry(node, struct vfio_pfn, node);
 344
 345		if (iova_end <= vpfn->iova)
 346			node = node->rb_left;
 347		else if (iova_start > vpfn->iova)
 348			node = node->rb_right;
 349		else
 350			return vpfn;
 351	}
 352	return NULL;
 353}
 354
 355static inline struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova)
 356{
 357	return vfio_find_vpfn_range(dma, iova, iova + 1);
 358}
 359
 360static void vfio_link_pfn(struct vfio_dma *dma,
 361			  struct vfio_pfn *new)
 362{
 363	struct rb_node **link, *parent = NULL;
 364	struct vfio_pfn *vpfn;
 365
 366	link = &dma->pfn_list.rb_node;
 367	while (*link) {
 368		parent = *link;
 369		vpfn = rb_entry(parent, struct vfio_pfn, node);
 370
 371		if (new->iova < vpfn->iova)
 372			link = &(*link)->rb_left;
 373		else
 374			link = &(*link)->rb_right;
 375	}
 376
 377	rb_link_node(&new->node, parent, link);
 378	rb_insert_color(&new->node, &dma->pfn_list);
 379}
 380
 381static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old)
 382{
 383	rb_erase(&old->node, &dma->pfn_list);
 384}
 385
 386static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova,
 387				unsigned long pfn)
 388{
 389	struct vfio_pfn *vpfn;
 390
 391	vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL);
 392	if (!vpfn)
 393		return -ENOMEM;
 394
 395	vpfn->iova = iova;
 396	vpfn->pfn = pfn;
 397	vpfn->ref_count = 1;
 398	vfio_link_pfn(dma, vpfn);
 399	return 0;
 400}
 401
 402static void vfio_remove_from_pfn_list(struct vfio_dma *dma,
 403				      struct vfio_pfn *vpfn)
 404{
 405	vfio_unlink_pfn(dma, vpfn);
 406	kfree(vpfn);
 407}
 408
 409static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
 410					       unsigned long iova)
 411{
 412	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
 413
 414	if (vpfn)
 415		vpfn->ref_count++;
 416	return vpfn;
 417}
 418
 419static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
 420{
 421	int ret = 0;
 422
 423	vpfn->ref_count--;
 424	if (!vpfn->ref_count) {
 425		ret = put_pfn(vpfn->pfn, dma->prot);
 426		vfio_remove_from_pfn_list(dma, vpfn);
 427	}
 428	return ret;
 429}
 430
 431static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm,
 432			bool lock_cap, long npage)
 433{
 434	int ret = mmap_write_lock_killable(mm);
 435
 436	if (ret)
 437		return ret;
 438
 439	ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap);
 440	mmap_write_unlock(mm);
 441	return ret;
 442}
 443
 444static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 445{
 446	struct mm_struct *mm;
 447	int ret;
 448
 449	if (!npage)
 450		return 0;
 451
 452	mm = dma->mm;
 453	if (async && !mmget_not_zero(mm))
 454		return -ESRCH; /* process exited */
 455
 456	ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage);
 457	if (!ret)
 458		dma->locked_vm += npage;
 459
 460	if (async)
 461		mmput(mm);
 462
 463	return ret;
 464}
 465
 466/*
 467 * Some mappings aren't backed by a struct page, for example an mmap'd
 468 * MMIO range for our own or another device.  These use a different
 469 * pfn conversion and shouldn't be tracked as locked pages.
 470 * For compound pages, any driver that sets the reserved bit in head
 471 * page needs to set the reserved bit in all subpages to be safe.
 472 */
 473static bool is_invalid_reserved_pfn(unsigned long pfn)
 474{
 475	if (pfn_valid(pfn))
 476		return PageReserved(pfn_to_page(pfn));
 477
 478	return true;
 479}
 480
 481static int put_pfn(unsigned long pfn, int prot)
 482{
 483	if (!is_invalid_reserved_pfn(pfn)) {
 484		struct page *page = pfn_to_page(pfn);
 485
 486		unpin_user_pages_dirty_lock(&page, 1, prot & IOMMU_WRITE);
 487		return 1;
 488	}
 489	return 0;
 490}
 491
 492#define VFIO_BATCH_MAX_CAPACITY (PAGE_SIZE / sizeof(struct page *))
 493
 494static void __vfio_batch_init(struct vfio_batch *batch, bool single)
 495{
 496	batch->size = 0;
 497	batch->offset = 0;
 498
 499	if (single || unlikely(disable_hugepages))
 500		goto fallback;
 501
 502	batch->pages = (struct page **) __get_free_page(GFP_KERNEL);
 503	if (!batch->pages)
 504		goto fallback;
 505
 506	batch->capacity = VFIO_BATCH_MAX_CAPACITY;
 507	return;
 508
 509fallback:
 510	batch->pages = &batch->fallback_page;
 511	batch->capacity = 1;
 512}
 513
 514static void vfio_batch_init(struct vfio_batch *batch)
 515{
 516	__vfio_batch_init(batch, false);
 517}
 518
 519static void vfio_batch_init_single(struct vfio_batch *batch)
 520{
 521	__vfio_batch_init(batch, true);
 522}
 523
 524static void vfio_batch_unpin(struct vfio_batch *batch, struct vfio_dma *dma)
 525{
 526	while (batch->size) {
 527		unsigned long pfn = page_to_pfn(batch->pages[batch->offset]);
 528
 529		put_pfn(pfn, dma->prot);
 530		batch->offset++;
 531		batch->size--;
 532	}
 533}
 534
 535static void vfio_batch_fini(struct vfio_batch *batch)
 536{
 537	if (batch->capacity == VFIO_BATCH_MAX_CAPACITY)
 538		free_page((unsigned long)batch->pages);
 539}
 540
 541static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
 542			    unsigned long vaddr, unsigned long *pfn,
 543			    unsigned long *addr_mask, bool write_fault)
 544{
 545	struct follow_pfnmap_args args = { .vma = vma, .address = vaddr };
 546	int ret;
 547
 548	ret = follow_pfnmap_start(&args);
 549	if (ret) {
 550		bool unlocked = false;
 551
 552		ret = fixup_user_fault(mm, vaddr,
 553				       FAULT_FLAG_REMOTE |
 554				       (write_fault ?  FAULT_FLAG_WRITE : 0),
 555				       &unlocked);
 556		if (unlocked)
 557			return -EAGAIN;
 558
 559		if (ret)
 560			return ret;
 561
 562		ret = follow_pfnmap_start(&args);
 563		if (ret)
 564			return ret;
 565	}
 566
 567	if (write_fault && !args.writable) {
 568		ret = -EFAULT;
 569	} else {
 570		*pfn = args.pfn;
 571		*addr_mask = args.addr_mask;
 572	}
 573
 574	follow_pfnmap_end(&args);
 575	return ret;
 576}
 577
 578/*
 579 * Returns the positive number of pfns successfully obtained or a negative
 580 * error code.  The initial pfn is stored in the pfn arg.  For page-backed
 581 * pfns, the provided batch is also updated to indicate the filled pages and
 582 * initial offset.  For VM_PFNMAP pfns, only the returned number of pfns and
 583 * returned initial pfn are provided; subsequent pfns are contiguous.
 584 */
 585static long vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr,
 586			   unsigned long npages, int prot, unsigned long *pfn,
 587			   struct vfio_batch *batch)
 588{
 589	unsigned long pin_pages = min_t(unsigned long, npages, batch->capacity);
 590	struct vm_area_struct *vma;
 591	unsigned int flags = 0;
 592	long ret;
 593
 594	if (prot & IOMMU_WRITE)
 595		flags |= FOLL_WRITE;
 596
 597	mmap_read_lock(mm);
 598	ret = pin_user_pages_remote(mm, vaddr, pin_pages, flags | FOLL_LONGTERM,
 599				    batch->pages, NULL);
 600	if (ret > 0) {
 601		*pfn = page_to_pfn(batch->pages[0]);
 602		batch->size = ret;
 603		batch->offset = 0;
 604		goto done;
 605	} else if (!ret) {
 606		ret = -EFAULT;
 607	}
 608
 609	vaddr = untagged_addr_remote(mm, vaddr);
 610
 611retry:
 612	vma = vma_lookup(mm, vaddr);
 613
 614	if (vma && vma->vm_flags & VM_PFNMAP) {
 615		unsigned long addr_mask;
 616
 617		ret = follow_fault_pfn(vma, mm, vaddr, pfn, &addr_mask,
 618				       prot & IOMMU_WRITE);
 619		if (ret == -EAGAIN)
 620			goto retry;
 621
 622		if (!ret) {
 623			if (is_invalid_reserved_pfn(*pfn)) {
 624				unsigned long epfn;
 625
 626				epfn = (*pfn | (~addr_mask >> PAGE_SHIFT)) + 1;
 627				ret = min_t(long, npages, epfn - *pfn);
 628			} else {
 629				ret = -EFAULT;
 630			}
 631		}
 632	}
 633done:
 634	mmap_read_unlock(mm);
 635	return ret;
 636}
 637
 638
 639static long vpfn_pages(struct vfio_dma *dma,
 640		dma_addr_t iova_start, long nr_pages)
 641{
 642	dma_addr_t iova_end = iova_start + (nr_pages << PAGE_SHIFT);
 643	struct vfio_pfn *top = vfio_find_vpfn_range(dma, iova_start, iova_end);
 644	long ret = 1;
 645	struct vfio_pfn *vpfn;
 646	struct rb_node *prev;
 647	struct rb_node *next;
 648
 649	if (likely(!top))
 650		return 0;
 651
 652	prev = next = &top->node;
 653
 654	while ((prev = rb_prev(prev))) {
 655		vpfn = rb_entry(prev, struct vfio_pfn, node);
 656		if (vpfn->iova < iova_start)
 657			break;
 658		ret++;
 659	}
 660
 661	while ((next = rb_next(next))) {
 662		vpfn = rb_entry(next, struct vfio_pfn, node);
 663		if (vpfn->iova >= iova_end)
 664			break;
 665		ret++;
 666	}
 667
 668	return ret;
 669}
 670
 671/*
 672 * Attempt to pin pages.  We really don't want to track all the pfns and
 673 * the iommu can only map chunks of consecutive pfns anyway, so get the
 674 * first page and all consecutive pages with the same locking.
 675 */
 676static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
 677				  unsigned long npage, unsigned long *pfn_base,
 678				  unsigned long limit, struct vfio_batch *batch)
 679{
 680	unsigned long pfn;
 681	struct mm_struct *mm = current->mm;
 682	long ret, pinned = 0, lock_acct = 0;
 683	bool rsvd;
 684	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
 685
 686	/* This code path is only user initiated */
 687	if (!mm)
 688		return -ENODEV;
 689
 690	if (batch->size) {
 691		/* Leftover pages in batch from an earlier call. */
 692		*pfn_base = page_to_pfn(batch->pages[batch->offset]);
 693		pfn = *pfn_base;
 694		rsvd = is_invalid_reserved_pfn(*pfn_base);
 695	} else {
 696		*pfn_base = 0;
 697	}
 698
 699	if (unlikely(disable_hugepages))
 700		npage = 1;
 701
 702	while (npage) {
 703		if (!batch->size) {
 704			/*
 705			 * Large mappings may take a while to repeatedly refill
 706			 * the batch, so conditionally relinquish the CPU when
 707			 * needed to avoid stalls.
 708			 */
 709			cond_resched();
 710
 711			/* Empty batch, so refill it. */
 712			ret = vaddr_get_pfns(mm, vaddr, npage, dma->prot,
 713					     &pfn, batch);
 714			if (ret < 0)
 715				goto unpin_out;
 716
 717			if (!*pfn_base) {
 718				*pfn_base = pfn;
 719				rsvd = is_invalid_reserved_pfn(*pfn_base);
 720			}
 721
 722			/* Handle pfnmap */
 723			if (!batch->size) {
 724				if (pfn != *pfn_base + pinned || !rsvd)
 725					goto out;
 726
 727				pinned += ret;
 728				npage -= ret;
 729				vaddr += (PAGE_SIZE * ret);
 730				iova += (PAGE_SIZE * ret);
 731				continue;
 732			}
 733		}
 734
 735		/*
 736		 * pfn is preset for the first iteration of this inner loop
 737		 * due to the fact that vaddr_get_pfns() needs to provide the
 738		 * initial pfn for pfnmaps.  Therefore to reduce redundancy,
 739		 * the next pfn is fetched at the end of the loop.
 740		 * A PageReserved() page could still qualify as page backed
 741		 * and rsvd here, and therefore continues to use the batch.
 742		 */
 743		while (true) {
 744			long nr_pages, acct_pages = 0;
 745
 746			if (pfn != *pfn_base + pinned ||
 747			    rsvd != is_invalid_reserved_pfn(pfn))
 748				goto out;
 749
 750			/*
 751			 * Using GUP with the FOLL_LONGTERM in
 752			 * vaddr_get_pfns() will not return invalid
 753			 * or reserved pages.
 754			 */
 755			nr_pages = num_pages_contiguous(
 756					&batch->pages[batch->offset],
 757					batch->size);
 758			if (!rsvd) {
 759				acct_pages = nr_pages;
 760				acct_pages -= vpfn_pages(dma, iova, nr_pages);
 761			}
 762
 763			/*
 764			 * Reserved pages aren't counted against the user,
 765			 * externally pinned pages are already counted against
 766			 * the user.
 767			 */
 768			if (acct_pages) {
 769				if (!dma->lock_cap &&
 770				    mm->locked_vm + lock_acct + acct_pages > limit) {
 771					pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
 772						__func__, limit << PAGE_SHIFT);
 773					ret = -ENOMEM;
 774					goto unpin_out;
 775				}
 776				lock_acct += acct_pages;
 777			}
 778
 779			pinned += nr_pages;
 780			npage -= nr_pages;
 781			vaddr += PAGE_SIZE * nr_pages;
 782			iova += PAGE_SIZE * nr_pages;
 783			batch->offset += nr_pages;
 784			batch->size -= nr_pages;
 785
 786			if (!batch->size)
 787				break;
 788
 789			pfn = page_to_pfn(batch->pages[batch->offset]);
 790		}
 791	}
 792
 793out:
 794	dma->has_rsvd |= rsvd;
 795	ret = vfio_lock_acct(dma, lock_acct, false);
 796
 797unpin_out:
 798	if (ret < 0) {
 799		if (pinned && !rsvd) {
 800			for (pfn = *pfn_base ; pinned ; pfn++, pinned--)
 801				put_pfn(pfn, dma->prot);
 802		}
 803		vfio_batch_unpin(batch, dma);
 804
 805		return ret;
 806	}
 807
 808	return pinned;
 809}
 810
 811static inline void put_valid_unreserved_pfns(unsigned long start_pfn,
 812		unsigned long npage, int prot)
 813{
 814	unpin_user_page_range_dirty_lock(pfn_to_page(start_pfn), npage,
 815					 prot & IOMMU_WRITE);
 816}
 817
 818static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
 819				    unsigned long pfn, unsigned long npage,
 820				    bool do_accounting)
 821{
 822	long unlocked = 0, locked = vpfn_pages(dma, iova, npage);
 823
 824	if (dma->has_rsvd) {
 825		unsigned long i;
 826
 827		for (i = 0; i < npage; i++)
 828			if (put_pfn(pfn++, dma->prot))
 829				unlocked++;
 830	} else {
 831		put_valid_unreserved_pfns(pfn, npage, dma->prot);
 832		unlocked = npage;
 833	}
 834	if (do_accounting)
 835		vfio_lock_acct(dma, locked - unlocked, true);
 836
 837	return unlocked;
 838}
 839
 840static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
 841				  unsigned long *pfn_base, bool do_accounting)
 842{
 843	struct vfio_batch batch;
 844	struct mm_struct *mm;
 845	int ret;
 846
 847	mm = dma->mm;
 848	if (!mmget_not_zero(mm))
 849		return -ENODEV;
 850
 851	vfio_batch_init_single(&batch);
 852
 853	ret = vaddr_get_pfns(mm, vaddr, 1, dma->prot, pfn_base, &batch);
 854	if (ret != 1)
 855		goto out;
 856
 857	ret = 0;
 858
 859	if (do_accounting && !is_invalid_reserved_pfn(*pfn_base)) {
 860		ret = vfio_lock_acct(dma, 1, false);
 861		if (ret) {
 862			put_pfn(*pfn_base, dma->prot);
 863			if (ret == -ENOMEM)
 864				pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK "
 865					"(%ld) exceeded\n", __func__,
 866					dma->task->comm, task_pid_nr(dma->task),
 867					task_rlimit(dma->task, RLIMIT_MEMLOCK));
 868		}
 869	}
 870
 871out:
 872	vfio_batch_fini(&batch);
 873	mmput(mm);
 874	return ret;
 875}
 876
 877static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
 878				    bool do_accounting)
 879{
 880	int unlocked;
 881	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
 882
 883	if (!vpfn)
 884		return 0;
 885
 886	unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
 887
 888	if (do_accounting)
 889		vfio_lock_acct(dma, -unlocked, true);
 890
 891	return unlocked;
 892}
 893
 894static int vfio_iommu_type1_pin_pages(void *iommu_data,
 895				      struct iommu_group *iommu_group,
 896				      dma_addr_t user_iova,
 897				      int npage, int prot,
 898				      struct page **pages)
 899{
 900	struct vfio_iommu *iommu = iommu_data;
 901	struct vfio_iommu_group *group;
 902	int i, j, ret;
 903	unsigned long remote_vaddr;
 904	struct vfio_dma *dma;
 905	bool do_accounting;
 906	dma_addr_t iova_end;
 907	size_t iova_size;
 908
 909	if (!iommu || !pages || npage <= 0)
 910		return -EINVAL;
 911
 912	/* Supported for v2 version only */
 913	if (!iommu->v2)
 914		return -EACCES;
 915
 916	if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) ||
 917	    check_add_overflow(user_iova, iova_size - 1, &iova_end))
 918		return -EOVERFLOW;
 919
 920	mutex_lock(&iommu->lock);
 921
 922	if (WARN_ONCE(iommu->vaddr_invalid_count,
 923		      "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) {
 924		ret = -EBUSY;
 925		goto pin_done;
 926	}
 927
 928	/* Fail if no dma_umap notifier is registered */
 929	if (list_empty(&iommu->device_list)) {
 930		ret = -EINVAL;
 931		goto pin_done;
 932	}
 933
 934	/*
 935	 * If iommu capable domain exist in the container then all pages are
 936	 * already pinned and accounted. Accounting should be done if there is no
 937	 * iommu capable domain in the container.
 938	 */
 939	do_accounting = list_empty(&iommu->domain_list);
 940
 941	for (i = 0; i < npage; i++) {
 942		unsigned long phys_pfn;
 943		dma_addr_t iova;
 944		struct vfio_pfn *vpfn;
 945
 946		iova = user_iova + PAGE_SIZE * i;
 947		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
 948		if (!dma) {
 949			ret = -EINVAL;
 950			goto pin_unwind;
 951		}
 952
 953		if ((dma->prot & prot) != prot) {
 954			ret = -EPERM;
 955			goto pin_unwind;
 956		}
 957
 958		vpfn = vfio_iova_get_vfio_pfn(dma, iova);
 959		if (vpfn) {
 960			pages[i] = pfn_to_page(vpfn->pfn);
 961			continue;
 962		}
 963
 964		remote_vaddr = dma->vaddr + (iova - dma->iova);
 965		ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn,
 966					     do_accounting);
 967		if (ret)
 968			goto pin_unwind;
 969
 970		if (!pfn_valid(phys_pfn)) {
 971			ret = -EINVAL;
 972			goto pin_unwind;
 973		}
 974
 975		ret = vfio_add_to_pfn_list(dma, iova, phys_pfn);
 976		if (ret) {
 977			if (put_pfn(phys_pfn, dma->prot) && do_accounting)
 978				vfio_lock_acct(dma, -1, true);
 979			goto pin_unwind;
 980		}
 981
 982		pages[i] = pfn_to_page(phys_pfn);
 983
 984		if (iommu->dirty_page_tracking) {
 985			unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
 986
 987			/*
 988			 * Bitmap populated with the smallest supported page
 989			 * size
 990			 */
 991			bitmap_set(dma->bitmap,
 992				   (iova - dma->iova) >> pgshift, 1);
 993		}
 994	}
 995	ret = i;
 996
 997	group = vfio_iommu_find_iommu_group(iommu, iommu_group);
 998	if (!group->pinned_page_dirty_scope) {
 999		group->pinned_page_dirty_scope = true;
1000		iommu->num_non_pinned_groups--;
1001	}
1002
1003	goto pin_done;
1004
1005pin_unwind:
1006	pages[i] = NULL;
1007	for (j = 0; j < i; j++) {
1008		dma_addr_t iova;
1009
1010		iova = user_iova + PAGE_SIZE * j;
1011		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
1012		vfio_unpin_page_external(dma, iova, do_accounting);
1013		pages[j] = NULL;
1014	}
1015pin_done:
1016	mutex_unlock(&iommu->lock);
1017	return ret;
1018}
1019
1020static void vfio_iommu_type1_unpin_pages(void *iommu_data,
1021					 dma_addr_t user_iova, int npage)
1022{
1023	struct vfio_iommu *iommu = iommu_data;
1024	bool do_accounting;
1025	dma_addr_t iova_end;
1026	size_t iova_size;
1027	int i;
1028
1029	/* Supported for v2 version only */
1030	if (WARN_ON(!iommu->v2))
1031		return;
1032
1033	if (WARN_ON(npage <= 0))
1034		return;
1035
1036	if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) ||
1037		    check_add_overflow(user_iova, iova_size - 1, &iova_end)))
1038		return;
1039
1040	mutex_lock(&iommu->lock);
1041
1042	do_accounting = list_empty(&iommu->domain_list);
1043	for (i = 0; i < npage; i++) {
1044		dma_addr_t iova = user_iova + PAGE_SIZE * i;
1045		struct vfio_dma *dma;
1046
1047		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
1048		if (!dma)
1049			break;
1050
1051		vfio_unpin_page_external(dma, iova, do_accounting);
1052	}
1053
1054	mutex_unlock(&iommu->lock);
1055
1056	WARN_ON(i != npage);
1057}
1058
1059static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain,
1060			    struct list_head *regions,
1061			    struct iommu_iotlb_gather *iotlb_gather)
1062{
1063	long unlocked = 0;
1064	struct vfio_regions *entry, *next;
1065
1066	iommu_iotlb_sync(domain->domain, iotlb_gather);
1067
1068	list_for_each_entry_safe(entry, next, regions, list) {
1069		unlocked += vfio_unpin_pages_remote(dma,
1070						    entry->iova,
1071						    entry->phys >> PAGE_SHIFT,
1072						    entry->len >> PAGE_SHIFT,
1073						    false);
1074		list_del(&entry->list);
1075		kfree(entry);
1076	}
1077
1078	cond_resched();
1079
1080	return unlocked;
1081}
1082
1083/*
1084 * Generally, VFIO needs to unpin remote pages after each IOTLB flush.
1085 * Therefore, when using IOTLB flush sync interface, VFIO need to keep track
1086 * of these regions (currently using a list).
1087 *
1088 * This value specifies maximum number of regions for each IOTLB flush sync.
1089 */
1090#define VFIO_IOMMU_TLB_SYNC_MAX		512
1091
1092static size_t unmap_unpin_fast(struct vfio_domain *domain,
1093			       struct vfio_dma *dma, dma_addr_t iova,
1094			       size_t len, phys_addr_t phys, long *unlocked,
1095			       struct list_head *unmapped_list,
1096			       int *unmapped_cnt,
1097			       struct iommu_iotlb_gather *iotlb_gather)
1098{
1099	size_t unmapped = 0;
1100	struct vfio_regions *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1101
1102	if (entry) {
1103		unmapped = iommu_unmap_fast(domain->domain, iova, len,
1104					    iotlb_gather);
1105
1106		if (!unmapped) {
1107			kfree(entry);
1108		} else {
1109			entry->iova = iova;
1110			entry->phys = phys;
1111			entry->len  = unmapped;
1112			list_add_tail(&entry->list, unmapped_list);
1113
1114			(*unmapped_cnt)++;
1115		}
1116	}
1117
1118	/*
1119	 * Sync if the number of fast-unmap regions hits the limit
1120	 * or in case of errors.
1121	 */
1122	if (*unmapped_cnt >= VFIO_IOMMU_TLB_SYNC_MAX || !unmapped) {
1123		*unlocked += vfio_sync_unpin(dma, domain, unmapped_list,
1124					     iotlb_gather);
1125		*unmapped_cnt = 0;
1126	}
1127
1128	return unmapped;
1129}
1130
1131static size_t unmap_unpin_slow(struct vfio_domain *domain,
1132			       struct vfio_dma *dma, dma_addr_t iova,
1133			       size_t len, phys_addr_t phys,
1134			       long *unlocked)
1135{
1136	size_t unmapped = iommu_unmap(domain->domain, iova, len);
1137
1138	if (unmapped) {
1139		*unlocked += vfio_unpin_pages_remote(dma, iova,
1140						     phys >> PAGE_SHIFT,
1141						     unmapped >> PAGE_SHIFT,
1142						     false);
1143		cond_resched();
1144	}
1145	return unmapped;
1146}
1147
1148static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
1149			     bool do_accounting)
1150{
1151	struct vfio_domain *domain, *d;
1152	LIST_HEAD(unmapped_region_list);
1153	struct iommu_iotlb_gather iotlb_gather;
1154	int unmapped_region_cnt = 0;
1155	long unlocked = 0;
1156	size_t pos = 0;
1157
1158	if (!dma->size)
1159		return 0;
1160
1161	if (list_empty(&iommu->domain_list))
1162		return 0;
1163
1164	/*
1165	 * We use the IOMMU to track the physical addresses, otherwise we'd
1166	 * need a much more complicated tracking system.  Unfortunately that
1167	 * means we need to use one of the iommu domains to figure out the
1168	 * pfns to unpin.  The rest need to be unmapped in advance so we have
1169	 * no iommu translations remaining when the pages are unpinned.
1170	 */
1171	domain = d = list_first_entry(&iommu->domain_list,
1172				      struct vfio_domain, next);
1173
1174	list_for_each_entry_continue(d, &iommu->domain_list, next) {
1175		iommu_unmap(d->domain, dma->iova, dma->size);
1176		cond_resched();
1177	}
1178
1179	iommu_iotlb_gather_init(&iotlb_gather);
1180	while (pos < dma->size) {
1181		size_t unmapped, len;
1182		phys_addr_t phys, next;
1183		dma_addr_t iova = dma->iova + pos;
1184
1185		phys = iommu_iova_to_phys(domain->domain, iova);
1186		if (WARN_ON(!phys)) {
1187			pos += PAGE_SIZE;
1188			continue;
1189		}
1190
1191		/*
1192		 * To optimize for fewer iommu_unmap() calls, each of which
1193		 * may require hardware cache flushing, try to find the
1194		 * largest contiguous physical memory chunk to unmap.
1195		 */
1196		for (len = PAGE_SIZE; pos + len < dma->size; len += PAGE_SIZE) {
1197			next = iommu_iova_to_phys(domain->domain, iova + len);
1198			if (next != phys + len)
1199				break;
1200		}
1201
1202		/*
1203		 * First, try to use fast unmap/unpin. In case of failure,
1204		 * switch to slow unmap/unpin path.
1205		 */
1206		unmapped = unmap_unpin_fast(domain, dma, iova, len, phys,
1207					    &unlocked, &unmapped_region_list,
1208					    &unmapped_region_cnt,
1209					    &iotlb_gather);
1210		if (!unmapped) {
1211			unmapped = unmap_unpin_slow(domain, dma, iova, len,
1212						    phys, &unlocked);
1213			if (WARN_ON(!unmapped))
1214				break;
1215		}
1216
1217		pos += unmapped;
1218	}
1219
1220	dma->iommu_mapped = false;
1221
1222	if (unmapped_region_cnt) {
1223		unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list,
1224					    &iotlb_gather);
1225	}
1226
1227	if (do_accounting) {
1228		vfio_lock_acct(dma, -unlocked, true);
1229		return 0;
1230	}
1231	return unlocked;
1232}
1233
1234static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
1235{
1236	WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list));
1237	vfio_unmap_unpin(iommu, dma, true);
1238	vfio_unlink_dma(iommu, dma);
1239	put_task_struct(dma->task);
1240	mmdrop(dma->mm);
1241	vfio_dma_bitmap_free(dma);
1242	if (dma->vaddr_invalid)
1243		iommu->vaddr_invalid_count--;
1244	kfree(dma);
1245	iommu->dma_avail++;
1246}
1247
1248static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu)
1249{
1250	struct vfio_domain *domain;
1251
1252	iommu->pgsize_bitmap = ULONG_MAX;
1253
1254	list_for_each_entry(domain, &iommu->domain_list, next)
1255		iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap;
1256
1257	/*
1258	 * In case the IOMMU supports page sizes smaller than PAGE_SIZE
1259	 * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes.
1260	 * That way the user will be able to map/unmap buffers whose size/
1261	 * start address is aligned with PAGE_SIZE. Pinning code uses that
1262	 * granularity while iommu driver can use the sub-PAGE_SIZE size
1263	 * to map the buffer.
1264	 */
1265	if (iommu->pgsize_bitmap & ~PAGE_MASK) {
1266		iommu->pgsize_bitmap &= PAGE_MASK;
1267		iommu->pgsize_bitmap |= PAGE_SIZE;
1268	}
1269}
1270
1271static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
1272			      struct vfio_dma *dma, dma_addr_t base_iova,
1273			      size_t pgsize)
1274{
1275	unsigned long pgshift = __ffs(pgsize);
1276	unsigned long nbits = dma->size >> pgshift;
1277	unsigned long bit_offset = (dma->iova - base_iova) >> pgshift;
1278	unsigned long copy_offset = bit_offset / BITS_PER_LONG;
1279	unsigned long shift = bit_offset % BITS_PER_LONG;
1280	unsigned long leftover;
1281
1282	/*
1283	 * mark all pages dirty if any IOMMU capable device is not able
1284	 * to report dirty pages and all pages are pinned and mapped.
1285	 */
1286	if (iommu->num_non_pinned_groups && dma->iommu_mapped)
1287		bitmap_set(dma->bitmap, 0, nbits);
1288
1289	if (shift) {
1290		bitmap_shift_left(dma->bitmap, dma->bitmap, shift,
1291				  nbits + shift);
1292
1293		if (copy_from_user(&leftover,
1294				   (void __user *)(bitmap + copy_offset),
1295				   sizeof(leftover)))
1296			return -EFAULT;
1297
1298		bitmap_or(dma->bitmap, dma->bitmap, &leftover, shift);
1299	}
1300
1301	if (copy_to_user((void __user *)(bitmap + copy_offset), dma->bitmap,
1302			 DIRTY_BITMAP_BYTES(nbits + shift)))
1303		return -EFAULT;
1304
1305	return 0;
1306}
1307
1308static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
1309				  dma_addr_t iova, dma_addr_t iova_end, size_t pgsize)
1310{
1311	struct vfio_dma *dma;
1312	struct rb_node *n;
1313	unsigned long pgshift = __ffs(pgsize);
1314	int ret;
1315
1316	/*
1317	 * GET_BITMAP request must fully cover vfio_dma mappings.  Multiple
1318	 * vfio_dma mappings may be clubbed by specifying large ranges, but
1319	 * there must not be any previous mappings bisected by the range.
1320	 * An error will be returned if these conditions are not met.
1321	 */
1322	dma = vfio_find_dma(iommu, iova, 1);
1323	if (dma && dma->iova != iova)
1324		return -EINVAL;
1325
1326	dma = vfio_find_dma(iommu, iova_end, 1);
1327	if (dma && dma->iova + dma->size - 1 != iova_end)
1328		return -EINVAL;
1329
1330	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
1331		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
1332
1333		if (dma->iova < iova)
1334			continue;
1335
1336		if (dma->iova > iova_end)
1337			break;
1338
1339		ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);
1340		if (ret)
1341			return ret;
1342
1343		/*
1344		 * Re-populate bitmap to include all pinned pages which are
1345		 * considered as dirty but exclude pages which are unpinned and
1346		 * pages which are marked dirty by vfio_dma_rw()
1347		 */
1348		bitmap_clear(dma->bitmap, 0, dma->size >> pgshift);
1349		vfio_dma_populate_bitmap(dma, pgsize);
1350	}
1351	return 0;
1352}
1353
1354static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size)
1355{
1356	if (!npages || !bitmap_size || (bitmap_size > DIRTY_BITMAP_SIZE_MAX) ||
1357	    (bitmap_size < DIRTY_BITMAP_BYTES(npages)))
1358		return -EINVAL;
1359
1360	return 0;
1361}
1362
1363/*
1364 * Notify VFIO drivers using vfio_register_emulated_iommu_dev() to invalidate
1365 * and unmap iovas within the range we're about to unmap. Drivers MUST unpin
1366 * pages in response to an invalidation.
1367 */
1368static void vfio_notify_dma_unmap(struct vfio_iommu *iommu,
1369				  struct vfio_dma *dma)
1370{
1371	struct vfio_device *device;
1372
1373	if (list_empty(&iommu->device_list))
1374		return;
1375
1376	/*
1377	 * The device is expected to call vfio_unpin_pages() for any IOVA it has
1378	 * pinned within the range. Since vfio_unpin_pages() will eventually
1379	 * call back down to this code and try to obtain the iommu->lock we must
1380	 * drop it.
1381	 */
1382	mutex_lock(&iommu->device_list_lock);
1383	mutex_unlock(&iommu->lock);
1384
1385	list_for_each_entry(device, &iommu->device_list, iommu_entry)
1386		device->ops->dma_unmap(device, dma->iova, dma->size);
1387
1388	mutex_unlock(&iommu->device_list_lock);
1389	mutex_lock(&iommu->lock);
1390}
1391
1392static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
1393			     struct vfio_iommu_type1_dma_unmap *unmap,
1394			     struct vfio_bitmap *bitmap)
1395{
1396	struct vfio_dma *dma, *dma_last = NULL;
1397	size_t unmapped = 0, pgsize;
1398	int ret = -EINVAL, retries = 0;
1399	unsigned long pgshift;
1400	dma_addr_t iova = unmap->iova;
1401	dma_addr_t iova_end;
1402	size_t size = unmap->size;
1403	bool unmap_all = unmap->flags & VFIO_DMA_UNMAP_FLAG_ALL;
1404	bool invalidate_vaddr = unmap->flags & VFIO_DMA_UNMAP_FLAG_VADDR;
1405	struct rb_node *n, *first_n;
1406
1407	mutex_lock(&iommu->lock);
1408
1409	/* Cannot update vaddr if mdev is present. */
1410	if (invalidate_vaddr && !list_empty(&iommu->emulated_iommu_groups)) {
1411		ret = -EBUSY;
1412		goto unlock;
1413	}
1414
1415	if (iova != unmap->iova || size != unmap->size) {
1416		ret = -EOVERFLOW;
1417		goto unlock;
1418	}
1419
1420	pgshift = __ffs(iommu->pgsize_bitmap);
1421	pgsize = (size_t)1 << pgshift;
1422
1423	if (iova & (pgsize - 1))
1424		goto unlock;
1425
1426	if (unmap_all) {
1427		if (iova || size)
1428			goto unlock;
1429		iova_end = ~(dma_addr_t)0;
1430	} else {
1431		if (!size || size & (pgsize - 1))
1432			goto unlock;
1433
1434		if (check_add_overflow(iova, size - 1, &iova_end)) {
1435			ret = -EOVERFLOW;
1436			goto unlock;
1437		}
1438	}
1439
1440	/* When dirty tracking is enabled, allow only min supported pgsize */
1441	if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
1442	    (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {
1443		goto unlock;
1444	}
1445
1446	WARN_ON((pgsize - 1) & PAGE_MASK);
1447again:
1448	/*
1449	 * vfio-iommu-type1 (v1) - User mappings were coalesced together to
1450	 * avoid tracking individual mappings.  This means that the granularity
1451	 * of the original mapping was lost and the user was allowed to attempt
1452	 * to unmap any range.  Depending on the contiguousness of physical
1453	 * memory and page sizes supported by the IOMMU, arbitrary unmaps may
1454	 * or may not have worked.  We only guaranteed unmap granularity
1455	 * matching the original mapping; even though it was untracked here,
1456	 * the original mappings are reflected in IOMMU mappings.  This
1457	 * resulted in a couple unusual behaviors.  First, if a range is not
1458	 * able to be unmapped, ex. a set of 4k pages that was mapped as a
1459	 * 2M hugepage into the IOMMU, the unmap ioctl returns success but with
1460	 * a zero sized unmap.  Also, if an unmap request overlaps the first
1461	 * address of a hugepage, the IOMMU will unmap the entire hugepage.
1462	 * This also returns success and the returned unmap size reflects the
1463	 * actual size unmapped.
1464	 *
1465	 * We attempt to maintain compatibility with this "v1" interface, but
1466	 * we take control out of the hands of the IOMMU.  Therefore, an unmap
1467	 * request offset from the beginning of the original mapping will
1468	 * return success with zero sized unmap.  And an unmap request covering
1469	 * the first iova of mapping will unmap the entire range.
1470	 *
1471	 * The v2 version of this interface intends to be more deterministic.
1472	 * Unmap requests must fully cover previous mappings.  Multiple
1473	 * mappings may still be unmaped by specifying large ranges, but there
1474	 * must not be any previous mappings bisected by the range.  An error
1475	 * will be returned if these conditions are not met.  The v2 interface
1476	 * will only return success and a size of zero if there were no
1477	 * mappings within the range.
1478	 */
1479	if (iommu->v2 && !unmap_all) {
1480		dma = vfio_find_dma(iommu, iova, 1);
1481		if (dma && dma->iova != iova)
1482			goto unlock;
1483
1484		dma = vfio_find_dma(iommu, iova_end, 1);
1485		if (dma && dma->iova + dma->size - 1 != iova_end)
1486			goto unlock;
1487	}
1488
1489	ret = 0;
1490	n = first_n = vfio_find_dma_first_node(iommu, iova, iova_end);
1491
1492	while (n) {
1493		dma = rb_entry(n, struct vfio_dma, node);
1494		if (dma->iova > iova_end)
1495			break;
1496
1497		if (!iommu->v2 && iova > dma->iova)
1498			break;
1499
1500		if (invalidate_vaddr) {
1501			if (dma->vaddr_invalid) {
1502				struct rb_node *last_n = n;
1503
1504				for (n = first_n; n != last_n; n = rb_next(n)) {
1505					dma = rb_entry(n,
1506						       struct vfio_dma, node);
1507					dma->vaddr_invalid = false;
1508					iommu->vaddr_invalid_count--;
1509				}
1510				ret = -EINVAL;
1511				unmapped = 0;
1512				break;
1513			}
1514			dma->vaddr_invalid = true;
1515			iommu->vaddr_invalid_count++;
1516			unmapped += dma->size;
1517			n = rb_next(n);
1518			continue;
1519		}
1520
1521		if (!RB_EMPTY_ROOT(&dma->pfn_list)) {
1522			if (dma_last == dma) {
1523				BUG_ON(++retries > 10);
1524			} else {
1525				dma_last = dma;
1526				retries = 0;
1527			}
1528
1529			vfio_notify_dma_unmap(iommu, dma);
1530			goto again;
1531		}
1532
1533		if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
1534			ret = update_user_bitmap(bitmap->data, iommu, dma,
1535						 iova, pgsize);
1536			if (ret)
1537				break;
1538		}
1539
1540		unmapped += dma->size;
1541		n = rb_next(n);
1542		vfio_remove_dma(iommu, dma);
1543	}
1544
1545unlock:
1546	mutex_unlock(&iommu->lock);
1547
1548	/* Report how much was unmapped */
1549	unmap->size = unmapped;
1550
1551	return ret;
1552}
1553
1554static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova,
1555			  unsigned long pfn, long npage, int prot)
1556{
1557	struct vfio_domain *d;
1558	int ret;
1559
1560	list_for_each_entry(d, &iommu->domain_list, next) {
1561		ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT,
1562				npage << PAGE_SHIFT, prot | IOMMU_CACHE,
1563				GFP_KERNEL_ACCOUNT);
1564		if (ret)
1565			goto unwind;
1566
1567		cond_resched();
1568	}
1569
1570	return 0;
1571
1572unwind:
1573	list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) {
1574		iommu_unmap(d->domain, iova, npage << PAGE_SHIFT);
1575		cond_resched();
1576	}
1577
1578	return ret;
1579}
1580
1581static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
1582			    size_t map_size)
1583{
1584	dma_addr_t iova = dma->iova;
1585	unsigned long vaddr = dma->vaddr;
1586	struct vfio_batch batch;
1587	size_t size = map_size;
1588	long npage;
1589	unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1590	int ret = 0;
1591
1592	vfio_batch_init(&batch);
1593
1594	while (size) {
1595		/* Pin a contiguous chunk of memory */
1596		npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
1597					      size >> PAGE_SHIFT, &pfn, limit,
1598					      &batch);
1599		if (npage <= 0) {
1600			WARN_ON(!npage);
1601			ret = (int)npage;
1602			break;
1603		}
1604
1605		/* Map it! */
1606		ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
1607				     dma->prot);
1608		if (ret) {
1609			vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
1610						npage, true);
1611			vfio_batch_unpin(&batch, dma);
1612			break;
1613		}
1614
1615		size -= npage << PAGE_SHIFT;
1616		dma->size += npage << PAGE_SHIFT;
1617	}
1618
1619	vfio_batch_fini(&batch);
1620	dma->iommu_mapped = true;
1621
1622	if (ret)
1623		vfio_remove_dma(iommu, dma);
1624
1625	return ret;
1626}
1627
1628/*
1629 * Check dma map request is within a valid iova range
1630 */
1631static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu,
1632				      dma_addr_t start, dma_addr_t end)
1633{
1634	struct list_head *iova = &iommu->iova_list;
1635	struct vfio_iova *node;
1636
1637	list_for_each_entry(node, iova, list) {
1638		if (start >= node->start && end <= node->end)
1639			return true;
1640	}
1641
1642	/*
1643	 * Check for list_empty() as well since a container with
1644	 * a single mdev device will have an empty list.
1645	 */
1646	return list_empty(iova);
1647}
1648
1649static int vfio_change_dma_owner(struct vfio_dma *dma)
1650{
1651	struct task_struct *task = current->group_leader;
1652	struct mm_struct *mm = current->mm;
1653	long npage = dma->locked_vm;
1654	bool lock_cap;
1655	int ret;
1656
1657	if (mm == dma->mm)
1658		return 0;
1659
1660	lock_cap = capable(CAP_IPC_LOCK);
1661	ret = mm_lock_acct(task, mm, lock_cap, npage);
1662	if (ret)
1663		return ret;
1664
1665	if (mmget_not_zero(dma->mm)) {
1666		mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage);
1667		mmput(dma->mm);
1668	}
1669
1670	if (dma->task != task) {
1671		put_task_struct(dma->task);
1672		dma->task = get_task_struct(task);
1673	}
1674	mmdrop(dma->mm);
1675	dma->mm = mm;
1676	mmgrab(dma->mm);
1677	dma->lock_cap = lock_cap;
1678	return 0;
1679}
1680
1681static int vfio_dma_do_map(struct vfio_iommu *iommu,
1682			   struct vfio_iommu_type1_dma_map *map)
1683{
1684	bool set_vaddr = map->flags & VFIO_DMA_MAP_FLAG_VADDR;
1685	dma_addr_t iova = map->iova;
1686	dma_addr_t iova_end;
1687	unsigned long vaddr = map->vaddr;
1688	unsigned long vaddr_end;
1689	size_t size = map->size;
1690	int ret = 0, prot = 0;
1691	size_t pgsize;
1692	struct vfio_dma *dma;
1693
1694	/* Verify that none of our __u64 fields overflow */
1695	if (map->size != size || map->vaddr != vaddr || map->iova != iova)
1696		return -EOVERFLOW;
1697
1698	if (!size)
1699		return -EINVAL;
1700
1701	if (check_add_overflow(iova, size - 1, &iova_end) ||
1702	    check_add_overflow(vaddr, size - 1, &vaddr_end))
1703		return -EOVERFLOW;
1704
1705	/* READ/WRITE from device perspective */
1706	if (map->flags & VFIO_DMA_MAP_FLAG_WRITE)
1707		prot |= IOMMU_WRITE;
1708	if (map->flags & VFIO_DMA_MAP_FLAG_READ)
1709		prot |= IOMMU_READ;
1710
1711	if ((prot && set_vaddr) || (!prot && !set_vaddr))
1712		return -EINVAL;
1713
1714	mutex_lock(&iommu->lock);
1715
1716	pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
1717
1718	WARN_ON((pgsize - 1) & PAGE_MASK);
1719
1720	if ((size | iova | vaddr) & (pgsize - 1)) {
1721		ret = -EINVAL;
1722		goto out_unlock;
1723	}
1724
1725	dma = vfio_find_dma(iommu, iova, size);
1726	if (set_vaddr) {
1727		if (!dma) {
1728			ret = -ENOENT;
1729		} else if (!dma->vaddr_invalid || dma->iova != iova ||
1730			   dma->size != size) {
1731			ret = -EINVAL;
1732		} else {
1733			ret = vfio_change_dma_owner(dma);
1734			if (ret)
1735				goto out_unlock;
1736			dma->vaddr = vaddr;
1737			dma->vaddr_invalid = false;
1738			iommu->vaddr_invalid_count--;
1739		}
1740		goto out_unlock;
1741	} else if (dma) {
1742		ret = -EEXIST;
1743		goto out_unlock;
1744	}
1745
1746	if (!iommu->dma_avail) {
1747		ret = -ENOSPC;
1748		goto out_unlock;
1749	}
1750
1751	if (!vfio_iommu_iova_dma_valid(iommu, iova, iova_end)) {
1752		ret = -EINVAL;
1753		goto out_unlock;
1754	}
1755
1756	dma = kzalloc(sizeof(*dma), GFP_KERNEL);
1757	if (!dma) {
1758		ret = -ENOMEM;
1759		goto out_unlock;
1760	}
1761
1762	iommu->dma_avail--;
1763	dma->iova = iova;
1764	dma->vaddr = vaddr;
1765	dma->prot = prot;
1766
1767	/*
1768	 * We need to be able to both add to a task's locked memory and test
1769	 * against the locked memory limit and we need to be able to do both
1770	 * outside of this call path as pinning can be asynchronous via the
1771	 * external interfaces for mdev devices.  RLIMIT_MEMLOCK requires a
1772	 * task_struct. Save the group_leader so that all DMA tracking uses
1773	 * the same task, to make debugging easier.  VM locked pages requires
1774	 * an mm_struct, so grab the mm in case the task dies.
1775	 */
1776	get_task_struct(current->group_leader);
1777	dma->task = current->group_leader;
1778	dma->lock_cap = capable(CAP_IPC_LOCK);
1779	dma->mm = current->mm;
1780	mmgrab(dma->mm);
1781
1782	dma->pfn_list = RB_ROOT;
1783
1784	/* Insert zero-sized and grow as we map chunks of it */
1785	vfio_link_dma(iommu, dma);
1786
1787	/* Don't pin and map if container doesn't contain IOMMU capable domain*/
1788	if (list_empty(&iommu->domain_list))
1789		dma->size = size;
1790	else
1791		ret = vfio_pin_map_dma(iommu, dma, size);
1792
1793	if (!ret && iommu->dirty_page_tracking) {
1794		ret = vfio_dma_bitmap_alloc(dma, pgsize);
1795		if (ret)
1796			vfio_remove_dma(iommu, dma);
1797	}
1798
1799out_unlock:
1800	mutex_unlock(&iommu->lock);
1801	return ret;
1802}
1803
1804static int vfio_iommu_replay(struct vfio_iommu *iommu,
1805			     struct vfio_domain *domain)
1806{
1807	struct vfio_batch batch;
1808	struct vfio_domain *d = NULL;
1809	struct rb_node *n;
1810	unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1811	int ret;
1812
1813	/* Arbitrarily pick the first domain in the list for lookups */
1814	if (!list_empty(&iommu->domain_list))
1815		d = list_first_entry(&iommu->domain_list,
1816				     struct vfio_domain, next);
1817
1818	vfio_batch_init(&batch);
1819
1820	n = rb_first(&iommu->dma_list);
1821
1822	for (; n; n = rb_next(n)) {
1823		struct vfio_dma *dma;
1824		size_t pos = 0;
1825
1826		dma = rb_entry(n, struct vfio_dma, node);
1827
1828		while (pos < dma->size) {
1829			dma_addr_t iova = dma->iova + pos;
1830			phys_addr_t phys;
1831			size_t size;
1832
1833			if (dma->iommu_mapped) {
1834				phys_addr_t p;
1835				dma_addr_t i;
1836
1837				if (WARN_ON(!d)) { /* mapped w/o a domain?! */
1838					ret = -EINVAL;
1839					goto unwind;
1840				}
1841
1842				phys = iommu_iova_to_phys(d->domain, iova);
1843
1844				if (WARN_ON(!phys)) {
1845					pos += PAGE_SIZE;
1846					continue;
1847				}
1848
1849				size = PAGE_SIZE;
1850				p = phys + size;
1851				i = iova + size;
1852				while (pos + size < dma->size &&
1853				       p == iommu_iova_to_phys(d->domain, i)) {
1854					size += PAGE_SIZE;
1855					p += PAGE_SIZE;
1856					i += PAGE_SIZE;
1857				}
1858			} else {
1859				unsigned long pfn;
1860				unsigned long vaddr = dma->vaddr + pos;
1861				size_t n = dma->size - pos;
1862				long npage;
1863
1864				npage = vfio_pin_pages_remote(dma, vaddr,
1865							      n >> PAGE_SHIFT,
1866							      &pfn, limit,
1867							      &batch);
1868				if (npage <= 0) {
1869					WARN_ON(!npage);
1870					ret = (int)npage;
1871					goto unwind;
1872				}
1873
1874				phys = pfn << PAGE_SHIFT;
1875				size = npage << PAGE_SHIFT;
1876			}
1877
1878			ret = iommu_map(domain->domain, iova, phys, size,
1879					dma->prot | IOMMU_CACHE,
1880					GFP_KERNEL_ACCOUNT);
1881			if (ret) {
1882				if (!dma->iommu_mapped) {
1883					vfio_unpin_pages_remote(dma, iova,
1884							phys >> PAGE_SHIFT,
1885							size >> PAGE_SHIFT,
1886							true);
1887					vfio_batch_unpin(&batch, dma);
1888				}
1889				goto unwind;
1890			}
1891
1892			pos += size;
1893		}
1894	}
1895
1896	/* All dmas are now mapped, defer to second tree walk for unwind */
1897	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
1898		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
1899
1900		dma->iommu_mapped = true;
1901	}
1902
1903	vfio_batch_fini(&batch);
1904	return 0;
1905
1906unwind:
1907	for (; n; n = rb_prev(n)) {
1908		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
1909		size_t pos = 0;
1910
1911		if (dma->iommu_mapped) {
1912			iommu_unmap(domain->domain, dma->iova, dma->size);
1913			continue;
1914		}
1915
1916		while (pos < dma->size) {
1917			dma_addr_t iova = dma->iova + pos;
1918			phys_addr_t phys, p;
1919			size_t size;
1920			dma_addr_t i;
1921
1922			phys = iommu_iova_to_phys(domain->domain, iova);
1923			if (!phys) {
1924				pos += PAGE_SIZE;
1925				continue;
1926			}
1927
1928			size = PAGE_SIZE;
1929			p = phys + size;
1930			i = iova + size;
1931			while (pos + size < dma->size &&
1932			       p == iommu_iova_to_phys(domain->domain, i)) {
1933				size += PAGE_SIZE;
1934				p += PAGE_SIZE;
1935				i += PAGE_SIZE;
1936			}
1937
1938			iommu_unmap(domain->domain, iova, size);
1939			vfio_unpin_pages_remote(dma, iova, phys >> PAGE_SHIFT,
1940						size >> PAGE_SHIFT, true);
1941		}
1942	}
1943
1944	vfio_batch_fini(&batch);
1945	return ret;
1946}
1947
1948static struct vfio_iommu_group *find_iommu_group(struct vfio_domain *domain,
1949						 struct iommu_group *iommu_group)
1950{
1951	struct vfio_iommu_group *g;
1952
1953	list_for_each_entry(g, &domain->group_list, next) {
1954		if (g->iommu_group == iommu_group)
1955			return g;
1956	}
1957
1958	return NULL;
1959}
1960
1961static struct vfio_iommu_group*
1962vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
1963			    struct iommu_group *iommu_group)
1964{
1965	struct vfio_iommu_group *group;
1966	struct vfio_domain *domain;
1967
1968	list_for_each_entry(domain, &iommu->domain_list, next) {
1969		group = find_iommu_group(domain, iommu_group);
1970		if (group)
1971			return group;
1972	}
1973
1974	list_for_each_entry(group, &iommu->emulated_iommu_groups, next)
1975		if (group->iommu_group == iommu_group)
1976			return group;
1977	return NULL;
1978}
1979
1980static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
1981				  phys_addr_t *base)
1982{
1983	struct iommu_resv_region *region;
1984	bool ret = false;
1985
1986	list_for_each_entry(region, group_resv_regions, list) {
1987		/*
1988		 * The presence of any 'real' MSI regions should take
1989		 * precedence over the software-managed one if the
1990		 * IOMMU driver happens to advertise both types.
1991		 */
1992		if (region->type == IOMMU_RESV_MSI) {
1993			ret = false;
1994			break;
1995		}
1996
1997		if (region->type == IOMMU_RESV_SW_MSI) {
1998			*base = region->start;
1999			ret = true;
2000		}
2001	}
2002
2003	return ret;
2004}
2005
2006/*
2007 * This is a helper function to insert an address range to iova list.
2008 * The list is initially created with a single entry corresponding to
2009 * the IOMMU domain geometry to which the device group is attached.
2010 * The list aperture gets modified when a new domain is added to the
2011 * container if the new aperture doesn't conflict with the current one
2012 * or with any existing dma mappings. The list is also modified to
2013 * exclude any reserved regions associated with the device group.
2014 */
2015static int vfio_iommu_iova_insert(struct list_head *head,
2016				  dma_addr_t start, dma_addr_t end)
2017{
2018	struct vfio_iova *region;
2019
2020	region = kmalloc(sizeof(*region), GFP_KERNEL);
2021	if (!region)
2022		return -ENOMEM;
2023
2024	INIT_LIST_HEAD(&region->list);
2025	region->start = start;
2026	region->end = end;
2027
2028	list_add_tail(&region->list, head);
2029	return 0;
2030}
2031
2032/*
2033 * Check the new iommu aperture conflicts with existing aper or with any
2034 * existing dma mappings.
2035 */
2036static bool vfio_iommu_aper_conflict(struct vfio_iommu *iommu,
2037				     dma_addr_t start, dma_addr_t end)
2038{
2039	struct vfio_iova *first, *last;
2040	struct list_head *iova = &iommu->iova_list;
2041
2042	if (list_empty(iova))
2043		return false;
2044
2045	/* Disjoint sets, return conflict */
2046	first = list_first_entry(iova, struct vfio_iova, list);
2047	last = list_last_entry(iova, struct vfio_iova, list);
2048	if (start > last->end || end < first->start)
2049		return true;
2050
2051	/* Check for any existing dma mappings below the new start */
2052	if (start > first->start) {
2053		if (vfio_find_dma(iommu, first->start, start - first->start))
2054			return true;
2055	}
2056
2057	/* Check for any existing dma mappings beyond the new end */
2058	if (end < last->end) {
2059		if (vfio_find_dma(iommu, end + 1, last->end - end))
2060			return true;
2061	}
2062
2063	return false;
2064}
2065
2066/*
2067 * Resize iommu iova aperture window. This is called only if the new
2068 * aperture has no conflict with existing aperture and dma mappings.
2069 */
2070static int vfio_iommu_aper_resize(struct list_head *iova,
2071				  dma_addr_t start, dma_addr_t end)
2072{
2073	struct vfio_iova *node, *next;
2074
2075	if (list_empty(iova))
2076		return vfio_iommu_iova_insert(iova, start, end);
2077
2078	/* Adjust iova list start */
2079	list_for_each_entry_safe(node, next, iova, list) {
2080		if (start < node->start)
2081			break;
2082		if (start >= node->start && start < node->end) {
2083			node->start = start;
2084			break;
2085		}
2086		/* Delete nodes before new start */
2087		list_del(&node->list);
2088		kfree(node);
2089	}
2090
2091	/* Adjust iova list end */
2092	list_for_each_entry_safe(node, next, iova, list) {
2093		if (end > node->end)
2094			continue;
2095		if (end > node->start && end <= node->end) {
2096			node->end = end;
2097			continue;
2098		}
2099		/* Delete nodes after new end */
2100		list_del(&node->list);
2101		kfree(node);
2102	}
2103
2104	return 0;
2105}
2106
2107/*
2108 * Check reserved region conflicts with existing dma mappings
2109 */
2110static bool vfio_iommu_resv_conflict(struct vfio_iommu *iommu,
2111				     struct list_head *resv_regions)
2112{
2113	struct iommu_resv_region *region;
2114
2115	/* Check for conflict with existing dma mappings */
2116	list_for_each_entry(region, resv_regions, list) {
2117		if (region->type == IOMMU_RESV_DIRECT_RELAXABLE)
2118			continue;
2119
2120		if (vfio_find_dma(iommu, region->start, region->length))
2121			return true;
2122	}
2123
2124	return false;
2125}
2126
2127/*
2128 * Check iova region overlap with  reserved regions and
2129 * exclude them from the iommu iova range
2130 */
2131static int vfio_iommu_resv_exclude(struct list_head *iova,
2132				   struct list_head *resv_regions)
2133{
2134	struct iommu_resv_region *resv;
2135	struct vfio_iova *n, *next;
2136
2137	list_for_each_entry(resv, resv_regions, list) {
2138		phys_addr_t start, end;
2139
2140		if (resv->type == IOMMU_RESV_DIRECT_RELAXABLE)
2141			continue;
2142
2143		start = resv->start;
2144		end = resv->start + resv->length - 1;
2145
2146		list_for_each_entry_safe(n, next, iova, list) {
2147			int ret = 0;
2148
2149			/* No overlap */
2150			if (start > n->end || end < n->start)
2151				continue;
2152			/*
2153			 * Insert a new node if current node overlaps with the
2154			 * reserve region to exclude that from valid iova range.
2155			 * Note that, new node is inserted before the current
2156			 * node and finally the current node is deleted keeping
2157			 * the list updated and sorted.
2158			 */
2159			if (start > n->start)
2160				ret = vfio_iommu_iova_insert(&n->list, n->start,
2161							     start - 1);
2162			if (!ret && end < n->end)
2163				ret = vfio_iommu_iova_insert(&n->list, end + 1,
2164							     n->end);
2165			if (ret)
2166				return ret;
2167
2168			list_del(&n->list);
2169			kfree(n);
2170		}
2171	}
2172
2173	if (list_empty(iova))
2174		return -EINVAL;
2175
2176	return 0;
2177}
2178
2179static void vfio_iommu_resv_free(struct list_head *resv_regions)
2180{
2181	struct iommu_resv_region *n, *next;
2182
2183	list_for_each_entry_safe(n, next, resv_regions, list) {
2184		list_del(&n->list);
2185		kfree(n);
2186	}
2187}
2188
2189static void vfio_iommu_iova_free(struct list_head *iova)
2190{
2191	struct vfio_iova *n, *next;
2192
2193	list_for_each_entry_safe(n, next, iova, list) {
2194		list_del(&n->list);
2195		kfree(n);
2196	}
2197}
2198
2199static int vfio_iommu_iova_get_copy(struct vfio_iommu *iommu,
2200				    struct list_head *iova_copy)
2201{
2202	struct list_head *iova = &iommu->iova_list;
2203	struct vfio_iova *n;
2204	int ret;
2205
2206	list_for_each_entry(n, iova, list) {
2207		ret = vfio_iommu_iova_insert(iova_copy, n->start, n->end);
2208		if (ret)
2209			goto out_free;
2210	}
2211
2212	return 0;
2213
2214out_free:
2215	vfio_iommu_iova_free(iova_copy);
2216	return ret;
2217}
2218
2219static void vfio_iommu_iova_insert_copy(struct vfio_iommu *iommu,
2220					struct list_head *iova_copy)
2221{
2222	struct list_head *iova = &iommu->iova_list;
2223
2224	vfio_iommu_iova_free(iova);
2225
2226	list_splice_tail(iova_copy, iova);
2227}
2228
2229static int vfio_iommu_domain_alloc(struct device *dev, void *data)
2230{
2231	struct iommu_domain **domain = data;
2232
2233	*domain = iommu_paging_domain_alloc(dev);
2234	return 1; /* Don't iterate */
2235}
2236
2237static int vfio_iommu_type1_attach_group(void *iommu_data,
2238		struct iommu_group *iommu_group, enum vfio_group_type type)
2239{
2240	struct vfio_iommu *iommu = iommu_data;
2241	struct vfio_iommu_group *group;
2242	struct vfio_domain *domain, *d;
2243	bool resv_msi;
2244	phys_addr_t resv_msi_base = 0;
2245	struct iommu_domain_geometry *geo;
2246	LIST_HEAD(iova_copy);
2247	LIST_HEAD(group_resv_regions);
2248	int ret = -EBUSY;
2249
2250	mutex_lock(&iommu->lock);
2251
2252	/* Attach could require pinning, so disallow while vaddr is invalid. */
2253	if (iommu->vaddr_invalid_count)
2254		goto out_unlock;
2255
2256	/* Check for duplicates */
2257	ret = -EINVAL;
2258	if (vfio_iommu_find_iommu_group(iommu, iommu_group))
2259		goto out_unlock;
2260
2261	ret = -ENOMEM;
2262	group = kzalloc(sizeof(*group), GFP_KERNEL);
2263	if (!group)
2264		goto out_unlock;
2265	group->iommu_group = iommu_group;
2266
2267	if (type == VFIO_EMULATED_IOMMU) {
2268		list_add(&group->next, &iommu->emulated_iommu_groups);
2269		/*
2270		 * An emulated IOMMU group cannot dirty memory directly, it can
2271		 * only use interfaces that provide dirty tracking.
2272		 * The iommu scope can only be promoted with the addition of a
2273		 * dirty tracking group.
2274		 */
2275		group->pinned_page_dirty_scope = true;
2276		ret = 0;
2277		goto out_unlock;
2278	}
2279
2280	ret = -ENOMEM;
2281	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2282	if (!domain)
2283		goto out_free_group;
2284
2285	/*
2286	 * Going via the iommu_group iterator avoids races, and trivially gives
2287	 * us a representative device for the IOMMU API call. We don't actually
2288	 * want to iterate beyond the first device (if any).
2289	 */
2290	iommu_group_for_each_dev(iommu_group, &domain->domain,
2291				 vfio_iommu_domain_alloc);
2292	if (IS_ERR(domain->domain)) {
2293		ret = PTR_ERR(domain->domain);
2294		goto out_free_domain;
2295	}
2296
2297	ret = iommu_attach_group(domain->domain, group->iommu_group);
2298	if (ret)
2299		goto out_domain;
2300
2301	/* Get aperture info */
2302	geo = &domain->domain->geometry;
2303	if (vfio_iommu_aper_conflict(iommu, geo->aperture_start,
2304				     geo->aperture_end)) {
2305		ret = -EINVAL;
2306		goto out_detach;
2307	}
2308
2309	ret = iommu_get_group_resv_regions(iommu_group, &group_resv_regions);
2310	if (ret)
2311		goto out_detach;
2312
2313	if (vfio_iommu_resv_conflict(iommu, &group_resv_regions)) {
2314		ret = -EINVAL;
2315		goto out_detach;
2316	}
2317
2318	/*
2319	 * We don't want to work on the original iova list as the list
2320	 * gets modified and in case of failure we have to retain the
2321	 * original list. Get a copy here.
2322	 */
2323	ret = vfio_iommu_iova_get_copy(iommu, &iova_copy);
2324	if (ret)
2325		goto out_detach;
2326
2327	ret = vfio_iommu_aper_resize(&iova_copy, geo->aperture_start,
2328				     geo->aperture_end);
2329	if (ret)
2330		goto out_detach;
2331
2332	ret = vfio_iommu_resv_exclude(&iova_copy, &group_resv_regions);
2333	if (ret)
2334		goto out_detach;
2335
2336	resv_msi = vfio_iommu_has_sw_msi(&group_resv_regions, &resv_msi_base);
2337
2338	INIT_LIST_HEAD(&domain->group_list);
2339	list_add(&group->next, &domain->group_list);
2340
2341	if (!allow_unsafe_interrupts &&
2342	    !iommu_group_has_isolated_msi(iommu_group)) {
2343		pr_warn("%s: No interrupt remapping support.  Use the module param \"allow_unsafe_interrupts\" to enable VFIO IOMMU support on this platform\n",
2344		       __func__);
2345		ret = -EPERM;
2346		goto out_detach;
2347	}
2348
2349	/*
2350	 * If the IOMMU can block non-coherent operations (ie PCIe TLPs with
2351	 * no-snoop set) then VFIO always turns this feature on because on Intel
2352	 * platforms it optimizes KVM to disable wbinvd emulation.
2353	 */
2354	if (domain->domain->ops->enforce_cache_coherency)
2355		domain->enforce_cache_coherency =
2356			domain->domain->ops->enforce_cache_coherency(
2357				domain->domain);
2358
2359	/*
2360	 * Try to match an existing compatible domain.  We don't want to
2361	 * preclude an IOMMU driver supporting multiple bus_types and being
2362	 * able to include different bus_types in the same IOMMU domain, so
2363	 * we test whether the domains use the same iommu_ops rather than
2364	 * testing if they're on the same bus_type.
2365	 */
2366	list_for_each_entry(d, &iommu->domain_list, next) {
2367		if (d->domain->ops == domain->domain->ops &&
2368		    d->enforce_cache_coherency ==
2369			    domain->enforce_cache_coherency) {
2370			iommu_detach_group(domain->domain, group->iommu_group);
2371			if (!iommu_attach_group(d->domain,
2372						group->iommu_group)) {
2373				list_add(&group->next, &d->group_list);
2374				iommu_domain_free(domain->domain);
2375				kfree(domain);
2376				goto done;
2377			}
2378
2379			ret = iommu_attach_group(domain->domain,
2380						 group->iommu_group);
2381			if (ret)
2382				goto out_domain;
2383		}
2384	}
2385
2386	/* replay mappings on new domains */
2387	ret = vfio_iommu_replay(iommu, domain);
2388	if (ret)
2389		goto out_detach;
2390
2391	if (resv_msi) {
2392		ret = iommu_get_msi_cookie(domain->domain, resv_msi_base);
2393		if (ret && ret != -ENODEV)
2394			goto out_detach;
2395	}
2396
2397	list_add(&domain->next, &iommu->domain_list);
2398	vfio_update_pgsize_bitmap(iommu);
2399done:
2400	/* Delete the old one and insert new iova list */
2401	vfio_iommu_iova_insert_copy(iommu, &iova_copy);
2402
2403	/*
2404	 * An iommu backed group can dirty memory directly and therefore
2405	 * demotes the iommu scope until it declares itself dirty tracking
2406	 * capable via the page pinning interface.
2407	 */
2408	iommu->num_non_pinned_groups++;
2409	mutex_unlock(&iommu->lock);
2410	vfio_iommu_resv_free(&group_resv_regions);
2411
2412	return 0;
2413
2414out_detach:
2415	iommu_detach_group(domain->domain, group->iommu_group);
2416out_domain:
2417	iommu_domain_free(domain->domain);
2418	vfio_iommu_iova_free(&iova_copy);
2419	vfio_iommu_resv_free(&group_resv_regions);
2420out_free_domain:
2421	kfree(domain);
2422out_free_group:
2423	kfree(group);
2424out_unlock:
2425	mutex_unlock(&iommu->lock);
2426	return ret;
2427}
2428
2429static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)
2430{
2431	struct rb_node *node;
2432
2433	while ((node = rb_first(&iommu->dma_list)))
2434		vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));
2435}
2436
2437static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
2438{
2439	struct rb_node *n, *p;
2440
2441	n = rb_first(&iommu->dma_list);
2442	for (; n; n = rb_next(n)) {
2443		struct vfio_dma *dma;
2444		long locked = 0, unlocked = 0;
2445
2446		dma = rb_entry(n, struct vfio_dma, node);
2447		unlocked += vfio_unmap_unpin(iommu, dma, false);
2448		p = rb_first(&dma->pfn_list);
2449		for (; p; p = rb_next(p)) {
2450			struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn,
2451							 node);
2452
2453			if (!is_invalid_reserved_pfn(vpfn->pfn))
2454				locked++;
2455		}
2456		vfio_lock_acct(dma, locked - unlocked, true);
2457	}
2458}
2459
2460/*
2461 * Called when a domain is removed in detach. It is possible that
2462 * the removed domain decided the iova aperture window. Modify the
2463 * iova aperture with the smallest window among existing domains.
2464 */
2465static void vfio_iommu_aper_expand(struct vfio_iommu *iommu,
2466				   struct list_head *iova_copy)
2467{
2468	struct vfio_domain *domain;
2469	struct vfio_iova *node;
2470	dma_addr_t start = 0;
2471	dma_addr_t end = (dma_addr_t)~0;
2472
2473	if (list_empty(iova_copy))
2474		return;
2475
2476	list_for_each_entry(domain, &iommu->domain_list, next) {
2477		struct iommu_domain_geometry *geo = &domain->domain->geometry;
2478
2479		if (geo->aperture_start > start)
2480			start = geo->aperture_start;
2481		if (geo->aperture_end < end)
2482			end = geo->aperture_end;
2483	}
2484
2485	/* Modify aperture limits. The new aper is either same or bigger */
2486	node = list_first_entry(iova_copy, struct vfio_iova, list);
2487	node->start = start;
2488	node = list_last_entry(iova_copy, struct vfio_iova, list);
2489	node->end = end;
2490}
2491
2492/*
2493 * Called when a group is detached. The reserved regions for that
2494 * group can be part of valid iova now. But since reserved regions
2495 * may be duplicated among groups, populate the iova valid regions
2496 * list again.
2497 */
2498static int vfio_iommu_resv_refresh(struct vfio_iommu *iommu,
2499				   struct list_head *iova_copy)
2500{
2501	struct vfio_domain *d;
2502	struct vfio_iommu_group *g;
2503	struct vfio_iova *node;
2504	dma_addr_t start, end;
2505	LIST_HEAD(resv_regions);
2506	int ret;
2507
2508	if (list_empty(iova_copy))
2509		return -EINVAL;
2510
2511	list_for_each_entry(d, &iommu->domain_list, next) {
2512		list_for_each_entry(g, &d->group_list, next) {
2513			ret = iommu_get_group_resv_regions(g->iommu_group,
2514							   &resv_regions);
2515			if (ret)
2516				goto done;
2517		}
2518	}
2519
2520	node = list_first_entry(iova_copy, struct vfio_iova, list);
2521	start = node->start;
2522	node = list_last_entry(iova_copy, struct vfio_iova, list);
2523	end = node->end;
2524
2525	/* purge the iova list and create new one */
2526	vfio_iommu_iova_free(iova_copy);
2527
2528	ret = vfio_iommu_aper_resize(iova_copy, start, end);
2529	if (ret)
2530		goto done;
2531
2532	/* Exclude current reserved regions from iova ranges */
2533	ret = vfio_iommu_resv_exclude(iova_copy, &resv_regions);
2534done:
2535	vfio_iommu_resv_free(&resv_regions);
2536	return ret;
2537}
2538
2539static void vfio_iommu_type1_detach_group(void *iommu_data,
2540					  struct iommu_group *iommu_group)
2541{
2542	struct vfio_iommu *iommu = iommu_data;
2543	struct vfio_domain *domain;
2544	struct vfio_iommu_group *group;
2545	bool update_dirty_scope = false;
2546	LIST_HEAD(iova_copy);
2547
2548	mutex_lock(&iommu->lock);
2549	list_for_each_entry(group, &iommu->emulated_iommu_groups, next) {
2550		if (group->iommu_group != iommu_group)
2551			continue;
2552		update_dirty_scope = !group->pinned_page_dirty_scope;
2553		list_del(&group->next);
2554		kfree(group);
2555
2556		if (list_empty(&iommu->emulated_iommu_groups) &&
2557		    list_empty(&iommu->domain_list)) {
2558			WARN_ON(!list_empty(&iommu->device_list));
2559			vfio_iommu_unmap_unpin_all(iommu);
2560		}
2561		goto detach_group_done;
2562	}
2563
2564	/*
2565	 * Get a copy of iova list. This will be used to update
2566	 * and to replace the current one later. Please note that
2567	 * we will leave the original list as it is if update fails.
2568	 */
2569	vfio_iommu_iova_get_copy(iommu, &iova_copy);
2570
2571	list_for_each_entry(domain, &iommu->domain_list, next) {
2572		group = find_iommu_group(domain, iommu_group);
2573		if (!group)
2574			continue;
2575
2576		iommu_detach_group(domain->domain, group->iommu_group);
2577		update_dirty_scope = !group->pinned_page_dirty_scope;
2578		list_del(&group->next);
2579		kfree(group);
2580		/*
2581		 * Group ownership provides privilege, if the group list is
2582		 * empty, the domain goes away. If it's the last domain with
2583		 * iommu and external domain doesn't exist, then all the
2584		 * mappings go away too. If it's the last domain with iommu and
2585		 * external domain exist, update accounting
2586		 */
2587		if (list_empty(&domain->group_list)) {
2588			if (list_is_singular(&iommu->domain_list)) {
2589				if (list_empty(&iommu->emulated_iommu_groups)) {
2590					WARN_ON(!list_empty(
2591						&iommu->device_list));
2592					vfio_iommu_unmap_unpin_all(iommu);
2593				} else {
2594					vfio_iommu_unmap_unpin_reaccount(iommu);
2595				}
2596			}
2597			iommu_domain_free(domain->domain);
2598			list_del(&domain->next);
2599			kfree(domain);
2600			vfio_iommu_aper_expand(iommu, &iova_copy);
2601			vfio_update_pgsize_bitmap(iommu);
2602		}
2603		break;
2604	}
2605
2606	if (!vfio_iommu_resv_refresh(iommu, &iova_copy))
2607		vfio_iommu_iova_insert_copy(iommu, &iova_copy);
2608	else
2609		vfio_iommu_iova_free(&iova_copy);
2610
2611detach_group_done:
2612	/*
2613	 * Removal of a group without dirty tracking may allow the iommu scope
2614	 * to be promoted.
2615	 */
2616	if (update_dirty_scope) {
2617		iommu->num_non_pinned_groups--;
2618		if (iommu->dirty_page_tracking)
2619			vfio_iommu_populate_bitmap_full(iommu);
2620	}
2621	mutex_unlock(&iommu->lock);
2622}
2623
2624static void *vfio_iommu_type1_open(unsigned long arg)
2625{
2626	struct vfio_iommu *iommu;
2627
2628	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
2629	if (!iommu)
2630		return ERR_PTR(-ENOMEM);
2631
2632	switch (arg) {
2633	case VFIO_TYPE1_IOMMU:
2634		break;
2635	case __VFIO_RESERVED_TYPE1_NESTING_IOMMU:
2636	case VFIO_TYPE1v2_IOMMU:
2637		iommu->v2 = true;
2638		break;
2639	default:
2640		kfree(iommu);
2641		return ERR_PTR(-EINVAL);
2642	}
2643
2644	INIT_LIST_HEAD(&iommu->domain_list);
2645	INIT_LIST_HEAD(&iommu->iova_list);
2646	iommu->dma_list = RB_ROOT;
2647	iommu->dma_avail = dma_entry_limit;
2648	mutex_init(&iommu->lock);
2649	mutex_init(&iommu->device_list_lock);
2650	INIT_LIST_HEAD(&iommu->device_list);
2651	iommu->pgsize_bitmap = PAGE_MASK;
2652	INIT_LIST_HEAD(&iommu->emulated_iommu_groups);
2653
2654	return iommu;
2655}
2656
2657static void vfio_release_domain(struct vfio_domain *domain)
2658{
2659	struct vfio_iommu_group *group, *group_tmp;
2660
2661	list_for_each_entry_safe(group, group_tmp,
2662				 &domain->group_list, next) {
2663		iommu_detach_group(domain->domain, group->iommu_group);
2664		list_del(&group->next);
2665		kfree(group);
2666	}
2667
2668	iommu_domain_free(domain->domain);
2669}
2670
2671static void vfio_iommu_type1_release(void *iommu_data)
2672{
2673	struct vfio_iommu *iommu = iommu_data;
2674	struct vfio_domain *domain, *domain_tmp;
2675	struct vfio_iommu_group *group, *next_group;
2676
2677	list_for_each_entry_safe(group, next_group,
2678			&iommu->emulated_iommu_groups, next) {
2679		list_del(&group->next);
2680		kfree(group);
2681	}
2682
2683	vfio_iommu_unmap_unpin_all(iommu);
2684
2685	list_for_each_entry_safe(domain, domain_tmp,
2686				 &iommu->domain_list, next) {
2687		vfio_release_domain(domain);
2688		list_del(&domain->next);
2689		kfree(domain);
2690	}
2691
2692	vfio_iommu_iova_free(&iommu->iova_list);
2693
2694	kfree(iommu);
2695}
2696
2697static int vfio_domains_have_enforce_cache_coherency(struct vfio_iommu *iommu)
2698{
2699	struct vfio_domain *domain;
2700	int ret = 1;
2701
2702	mutex_lock(&iommu->lock);
2703	list_for_each_entry(domain, &iommu->domain_list, next) {
2704		if (!(domain->enforce_cache_coherency)) {
2705			ret = 0;
2706			break;
2707		}
2708	}
2709	mutex_unlock(&iommu->lock);
2710
2711	return ret;
2712}
2713
2714static bool vfio_iommu_has_emulated(struct vfio_iommu *iommu)
2715{
2716	bool ret;
2717
2718	mutex_lock(&iommu->lock);
2719	ret = !list_empty(&iommu->emulated_iommu_groups);
2720	mutex_unlock(&iommu->lock);
2721	return ret;
2722}
2723
2724static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu,
2725					    unsigned long arg)
2726{
2727	switch (arg) {
2728	case VFIO_TYPE1_IOMMU:
2729	case VFIO_TYPE1v2_IOMMU:
2730	case VFIO_UNMAP_ALL:
2731		return 1;
2732	case VFIO_UPDATE_VADDR:
2733		/*
2734		 * Disable this feature if mdevs are present.  They cannot
2735		 * safely pin/unpin/rw while vaddrs are being updated.
2736		 */
2737		return iommu && !vfio_iommu_has_emulated(iommu);
2738	case VFIO_DMA_CC_IOMMU:
2739		if (!iommu)
2740			return 0;
2741		return vfio_domains_have_enforce_cache_coherency(iommu);
2742	default:
2743		return 0;
2744	}
2745}
2746
2747static int vfio_iommu_iova_add_cap(struct vfio_info_cap *caps,
2748		 struct vfio_iommu_type1_info_cap_iova_range *cap_iovas,
2749		 size_t size)
2750{
2751	struct vfio_info_cap_header *header;
2752	struct vfio_iommu_type1_info_cap_iova_range *iova_cap;
2753
2754	header = vfio_info_cap_add(caps, size,
2755				   VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE, 1);
2756	if (IS_ERR(header))
2757		return PTR_ERR(header);
2758
2759	iova_cap = container_of(header,
2760				struct vfio_iommu_type1_info_cap_iova_range,
2761				header);
2762	iova_cap->nr_iovas = cap_iovas->nr_iovas;
2763	memcpy(iova_cap->iova_ranges, cap_iovas->iova_ranges,
2764	       cap_iovas->nr_iovas * sizeof(*cap_iovas->iova_ranges));
2765	return 0;
2766}
2767
2768static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
2769				      struct vfio_info_cap *caps)
2770{
2771	struct vfio_iommu_type1_info_cap_iova_range *cap_iovas;
2772	struct vfio_iova *iova;
2773	size_t size;
2774	int iovas = 0, i = 0, ret;
2775
2776	list_for_each_entry(iova, &iommu->iova_list, list)
2777		iovas++;
2778
2779	if (!iovas) {
2780		/*
2781		 * Return 0 as a container with a single mdev device
2782		 * will have an empty list
2783		 */
2784		return 0;
2785	}
2786
2787	size = struct_size(cap_iovas, iova_ranges, iovas);
2788
2789	cap_iovas = kzalloc(size, GFP_KERNEL);
2790	if (!cap_iovas)
2791		return -ENOMEM;
2792
2793	cap_iovas->nr_iovas = iovas;
2794
2795	list_for_each_entry(iova, &iommu->iova_list, list) {
2796		cap_iovas->iova_ranges[i].start = iova->start;
2797		cap_iovas->iova_ranges[i].end = iova->end;
2798		i++;
2799	}
2800
2801	ret = vfio_iommu_iova_add_cap(caps, cap_iovas, size);
2802
2803	kfree(cap_iovas);
2804	return ret;
2805}
2806
2807static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
2808					   struct vfio_info_cap *caps)
2809{
2810	struct vfio_iommu_type1_info_cap_migration cap_mig = {};
2811
2812	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
2813	cap_mig.header.version = 1;
2814
2815	cap_mig.flags = 0;
2816	/* support minimum pgsize */
2817	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
2818	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
2819
2820	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
2821}
2822
2823static int vfio_iommu_dma_avail_build_caps(struct vfio_iommu *iommu,
2824					   struct vfio_info_cap *caps)
2825{
2826	struct vfio_iommu_type1_info_dma_avail cap_dma_avail;
2827
2828	cap_dma_avail.header.id = VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL;
2829	cap_dma_avail.header.version = 1;
2830
2831	cap_dma_avail.avail = iommu->dma_avail;
2832
2833	return vfio_info_add_capability(caps, &cap_dma_avail.header,
2834					sizeof(cap_dma_avail));
2835}
2836
2837static int vfio_iommu_type1_get_info(struct vfio_iommu *iommu,
2838				     unsigned long arg)
2839{
2840	struct vfio_iommu_type1_info info = {};
2841	unsigned long minsz;
2842	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
2843	int ret;
2844
2845	minsz = offsetofend(struct vfio_iommu_type1_info, iova_pgsizes);
2846
2847	if (copy_from_user(&info, (void __user *)arg, minsz))
2848		return -EFAULT;
2849
2850	if (info.argsz < minsz)
2851		return -EINVAL;
2852
2853	minsz = min_t(size_t, info.argsz, sizeof(info));
2854
2855	mutex_lock(&iommu->lock);
2856	info.flags = VFIO_IOMMU_INFO_PGSIZES;
2857
2858	info.iova_pgsizes = iommu->pgsize_bitmap;
2859
2860	ret = vfio_iommu_migration_build_caps(iommu, &caps);
2861
2862	if (!ret)
2863		ret = vfio_iommu_dma_avail_build_caps(iommu, &caps);
2864
2865	if (!ret)
2866		ret = vfio_iommu_iova_build_caps(iommu, &caps);
2867
2868	mutex_unlock(&iommu->lock);
2869
2870	if (ret)
2871		return ret;
2872
2873	if (caps.size) {
2874		info.flags |= VFIO_IOMMU_INFO_CAPS;
2875
2876		if (info.argsz < sizeof(info) + caps.size) {
2877			info.argsz = sizeof(info) + caps.size;
2878		} else {
2879			vfio_info_cap_shift(&caps, sizeof(info));
2880			if (copy_to_user((void __user *)arg +
2881					sizeof(info), caps.buf,
2882					caps.size)) {
2883				kfree(caps.buf);
2884				return -EFAULT;
2885			}
2886			info.cap_offset = sizeof(info);
2887		}
2888
2889		kfree(caps.buf);
2890	}
2891
2892	return copy_to_user((void __user *)arg, &info, minsz) ?
2893			-EFAULT : 0;
2894}
2895
2896static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu,
2897				    unsigned long arg)
2898{
2899	struct vfio_iommu_type1_dma_map map;
2900	unsigned long minsz;
2901	uint32_t mask = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE |
2902			VFIO_DMA_MAP_FLAG_VADDR;
2903
2904	minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
2905
2906	if (copy_from_user(&map, (void __user *)arg, minsz))
2907		return -EFAULT;
2908
2909	if (map.argsz < minsz || map.flags & ~mask)
2910		return -EINVAL;
2911
2912	return vfio_dma_do_map(iommu, &map);
2913}
2914
2915static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu,
2916				      unsigned long arg)
2917{
2918	struct vfio_iommu_type1_dma_unmap unmap;
2919	struct vfio_bitmap bitmap = { 0 };
2920	uint32_t mask = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP |
2921			VFIO_DMA_UNMAP_FLAG_VADDR |
2922			VFIO_DMA_UNMAP_FLAG_ALL;
2923	unsigned long minsz;
2924	int ret;
2925
2926	minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
2927
2928	if (copy_from_user(&unmap, (void __user *)arg, minsz))
2929		return -EFAULT;
2930
2931	if (unmap.argsz < minsz || unmap.flags & ~mask)
2932		return -EINVAL;
2933
2934	if ((unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
2935	    (unmap.flags & (VFIO_DMA_UNMAP_FLAG_ALL |
2936			    VFIO_DMA_UNMAP_FLAG_VADDR)))
2937		return -EINVAL;
2938
2939	if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
2940		unsigned long pgshift;
2941
2942		if (unmap.argsz < (minsz + sizeof(bitmap)))
2943			return -EINVAL;
2944
2945		if (copy_from_user(&bitmap,
2946				   (void __user *)(arg + minsz),
2947				   sizeof(bitmap)))
2948			return -EFAULT;
2949
2950		if (!access_ok((void __user *)bitmap.data, bitmap.size))
2951			return -EINVAL;
2952
2953		pgshift = __ffs(bitmap.pgsize);
2954		ret = verify_bitmap_size(unmap.size >> pgshift,
2955					 bitmap.size);
2956		if (ret)
2957			return ret;
2958	}
2959
2960	ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap);
2961	if (ret)
2962		return ret;
2963
2964	return copy_to_user((void __user *)arg, &unmap, minsz) ?
2965			-EFAULT : 0;
2966}
2967
2968static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu,
2969					unsigned long arg)
2970{
2971	struct vfio_iommu_type1_dirty_bitmap dirty;
2972	uint32_t mask = VFIO_IOMMU_DIRTY_PAGES_FLAG_START |
2973			VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP |
2974			VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
2975	unsigned long minsz;
2976	int ret = 0;
2977
2978	if (!iommu->v2)
2979		return -EACCES;
2980
2981	minsz = offsetofend(struct vfio_iommu_type1_dirty_bitmap, flags);
2982
2983	if (copy_from_user(&dirty, (void __user *)arg, minsz))
2984		return -EFAULT;
2985
2986	if (dirty.argsz < minsz || dirty.flags & ~mask)
2987		return -EINVAL;
2988
2989	/* only one flag should be set at a time */
2990	if (__ffs(dirty.flags) != __fls(dirty.flags))
2991		return -EINVAL;
2992
2993	if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_START) {
2994		size_t pgsize;
2995
2996		mutex_lock(&iommu->lock);
2997		pgsize = 1 << __ffs(iommu->pgsize_bitmap);
2998		if (!iommu->dirty_page_tracking) {
2999			ret = vfio_dma_bitmap_alloc_all(iommu, pgsize);
3000			if (!ret)
3001				iommu->dirty_page_tracking = true;
3002		}
3003		mutex_unlock(&iommu->lock);
3004		return ret;
3005	} else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP) {
3006		mutex_lock(&iommu->lock);
3007		if (iommu->dirty_page_tracking) {
3008			iommu->dirty_page_tracking = false;
3009			vfio_dma_bitmap_free_all(iommu);
3010		}
3011		mutex_unlock(&iommu->lock);
3012		return 0;
3013	} else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP) {
3014		struct vfio_iommu_type1_dirty_bitmap_get range;
3015		unsigned long pgshift;
3016		size_t data_size = dirty.argsz - minsz;
3017		size_t size, iommu_pgsize;
3018		dma_addr_t iova, iova_end;
3019
3020		if (!data_size || data_size < sizeof(range))
3021			return -EINVAL;
3022
3023		if (copy_from_user(&range, (void __user *)(arg + minsz),
3024				   sizeof(range)))
3025			return -EFAULT;
3026
3027		iova = range.iova;
3028		size = range.size;
3029
3030		if (iova != range.iova || size != range.size)
3031			return -EOVERFLOW;
3032
3033		if (!size)
3034			return -EINVAL;
3035
3036		if (check_add_overflow(iova, size - 1, &iova_end))
3037			return -EOVERFLOW;
3038
3039		if (!access_ok((void __user *)range.bitmap.data,
3040			       range.bitmap.size))
3041			return -EINVAL;
3042
3043		pgshift = __ffs(range.bitmap.pgsize);
3044		ret = verify_bitmap_size(size >> pgshift,
3045					 range.bitmap.size);
3046		if (ret)
3047			return ret;
3048
3049		mutex_lock(&iommu->lock);
3050
3051		iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
3052
3053		/* allow only smallest supported pgsize */
3054		if (range.bitmap.pgsize != iommu_pgsize) {
3055			ret = -EINVAL;
3056			goto out_unlock;
3057		}
3058		if (iova & (iommu_pgsize - 1)) {
3059			ret = -EINVAL;
3060			goto out_unlock;
3061		}
3062		if (size & (iommu_pgsize - 1)) {
3063			ret = -EINVAL;
3064			goto out_unlock;
3065		}
3066
3067		if (iommu->dirty_page_tracking)
3068			ret = vfio_iova_dirty_bitmap(range.bitmap.data,
3069						     iommu, iova, iova_end,
3070						     range.bitmap.pgsize);
3071		else
3072			ret = -EINVAL;
3073out_unlock:
3074		mutex_unlock(&iommu->lock);
3075
3076		return ret;
3077	}
3078
3079	return -EINVAL;
3080}
3081
3082static long vfio_iommu_type1_ioctl(void *iommu_data,
3083				   unsigned int cmd, unsigned long arg)
3084{
3085	struct vfio_iommu *iommu = iommu_data;
3086
3087	switch (cmd) {
3088	case VFIO_CHECK_EXTENSION:
3089		return vfio_iommu_type1_check_extension(iommu, arg);
3090	case VFIO_IOMMU_GET_INFO:
3091		return vfio_iommu_type1_get_info(iommu, arg);
3092	case VFIO_IOMMU_MAP_DMA:
3093		return vfio_iommu_type1_map_dma(iommu, arg);
3094	case VFIO_IOMMU_UNMAP_DMA:
3095		return vfio_iommu_type1_unmap_dma(iommu, arg);
3096	case VFIO_IOMMU_DIRTY_PAGES:
3097		return vfio_iommu_type1_dirty_pages(iommu, arg);
3098	default:
3099		return -ENOTTY;
3100	}
3101}
3102
3103static void vfio_iommu_type1_register_device(void *iommu_data,
3104					     struct vfio_device *vdev)
3105{
3106	struct vfio_iommu *iommu = iommu_data;
3107
3108	if (!vdev->ops->dma_unmap)
3109		return;
3110
3111	/*
3112	 * list_empty(&iommu->device_list) is tested under the iommu->lock while
3113	 * iteration for dma_unmap must be done under the device_list_lock.
3114	 * Holding both locks here allows avoiding the device_list_lock in
3115	 * several fast paths. See vfio_notify_dma_unmap()
3116	 */
3117	mutex_lock(&iommu->lock);
3118	mutex_lock(&iommu->device_list_lock);
3119	list_add(&vdev->iommu_entry, &iommu->device_list);
3120	mutex_unlock(&iommu->device_list_lock);
3121	mutex_unlock(&iommu->lock);
3122}
3123
3124static void vfio_iommu_type1_unregister_device(void *iommu_data,
3125					       struct vfio_device *vdev)
3126{
3127	struct vfio_iommu *iommu = iommu_data;
3128
3129	if (!vdev->ops->dma_unmap)
3130		return;
3131
3132	mutex_lock(&iommu->lock);
3133	mutex_lock(&iommu->device_list_lock);
3134	list_del(&vdev->iommu_entry);
3135	mutex_unlock(&iommu->device_list_lock);
3136	mutex_unlock(&iommu->lock);
3137}
3138
3139static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,
3140					 dma_addr_t user_iova, void *data,
3141					 size_t count, bool write,
3142					 size_t *copied)
3143{
3144	struct mm_struct *mm;
3145	unsigned long vaddr;
3146	struct vfio_dma *dma;
3147	bool kthread = current->mm == NULL;
3148	size_t offset;
3149
3150	*copied = 0;
3151
3152	dma = vfio_find_dma(iommu, user_iova, 1);
3153	if (!dma)
3154		return -EINVAL;
3155
3156	if ((write && !(dma->prot & IOMMU_WRITE)) ||
3157			!(dma->prot & IOMMU_READ))
3158		return -EPERM;
3159
3160	mm = dma->mm;
3161	if (!mmget_not_zero(mm))
3162		return -EPERM;
3163
3164	if (kthread)
3165		kthread_use_mm(mm);
3166	else if (current->mm != mm)
3167		goto out;
3168
3169	offset = user_iova - dma->iova;
3170
3171	if (count > dma->size - offset)
3172		count = dma->size - offset;
3173
3174	vaddr = dma->vaddr + offset;
3175
3176	if (write) {
3177		*copied = copy_to_user((void __user *)vaddr, data,
3178					 count) ? 0 : count;
3179		if (*copied && iommu->dirty_page_tracking) {
3180			unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
3181			/*
3182			 * Bitmap populated with the smallest supported page
3183			 * size
3184			 */
3185			bitmap_set(dma->bitmap, offset >> pgshift,
3186				   ((offset + *copied - 1) >> pgshift) -
3187				   (offset >> pgshift) + 1);
3188		}
3189	} else
3190		*copied = copy_from_user(data, (void __user *)vaddr,
3191					   count) ? 0 : count;
3192	if (kthread)
3193		kthread_unuse_mm(mm);
3194out:
3195	mmput(mm);
3196	return *copied ? 0 : -EFAULT;
3197}
3198
3199static int vfio_iommu_type1_dma_rw(void *iommu_data, dma_addr_t user_iova,
3200				   void *data, size_t count, bool write)
3201{
3202	struct vfio_iommu *iommu = iommu_data;
3203	int ret = 0;
3204	size_t done;
3205
3206	mutex_lock(&iommu->lock);
3207
3208	if (WARN_ONCE(iommu->vaddr_invalid_count,
3209		      "vfio_dma_rw not allowed with VFIO_UPDATE_VADDR\n")) {
3210		ret = -EBUSY;
3211		goto out;
3212	}
3213
3214	while (count > 0) {
3215		ret = vfio_iommu_type1_dma_rw_chunk(iommu, user_iova, data,
3216						    count, write, &done);
3217		if (ret)
3218			break;
3219
3220		count -= done;
3221		data += done;
3222		user_iova += done;
3223	}
3224
3225out:
3226	mutex_unlock(&iommu->lock);
3227	return ret;
3228}
3229
3230static struct iommu_domain *
3231vfio_iommu_type1_group_iommu_domain(void *iommu_data,
3232				    struct iommu_group *iommu_group)
3233{
3234	struct iommu_domain *domain = ERR_PTR(-ENODEV);
3235	struct vfio_iommu *iommu = iommu_data;
3236	struct vfio_domain *d;
3237
3238	if (!iommu || !iommu_group)
3239		return ERR_PTR(-EINVAL);
3240
3241	mutex_lock(&iommu->lock);
3242	list_for_each_entry(d, &iommu->domain_list, next) {
3243		if (find_iommu_group(d, iommu_group)) {
3244			domain = d->domain;
3245			break;
3246		}
3247	}
3248	mutex_unlock(&iommu->lock);
3249
3250	return domain;
3251}
3252
3253static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
3254	.name			= "vfio-iommu-type1",
3255	.owner			= THIS_MODULE,
3256	.open			= vfio_iommu_type1_open,
3257	.release		= vfio_iommu_type1_release,
3258	.ioctl			= vfio_iommu_type1_ioctl,
3259	.attach_group		= vfio_iommu_type1_attach_group,
3260	.detach_group		= vfio_iommu_type1_detach_group,
3261	.pin_pages		= vfio_iommu_type1_pin_pages,
3262	.unpin_pages		= vfio_iommu_type1_unpin_pages,
3263	.register_device	= vfio_iommu_type1_register_device,
3264	.unregister_device	= vfio_iommu_type1_unregister_device,
3265	.dma_rw			= vfio_iommu_type1_dma_rw,
3266	.group_iommu_domain	= vfio_iommu_type1_group_iommu_domain,
3267};
3268
3269static int __init vfio_iommu_type1_init(void)
3270{
3271	return vfio_register_iommu_driver(&vfio_iommu_driver_ops_type1);
3272}
3273
3274static void __exit vfio_iommu_type1_cleanup(void)
3275{
3276	vfio_unregister_iommu_driver(&vfio_iommu_driver_ops_type1);
3277}
3278
3279module_init(vfio_iommu_type1_init);
3280module_exit(vfio_iommu_type1_cleanup);
3281
3282MODULE_VERSION(DRIVER_VERSION);
3283MODULE_LICENSE("GPL v2");
3284MODULE_AUTHOR(DRIVER_AUTHOR);
3285MODULE_DESCRIPTION(DRIVER_DESC);