vfio_iommu_spapr_tce.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * VFIO: IOMMU DMA mapping support for TCE on POWER
   4 *
   5 * Copyright (C) 2013 IBM Corp.  All rights reserved.
   6 *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
   7 * Copyright Gavin Shan, IBM Corporation 2014.
   8 *
   9 * Derived from original vfio_iommu_type1.c:
  10 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
  11 *     Author: Alex Williamson <alex.williamson@redhat.com>
  12 */
  13
  14#include <linux/module.h>
  15#include <linux/pci.h>
  16#include <linux/slab.h>
  17#include <linux/uaccess.h>
  18#include <linux/err.h>
  19#include <linux/vfio.h>
  20#include <linux/vmalloc.h>
  21#include <linux/sched/mm.h>
  22#include <linux/sched/signal.h>
  23#include <linux/mm.h>
  24#include "vfio.h"
  25
  26#include <asm/iommu.h>
  27#include <asm/tce.h>
  28#include <asm/mmu_context.h>
  29
  30#define DRIVER_VERSION  "0.1"
  31#define DRIVER_AUTHOR   "aik@ozlabs.ru"
  32#define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"
  33
  34static void tce_iommu_detach_group(void *iommu_data,
  35		struct iommu_group *iommu_group);
  36
  37/*
  38 * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
  39 *
  40 * This code handles mapping and unmapping of user data buffers
  41 * into DMA'ble space using the IOMMU
  42 */
  43
  44struct tce_iommu_group {
  45	struct list_head next;
  46	struct iommu_group *grp;
  47};
  48
  49/*
  50 * A container needs to remember which preregistered region  it has
  51 * referenced to do proper cleanup at the userspace process exit.
  52 */
  53struct tce_iommu_prereg {
  54	struct list_head next;
  55	struct mm_iommu_table_group_mem_t *mem;
  56};
  57
  58/*
  59 * The container descriptor supports only a single group per container.
  60 * Required by the API as the container is not supplied with the IOMMU group
  61 * at the moment of initialization.
  62 */
  63struct tce_container {
  64	struct mutex lock;
  65	bool enabled;
  66	bool v2;
  67	bool def_window_pending;
  68	unsigned long locked_pages;
  69	struct mm_struct *mm;
  70	struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
  71	struct list_head group_list;
  72	struct list_head prereg_list;
  73};
  74
  75static long tce_iommu_mm_set(struct tce_container *container)
  76{
  77	if (container->mm) {
  78		if (container->mm == current->mm)
  79			return 0;
  80		return -EPERM;
  81	}
  82	BUG_ON(!current->mm);
  83	container->mm = current->mm;
  84	mmgrab(container->mm);
  85
  86	return 0;
  87}
  88
  89static long tce_iommu_prereg_free(struct tce_container *container,
  90		struct tce_iommu_prereg *tcemem)
  91{
  92	long ret;
  93
  94	ret = mm_iommu_put(container->mm, tcemem->mem);
  95	if (ret)
  96		return ret;
  97
  98	list_del(&tcemem->next);
  99	kfree(tcemem);
 100
 101	return 0;
 102}
 103
 104static long tce_iommu_unregister_pages(struct tce_container *container,
 105		__u64 vaddr, __u64 size)
 106{
 107	struct mm_iommu_table_group_mem_t *mem;
 108	struct tce_iommu_prereg *tcemem;
 109	bool found = false;
 110	long ret;
 111
 112	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
 113		return -EINVAL;
 114
 115	mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
 116	if (!mem)
 117		return -ENOENT;
 118
 119	list_for_each_entry(tcemem, &container->prereg_list, next) {
 120		if (tcemem->mem == mem) {
 121			found = true;
 122			break;
 123		}
 124	}
 125
 126	if (!found)
 127		ret = -ENOENT;
 128	else
 129		ret = tce_iommu_prereg_free(container, tcemem);
 130
 131	mm_iommu_put(container->mm, mem);
 132
 133	return ret;
 134}
 135
 136static long tce_iommu_register_pages(struct tce_container *container,
 137		__u64 vaddr, __u64 size)
 138{
 139	long ret = 0;
 140	struct mm_iommu_table_group_mem_t *mem = NULL;
 141	struct tce_iommu_prereg *tcemem;
 142	unsigned long entries = size >> PAGE_SHIFT;
 143
 144	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
 145			((vaddr + size) < vaddr))
 146		return -EINVAL;
 147
 148	mem = mm_iommu_get(container->mm, vaddr, entries);
 149	if (mem) {
 150		list_for_each_entry(tcemem, &container->prereg_list, next) {
 151			if (tcemem->mem == mem) {
 152				ret = -EBUSY;
 153				goto put_exit;
 154			}
 155		}
 156	} else {
 157		ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
 158		if (ret)
 159			return ret;
 160	}
 161
 162	tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
 163	if (!tcemem) {
 164		ret = -ENOMEM;
 165		goto put_exit;
 166	}
 167
 168	tcemem->mem = mem;
 169	list_add(&tcemem->next, &container->prereg_list);
 170
 171	container->enabled = true;
 172
 173	return 0;
 174
 175put_exit:
 176	mm_iommu_put(container->mm, mem);
 177	return ret;
 178}
 179
 180static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
 181		unsigned int it_page_shift)
 182{
 183	struct page *page;
 184	unsigned long size = 0;
 185
 186	if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
 187		return size == (1UL << it_page_shift);
 188
 189	page = pfn_to_page(hpa >> PAGE_SHIFT);
 190	/*
 191	 * Check that the TCE table granularity is not bigger than the size of
 192	 * a page we just found. Otherwise the hardware can get access to
 193	 * a bigger memory chunk that it should.
 194	 */
 195	return page_shift(compound_head(page)) >= it_page_shift;
 196}
 197
 198static inline bool tce_groups_attached(struct tce_container *container)
 199{
 200	return !list_empty(&container->group_list);
 201}
 202
 203static long tce_iommu_find_table(struct tce_container *container,
 204		phys_addr_t ioba, struct iommu_table **ptbl)
 205{
 206	long i;
 207
 208	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
 209		struct iommu_table *tbl = container->tables[i];
 210
 211		if (tbl) {
 212			unsigned long entry = ioba >> tbl->it_page_shift;
 213			unsigned long start = tbl->it_offset;
 214			unsigned long end = start + tbl->it_size;
 215
 216			if ((start <= entry) && (entry < end)) {
 217				*ptbl = tbl;
 218				return i;
 219			}
 220		}
 221	}
 222
 223	return -1;
 224}
 225
 226static int tce_iommu_find_free_table(struct tce_container *container)
 227{
 228	int i;
 229
 230	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
 231		if (!container->tables[i])
 232			return i;
 233	}
 234
 235	return -ENOSPC;
 236}
 237
 238static int tce_iommu_enable(struct tce_container *container)
 239{
 240	int ret = 0;
 241	unsigned long locked;
 242	struct iommu_table_group *table_group;
 243	struct tce_iommu_group *tcegrp;
 244
 245	if (container->enabled)
 246		return -EBUSY;
 247
 248	/*
 249	 * When userspace pages are mapped into the IOMMU, they are effectively
 250	 * locked memory, so, theoretically, we need to update the accounting
 251	 * of locked pages on each map and unmap.  For powerpc, the map unmap
 252	 * paths can be very hot, though, and the accounting would kill
 253	 * performance, especially since it would be difficult to impossible
 254	 * to handle the accounting in real mode only.
 255	 *
 256	 * To address that, rather than precisely accounting every page, we
 257	 * instead account for a worst case on locked memory when the iommu is
 258	 * enabled and disabled.  The worst case upper bound on locked memory
 259	 * is the size of the whole iommu window, which is usually relatively
 260	 * small (compared to total memory sizes) on POWER hardware.
 261	 *
 262	 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
 263	 * that would effectively kill the guest at random points, much better
 264	 * enforcing the limit based on the max that the guest can map.
 265	 *
 266	 * Unfortunately at the moment it counts whole tables, no matter how
 267	 * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
 268	 * each with 2GB DMA window, 8GB will be counted here. The reason for
 269	 * this is that we cannot tell here the amount of RAM used by the guest
 270	 * as this information is only available from KVM and VFIO is
 271	 * KVM agnostic.
 272	 *
 273	 * So we do not allow enabling a container without a group attached
 274	 * as there is no way to know how much we should increment
 275	 * the locked_vm counter.
 276	 */
 277	if (!tce_groups_attached(container))
 278		return -ENODEV;
 279
 280	tcegrp = list_first_entry(&container->group_list,
 281			struct tce_iommu_group, next);
 282	table_group = iommu_group_get_iommudata(tcegrp->grp);
 283	if (!table_group)
 284		return -ENODEV;
 285
 286	if (!table_group->tce32_size)
 287		return -EPERM;
 288
 289	ret = tce_iommu_mm_set(container);
 290	if (ret)
 291		return ret;
 292
 293	locked = table_group->tce32_size >> PAGE_SHIFT;
 294	ret = account_locked_vm(container->mm, locked, true);
 295	if (ret)
 296		return ret;
 297
 298	container->locked_pages = locked;
 299
 300	container->enabled = true;
 301
 302	return ret;
 303}
 304
 305static void tce_iommu_disable(struct tce_container *container)
 306{
 307	if (!container->enabled)
 308		return;
 309
 310	container->enabled = false;
 311
 312	BUG_ON(!container->mm);
 313	account_locked_vm(container->mm, container->locked_pages, false);
 314}
 315
 316static void *tce_iommu_open(unsigned long arg)
 317{
 318	struct tce_container *container;
 319
 320	if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
 321		pr_err("tce_vfio: Wrong IOMMU type\n");
 322		return ERR_PTR(-EINVAL);
 323	}
 324
 325	container = kzalloc(sizeof(*container), GFP_KERNEL);
 326	if (!container)
 327		return ERR_PTR(-ENOMEM);
 328
 329	mutex_init(&container->lock);
 330	INIT_LIST_HEAD_RCU(&container->group_list);
 331	INIT_LIST_HEAD_RCU(&container->prereg_list);
 332
 333	container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
 334
 335	return container;
 336}
 337
 338static int tce_iommu_clear(struct tce_container *container,
 339		struct iommu_table *tbl,
 340		unsigned long entry, unsigned long pages);
 341static void tce_iommu_free_table(struct tce_container *container,
 342		struct iommu_table *tbl);
 343
 344static void tce_iommu_release(void *iommu_data)
 345{
 346	struct tce_container *container = iommu_data;
 347	struct tce_iommu_group *tcegrp;
 348	struct tce_iommu_prereg *tcemem, *tmtmp;
 349	long i;
 350
 351	while (tce_groups_attached(container)) {
 352		tcegrp = list_first_entry(&container->group_list,
 353				struct tce_iommu_group, next);
 354		tce_iommu_detach_group(iommu_data, tcegrp->grp);
 355	}
 356
 357	/*
 358	 * If VFIO created a table, it was not disposed
 359	 * by tce_iommu_detach_group() so do it now.
 360	 */
 361	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
 362		struct iommu_table *tbl = container->tables[i];
 363
 364		if (!tbl)
 365			continue;
 366
 367		tce_iommu_free_table(container, tbl);
 368	}
 369
 370	list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
 371		WARN_ON(tce_iommu_prereg_free(container, tcemem));
 372
 373	tce_iommu_disable(container);
 374	if (container->mm)
 375		mmdrop(container->mm);
 376	mutex_destroy(&container->lock);
 377
 378	kfree(container);
 379}
 380
 381static void tce_iommu_unuse_page(unsigned long hpa)
 382{
 383	struct page *page;
 384
 385	page = pfn_to_page(hpa >> PAGE_SHIFT);
 386	unpin_user_page(page);
 387}
 388
 389static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
 390		unsigned long tce, unsigned long shift,
 391		unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
 392{
 393	long ret = 0;
 394	struct mm_iommu_table_group_mem_t *mem;
 395
 396	mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
 397	if (!mem)
 398		return -EINVAL;
 399
 400	ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
 401	if (ret)
 402		return -EINVAL;
 403
 404	*pmem = mem;
 405
 406	return 0;
 407}
 408
 409static void tce_iommu_unuse_page_v2(struct tce_container *container,
 410		struct iommu_table *tbl, unsigned long entry)
 411{
 412	struct mm_iommu_table_group_mem_t *mem = NULL;
 413	int ret;
 414	unsigned long hpa = 0;
 415	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
 416
 417	if (!pua)
 418		return;
 419
 420	ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
 421			tbl->it_page_shift, &hpa, &mem);
 422	if (ret)
 423		pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
 424				__func__, be64_to_cpu(*pua), entry, ret);
 425	if (mem)
 426		mm_iommu_mapped_dec(mem);
 427
 428	*pua = cpu_to_be64(0);
 429}
 430
 431static int tce_iommu_clear(struct tce_container *container,
 432		struct iommu_table *tbl,
 433		unsigned long entry, unsigned long pages)
 434{
 435	unsigned long oldhpa;
 436	long ret;
 437	enum dma_data_direction direction;
 438	unsigned long lastentry = entry + pages, firstentry = entry;
 439
 440	for ( ; entry < lastentry; ++entry) {
 441		if (tbl->it_indirect_levels && tbl->it_userspace) {
 442			/*
 443			 * For multilevel tables, we can take a shortcut here
 444			 * and skip some TCEs as we know that the userspace
 445			 * addresses cache is a mirror of the real TCE table
 446			 * and if it is missing some indirect levels, then
 447			 * the hardware table does not have them allocated
 448			 * either and therefore does not require updating.
 449			 */
 450			__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
 451					entry);
 452			if (!pua) {
 453				/* align to level_size which is power of two */
 454				entry |= tbl->it_level_size - 1;
 455				continue;
 456			}
 457		}
 458
 459		cond_resched();
 460
 461		direction = DMA_NONE;
 462		oldhpa = 0;
 463		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
 464				&direction);
 465		if (ret)
 466			continue;
 467
 468		if (direction == DMA_NONE)
 469			continue;
 470
 471		if (container->v2) {
 472			tce_iommu_unuse_page_v2(container, tbl, entry);
 473			continue;
 474		}
 475
 476		tce_iommu_unuse_page(oldhpa);
 477	}
 478
 479	iommu_tce_kill(tbl, firstentry, pages);
 480
 481	return 0;
 482}
 483
 484static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
 485{
 486	struct page *page = NULL;
 487	enum dma_data_direction direction = iommu_tce_direction(tce);
 488
 489	if (pin_user_pages_fast(tce & PAGE_MASK, 1,
 490			direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
 491			&page) != 1)
 492		return -EFAULT;
 493
 494	*hpa = __pa((unsigned long) page_address(page));
 495
 496	return 0;
 497}
 498
 499static long tce_iommu_build(struct tce_container *container,
 500		struct iommu_table *tbl,
 501		unsigned long entry, unsigned long tce, unsigned long pages,
 502		enum dma_data_direction direction)
 503{
 504	long i, ret = 0;
 505	unsigned long hpa;
 506	enum dma_data_direction dirtmp;
 507
 508	for (i = 0; i < pages; ++i) {
 509		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
 510
 511		ret = tce_iommu_use_page(tce, &hpa);
 512		if (ret)
 513			break;
 514
 515		if (!tce_page_is_contained(container->mm, hpa,
 516				tbl->it_page_shift)) {
 517			ret = -EPERM;
 518			break;
 519		}
 520
 521		hpa |= offset;
 522		dirtmp = direction;
 523		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
 524				&hpa, &dirtmp);
 525		if (ret) {
 526			tce_iommu_unuse_page(hpa);
 527			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
 528					__func__, entry << tbl->it_page_shift,
 529					tce, ret);
 530			break;
 531		}
 532
 533		if (dirtmp != DMA_NONE)
 534			tce_iommu_unuse_page(hpa);
 535
 536		tce += IOMMU_PAGE_SIZE(tbl);
 537	}
 538
 539	if (ret)
 540		tce_iommu_clear(container, tbl, entry, i);
 541	else
 542		iommu_tce_kill(tbl, entry, pages);
 543
 544	return ret;
 545}
 546
 547static long tce_iommu_build_v2(struct tce_container *container,
 548		struct iommu_table *tbl,
 549		unsigned long entry, unsigned long tce, unsigned long pages,
 550		enum dma_data_direction direction)
 551{
 552	long i, ret = 0;
 553	unsigned long hpa;
 554	enum dma_data_direction dirtmp;
 555
 556	for (i = 0; i < pages; ++i) {
 557		struct mm_iommu_table_group_mem_t *mem = NULL;
 558		__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
 559
 560		ret = tce_iommu_prereg_ua_to_hpa(container,
 561				tce, tbl->it_page_shift, &hpa, &mem);
 562		if (ret)
 563			break;
 564
 565		if (!tce_page_is_contained(container->mm, hpa,
 566				tbl->it_page_shift)) {
 567			ret = -EPERM;
 568			break;
 569		}
 570
 571		/* Preserve offset within IOMMU page */
 572		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
 573		dirtmp = direction;
 574
 575		/* The registered region is being unregistered */
 576		if (mm_iommu_mapped_inc(mem))
 577			break;
 578
 579		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
 580				&hpa, &dirtmp);
 581		if (ret) {
 582			/* dirtmp cannot be DMA_NONE here */
 583			tce_iommu_unuse_page_v2(container, tbl, entry + i);
 584			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
 585					__func__, entry << tbl->it_page_shift,
 586					tce, ret);
 587			break;
 588		}
 589
 590		if (dirtmp != DMA_NONE)
 591			tce_iommu_unuse_page_v2(container, tbl, entry + i);
 592
 593		*pua = cpu_to_be64(tce);
 594
 595		tce += IOMMU_PAGE_SIZE(tbl);
 596	}
 597
 598	if (ret)
 599		tce_iommu_clear(container, tbl, entry, i);
 600	else
 601		iommu_tce_kill(tbl, entry, pages);
 602
 603	return ret;
 604}
 605
 606static long tce_iommu_create_table(struct tce_container *container,
 607			struct iommu_table_group *table_group,
 608			int num,
 609			__u32 page_shift,
 610			__u64 window_size,
 611			__u32 levels,
 612			struct iommu_table **ptbl)
 613{
 614	long ret, table_size;
 615
 616	table_size = table_group->ops->get_table_size(page_shift, window_size,
 617			levels);
 618	if (!table_size)
 619		return -EINVAL;
 620
 621	ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
 622	if (ret)
 623		return ret;
 624
 625	ret = table_group->ops->create_table(table_group, num,
 626			page_shift, window_size, levels, ptbl);
 627
 628	WARN_ON(!ret && !(*ptbl)->it_ops->free);
 629	WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
 630
 631	return ret;
 632}
 633
 634static void tce_iommu_free_table(struct tce_container *container,
 635		struct iommu_table *tbl)
 636{
 637	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
 638
 639	iommu_tce_table_put(tbl);
 640	account_locked_vm(container->mm, pages, false);
 641}
 642
 643static long tce_iommu_create_window(struct tce_container *container,
 644		__u32 page_shift, __u64 window_size, __u32 levels,
 645		__u64 *start_addr)
 646{
 647	struct tce_iommu_group *tcegrp;
 648	struct iommu_table_group *table_group;
 649	struct iommu_table *tbl = NULL;
 650	long ret, num;
 651
 652	num = tce_iommu_find_free_table(container);
 653	if (num < 0)
 654		return num;
 655
 656	/* Get the first group for ops::create_table */
 657	tcegrp = list_first_entry(&container->group_list,
 658			struct tce_iommu_group, next);
 659	table_group = iommu_group_get_iommudata(tcegrp->grp);
 660	if (!table_group)
 661		return -EFAULT;
 662
 663	if (!(table_group->pgsizes & (1ULL << page_shift)))
 664		return -EINVAL;
 665
 666	if (!table_group->ops->set_window || !table_group->ops->unset_window ||
 667			!table_group->ops->get_table_size ||
 668			!table_group->ops->create_table)
 669		return -EPERM;
 670
 671	/* Create TCE table */
 672	ret = tce_iommu_create_table(container, table_group, num,
 673			page_shift, window_size, levels, &tbl);
 674	if (ret)
 675		return ret;
 676
 677	BUG_ON(!tbl->it_ops->free);
 678
 679	/*
 680	 * Program the table to every group.
 681	 * Groups have been tested for compatibility at the attach time.
 682	 */
 683	list_for_each_entry(tcegrp, &container->group_list, next) {
 684		table_group = iommu_group_get_iommudata(tcegrp->grp);
 685
 686		ret = table_group->ops->set_window(table_group, num, tbl);
 687		if (ret)
 688			goto unset_exit;
 689	}
 690
 691	container->tables[num] = tbl;
 692
 693	/* Return start address assigned by platform in create_table() */
 694	*start_addr = tbl->it_offset << tbl->it_page_shift;
 695
 696	return 0;
 697
 698unset_exit:
 699	list_for_each_entry(tcegrp, &container->group_list, next) {
 700		table_group = iommu_group_get_iommudata(tcegrp->grp);
 701		table_group->ops->unset_window(table_group, num);
 702	}
 703	tce_iommu_free_table(container, tbl);
 704
 705	return ret;
 706}
 707
 708static long tce_iommu_remove_window(struct tce_container *container,
 709		__u64 start_addr)
 710{
 711	struct iommu_table_group *table_group = NULL;
 712	struct iommu_table *tbl;
 713	struct tce_iommu_group *tcegrp;
 714	int num;
 715
 716	num = tce_iommu_find_table(container, start_addr, &tbl);
 717	if (num < 0)
 718		return -EINVAL;
 719
 720	BUG_ON(!tbl->it_size);
 721
 722	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
 723
 724	/* Detach groups from IOMMUs */
 725	list_for_each_entry(tcegrp, &container->group_list, next) {
 726		table_group = iommu_group_get_iommudata(tcegrp->grp);
 727
 728		/*
 729		 * SPAPR TCE IOMMU exposes the default DMA window to
 730		 * the guest via dma32_window_start/size of
 731		 * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
 732		 * the userspace to remove this window, some do not so
 733		 * here we check for the platform capability.
 734		 */
 735		if (!table_group->ops || !table_group->ops->unset_window)
 736			return -EPERM;
 737
 738		table_group->ops->unset_window(table_group, num);
 739	}
 740
 741	/* Free table */
 742	tce_iommu_free_table(container, tbl);
 743	container->tables[num] = NULL;
 744
 745	return 0;
 746}
 747
 748static long tce_iommu_create_default_window(struct tce_container *container)
 749{
 750	long ret;
 751	__u64 start_addr = 0;
 752	struct tce_iommu_group *tcegrp;
 753	struct iommu_table_group *table_group;
 754
 755	if (!container->def_window_pending)
 756		return 0;
 757
 758	if (!tce_groups_attached(container))
 759		return -ENODEV;
 760
 761	tcegrp = list_first_entry(&container->group_list,
 762			struct tce_iommu_group, next);
 763	table_group = iommu_group_get_iommudata(tcegrp->grp);
 764	if (!table_group)
 765		return -ENODEV;
 766
 767	ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
 768			table_group->tce32_size, 1, &start_addr);
 769	WARN_ON_ONCE(!ret && start_addr);
 770
 771	if (!ret)
 772		container->def_window_pending = false;
 773
 774	return ret;
 775}
 776
 777static long vfio_spapr_ioctl_eeh_pe_op(struct iommu_group *group,
 778				       unsigned long arg)
 779{
 780	struct eeh_pe *pe;
 781	struct vfio_eeh_pe_op op;
 782	unsigned long minsz;
 783
 784	pe = eeh_iommu_group_to_pe(group);
 785	if (!pe)
 786		return -ENODEV;
 787
 788	minsz = offsetofend(struct vfio_eeh_pe_op, op);
 789	if (copy_from_user(&op, (void __user *)arg, minsz))
 790		return -EFAULT;
 791	if (op.argsz < minsz || op.flags)
 792		return -EINVAL;
 793
 794	switch (op.op) {
 795	case VFIO_EEH_PE_DISABLE:
 796		return eeh_pe_set_option(pe, EEH_OPT_DISABLE);
 797	case VFIO_EEH_PE_ENABLE:
 798		return eeh_pe_set_option(pe, EEH_OPT_ENABLE);
 799	case VFIO_EEH_PE_UNFREEZE_IO:
 800		return eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
 801	case VFIO_EEH_PE_UNFREEZE_DMA:
 802		return eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
 803	case VFIO_EEH_PE_GET_STATE:
 804		return eeh_pe_get_state(pe);
 805		break;
 806	case VFIO_EEH_PE_RESET_DEACTIVATE:
 807		return eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, true);
 808	case VFIO_EEH_PE_RESET_HOT:
 809		return eeh_pe_reset(pe, EEH_RESET_HOT, true);
 810	case VFIO_EEH_PE_RESET_FUNDAMENTAL:
 811		return eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL, true);
 812	case VFIO_EEH_PE_CONFIGURE:
 813		return eeh_pe_configure(pe);
 814	case VFIO_EEH_PE_INJECT_ERR:
 815		minsz = offsetofend(struct vfio_eeh_pe_op, err.mask);
 816		if (op.argsz < minsz)
 817			return -EINVAL;
 818		if (copy_from_user(&op, (void __user *)arg, minsz))
 819			return -EFAULT;
 820
 821		return eeh_pe_inject_err(pe, op.err.type, op.err.func,
 822					 op.err.addr, op.err.mask);
 823	default:
 824		return -EINVAL;
 825	}
 826}
 827
 828static long tce_iommu_ioctl(void *iommu_data,
 829				 unsigned int cmd, unsigned long arg)
 830{
 831	struct tce_container *container = iommu_data;
 832	unsigned long minsz, ddwsz;
 833	long ret;
 834
 835	switch (cmd) {
 836	case VFIO_CHECK_EXTENSION:
 837		switch (arg) {
 838		case VFIO_SPAPR_TCE_IOMMU:
 839		case VFIO_SPAPR_TCE_v2_IOMMU:
 840			return 1;
 841		case VFIO_EEH:
 842			return eeh_enabled();
 843		default:
 844			return 0;
 845		}
 846	}
 847
 848	/*
 849	 * Sanity check to prevent one userspace from manipulating
 850	 * another userspace mm.
 851	 */
 852	BUG_ON(!container);
 853	if (container->mm && container->mm != current->mm)
 854		return -EPERM;
 855
 856	switch (cmd) {
 857	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
 858		struct vfio_iommu_spapr_tce_info info;
 859		struct tce_iommu_group *tcegrp;
 860		struct iommu_table_group *table_group;
 861
 862		if (!tce_groups_attached(container))
 863			return -ENXIO;
 864
 865		tcegrp = list_first_entry(&container->group_list,
 866				struct tce_iommu_group, next);
 867		table_group = iommu_group_get_iommudata(tcegrp->grp);
 868
 869		if (!table_group)
 870			return -ENXIO;
 871
 872		minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
 873				dma32_window_size);
 874
 875		if (copy_from_user(&info, (void __user *)arg, minsz))
 876			return -EFAULT;
 877
 878		if (info.argsz < minsz)
 879			return -EINVAL;
 880
 881		info.dma32_window_start = table_group->tce32_start;
 882		info.dma32_window_size = table_group->tce32_size;
 883		info.flags = 0;
 884		memset(&info.ddw, 0, sizeof(info.ddw));
 885
 886		if (table_group->max_dynamic_windows_supported &&
 887				container->v2) {
 888			info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
 889			info.ddw.pgsizes = table_group->pgsizes;
 890			info.ddw.max_dynamic_windows_supported =
 891				table_group->max_dynamic_windows_supported;
 892			info.ddw.levels = table_group->max_levels;
 893		}
 894
 895		ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
 896
 897		if (info.argsz >= ddwsz)
 898			minsz = ddwsz;
 899
 900		if (copy_to_user((void __user *)arg, &info, minsz))
 901			return -EFAULT;
 902
 903		return 0;
 904	}
 905	case VFIO_IOMMU_MAP_DMA: {
 906		struct vfio_iommu_type1_dma_map param;
 907		struct iommu_table *tbl = NULL;
 908		long num;
 909		enum dma_data_direction direction;
 910
 911		if (!container->enabled)
 912			return -EPERM;
 913
 914		minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
 915
 916		if (copy_from_user(&param, (void __user *)arg, minsz))
 917			return -EFAULT;
 918
 919		if (param.argsz < minsz)
 920			return -EINVAL;
 921
 922		if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
 923				VFIO_DMA_MAP_FLAG_WRITE))
 924			return -EINVAL;
 925
 926		ret = tce_iommu_create_default_window(container);
 927		if (ret)
 928			return ret;
 929
 930		num = tce_iommu_find_table(container, param.iova, &tbl);
 931		if (num < 0)
 932			return -ENXIO;
 933
 934		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
 935				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
 936			return -EINVAL;
 937
 938		/* iova is checked by the IOMMU API */
 939		if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
 940			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
 941				direction = DMA_BIDIRECTIONAL;
 942			else
 943				direction = DMA_TO_DEVICE;
 944		} else {
 945			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
 946				direction = DMA_FROM_DEVICE;
 947			else
 948				return -EINVAL;
 949		}
 950
 951		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
 952		if (ret)
 953			return ret;
 954
 955		if (container->v2)
 956			ret = tce_iommu_build_v2(container, tbl,
 957					param.iova >> tbl->it_page_shift,
 958					param.vaddr,
 959					param.size >> tbl->it_page_shift,
 960					direction);
 961		else
 962			ret = tce_iommu_build(container, tbl,
 963					param.iova >> tbl->it_page_shift,
 964					param.vaddr,
 965					param.size >> tbl->it_page_shift,
 966					direction);
 967
 968		iommu_flush_tce(tbl);
 969
 970		return ret;
 971	}
 972	case VFIO_IOMMU_UNMAP_DMA: {
 973		struct vfio_iommu_type1_dma_unmap param;
 974		struct iommu_table *tbl = NULL;
 975		long num;
 976
 977		if (!container->enabled)
 978			return -EPERM;
 979
 980		minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
 981				size);
 982
 983		if (copy_from_user(&param, (void __user *)arg, minsz))
 984			return -EFAULT;
 985
 986		if (param.argsz < minsz)
 987			return -EINVAL;
 988
 989		/* No flag is supported now */
 990		if (param.flags)
 991			return -EINVAL;
 992
 993		ret = tce_iommu_create_default_window(container);
 994		if (ret)
 995			return ret;
 996
 997		num = tce_iommu_find_table(container, param.iova, &tbl);
 998		if (num < 0)
 999			return -ENXIO;
1000
1001		if (param.size & ~IOMMU_PAGE_MASK(tbl))
1002			return -EINVAL;
1003
1004		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
1005				param.size >> tbl->it_page_shift);
1006		if (ret)
1007			return ret;
1008
1009		ret = tce_iommu_clear(container, tbl,
1010				param.iova >> tbl->it_page_shift,
1011				param.size >> tbl->it_page_shift);
1012		iommu_flush_tce(tbl);
1013
1014		return ret;
1015	}
1016	case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
1017		struct vfio_iommu_spapr_register_memory param;
1018
1019		if (!container->v2)
1020			break;
1021
1022		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1023				size);
1024
1025		ret = tce_iommu_mm_set(container);
1026		if (ret)
1027			return ret;
1028
1029		if (copy_from_user(&param, (void __user *)arg, minsz))
1030			return -EFAULT;
1031
1032		if (param.argsz < minsz)
1033			return -EINVAL;
1034
1035		/* No flag is supported now */
1036		if (param.flags)
1037			return -EINVAL;
1038
1039		mutex_lock(&container->lock);
1040		ret = tce_iommu_register_pages(container, param.vaddr,
1041				param.size);
1042		mutex_unlock(&container->lock);
1043
1044		return ret;
1045	}
1046	case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
1047		struct vfio_iommu_spapr_register_memory param;
1048
1049		if (!container->v2)
1050			break;
1051
1052		if (!container->mm)
1053			return -EPERM;
1054
1055		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1056				size);
1057
1058		if (copy_from_user(&param, (void __user *)arg, minsz))
1059			return -EFAULT;
1060
1061		if (param.argsz < minsz)
1062			return -EINVAL;
1063
1064		/* No flag is supported now */
1065		if (param.flags)
1066			return -EINVAL;
1067
1068		mutex_lock(&container->lock);
1069		ret = tce_iommu_unregister_pages(container, param.vaddr,
1070				param.size);
1071		mutex_unlock(&container->lock);
1072
1073		return ret;
1074	}
1075	case VFIO_IOMMU_ENABLE:
1076		if (container->v2)
1077			break;
1078
1079		mutex_lock(&container->lock);
1080		ret = tce_iommu_enable(container);
1081		mutex_unlock(&container->lock);
1082		return ret;
1083
1084
1085	case VFIO_IOMMU_DISABLE:
1086		if (container->v2)
1087			break;
1088
1089		mutex_lock(&container->lock);
1090		tce_iommu_disable(container);
1091		mutex_unlock(&container->lock);
1092		return 0;
1093
1094	case VFIO_EEH_PE_OP: {
1095		struct tce_iommu_group *tcegrp;
1096
1097		ret = 0;
1098		list_for_each_entry(tcegrp, &container->group_list, next) {
1099			ret = vfio_spapr_ioctl_eeh_pe_op(tcegrp->grp, arg);
1100			if (ret)
1101				return ret;
1102		}
1103		return ret;
1104	}
1105
1106	case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1107		struct vfio_iommu_spapr_tce_create create;
1108
1109		if (!container->v2)
1110			break;
1111
1112		ret = tce_iommu_mm_set(container);
1113		if (ret)
1114			return ret;
1115
1116		if (!tce_groups_attached(container))
1117			return -ENXIO;
1118
1119		minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1120				start_addr);
1121
1122		if (copy_from_user(&create, (void __user *)arg, minsz))
1123			return -EFAULT;
1124
1125		if (create.argsz < minsz)
1126			return -EINVAL;
1127
1128		if (create.flags)
1129			return -EINVAL;
1130
1131		mutex_lock(&container->lock);
1132
1133		ret = tce_iommu_create_default_window(container);
1134		if (!ret)
1135			ret = tce_iommu_create_window(container,
1136					create.page_shift,
1137					create.window_size, create.levels,
1138					&create.start_addr);
1139
1140		mutex_unlock(&container->lock);
1141
1142		if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1143			ret = -EFAULT;
1144
1145		return ret;
1146	}
1147	case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1148		struct vfio_iommu_spapr_tce_remove remove;
1149
1150		if (!container->v2)
1151			break;
1152
1153		ret = tce_iommu_mm_set(container);
1154		if (ret)
1155			return ret;
1156
1157		if (!tce_groups_attached(container))
1158			return -ENXIO;
1159
1160		minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1161				start_addr);
1162
1163		if (copy_from_user(&remove, (void __user *)arg, minsz))
1164			return -EFAULT;
1165
1166		if (remove.argsz < minsz)
1167			return -EINVAL;
1168
1169		if (remove.flags)
1170			return -EINVAL;
1171
1172		if (container->def_window_pending && !remove.start_addr) {
1173			container->def_window_pending = false;
1174			return 0;
1175		}
1176
1177		mutex_lock(&container->lock);
1178
1179		ret = tce_iommu_remove_window(container, remove.start_addr);
1180
1181		mutex_unlock(&container->lock);
1182
1183		return ret;
1184	}
1185	}
1186
1187	return -ENOTTY;
1188}
1189
1190static void tce_iommu_release_ownership(struct tce_container *container,
1191		struct iommu_table_group *table_group)
1192{
1193	long i;
1194
1195	if (!table_group->ops->unset_window) {
1196		WARN_ON_ONCE(1);
1197		return;
1198	}
1199
1200	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1201		if (container->tables[i]) {
1202			tce_iommu_clear(container, container->tables[i],
1203					container->tables[i]->it_offset,
1204					container->tables[i]->it_size);
1205			table_group->ops->unset_window(table_group, i);
1206		}
1207	}
1208}
1209
1210static long tce_iommu_take_ownership(struct tce_container *container,
1211		struct iommu_table_group *table_group)
1212{
1213	long i, ret = 0;
1214
1215	/* Set all windows to the new group */
1216	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1217		struct iommu_table *tbl = container->tables[i];
1218
1219		if (!tbl)
1220			continue;
1221
1222		ret = table_group->ops->set_window(table_group, i, tbl);
1223		if (ret)
1224			goto release_exit;
1225	}
1226
1227	return 0;
1228
1229release_exit:
1230	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1231		table_group->ops->unset_window(table_group, i);
1232
1233	return ret;
1234}
1235
1236static int tce_iommu_attach_group(void *iommu_data,
1237		struct iommu_group *iommu_group, enum vfio_group_type type)
1238{
1239	int ret = 0;
1240	struct tce_container *container = iommu_data;
1241	struct iommu_table_group *table_group;
1242	struct tce_iommu_group *tcegrp = NULL;
1243
1244	if (type == VFIO_EMULATED_IOMMU)
1245		return -EINVAL;
1246
1247	mutex_lock(&container->lock);
1248
1249	/* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
1250			iommu_group_id(iommu_group), iommu_group); */
1251	table_group = iommu_group_get_iommudata(iommu_group);
1252	if (!table_group) {
1253		ret = -ENODEV;
1254		goto unlock_exit;
1255	}
1256
1257	/* v2 requires full support of dynamic DMA windows */
1258	if (container->v2 && table_group->max_dynamic_windows_supported == 0) {
1259		ret = -EINVAL;
1260		goto unlock_exit;
1261	}
1262
1263	/* v1 reuses TCE tables and does not share them among PEs */
1264	if (!container->v2 && tce_groups_attached(container)) {
1265		ret = -EBUSY;
1266		goto unlock_exit;
1267	}
1268
1269	/*
1270	 * Check if new group has the same iommu_table_group_ops
1271	 * (i.e. compatible)
1272	 */
1273	list_for_each_entry(tcegrp, &container->group_list, next) {
1274		struct iommu_table_group *table_group_tmp;
1275
1276		if (tcegrp->grp == iommu_group) {
1277			pr_warn("tce_vfio: Group %d is already attached\n",
1278					iommu_group_id(iommu_group));
1279			ret = -EBUSY;
1280			goto unlock_exit;
1281		}
1282		table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
1283		if (table_group_tmp->ops->create_table !=
1284				table_group->ops->create_table) {
1285			pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
1286					iommu_group_id(iommu_group),
1287					iommu_group_id(tcegrp->grp));
1288			ret = -EPERM;
1289			goto unlock_exit;
1290		}
1291	}
1292
1293	tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
1294	if (!tcegrp) {
1295		ret = -ENOMEM;
1296		goto unlock_exit;
1297	}
1298
1299	ret = tce_iommu_take_ownership(container, table_group);
1300	if (!tce_groups_attached(container) && !container->tables[0])
1301		container->def_window_pending = true;
1302
1303	if (!ret) {
1304		tcegrp->grp = iommu_group;
1305		list_add(&tcegrp->next, &container->group_list);
1306	}
1307
1308	if (ret && tcegrp)
1309		kfree(tcegrp);
1310
1311unlock_exit:
1312	mutex_unlock(&container->lock);
1313
1314	return ret;
1315}
1316
1317static void tce_iommu_detach_group(void *iommu_data,
1318		struct iommu_group *iommu_group)
1319{
1320	struct tce_container *container = iommu_data;
1321	struct iommu_table_group *table_group;
1322	bool found = false;
1323	struct tce_iommu_group *tcegrp;
1324
1325	mutex_lock(&container->lock);
1326
1327	list_for_each_entry(tcegrp, &container->group_list, next) {
1328		if (tcegrp->grp == iommu_group) {
1329			found = true;
1330			break;
1331		}
1332	}
1333
1334	if (!found) {
1335		pr_warn("tce_vfio: detaching unattached group #%u\n",
1336				iommu_group_id(iommu_group));
1337		goto unlock_exit;
1338	}
1339
1340	list_del(&tcegrp->next);
1341	kfree(tcegrp);
1342
1343	table_group = iommu_group_get_iommudata(iommu_group);
1344	BUG_ON(!table_group);
1345
1346	tce_iommu_release_ownership(container, table_group);
1347
1348unlock_exit:
1349	mutex_unlock(&container->lock);
1350}
1351
1352static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
1353	.name		= "iommu-vfio-powerpc",
1354	.owner		= THIS_MODULE,
1355	.open		= tce_iommu_open,
1356	.release	= tce_iommu_release,
1357	.ioctl		= tce_iommu_ioctl,
1358	.attach_group	= tce_iommu_attach_group,
1359	.detach_group	= tce_iommu_detach_group,
1360};
1361
1362static int __init tce_iommu_init(void)
1363{
1364	return vfio_register_iommu_driver(&tce_iommu_driver_ops);
1365}
1366
1367static void __exit tce_iommu_cleanup(void)
1368{
1369	vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
1370}
1371
1372module_init(tce_iommu_init);
1373module_exit(tce_iommu_cleanup);
1374
1375MODULE_VERSION(DRIVER_VERSION);
1376MODULE_LICENSE("GPL v2");
1377MODULE_AUTHOR(DRIVER_AUTHOR);
1378MODULE_DESCRIPTION(DRIVER_DESC);
1379