privcmd.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/******************************************************************************
   3 * privcmd.c
   4 *
   5 * Interface to privileged domain-0 commands.
   6 *
   7 * Copyright (c) 2002-2004, K A Fraser, B Dragovic
   8 */
   9
  10#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  11
  12#include <linux/eventfd.h>
  13#include <linux/file.h>
  14#include <linux/kernel.h>
  15#include <linux/kstrtox.h>
  16#include <linux/module.h>
  17#include <linux/mutex.h>
  18#include <linux/poll.h>
  19#include <linux/sched.h>
  20#include <linux/slab.h>
  21#include <linux/srcu.h>
  22#include <linux/string.h>
  23#include <linux/workqueue.h>
  24#include <linux/errno.h>
  25#include <linux/mm.h>
  26#include <linux/mman.h>
  27#include <linux/uaccess.h>
  28#include <linux/swap.h>
  29#include <linux/highmem.h>
  30#include <linux/pagemap.h>
  31#include <linux/seq_file.h>
  32#include <linux/miscdevice.h>
  33#include <linux/moduleparam.h>
  34#include <linux/notifier.h>
  35#include <linux/security.h>
  36#include <linux/virtio_mmio.h>
  37#include <linux/wait.h>
  38
  39#include <asm/xen/hypervisor.h>
  40#include <asm/xen/hypercall.h>
  41
  42#include <xen/xen.h>
  43#include <xen/events.h>
  44#include <xen/privcmd.h>
  45#include <xen/interface/xen.h>
  46#include <xen/interface/memory.h>
  47#include <xen/interface/hvm/dm_op.h>
  48#include <xen/interface/hvm/ioreq.h>
  49#include <xen/features.h>
  50#include <xen/page.h>
  51#include <xen/xen-ops.h>
  52#include <xen/balloon.h>
  53#include <xen/xenbus.h>
  54#ifdef CONFIG_XEN_ACPI
  55#include <xen/acpi.h>
  56#endif
  57
  58#include "privcmd.h"
  59
  60MODULE_DESCRIPTION("Xen hypercall passthrough driver");
  61MODULE_LICENSE("GPL");
  62
  63#define PRIV_VMA_LOCKED ((void *)1)
  64
  65static unsigned int privcmd_dm_op_max_num = 16;
  66module_param_named(dm_op_max_nr_bufs, privcmd_dm_op_max_num, uint, 0644);
  67MODULE_PARM_DESC(dm_op_max_nr_bufs,
  68		 "Maximum number of buffers per dm_op hypercall");
  69
  70static unsigned int privcmd_dm_op_buf_max_size = 4096;
  71module_param_named(dm_op_buf_max_size, privcmd_dm_op_buf_max_size, uint,
  72		   0644);
  73MODULE_PARM_DESC(dm_op_buf_max_size,
  74		 "Maximum size of a dm_op hypercall buffer");
  75
  76static bool unrestricted;
  77module_param(unrestricted, bool, 0);
  78MODULE_PARM_DESC(unrestricted,
  79	"Don't restrict hypercalls to target domain if running in a domU");
  80
  81struct privcmd_data {
  82	domid_t domid;
  83};
  84
  85/* DOMID_INVALID implies no restriction */
  86static domid_t target_domain = DOMID_INVALID;
  87static bool restrict_wait;
  88static DECLARE_WAIT_QUEUE_HEAD(restrict_wait_wq);
  89
  90static int privcmd_vma_range_is_mapped(
  91               struct vm_area_struct *vma,
  92               unsigned long addr,
  93               unsigned long nr_pages);
  94
  95static long privcmd_ioctl_hypercall(struct file *file, void __user *udata)
  96{
  97	struct privcmd_data *data = file->private_data;
  98	struct privcmd_hypercall hypercall;
  99	long ret;
 100
 101	/* Disallow arbitrary hypercalls if restricted */
 102	if (data->domid != DOMID_INVALID)
 103		return -EPERM;
 104
 105	if (copy_from_user(&hypercall, udata, sizeof(hypercall)))
 106		return -EFAULT;
 107
 108	xen_preemptible_hcall_begin();
 109	ret = privcmd_call(hypercall.op,
 110			   hypercall.arg[0], hypercall.arg[1],
 111			   hypercall.arg[2], hypercall.arg[3],
 112			   hypercall.arg[4]);
 113	xen_preemptible_hcall_end();
 114
 115	return ret;
 116}
 117
 118static void free_page_list(struct list_head *pages)
 119{
 120	struct page *p, *n;
 121
 122	list_for_each_entry_safe(p, n, pages, lru)
 123		__free_page(p);
 124
 125	INIT_LIST_HEAD(pages);
 126}
 127
 128/*
 129 * Given an array of items in userspace, return a list of pages
 130 * containing the data.  If copying fails, either because of memory
 131 * allocation failure or a problem reading user memory, return an
 132 * error code; its up to the caller to dispose of any partial list.
 133 */
 134static int gather_array(struct list_head *pagelist,
 135			unsigned nelem, size_t size,
 136			const void __user *data)
 137{
 138	unsigned pageidx;
 139	void *pagedata;
 140	int ret;
 141
 142	if (size > PAGE_SIZE)
 143		return 0;
 144
 145	pageidx = PAGE_SIZE;
 146	pagedata = NULL;	/* quiet, gcc */
 147	while (nelem--) {
 148		if (pageidx > PAGE_SIZE-size) {
 149			struct page *page = alloc_page(GFP_KERNEL);
 150
 151			ret = -ENOMEM;
 152			if (page == NULL)
 153				goto fail;
 154
 155			pagedata = page_address(page);
 156
 157			list_add_tail(&page->lru, pagelist);
 158			pageidx = 0;
 159		}
 160
 161		ret = -EFAULT;
 162		if (copy_from_user(pagedata + pageidx, data, size))
 163			goto fail;
 164
 165		data += size;
 166		pageidx += size;
 167	}
 168
 169	ret = 0;
 170
 171fail:
 172	return ret;
 173}
 174
 175/*
 176 * Call function "fn" on each element of the array fragmented
 177 * over a list of pages.
 178 */
 179static int traverse_pages(unsigned nelem, size_t size,
 180			  struct list_head *pos,
 181			  int (*fn)(void *data, void *state),
 182			  void *state)
 183{
 184	void *pagedata;
 185	unsigned pageidx;
 186	int ret = 0;
 187
 188	BUG_ON(size > PAGE_SIZE);
 189
 190	pageidx = PAGE_SIZE;
 191	pagedata = NULL;	/* hush, gcc */
 192
 193	while (nelem--) {
 194		if (pageidx > PAGE_SIZE-size) {
 195			struct page *page;
 196			pos = pos->next;
 197			page = list_entry(pos, struct page, lru);
 198			pagedata = page_address(page);
 199			pageidx = 0;
 200		}
 201
 202		ret = (*fn)(pagedata + pageidx, state);
 203		if (ret)
 204			break;
 205		pageidx += size;
 206	}
 207
 208	return ret;
 209}
 210
 211/*
 212 * Similar to traverse_pages, but use each page as a "block" of
 213 * data to be processed as one unit.
 214 */
 215static int traverse_pages_block(unsigned nelem, size_t size,
 216				struct list_head *pos,
 217				int (*fn)(void *data, int nr, void *state),
 218				void *state)
 219{
 220	void *pagedata;
 221	int ret = 0;
 222
 223	BUG_ON(size > PAGE_SIZE);
 224
 225	while (nelem) {
 226		int nr = (PAGE_SIZE/size);
 227		struct page *page;
 228		if (nr > nelem)
 229			nr = nelem;
 230		pos = pos->next;
 231		page = list_entry(pos, struct page, lru);
 232		pagedata = page_address(page);
 233		ret = (*fn)(pagedata, nr, state);
 234		if (ret)
 235			break;
 236		nelem -= nr;
 237	}
 238
 239	return ret;
 240}
 241
 242struct mmap_gfn_state {
 243	unsigned long va;
 244	struct vm_area_struct *vma;
 245	domid_t domain;
 246};
 247
 248static int mmap_gfn_range(void *data, void *state)
 249{
 250	struct privcmd_mmap_entry *msg = data;
 251	struct mmap_gfn_state *st = state;
 252	struct vm_area_struct *vma = st->vma;
 253	int rc;
 254
 255	/* Do not allow range to wrap the address space. */
 256	if ((msg->npages > (LONG_MAX >> PAGE_SHIFT)) ||
 257	    ((unsigned long)(msg->npages << PAGE_SHIFT) >= -st->va))
 258		return -EINVAL;
 259
 260	/* Range chunks must be contiguous in va space. */
 261	if ((msg->va != st->va) ||
 262	    ((msg->va+(msg->npages<<PAGE_SHIFT)) > vma->vm_end))
 263		return -EINVAL;
 264
 265	rc = xen_remap_domain_gfn_range(vma,
 266					msg->va & PAGE_MASK,
 267					msg->mfn, msg->npages,
 268					vma->vm_page_prot,
 269					st->domain, NULL);
 270	if (rc < 0)
 271		return rc;
 272
 273	st->va += msg->npages << PAGE_SHIFT;
 274
 275	return 0;
 276}
 277
 278static long privcmd_ioctl_mmap(struct file *file, void __user *udata)
 279{
 280	struct privcmd_data *data = file->private_data;
 281	struct privcmd_mmap mmapcmd;
 282	struct mm_struct *mm = current->mm;
 283	struct vm_area_struct *vma;
 284	int rc;
 285	LIST_HEAD(pagelist);
 286	struct mmap_gfn_state state;
 287
 288	/* We only support privcmd_ioctl_mmap_batch for non-auto-translated. */
 289	if (!xen_pv_domain())
 290		return -ENOSYS;
 291
 292	if (copy_from_user(&mmapcmd, udata, sizeof(mmapcmd)))
 293		return -EFAULT;
 294
 295	/* If restriction is in place, check the domid matches */
 296	if (data->domid != DOMID_INVALID && data->domid != mmapcmd.dom)
 297		return -EPERM;
 298
 299	rc = gather_array(&pagelist,
 300			  mmapcmd.num, sizeof(struct privcmd_mmap_entry),
 301			  mmapcmd.entry);
 302
 303	if (rc || list_empty(&pagelist))
 304		goto out;
 305
 306	mmap_write_lock(mm);
 307
 308	{
 309		struct page *page = list_first_entry(&pagelist,
 310						     struct page, lru);
 311		struct privcmd_mmap_entry *msg = page_address(page);
 312
 313		vma = vma_lookup(mm, msg->va);
 314		rc = -EINVAL;
 315
 316		if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data)
 317			goto out_up;
 318		vma->vm_private_data = PRIV_VMA_LOCKED;
 319	}
 320
 321	state.va = vma->vm_start;
 322	state.vma = vma;
 323	state.domain = mmapcmd.dom;
 324
 325	rc = traverse_pages(mmapcmd.num, sizeof(struct privcmd_mmap_entry),
 326			    &pagelist,
 327			    mmap_gfn_range, &state);
 328
 329
 330out_up:
 331	mmap_write_unlock(mm);
 332
 333out:
 334	free_page_list(&pagelist);
 335
 336	return rc;
 337}
 338
 339struct mmap_batch_state {
 340	domid_t domain;
 341	unsigned long va;
 342	struct vm_area_struct *vma;
 343	int index;
 344	/* A tristate:
 345	 *      0 for no errors
 346	 *      1 if at least one error has happened (and no
 347	 *          -ENOENT errors have happened)
 348	 *      -ENOENT if at least 1 -ENOENT has happened.
 349	 */
 350	int global_error;
 351	int version;
 352
 353	/* User-space gfn array to store errors in the second pass for V1. */
 354	xen_pfn_t __user *user_gfn;
 355	/* User-space int array to store errors in the second pass for V2. */
 356	int __user *user_err;
 357};
 358
 359/* auto translated dom0 note: if domU being created is PV, then gfn is
 360 * mfn(addr on bus). If it's auto xlated, then gfn is pfn (input to HAP).
 361 */
 362static int mmap_batch_fn(void *data, int nr, void *state)
 363{
 364	xen_pfn_t *gfnp = data;
 365	struct mmap_batch_state *st = state;
 366	struct vm_area_struct *vma = st->vma;
 367	struct page **pages = vma->vm_private_data;
 368	struct page **cur_pages = NULL;
 369	int ret;
 370
 371	if (!xen_pv_domain())
 372		cur_pages = &pages[st->index];
 373
 374	BUG_ON(nr < 0);
 375	ret = xen_remap_domain_gfn_array(st->vma, st->va & PAGE_MASK, gfnp, nr,
 376					 (int *)gfnp, st->vma->vm_page_prot,
 377					 st->domain, cur_pages);
 378
 379	/* Adjust the global_error? */
 380	if (ret != nr) {
 381		if (ret == -ENOENT)
 382			st->global_error = -ENOENT;
 383		else {
 384			/* Record that at least one error has happened. */
 385			if (st->global_error == 0)
 386				st->global_error = 1;
 387		}
 388	}
 389	st->va += XEN_PAGE_SIZE * nr;
 390	st->index += nr / XEN_PFN_PER_PAGE;
 391
 392	return 0;
 393}
 394
 395static int mmap_return_error(int err, struct mmap_batch_state *st)
 396{
 397	int ret;
 398
 399	if (st->version == 1) {
 400		if (err) {
 401			xen_pfn_t gfn;
 402
 403			ret = get_user(gfn, st->user_gfn);
 404			if (ret < 0)
 405				return ret;
 406			/*
 407			 * V1 encodes the error codes in the 32bit top
 408			 * nibble of the gfn (with its known
 409			 * limitations vis-a-vis 64 bit callers).
 410			 */
 411			gfn |= (err == -ENOENT) ?
 412				PRIVCMD_MMAPBATCH_PAGED_ERROR :
 413				PRIVCMD_MMAPBATCH_MFN_ERROR;
 414			return __put_user(gfn, st->user_gfn++);
 415		} else
 416			st->user_gfn++;
 417	} else { /* st->version == 2 */
 418		if (err)
 419			return __put_user(err, st->user_err++);
 420		else
 421			st->user_err++;
 422	}
 423
 424	return 0;
 425}
 426
 427static int mmap_return_errors(void *data, int nr, void *state)
 428{
 429	struct mmap_batch_state *st = state;
 430	int *errs = data;
 431	int i;
 432	int ret;
 433
 434	for (i = 0; i < nr; i++) {
 435		ret = mmap_return_error(errs[i], st);
 436		if (ret < 0)
 437			return ret;
 438	}
 439	return 0;
 440}
 441
 442/* Allocate pfns that are then mapped with gfns from foreign domid. Update
 443 * the vma with the page info to use later.
 444 * Returns: 0 if success, otherwise -errno
 445 */
 446static int alloc_empty_pages(struct vm_area_struct *vma, int numpgs)
 447{
 448	int rc;
 449	struct page **pages;
 450
 451	pages = kvcalloc(numpgs, sizeof(pages[0]), GFP_KERNEL);
 452	if (pages == NULL)
 453		return -ENOMEM;
 454
 455	rc = xen_alloc_unpopulated_pages(numpgs, pages);
 456	if (rc != 0) {
 457		pr_warn("%s Could not alloc %d pfns rc:%d\n", __func__,
 458			numpgs, rc);
 459		kvfree(pages);
 460		return -ENOMEM;
 461	}
 462	BUG_ON(vma->vm_private_data != NULL);
 463	vma->vm_private_data = pages;
 464
 465	return 0;
 466}
 467
 468static const struct vm_operations_struct privcmd_vm_ops;
 469
 470static long privcmd_ioctl_mmap_batch(
 471	struct file *file, void __user *udata, int version)
 472{
 473	struct privcmd_data *data = file->private_data;
 474	int ret;
 475	struct privcmd_mmapbatch_v2 m;
 476	struct mm_struct *mm = current->mm;
 477	struct vm_area_struct *vma;
 478	unsigned long nr_pages;
 479	LIST_HEAD(pagelist);
 480	struct mmap_batch_state state;
 481
 482	switch (version) {
 483	case 1:
 484		if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch)))
 485			return -EFAULT;
 486		/* Returns per-frame error in m.arr. */
 487		m.err = NULL;
 488		if (!access_ok(m.arr, m.num * sizeof(*m.arr)))
 489			return -EFAULT;
 490		break;
 491	case 2:
 492		if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch_v2)))
 493			return -EFAULT;
 494		/* Returns per-frame error code in m.err. */
 495		if (!access_ok(m.err, m.num * (sizeof(*m.err))))
 496			return -EFAULT;
 497		break;
 498	default:
 499		return -EINVAL;
 500	}
 501
 502	/* If restriction is in place, check the domid matches */
 503	if (data->domid != DOMID_INVALID && data->domid != m.dom)
 504		return -EPERM;
 505
 506	nr_pages = DIV_ROUND_UP(m.num, XEN_PFN_PER_PAGE);
 507	if ((m.num <= 0) || (nr_pages > (LONG_MAX >> PAGE_SHIFT)))
 508		return -EINVAL;
 509
 510	ret = gather_array(&pagelist, m.num, sizeof(xen_pfn_t), m.arr);
 511
 512	if (ret)
 513		goto out;
 514	if (list_empty(&pagelist)) {
 515		ret = -EINVAL;
 516		goto out;
 517	}
 518
 519	if (version == 2) {
 520		/* Zero error array now to only copy back actual errors. */
 521		if (clear_user(m.err, sizeof(int) * m.num)) {
 522			ret = -EFAULT;
 523			goto out;
 524		}
 525	}
 526
 527	mmap_write_lock(mm);
 528
 529	vma = find_vma(mm, m.addr);
 530	if (!vma ||
 531	    vma->vm_ops != &privcmd_vm_ops) {
 532		ret = -EINVAL;
 533		goto out_unlock;
 534	}
 535
 536	/*
 537	 * Caller must either:
 538	 *
 539	 * Map the whole VMA range, which will also allocate all the
 540	 * pages required for the auto_translated_physmap case.
 541	 *
 542	 * Or
 543	 *
 544	 * Map unmapped holes left from a previous map attempt (e.g.,
 545	 * because those foreign frames were previously paged out).
 546	 */
 547	if (vma->vm_private_data == NULL) {
 548		if (m.addr != vma->vm_start ||
 549		    m.addr + (nr_pages << PAGE_SHIFT) != vma->vm_end) {
 550			ret = -EINVAL;
 551			goto out_unlock;
 552		}
 553		if (!xen_pv_domain()) {
 554			ret = alloc_empty_pages(vma, nr_pages);
 555			if (ret < 0)
 556				goto out_unlock;
 557		} else
 558			vma->vm_private_data = PRIV_VMA_LOCKED;
 559	} else {
 560		if (m.addr < vma->vm_start ||
 561		    m.addr + (nr_pages << PAGE_SHIFT) > vma->vm_end) {
 562			ret = -EINVAL;
 563			goto out_unlock;
 564		}
 565		if (privcmd_vma_range_is_mapped(vma, m.addr, nr_pages)) {
 566			ret = -EINVAL;
 567			goto out_unlock;
 568		}
 569	}
 570
 571	state.domain        = m.dom;
 572	state.vma           = vma;
 573	state.va            = m.addr;
 574	state.index         = 0;
 575	state.global_error  = 0;
 576	state.version       = version;
 577
 578	BUILD_BUG_ON(((PAGE_SIZE / sizeof(xen_pfn_t)) % XEN_PFN_PER_PAGE) != 0);
 579	/* mmap_batch_fn guarantees ret == 0 */
 580	BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t),
 581				    &pagelist, mmap_batch_fn, &state));
 582
 583	mmap_write_unlock(mm);
 584
 585	if (state.global_error) {
 586		/* Write back errors in second pass. */
 587		state.user_gfn = (xen_pfn_t *)m.arr;
 588		state.user_err = m.err;
 589		ret = traverse_pages_block(m.num, sizeof(xen_pfn_t),
 590					   &pagelist, mmap_return_errors, &state);
 591	} else
 592		ret = 0;
 593
 594	/* If we have not had any EFAULT-like global errors then set the global
 595	 * error to -ENOENT if necessary. */
 596	if ((ret == 0) && (state.global_error == -ENOENT))
 597		ret = -ENOENT;
 598
 599out:
 600	free_page_list(&pagelist);
 601	return ret;
 602
 603out_unlock:
 604	mmap_write_unlock(mm);
 605	goto out;
 606}
 607
 608static int lock_pages(
 609	struct privcmd_dm_op_buf kbufs[], unsigned int num,
 610	struct page *pages[], unsigned int nr_pages, unsigned int *pinned)
 611{
 612	unsigned int i, off = 0;
 613
 614	for (i = 0; i < num; ) {
 615		unsigned int requested;
 616		int page_count;
 617
 618		requested = DIV_ROUND_UP(
 619			offset_in_page(kbufs[i].uptr) + kbufs[i].size,
 620			PAGE_SIZE) - off;
 621		if (requested > nr_pages)
 622			return -ENOSPC;
 623
 624		page_count = pin_user_pages_fast(
 625			(unsigned long)kbufs[i].uptr + off * PAGE_SIZE,
 626			requested, FOLL_WRITE, pages);
 627		if (page_count <= 0)
 628			return page_count ? : -EFAULT;
 629
 630		*pinned += page_count;
 631		nr_pages -= page_count;
 632		pages += page_count;
 633
 634		off = (requested == page_count) ? 0 : off + page_count;
 635		i += !off;
 636	}
 637
 638	return 0;
 639}
 640
 641static void unlock_pages(struct page *pages[], unsigned int nr_pages)
 642{
 643	unpin_user_pages_dirty_lock(pages, nr_pages, true);
 644}
 645
 646static long privcmd_ioctl_dm_op(struct file *file, void __user *udata)
 647{
 648	struct privcmd_data *data = file->private_data;
 649	struct privcmd_dm_op kdata;
 650	struct privcmd_dm_op_buf *kbufs;
 651	unsigned int nr_pages = 0;
 652	struct page **pages = NULL;
 653	struct xen_dm_op_buf *xbufs = NULL;
 654	unsigned int i;
 655	long rc;
 656	unsigned int pinned = 0;
 657
 658	if (copy_from_user(&kdata, udata, sizeof(kdata)))
 659		return -EFAULT;
 660
 661	/* If restriction is in place, check the domid matches */
 662	if (data->domid != DOMID_INVALID && data->domid != kdata.dom)
 663		return -EPERM;
 664
 665	if (kdata.num == 0)
 666		return 0;
 667
 668	if (kdata.num > privcmd_dm_op_max_num)
 669		return -E2BIG;
 670
 671	kbufs = kcalloc(kdata.num, sizeof(*kbufs), GFP_KERNEL);
 672	if (!kbufs)
 673		return -ENOMEM;
 674
 675	if (copy_from_user(kbufs, kdata.ubufs,
 676			   sizeof(*kbufs) * kdata.num)) {
 677		rc = -EFAULT;
 678		goto out;
 679	}
 680
 681	for (i = 0; i < kdata.num; i++) {
 682		if (kbufs[i].size > privcmd_dm_op_buf_max_size) {
 683			rc = -E2BIG;
 684			goto out;
 685		}
 686
 687		if (!access_ok(kbufs[i].uptr,
 688			       kbufs[i].size)) {
 689			rc = -EFAULT;
 690			goto out;
 691		}
 692
 693		nr_pages += DIV_ROUND_UP(
 694			offset_in_page(kbufs[i].uptr) + kbufs[i].size,
 695			PAGE_SIZE);
 696	}
 697
 698	pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
 699	if (!pages) {
 700		rc = -ENOMEM;
 701		goto out;
 702	}
 703
 704	xbufs = kcalloc(kdata.num, sizeof(*xbufs), GFP_KERNEL);
 705	if (!xbufs) {
 706		rc = -ENOMEM;
 707		goto out;
 708	}
 709
 710	rc = lock_pages(kbufs, kdata.num, pages, nr_pages, &pinned);
 711	if (rc < 0)
 712		goto out;
 713
 714	for (i = 0; i < kdata.num; i++) {
 715		set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr);
 716		xbufs[i].size = kbufs[i].size;
 717	}
 718
 719	xen_preemptible_hcall_begin();
 720	rc = HYPERVISOR_dm_op(kdata.dom, kdata.num, xbufs);
 721	xen_preemptible_hcall_end();
 722
 723out:
 724	unlock_pages(pages, pinned);
 725	kfree(xbufs);
 726	kfree(pages);
 727	kfree(kbufs);
 728
 729	return rc;
 730}
 731
 732static long privcmd_ioctl_restrict(struct file *file, void __user *udata)
 733{
 734	struct privcmd_data *data = file->private_data;
 735	domid_t dom;
 736
 737	if (copy_from_user(&dom, udata, sizeof(dom)))
 738		return -EFAULT;
 739
 740	/* Set restriction to the specified domain, or check it matches */
 741	if (data->domid == DOMID_INVALID)
 742		data->domid = dom;
 743	else if (data->domid != dom)
 744		return -EINVAL;
 745
 746	return 0;
 747}
 748
 749static long privcmd_ioctl_mmap_resource(struct file *file,
 750				struct privcmd_mmap_resource __user *udata)
 751{
 752	struct privcmd_data *data = file->private_data;
 753	struct mm_struct *mm = current->mm;
 754	struct vm_area_struct *vma;
 755	struct privcmd_mmap_resource kdata;
 756	xen_pfn_t *pfns = NULL;
 757	struct xen_mem_acquire_resource xdata = { };
 758	int rc;
 759
 760	if (copy_from_user(&kdata, udata, sizeof(kdata)))
 761		return -EFAULT;
 762
 763	/* If restriction is in place, check the domid matches */
 764	if (data->domid != DOMID_INVALID && data->domid != kdata.dom)
 765		return -EPERM;
 766
 767	/* Both fields must be set or unset */
 768	if (!!kdata.addr != !!kdata.num)
 769		return -EINVAL;
 770
 771	xdata.domid = kdata.dom;
 772	xdata.type = kdata.type;
 773	xdata.id = kdata.id;
 774
 775	if (!kdata.addr && !kdata.num) {
 776		/* Query the size of the resource. */
 777		rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, &xdata);
 778		if (rc)
 779			return rc;
 780		return __put_user(xdata.nr_frames, &udata->num);
 781	}
 782
 783	mmap_write_lock(mm);
 784
 785	vma = find_vma(mm, kdata.addr);
 786	if (!vma || vma->vm_ops != &privcmd_vm_ops) {
 787		rc = -EINVAL;
 788		goto out;
 789	}
 790
 791	pfns = kcalloc(kdata.num, sizeof(*pfns), GFP_KERNEL | __GFP_NOWARN);
 792	if (!pfns) {
 793		rc = -ENOMEM;
 794		goto out;
 795	}
 796
 797	if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && !xen_pv_domain()) {
 798		unsigned int nr = DIV_ROUND_UP(kdata.num, XEN_PFN_PER_PAGE);
 799		struct page **pages;
 800		unsigned int i;
 801
 802		rc = alloc_empty_pages(vma, nr);
 803		if (rc < 0)
 804			goto out;
 805
 806		pages = vma->vm_private_data;
 807
 808		for (i = 0; i < kdata.num; i++) {
 809			xen_pfn_t pfn =
 810				page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
 811
 812			pfns[i] = pfn + (i % XEN_PFN_PER_PAGE);
 813		}
 814	} else
 815		vma->vm_private_data = PRIV_VMA_LOCKED;
 816
 817	xdata.frame = kdata.idx;
 818	xdata.nr_frames = kdata.num;
 819	set_xen_guest_handle(xdata.frame_list, pfns);
 820
 821	xen_preemptible_hcall_begin();
 822	rc = HYPERVISOR_memory_op(XENMEM_acquire_resource, &xdata);
 823	xen_preemptible_hcall_end();
 824
 825	if (rc)
 826		goto out;
 827
 828	if (IS_ENABLED(CONFIG_XEN_AUTO_XLATE) && !xen_pv_domain()) {
 829		rc = xen_remap_vma_range(vma, kdata.addr, kdata.num << PAGE_SHIFT);
 830	} else {
 831		unsigned int domid =
 832			(xdata.flags & XENMEM_rsrc_acq_caller_owned) ?
 833			DOMID_SELF : kdata.dom;
 834		int num, *errs = (int *)pfns;
 835
 836		BUILD_BUG_ON(sizeof(*errs) > sizeof(*pfns));
 837		num = xen_remap_domain_mfn_array(vma,
 838						 kdata.addr & PAGE_MASK,
 839						 pfns, kdata.num, errs,
 840						 vma->vm_page_prot,
 841						 domid);
 842		if (num < 0)
 843			rc = num;
 844		else if (num != kdata.num) {
 845			unsigned int i;
 846
 847			for (i = 0; i < num; i++) {
 848				rc = errs[i];
 849				if (rc < 0)
 850					break;
 851			}
 852		} else
 853			rc = 0;
 854	}
 855
 856out:
 857	mmap_write_unlock(mm);
 858	kfree(pfns);
 859
 860	return rc;
 861}
 862
 863static long privcmd_ioctl_pcidev_get_gsi(struct file *file, void __user *udata)
 864{
 865#if defined(CONFIG_XEN_ACPI)
 866	int rc;
 867	struct privcmd_pcidev_get_gsi kdata;
 868
 869	if (copy_from_user(&kdata, udata, sizeof(kdata)))
 870		return -EFAULT;
 871
 872	rc = xen_acpi_get_gsi_from_sbdf(kdata.sbdf);
 873	if (rc < 0)
 874		return rc;
 875
 876	kdata.gsi = rc;
 877	if (copy_to_user(udata, &kdata, sizeof(kdata)))
 878		return -EFAULT;
 879
 880	return 0;
 881#else
 882	return -EINVAL;
 883#endif
 884}
 885
 886#ifdef CONFIG_XEN_PRIVCMD_EVENTFD
 887/* Irqfd support */
 888static struct workqueue_struct *irqfd_cleanup_wq;
 889static DEFINE_SPINLOCK(irqfds_lock);
 890DEFINE_STATIC_SRCU(irqfds_srcu);
 891static LIST_HEAD(irqfds_list);
 892
 893struct privcmd_kernel_irqfd {
 894	struct xen_dm_op_buf xbufs;
 895	domid_t dom;
 896	bool error;
 897	struct eventfd_ctx *eventfd;
 898	struct work_struct shutdown;
 899	wait_queue_entry_t wait;
 900	struct list_head list;
 901	poll_table pt;
 902};
 903
 904static void irqfd_deactivate(struct privcmd_kernel_irqfd *kirqfd)
 905{
 906	lockdep_assert_held(&irqfds_lock);
 907
 908	list_del_init(&kirqfd->list);
 909	queue_work(irqfd_cleanup_wq, &kirqfd->shutdown);
 910}
 911
 912static void irqfd_shutdown(struct work_struct *work)
 913{
 914	struct privcmd_kernel_irqfd *kirqfd =
 915		container_of(work, struct privcmd_kernel_irqfd, shutdown);
 916	u64 cnt;
 917
 918	/* Make sure irqfd has been initialized in assign path */
 919	synchronize_srcu(&irqfds_srcu);
 920
 921	eventfd_ctx_remove_wait_queue(kirqfd->eventfd, &kirqfd->wait, &cnt);
 922	eventfd_ctx_put(kirqfd->eventfd);
 923	kfree(kirqfd);
 924}
 925
 926static void irqfd_inject(struct privcmd_kernel_irqfd *kirqfd)
 927{
 928	u64 cnt;
 929	long rc;
 930
 931	eventfd_ctx_do_read(kirqfd->eventfd, &cnt);
 932
 933	xen_preemptible_hcall_begin();
 934	rc = HYPERVISOR_dm_op(kirqfd->dom, 1, &kirqfd->xbufs);
 935	xen_preemptible_hcall_end();
 936
 937	/* Don't repeat the error message for consecutive failures */
 938	if (rc && !kirqfd->error) {
 939		pr_err("Failed to configure irq for guest domain: %d\n",
 940		       kirqfd->dom);
 941	}
 942
 943	kirqfd->error = rc;
 944}
 945
 946static int
 947irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key)
 948{
 949	struct privcmd_kernel_irqfd *kirqfd =
 950		container_of(wait, struct privcmd_kernel_irqfd, wait);
 951	__poll_t flags = key_to_poll(key);
 952
 953	if (flags & EPOLLIN)
 954		irqfd_inject(kirqfd);
 955
 956	if (flags & EPOLLHUP) {
 957		unsigned long flags;
 958
 959		spin_lock_irqsave(&irqfds_lock, flags);
 960		irqfd_deactivate(kirqfd);
 961		spin_unlock_irqrestore(&irqfds_lock, flags);
 962	}
 963
 964	return 0;
 965}
 966
 967static void
 968irqfd_poll_func(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
 969{
 970	struct privcmd_kernel_irqfd *kirqfd =
 971		container_of(pt, struct privcmd_kernel_irqfd, pt);
 972
 973	add_wait_queue_priority(wqh, &kirqfd->wait);
 974}
 975
 976static int privcmd_irqfd_assign(struct privcmd_irqfd *irqfd)
 977{
 978	struct privcmd_kernel_irqfd *kirqfd, *tmp;
 979	unsigned long flags;
 980	__poll_t events;
 981	void *dm_op;
 982	int ret, idx;
 983
 984	CLASS(fd, f)(irqfd->fd);
 985
 986	kirqfd = kzalloc(sizeof(*kirqfd) + irqfd->size, GFP_KERNEL);
 987	if (!kirqfd)
 988		return -ENOMEM;
 989	dm_op = kirqfd + 1;
 990
 991	if (copy_from_user(dm_op, u64_to_user_ptr(irqfd->dm_op), irqfd->size)) {
 992		ret = -EFAULT;
 993		goto error_kfree;
 994	}
 995
 996	kirqfd->xbufs.size = irqfd->size;
 997	set_xen_guest_handle(kirqfd->xbufs.h, dm_op);
 998	kirqfd->dom = irqfd->dom;
 999	INIT_WORK(&kirqfd->shutdown, irqfd_shutdown);
1000
1001	if (fd_empty(f)) {
1002		ret = -EBADF;
1003		goto error_kfree;
1004	}
1005
1006	kirqfd->eventfd = eventfd_ctx_fileget(fd_file(f));
1007	if (IS_ERR(kirqfd->eventfd)) {
1008		ret = PTR_ERR(kirqfd->eventfd);
1009		goto error_kfree;
1010	}
1011
1012	/*
1013	 * Install our own custom wake-up handling so we are notified via a
1014	 * callback whenever someone signals the underlying eventfd.
1015	 */
1016	init_waitqueue_func_entry(&kirqfd->wait, irqfd_wakeup);
1017	init_poll_funcptr(&kirqfd->pt, irqfd_poll_func);
1018
1019	spin_lock_irqsave(&irqfds_lock, flags);
1020
1021	list_for_each_entry(tmp, &irqfds_list, list) {
1022		if (kirqfd->eventfd == tmp->eventfd) {
1023			ret = -EBUSY;
1024			spin_unlock_irqrestore(&irqfds_lock, flags);
1025			goto error_eventfd;
1026		}
1027	}
1028
1029	idx = srcu_read_lock(&irqfds_srcu);
1030	list_add_tail(&kirqfd->list, &irqfds_list);
1031	spin_unlock_irqrestore(&irqfds_lock, flags);
1032
1033	/*
1034	 * Check if there was an event already pending on the eventfd before we
1035	 * registered, and trigger it as if we didn't miss it.
1036	 */
1037	events = vfs_poll(fd_file(f), &kirqfd->pt);
1038	if (events & EPOLLIN)
1039		irqfd_inject(kirqfd);
1040
1041	srcu_read_unlock(&irqfds_srcu, idx);
1042	return 0;
1043
1044error_eventfd:
1045	eventfd_ctx_put(kirqfd->eventfd);
1046
1047error_kfree:
1048	kfree(kirqfd);
1049	return ret;
1050}
1051
1052static int privcmd_irqfd_deassign(struct privcmd_irqfd *irqfd)
1053{
1054	struct privcmd_kernel_irqfd *kirqfd;
1055	struct eventfd_ctx *eventfd;
1056	unsigned long flags;
1057
1058	eventfd = eventfd_ctx_fdget(irqfd->fd);
1059	if (IS_ERR(eventfd))
1060		return PTR_ERR(eventfd);
1061
1062	spin_lock_irqsave(&irqfds_lock, flags);
1063
1064	list_for_each_entry(kirqfd, &irqfds_list, list) {
1065		if (kirqfd->eventfd == eventfd) {
1066			irqfd_deactivate(kirqfd);
1067			break;
1068		}
1069	}
1070
1071	spin_unlock_irqrestore(&irqfds_lock, flags);
1072
1073	eventfd_ctx_put(eventfd);
1074
1075	/*
1076	 * Block until we know all outstanding shutdown jobs have completed so
1077	 * that we guarantee there will not be any more interrupts once this
1078	 * deassign function returns.
1079	 */
1080	flush_workqueue(irqfd_cleanup_wq);
1081
1082	return 0;
1083}
1084
1085static long privcmd_ioctl_irqfd(struct file *file, void __user *udata)
1086{
1087	struct privcmd_data *data = file->private_data;
1088	struct privcmd_irqfd irqfd;
1089
1090	if (copy_from_user(&irqfd, udata, sizeof(irqfd)))
1091		return -EFAULT;
1092
1093	/* No other flags should be set */
1094	if (irqfd.flags & ~PRIVCMD_IRQFD_FLAG_DEASSIGN)
1095		return -EINVAL;
1096
1097	/* If restriction is in place, check the domid matches */
1098	if (data->domid != DOMID_INVALID && data->domid != irqfd.dom)
1099		return -EPERM;
1100
1101	if (irqfd.flags & PRIVCMD_IRQFD_FLAG_DEASSIGN)
1102		return privcmd_irqfd_deassign(&irqfd);
1103
1104	return privcmd_irqfd_assign(&irqfd);
1105}
1106
1107static int privcmd_irqfd_init(void)
1108{
1109	irqfd_cleanup_wq = alloc_workqueue("privcmd-irqfd-cleanup", 0, 0);
1110	if (!irqfd_cleanup_wq)
1111		return -ENOMEM;
1112
1113	return 0;
1114}
1115
1116static void privcmd_irqfd_exit(void)
1117{
1118	struct privcmd_kernel_irqfd *kirqfd, *tmp;
1119	unsigned long flags;
1120
1121	spin_lock_irqsave(&irqfds_lock, flags);
1122
1123	list_for_each_entry_safe(kirqfd, tmp, &irqfds_list, list)
1124		irqfd_deactivate(kirqfd);
1125
1126	spin_unlock_irqrestore(&irqfds_lock, flags);
1127
1128	destroy_workqueue(irqfd_cleanup_wq);
1129}
1130
1131/* Ioeventfd Support */
1132#define QUEUE_NOTIFY_VQ_MASK 0xFFFF
1133
1134static DEFINE_MUTEX(ioreq_lock);
1135static LIST_HEAD(ioreq_list);
1136
1137/* per-eventfd structure */
1138struct privcmd_kernel_ioeventfd {
1139	struct eventfd_ctx *eventfd;
1140	struct list_head list;
1141	u64 addr;
1142	unsigned int addr_len;
1143	unsigned int vq;
1144};
1145
1146/* per-guest CPU / port structure */
1147struct ioreq_port {
1148	int vcpu;
1149	unsigned int port;
1150	struct privcmd_kernel_ioreq *kioreq;
1151};
1152
1153/* per-guest structure */
1154struct privcmd_kernel_ioreq {
1155	domid_t dom;
1156	unsigned int vcpus;
1157	u64 uioreq;
1158	struct ioreq *ioreq;
1159	spinlock_t lock; /* Protects ioeventfds list */
1160	struct list_head ioeventfds;
1161	struct list_head list;
1162	struct ioreq_port ports[] __counted_by(vcpus);
1163};
1164
1165static irqreturn_t ioeventfd_interrupt(int irq, void *dev_id)
1166{
1167	struct ioreq_port *port = dev_id;
1168	struct privcmd_kernel_ioreq *kioreq = port->kioreq;
1169	struct ioreq *ioreq = &kioreq->ioreq[port->vcpu];
1170	struct privcmd_kernel_ioeventfd *kioeventfd;
1171	unsigned int state = STATE_IOREQ_READY;
1172
1173	if (ioreq->state != STATE_IOREQ_READY ||
1174	    ioreq->type != IOREQ_TYPE_COPY || ioreq->dir != IOREQ_WRITE)
1175		return IRQ_NONE;
1176
1177	/*
1178	 * We need a barrier, smp_mb(), here to ensure reads are finished before
1179	 * `state` is updated. Since the lock implementation ensures that
1180	 * appropriate barrier will be added anyway, we can avoid adding
1181	 * explicit barrier here.
1182	 *
1183	 * Ideally we don't need to update `state` within the locks, but we do
1184	 * that here to avoid adding explicit barrier.
1185	 */
1186
1187	spin_lock(&kioreq->lock);
1188	ioreq->state = STATE_IOREQ_INPROCESS;
1189
1190	list_for_each_entry(kioeventfd, &kioreq->ioeventfds, list) {
1191		if (ioreq->addr == kioeventfd->addr + VIRTIO_MMIO_QUEUE_NOTIFY &&
1192		    ioreq->size == kioeventfd->addr_len &&
1193		    (ioreq->data & QUEUE_NOTIFY_VQ_MASK) == kioeventfd->vq) {
1194			eventfd_signal(kioeventfd->eventfd);
1195			state = STATE_IORESP_READY;
1196			break;
1197		}
1198	}
1199	spin_unlock(&kioreq->lock);
1200
1201	/*
1202	 * We need a barrier, smp_mb(), here to ensure writes are finished
1203	 * before `state` is updated. Since the lock implementation ensures that
1204	 * appropriate barrier will be added anyway, we can avoid adding
1205	 * explicit barrier here.
1206	 */
1207
1208	ioreq->state = state;
1209
1210	if (state == STATE_IORESP_READY) {
1211		notify_remote_via_evtchn(port->port);
1212		return IRQ_HANDLED;
1213	}
1214
1215	return IRQ_NONE;
1216}
1217
1218static void ioreq_free(struct privcmd_kernel_ioreq *kioreq)
1219{
1220	struct ioreq_port *ports = kioreq->ports;
1221	int i;
1222
1223	lockdep_assert_held(&ioreq_lock);
1224
1225	list_del(&kioreq->list);
1226
1227	for (i = kioreq->vcpus - 1; i >= 0; i--)
1228		unbind_from_irqhandler(irq_from_evtchn(ports[i].port), &ports[i]);
1229
1230	kfree(kioreq);
1231}
1232
1233static
1234struct privcmd_kernel_ioreq *alloc_ioreq(struct privcmd_ioeventfd *ioeventfd)
1235{
1236	struct privcmd_kernel_ioreq *kioreq;
1237	struct mm_struct *mm = current->mm;
1238	struct vm_area_struct *vma;
1239	struct page **pages;
1240	unsigned int *ports;
1241	int ret, size, i;
1242
1243	lockdep_assert_held(&ioreq_lock);
1244
1245	size = struct_size(kioreq, ports, ioeventfd->vcpus);
1246	kioreq = kzalloc(size, GFP_KERNEL);
1247	if (!kioreq)
1248		return ERR_PTR(-ENOMEM);
1249
1250	kioreq->dom = ioeventfd->dom;
1251	kioreq->vcpus = ioeventfd->vcpus;
1252	kioreq->uioreq = ioeventfd->ioreq;
1253	spin_lock_init(&kioreq->lock);
1254	INIT_LIST_HEAD(&kioreq->ioeventfds);
1255
1256	/* The memory for ioreq server must have been mapped earlier */
1257	mmap_write_lock(mm);
1258	vma = find_vma(mm, (unsigned long)ioeventfd->ioreq);
1259	if (!vma) {
1260		pr_err("Failed to find vma for ioreq page!\n");
1261		mmap_write_unlock(mm);
1262		ret = -EFAULT;
1263		goto error_kfree;
1264	}
1265
1266	pages = vma->vm_private_data;
1267	kioreq->ioreq = (struct ioreq *)(page_to_virt(pages[0]));
1268	mmap_write_unlock(mm);
1269
1270	ports = memdup_array_user(u64_to_user_ptr(ioeventfd->ports),
1271				  kioreq->vcpus, sizeof(*ports));
1272	if (IS_ERR(ports)) {
1273		ret = PTR_ERR(ports);
1274		goto error_kfree;
1275	}
1276
1277	for (i = 0; i < kioreq->vcpus; i++) {
1278		kioreq->ports[i].vcpu = i;
1279		kioreq->ports[i].port = ports[i];
1280		kioreq->ports[i].kioreq = kioreq;
1281
1282		ret = bind_evtchn_to_irqhandler_lateeoi(ports[i],
1283				ioeventfd_interrupt, IRQF_SHARED, "ioeventfd",
1284				&kioreq->ports[i]);
1285		if (ret < 0)
1286			goto error_unbind;
1287	}
1288
1289	kfree(ports);
1290
1291	list_add_tail(&kioreq->list, &ioreq_list);
1292
1293	return kioreq;
1294
1295error_unbind:
1296	while (--i >= 0)
1297		unbind_from_irqhandler(irq_from_evtchn(ports[i]), &kioreq->ports[i]);
1298
1299	kfree(ports);
1300error_kfree:
1301	kfree(kioreq);
1302	return ERR_PTR(ret);
1303}
1304
1305static struct privcmd_kernel_ioreq *
1306get_ioreq(struct privcmd_ioeventfd *ioeventfd, struct eventfd_ctx *eventfd)
1307{
1308	struct privcmd_kernel_ioreq *kioreq;
1309	unsigned long flags;
1310
1311	list_for_each_entry(kioreq, &ioreq_list, list) {
1312		struct privcmd_kernel_ioeventfd *kioeventfd;
1313
1314		/*
1315		 * kioreq fields can be accessed here without a lock as they are
1316		 * never updated after being added to the ioreq_list.
1317		 */
1318		if (kioreq->uioreq != ioeventfd->ioreq) {
1319			continue;
1320		} else if (kioreq->dom != ioeventfd->dom ||
1321			   kioreq->vcpus != ioeventfd->vcpus) {
1322			pr_err("Invalid ioeventfd configuration mismatch, dom (%u vs %u), vcpus (%u vs %u)\n",
1323			       kioreq->dom, ioeventfd->dom, kioreq->vcpus,
1324			       ioeventfd->vcpus);
1325			return ERR_PTR(-EINVAL);
1326		}
1327
1328		/* Look for a duplicate eventfd for the same guest */
1329		spin_lock_irqsave(&kioreq->lock, flags);
1330		list_for_each_entry(kioeventfd, &kioreq->ioeventfds, list) {
1331			if (eventfd == kioeventfd->eventfd) {
1332				spin_unlock_irqrestore(&kioreq->lock, flags);
1333				return ERR_PTR(-EBUSY);
1334			}
1335		}
1336		spin_unlock_irqrestore(&kioreq->lock, flags);
1337
1338		return kioreq;
1339	}
1340
1341	/* Matching kioreq isn't found, allocate a new one */
1342	return alloc_ioreq(ioeventfd);
1343}
1344
1345static void ioeventfd_free(struct privcmd_kernel_ioeventfd *kioeventfd)
1346{
1347	list_del(&kioeventfd->list);
1348	eventfd_ctx_put(kioeventfd->eventfd);
1349	kfree(kioeventfd);
1350}
1351
1352static int privcmd_ioeventfd_assign(struct privcmd_ioeventfd *ioeventfd)
1353{
1354	struct privcmd_kernel_ioeventfd *kioeventfd;
1355	struct privcmd_kernel_ioreq *kioreq;
1356	unsigned long flags;
1357	int ret;
1358
1359	/* Check for range overflow */
1360	if (ioeventfd->addr + ioeventfd->addr_len < ioeventfd->addr)
1361		return -EINVAL;
1362
1363	/* Vhost requires us to support length 1, 2, 4, and 8 */
1364	if (!(ioeventfd->addr_len == 1 || ioeventfd->addr_len == 2 ||
1365	      ioeventfd->addr_len == 4 || ioeventfd->addr_len == 8))
1366		return -EINVAL;
1367
1368	/* 4096 vcpus limit enough ? */
1369	if (!ioeventfd->vcpus || ioeventfd->vcpus > 4096)
1370		return -EINVAL;
1371
1372	kioeventfd = kzalloc(sizeof(*kioeventfd), GFP_KERNEL);
1373	if (!kioeventfd)
1374		return -ENOMEM;
1375
1376	kioeventfd->eventfd = eventfd_ctx_fdget(ioeventfd->event_fd);
1377	if (IS_ERR(kioeventfd->eventfd)) {
1378		ret = PTR_ERR(kioeventfd->eventfd);
1379		goto error_kfree;
1380	}
1381
1382	kioeventfd->addr = ioeventfd->addr;
1383	kioeventfd->addr_len = ioeventfd->addr_len;
1384	kioeventfd->vq = ioeventfd->vq;
1385
1386	mutex_lock(&ioreq_lock);
1387	kioreq = get_ioreq(ioeventfd, kioeventfd->eventfd);
1388	if (IS_ERR(kioreq)) {
1389		mutex_unlock(&ioreq_lock);
1390		ret = PTR_ERR(kioreq);
1391		goto error_eventfd;
1392	}
1393
1394	spin_lock_irqsave(&kioreq->lock, flags);
1395	list_add_tail(&kioeventfd->list, &kioreq->ioeventfds);
1396	spin_unlock_irqrestore(&kioreq->lock, flags);
1397
1398	mutex_unlock(&ioreq_lock);
1399
1400	return 0;
1401
1402error_eventfd:
1403	eventfd_ctx_put(kioeventfd->eventfd);
1404
1405error_kfree:
1406	kfree(kioeventfd);
1407	return ret;
1408}
1409
1410static int privcmd_ioeventfd_deassign(struct privcmd_ioeventfd *ioeventfd)
1411{
1412	struct privcmd_kernel_ioreq *kioreq, *tkioreq;
1413	struct eventfd_ctx *eventfd;
1414	unsigned long flags;
1415	int ret = 0;
1416
1417	eventfd = eventfd_ctx_fdget(ioeventfd->event_fd);
1418	if (IS_ERR(eventfd))
1419		return PTR_ERR(eventfd);
1420
1421	mutex_lock(&ioreq_lock);
1422	list_for_each_entry_safe(kioreq, tkioreq, &ioreq_list, list) {
1423		struct privcmd_kernel_ioeventfd *kioeventfd, *tmp;
1424		/*
1425		 * kioreq fields can be accessed here without a lock as they are
1426		 * never updated after being added to the ioreq_list.
1427		 */
1428		if (kioreq->dom != ioeventfd->dom ||
1429		    kioreq->uioreq != ioeventfd->ioreq ||
1430		    kioreq->vcpus != ioeventfd->vcpus)
1431			continue;
1432
1433		spin_lock_irqsave(&kioreq->lock, flags);
1434		list_for_each_entry_safe(kioeventfd, tmp, &kioreq->ioeventfds, list) {
1435			if (eventfd == kioeventfd->eventfd) {
1436				ioeventfd_free(kioeventfd);
1437				spin_unlock_irqrestore(&kioreq->lock, flags);
1438
1439				if (list_empty(&kioreq->ioeventfds))
1440					ioreq_free(kioreq);
1441				goto unlock;
1442			}
1443		}
1444		spin_unlock_irqrestore(&kioreq->lock, flags);
1445		break;
1446	}
1447
1448	pr_err("Ioeventfd isn't already assigned, dom: %u, addr: %llu\n",
1449	       ioeventfd->dom, ioeventfd->addr);
1450	ret = -ENODEV;
1451
1452unlock:
1453	mutex_unlock(&ioreq_lock);
1454	eventfd_ctx_put(eventfd);
1455
1456	return ret;
1457}
1458
1459static long privcmd_ioctl_ioeventfd(struct file *file, void __user *udata)
1460{
1461	struct privcmd_data *data = file->private_data;
1462	struct privcmd_ioeventfd ioeventfd;
1463
1464	if (copy_from_user(&ioeventfd, udata, sizeof(ioeventfd)))
1465		return -EFAULT;
1466
1467	/* No other flags should be set */
1468	if (ioeventfd.flags & ~PRIVCMD_IOEVENTFD_FLAG_DEASSIGN)
1469		return -EINVAL;
1470
1471	/* If restriction is in place, check the domid matches */
1472	if (data->domid != DOMID_INVALID && data->domid != ioeventfd.dom)
1473		return -EPERM;
1474
1475	if (ioeventfd.flags & PRIVCMD_IOEVENTFD_FLAG_DEASSIGN)
1476		return privcmd_ioeventfd_deassign(&ioeventfd);
1477
1478	return privcmd_ioeventfd_assign(&ioeventfd);
1479}
1480
1481static void privcmd_ioeventfd_exit(void)
1482{
1483	struct privcmd_kernel_ioreq *kioreq, *tmp;
1484	unsigned long flags;
1485
1486	mutex_lock(&ioreq_lock);
1487	list_for_each_entry_safe(kioreq, tmp, &ioreq_list, list) {
1488		struct privcmd_kernel_ioeventfd *kioeventfd, *tmp;
1489
1490		spin_lock_irqsave(&kioreq->lock, flags);
1491		list_for_each_entry_safe(kioeventfd, tmp, &kioreq->ioeventfds, list)
1492			ioeventfd_free(kioeventfd);
1493		spin_unlock_irqrestore(&kioreq->lock, flags);
1494
1495		ioreq_free(kioreq);
1496	}
1497	mutex_unlock(&ioreq_lock);
1498}
1499#else
1500static inline long privcmd_ioctl_irqfd(struct file *file, void __user *udata)
1501{
1502	return -EOPNOTSUPP;
1503}
1504
1505static inline int privcmd_irqfd_init(void)
1506{
1507	return 0;
1508}
1509
1510static inline void privcmd_irqfd_exit(void)
1511{
1512}
1513
1514static inline long privcmd_ioctl_ioeventfd(struct file *file, void __user *udata)
1515{
1516	return -EOPNOTSUPP;
1517}
1518
1519static inline void privcmd_ioeventfd_exit(void)
1520{
1521}
1522#endif /* CONFIG_XEN_PRIVCMD_EVENTFD */
1523
1524static long privcmd_ioctl(struct file *file,
1525			  unsigned int cmd, unsigned long data)
1526{
1527	int ret = -ENOTTY;
1528	void __user *udata = (void __user *) data;
1529
1530	switch (cmd) {
1531	case IOCTL_PRIVCMD_HYPERCALL:
1532		ret = privcmd_ioctl_hypercall(file, udata);
1533		break;
1534
1535	case IOCTL_PRIVCMD_MMAP:
1536		ret = privcmd_ioctl_mmap(file, udata);
1537		break;
1538
1539	case IOCTL_PRIVCMD_MMAPBATCH:
1540		ret = privcmd_ioctl_mmap_batch(file, udata, 1);
1541		break;
1542
1543	case IOCTL_PRIVCMD_MMAPBATCH_V2:
1544		ret = privcmd_ioctl_mmap_batch(file, udata, 2);
1545		break;
1546
1547	case IOCTL_PRIVCMD_DM_OP:
1548		ret = privcmd_ioctl_dm_op(file, udata);
1549		break;
1550
1551	case IOCTL_PRIVCMD_RESTRICT:
1552		ret = privcmd_ioctl_restrict(file, udata);
1553		break;
1554
1555	case IOCTL_PRIVCMD_MMAP_RESOURCE:
1556		ret = privcmd_ioctl_mmap_resource(file, udata);
1557		break;
1558
1559	case IOCTL_PRIVCMD_IRQFD:
1560		ret = privcmd_ioctl_irqfd(file, udata);
1561		break;
1562
1563	case IOCTL_PRIVCMD_IOEVENTFD:
1564		ret = privcmd_ioctl_ioeventfd(file, udata);
1565		break;
1566
1567	case IOCTL_PRIVCMD_PCIDEV_GET_GSI:
1568		ret = privcmd_ioctl_pcidev_get_gsi(file, udata);
1569		break;
1570
1571	default:
1572		break;
1573	}
1574
1575	return ret;
1576}
1577
1578static int privcmd_open(struct inode *ino, struct file *file)
1579{
1580	struct privcmd_data *data;
1581
1582	if (wait_event_interruptible(restrict_wait_wq, !restrict_wait) < 0)
1583		return -EINTR;
1584
1585	data = kzalloc(sizeof(*data), GFP_KERNEL);
1586	if (!data)
1587		return -ENOMEM;
1588
1589	data->domid = target_domain;
1590
1591	file->private_data = data;
1592	return 0;
1593}
1594
1595static int privcmd_release(struct inode *ino, struct file *file)
1596{
1597	struct privcmd_data *data = file->private_data;
1598
1599	kfree(data);
1600	return 0;
1601}
1602
1603static void privcmd_close(struct vm_area_struct *vma)
1604{
1605	struct page **pages = vma->vm_private_data;
1606	int numpgs = vma_pages(vma);
1607	int numgfns = (vma->vm_end - vma->vm_start) >> XEN_PAGE_SHIFT;
1608	int rc;
1609
1610	if (xen_pv_domain() || !numpgs || !pages)
1611		return;
1612
1613	rc = xen_unmap_domain_gfn_range(vma, numgfns, pages);
1614	if (rc == 0)
1615		xen_free_unpopulated_pages(numpgs, pages);
1616	else
1617		pr_crit("unable to unmap MFN range: leaking %d pages. rc=%d\n",
1618			numpgs, rc);
1619	kvfree(pages);
1620}
1621
1622static vm_fault_t privcmd_fault(struct vm_fault *vmf)
1623{
1624	printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n",
1625	       vmf->vma, vmf->vma->vm_start, vmf->vma->vm_end,
1626	       vmf->pgoff, (void *)vmf->address);
1627
1628	return VM_FAULT_SIGBUS;
1629}
1630
1631static const struct vm_operations_struct privcmd_vm_ops = {
1632	.close = privcmd_close,
1633	.fault = privcmd_fault
1634};
1635
1636static int privcmd_mmap(struct file *file, struct vm_area_struct *vma)
1637{
1638	/* DONTCOPY is essential for Xen because copy_page_range doesn't know
1639	 * how to recreate these mappings */
1640	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTCOPY |
1641			 VM_DONTEXPAND | VM_DONTDUMP);
1642	vma->vm_ops = &privcmd_vm_ops;
1643	vma->vm_private_data = NULL;
1644
1645	return 0;
1646}
1647
1648/*
1649 * For MMAPBATCH*. This allows asserting the singleshot mapping
1650 * on a per pfn/pte basis. Mapping calls that fail with ENOENT
1651 * can be then retried until success.
1652 */
1653static int is_mapped_fn(pte_t *pte, unsigned long addr, void *data)
1654{
1655	return pte_none(ptep_get(pte)) ? 0 : -EBUSY;
1656}
1657
1658static int privcmd_vma_range_is_mapped(
1659	           struct vm_area_struct *vma,
1660	           unsigned long addr,
1661	           unsigned long nr_pages)
1662{
1663	return apply_to_page_range(vma->vm_mm, addr, nr_pages << PAGE_SHIFT,
1664				   is_mapped_fn, NULL) != 0;
1665}
1666
1667const struct file_operations xen_privcmd_fops = {
1668	.owner = THIS_MODULE,
1669	.unlocked_ioctl = privcmd_ioctl,
1670	.open = privcmd_open,
1671	.release = privcmd_release,
1672	.mmap = privcmd_mmap,
1673};
1674EXPORT_SYMBOL_GPL(xen_privcmd_fops);
1675
1676static struct miscdevice privcmd_dev = {
1677	.minor = MISC_DYNAMIC_MINOR,
1678	.name = "xen/privcmd",
1679	.fops = &xen_privcmd_fops,
1680};
1681
1682static int init_restrict(struct notifier_block *notifier,
1683			 unsigned long event,
1684			 void *data)
1685{
1686	char *target;
1687	unsigned int domid;
1688
1689	/* Default to an guaranteed unused domain-id. */
1690	target_domain = DOMID_IDLE;
1691
1692	target = xenbus_read(XBT_NIL, "target", "", NULL);
1693	if (IS_ERR(target) || kstrtouint(target, 10, &domid)) {
1694		pr_err("No target domain found, blocking all hypercalls\n");
1695		goto out;
1696	}
1697
1698	target_domain = domid;
1699
1700 out:
1701	if (!IS_ERR(target))
1702		kfree(target);
1703
1704	restrict_wait = false;
1705	wake_up_all(&restrict_wait_wq);
1706
1707	return NOTIFY_DONE;
1708}
1709
1710static struct notifier_block xenstore_notifier = {
1711	.notifier_call = init_restrict,
1712};
1713
1714static void __init restrict_driver(void)
1715{
1716	if (unrestricted) {
1717		if (security_locked_down(LOCKDOWN_XEN_USER_ACTIONS))
1718			pr_warn("Kernel is locked down, parameter \"unrestricted\" ignored\n");
1719		else
1720			return;
1721	}
1722
1723	restrict_wait = true;
1724
1725	register_xenstore_notifier(&xenstore_notifier);
1726}
1727
1728static int __init privcmd_init(void)
1729{
1730	int err;
1731
1732	if (!xen_domain())
1733		return -ENODEV;
1734
1735	if (!xen_initial_domain())
1736		restrict_driver();
1737
1738	err = misc_register(&privcmd_dev);
1739	if (err != 0) {
1740		pr_err("Could not register Xen privcmd device\n");
1741		return err;
1742	}
1743
1744	err = misc_register(&xen_privcmdbuf_dev);
1745	if (err != 0) {
1746		pr_err("Could not register Xen hypercall-buf device\n");
1747		goto err_privcmdbuf;
1748	}
1749
1750	err = privcmd_irqfd_init();
1751	if (err != 0) {
1752		pr_err("irqfd init failed\n");
1753		goto err_irqfd;
1754	}
1755
1756	return 0;
1757
1758err_irqfd:
1759	misc_deregister(&xen_privcmdbuf_dev);
1760err_privcmdbuf:
1761	misc_deregister(&privcmd_dev);
1762	return err;
1763}
1764
1765static void __exit privcmd_exit(void)
1766{
1767	if (!xen_initial_domain())
1768		unregister_xenstore_notifier(&xenstore_notifier);
1769
1770	privcmd_ioeventfd_exit();
1771	privcmd_irqfd_exit();
1772	misc_deregister(&privcmd_dev);
1773	misc_deregister(&xen_privcmdbuf_dev);
1774}
1775
1776module_init(privcmd_init);
1777module_exit(privcmd_exit);