vhost.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (C) 2009 Red Hat, Inc.
   3 * Copyright (C) 2006 Rusty Russell IBM Corporation
   4 *
   5 * Author: Michael S. Tsirkin <mst@redhat.com>
   6 *
   7 * Inspiration, some code, and most witty comments come from
   8 * Documentation/virtual/lguest/lguest.c, by Rusty Russell
   9 *
  10 * Generic code for virtio server in host kernel.
  11 */
  12
  13#include <linux/eventfd.h>
  14#include <linux/vhost.h>
  15#include <linux/uio.h>
  16#include <linux/mm.h>
  17#include <linux/miscdevice.h>
  18#include <linux/mutex.h>
  19#include <linux/poll.h>
  20#include <linux/file.h>
  21#include <linux/highmem.h>
  22#include <linux/slab.h>
  23#include <linux/vmalloc.h>
  24#include <linux/kthread.h>
  25#include <linux/cgroup.h>
  26#include <linux/module.h>
  27#include <linux/sort.h>
  28#include <linux/sched/mm.h>
  29#include <linux/sched/signal.h>
  30#include <linux/sched/vhost_task.h>
  31#include <linux/interval_tree_generic.h>
  32#include <linux/nospec.h>
  33#include <linux/kcov.h>
  34
  35#include "vhost.h"
  36
  37static ushort max_mem_regions = 64;
  38module_param(max_mem_regions, ushort, 0444);
  39MODULE_PARM_DESC(max_mem_regions,
  40	"Maximum number of memory regions in memory map. (default: 64)");
  41static int max_iotlb_entries = 2048;
  42module_param(max_iotlb_entries, int, 0444);
  43MODULE_PARM_DESC(max_iotlb_entries,
  44	"Maximum number of iotlb entries. (default: 2048)");
  45static bool fork_from_owner_default = VHOST_FORK_OWNER_TASK;
  46
  47#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL
  48module_param(fork_from_owner_default, bool, 0444);
  49MODULE_PARM_DESC(fork_from_owner_default,
  50		 "Set task mode as the default(default: Y)");
  51#endif
  52
  53enum {
  54	VHOST_MEMORY_F_LOG = 0x1,
  55};
  56
  57#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
  58#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
  59
  60#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
  61static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
  62{
  63	vq->user_be = !virtio_legacy_is_little_endian();
  64}
  65
  66static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
  67{
  68	vq->user_be = true;
  69}
  70
  71static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
  72{
  73	vq->user_be = false;
  74}
  75
  76static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
  77{
  78	struct vhost_vring_state s;
  79
  80	if (vq->private_data)
  81		return -EBUSY;
  82
  83	if (copy_from_user(&s, argp, sizeof(s)))
  84		return -EFAULT;
  85
  86	if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
  87	    s.num != VHOST_VRING_BIG_ENDIAN)
  88		return -EINVAL;
  89
  90	if (s.num == VHOST_VRING_BIG_ENDIAN)
  91		vhost_enable_cross_endian_big(vq);
  92	else
  93		vhost_enable_cross_endian_little(vq);
  94
  95	return 0;
  96}
  97
  98static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
  99				   int __user *argp)
 100{
 101	struct vhost_vring_state s = {
 102		.index = idx,
 103		.num = vq->user_be
 104	};
 105
 106	if (copy_to_user(argp, &s, sizeof(s)))
 107		return -EFAULT;
 108
 109	return 0;
 110}
 111
 112static void vhost_init_is_le(struct vhost_virtqueue *vq)
 113{
 114	/* Note for legacy virtio: user_be is initialized at reset time
 115	 * according to the host endianness. If userspace does not set an
 116	 * explicit endianness, the default behavior is native endian, as
 117	 * expected by legacy virtio.
 118	 */
 119	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
 120}
 121#else
 122static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
 123{
 124}
 125
 126static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
 127{
 128	return -ENOIOCTLCMD;
 129}
 130
 131static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
 132				   int __user *argp)
 133{
 134	return -ENOIOCTLCMD;
 135}
 136
 137static void vhost_init_is_le(struct vhost_virtqueue *vq)
 138{
 139	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1)
 140		|| virtio_legacy_is_little_endian();
 141}
 142#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
 143
 144static void vhost_reset_is_le(struct vhost_virtqueue *vq)
 145{
 146	vhost_init_is_le(vq);
 147}
 148
 149struct vhost_flush_struct {
 150	struct vhost_work work;
 151	struct completion wait_event;
 152};
 153
 154static void vhost_flush_work(struct vhost_work *work)
 155{
 156	struct vhost_flush_struct *s;
 157
 158	s = container_of(work, struct vhost_flush_struct, work);
 159	complete(&s->wait_event);
 160}
 161
 162static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
 163			    poll_table *pt)
 164{
 165	struct vhost_poll *poll;
 166
 167	poll = container_of(pt, struct vhost_poll, table);
 168	poll->wqh = wqh;
 169	add_wait_queue(wqh, &poll->wait);
 170}
 171
 172static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,
 173			     void *key)
 174{
 175	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
 176	struct vhost_work *work = &poll->work;
 177
 178	if (!(key_to_poll(key) & poll->mask))
 179		return 0;
 180
 181	if (!poll->dev->use_worker)
 182		work->fn(work);
 183	else
 184		vhost_poll_queue(poll);
 185
 186	return 0;
 187}
 188
 189void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
 190{
 191	clear_bit(VHOST_WORK_QUEUED, &work->flags);
 192	work->fn = fn;
 193}
 194EXPORT_SYMBOL_GPL(vhost_work_init);
 195
 196/* Init poll structure */
 197void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
 198		     __poll_t mask, struct vhost_dev *dev,
 199		     struct vhost_virtqueue *vq)
 200{
 201	init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
 202	init_poll_funcptr(&poll->table, vhost_poll_func);
 203	poll->mask = mask;
 204	poll->dev = dev;
 205	poll->wqh = NULL;
 206	poll->vq = vq;
 207
 208	vhost_work_init(&poll->work, fn);
 209}
 210EXPORT_SYMBOL_GPL(vhost_poll_init);
 211
 212/* Start polling a file. We add ourselves to file's wait queue. The caller must
 213 * keep a reference to a file until after vhost_poll_stop is called. */
 214int vhost_poll_start(struct vhost_poll *poll, struct file *file)
 215{
 216	__poll_t mask;
 217
 218	if (poll->wqh)
 219		return 0;
 220
 221	mask = vfs_poll(file, &poll->table);
 222	if (mask)
 223		vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask));
 224	if (mask & EPOLLERR) {
 225		vhost_poll_stop(poll);
 226		return -EINVAL;
 227	}
 228
 229	return 0;
 230}
 231EXPORT_SYMBOL_GPL(vhost_poll_start);
 232
 233/* Stop polling a file. After this function returns, it becomes safe to drop the
 234 * file reference. You must also flush afterwards. */
 235void vhost_poll_stop(struct vhost_poll *poll)
 236{
 237	if (poll->wqh) {
 238		remove_wait_queue(poll->wqh, &poll->wait);
 239		poll->wqh = NULL;
 240	}
 241}
 242EXPORT_SYMBOL_GPL(vhost_poll_stop);
 243
 244static void vhost_worker_queue(struct vhost_worker *worker,
 245			       struct vhost_work *work)
 246{
 247	if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
 248		/* We can only add the work to the list after we're
 249		 * sure it was not in the list.
 250		 * test_and_set_bit() implies a memory barrier.
 251		 */
 252		llist_add(&work->node, &worker->work_list);
 253		worker->ops->wakeup(worker);
 254	}
 255}
 256
 257bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work)
 258{
 259	struct vhost_worker *worker;
 260	bool queued = false;
 261
 262	rcu_read_lock();
 263	worker = rcu_dereference(vq->worker);
 264	if (worker) {
 265		queued = true;
 266		vhost_worker_queue(worker, work);
 267	}
 268	rcu_read_unlock();
 269
 270	return queued;
 271}
 272EXPORT_SYMBOL_GPL(vhost_vq_work_queue);
 273
 274/**
 275 * __vhost_worker_flush - flush a worker
 276 * @worker: worker to flush
 277 *
 278 * The worker's flush_mutex must be held.
 279 */
 280static void __vhost_worker_flush(struct vhost_worker *worker)
 281{
 282	struct vhost_flush_struct flush;
 283
 284	if (!worker->attachment_cnt || worker->killed)
 285		return;
 286
 287	init_completion(&flush.wait_event);
 288	vhost_work_init(&flush.work, vhost_flush_work);
 289
 290	vhost_worker_queue(worker, &flush.work);
 291	/*
 292	 * Drop mutex in case our worker is killed and it needs to take the
 293	 * mutex to force cleanup.
 294	 */
 295	mutex_unlock(&worker->mutex);
 296	wait_for_completion(&flush.wait_event);
 297	mutex_lock(&worker->mutex);
 298}
 299
 300static void vhost_worker_flush(struct vhost_worker *worker)
 301{
 302	mutex_lock(&worker->mutex);
 303	__vhost_worker_flush(worker);
 304	mutex_unlock(&worker->mutex);
 305}
 306
 307void vhost_dev_flush(struct vhost_dev *dev)
 308{
 309	struct vhost_worker *worker;
 310	unsigned long i;
 311
 312	xa_for_each(&dev->worker_xa, i, worker)
 313		vhost_worker_flush(worker);
 314}
 315EXPORT_SYMBOL_GPL(vhost_dev_flush);
 316
 317/* A lockless hint for busy polling code to exit the loop */
 318bool vhost_vq_has_work(struct vhost_virtqueue *vq)
 319{
 320	struct vhost_worker *worker;
 321	bool has_work = false;
 322
 323	rcu_read_lock();
 324	worker = rcu_dereference(vq->worker);
 325	if (worker && !llist_empty(&worker->work_list))
 326		has_work = true;
 327	rcu_read_unlock();
 328
 329	return has_work;
 330}
 331EXPORT_SYMBOL_GPL(vhost_vq_has_work);
 332
 333void vhost_poll_queue(struct vhost_poll *poll)
 334{
 335	vhost_vq_work_queue(poll->vq, &poll->work);
 336}
 337EXPORT_SYMBOL_GPL(vhost_poll_queue);
 338
 339static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
 340{
 341	int j;
 342
 343	for (j = 0; j < VHOST_NUM_ADDRS; j++)
 344		vq->meta_iotlb[j] = NULL;
 345}
 346
 347static void vhost_vq_meta_reset(struct vhost_dev *d)
 348{
 349	int i;
 350
 351	for (i = 0; i < d->nvqs; ++i)
 352		__vhost_vq_meta_reset(d->vqs[i]);
 353}
 354
 355static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx)
 356{
 357	call_ctx->ctx = NULL;
 358	memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer));
 359}
 360
 361bool vhost_vq_is_setup(struct vhost_virtqueue *vq)
 362{
 363	return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq);
 364}
 365EXPORT_SYMBOL_GPL(vhost_vq_is_setup);
 366
 367static void vhost_vq_reset(struct vhost_dev *dev,
 368			   struct vhost_virtqueue *vq)
 369{
 370	vq->num = 1;
 371	vq->desc = NULL;
 372	vq->avail = NULL;
 373	vq->used = NULL;
 374	vq->last_avail_idx = 0;
 375	vq->next_avail_head = 0;
 376	vq->avail_idx = 0;
 377	vq->last_used_idx = 0;
 378	vq->signalled_used = 0;
 379	vq->signalled_used_valid = false;
 380	vq->used_flags = 0;
 381	vq->log_used = false;
 382	vq->log_addr = -1ull;
 383	vq->private_data = NULL;
 384	virtio_features_zero(vq->acked_features_array);
 385	vq->acked_backend_features = 0;
 386	vq->log_base = NULL;
 387	vq->error_ctx = NULL;
 388	vq->kick = NULL;
 389	vq->log_ctx = NULL;
 390	vhost_disable_cross_endian(vq);
 391	vhost_reset_is_le(vq);
 392	vq->busyloop_timeout = 0;
 393	vq->umem = NULL;
 394	vq->iotlb = NULL;
 395	rcu_assign_pointer(vq->worker, NULL);
 396	vhost_vring_call_reset(&vq->call_ctx);
 397	__vhost_vq_meta_reset(vq);
 398}
 399
 400static int vhost_run_work_kthread_list(void *data)
 401{
 402	struct vhost_worker *worker = data;
 403	struct vhost_work *work, *work_next;
 404	struct vhost_dev *dev = worker->dev;
 405	struct llist_node *node;
 406
 407	kthread_use_mm(dev->mm);
 408
 409	for (;;) {
 410		/* mb paired w/ kthread_stop */
 411		set_current_state(TASK_INTERRUPTIBLE);
 412
 413		if (kthread_should_stop()) {
 414			__set_current_state(TASK_RUNNING);
 415			break;
 416		}
 417		node = llist_del_all(&worker->work_list);
 418		if (!node)
 419			schedule();
 420
 421		node = llist_reverse_order(node);
 422		/* make sure flag is seen after deletion */
 423		smp_wmb();
 424		llist_for_each_entry_safe(work, work_next, node, node) {
 425			clear_bit(VHOST_WORK_QUEUED, &work->flags);
 426			__set_current_state(TASK_RUNNING);
 427			kcov_remote_start_common(worker->kcov_handle);
 428			work->fn(work);
 429			kcov_remote_stop();
 430			cond_resched();
 431		}
 432	}
 433	kthread_unuse_mm(dev->mm);
 434
 435	return 0;
 436}
 437
 438static bool vhost_run_work_list(void *data)
 439{
 440	struct vhost_worker *worker = data;
 441	struct vhost_work *work, *work_next;
 442	struct llist_node *node;
 443
 444	node = llist_del_all(&worker->work_list);
 445	if (node) {
 446		__set_current_state(TASK_RUNNING);
 447
 448		node = llist_reverse_order(node);
 449		/* make sure flag is seen after deletion */
 450		smp_wmb();
 451		llist_for_each_entry_safe(work, work_next, node, node) {
 452			clear_bit(VHOST_WORK_QUEUED, &work->flags);
 453			kcov_remote_start_common(worker->kcov_handle);
 454			work->fn(work);
 455			kcov_remote_stop();
 456			cond_resched();
 457		}
 458	}
 459
 460	return !!node;
 461}
 462
 463static void vhost_worker_killed(void *data)
 464{
 465	struct vhost_worker *worker = data;
 466	struct vhost_dev *dev = worker->dev;
 467	struct vhost_virtqueue *vq;
 468	int i, attach_cnt = 0;
 469
 470	mutex_lock(&worker->mutex);
 471	worker->killed = true;
 472
 473	for (i = 0; i < dev->nvqs; i++) {
 474		vq = dev->vqs[i];
 475
 476		mutex_lock(&vq->mutex);
 477		if (worker ==
 478		    rcu_dereference_check(vq->worker,
 479					  lockdep_is_held(&vq->mutex))) {
 480			rcu_assign_pointer(vq->worker, NULL);
 481			attach_cnt++;
 482		}
 483		mutex_unlock(&vq->mutex);
 484	}
 485
 486	worker->attachment_cnt -= attach_cnt;
 487	if (attach_cnt)
 488		synchronize_rcu();
 489	/*
 490	 * Finish vhost_worker_flush calls and any other works that snuck in
 491	 * before the synchronize_rcu.
 492	 */
 493	vhost_run_work_list(worker);
 494	mutex_unlock(&worker->mutex);
 495}
 496
 497static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
 498{
 499	kfree(vq->indirect);
 500	vq->indirect = NULL;
 501	kfree(vq->log);
 502	vq->log = NULL;
 503	kfree(vq->heads);
 504	vq->heads = NULL;
 505	kfree(vq->nheads);
 506	vq->nheads = NULL;
 507}
 508
 509/* Helper to allocate iovec buffers for all vqs. */
 510static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
 511{
 512	struct vhost_virtqueue *vq;
 513	int i;
 514
 515	for (i = 0; i < dev->nvqs; ++i) {
 516		vq = dev->vqs[i];
 517		vq->indirect = kmalloc_array(UIO_MAXIOV,
 518					     sizeof(*vq->indirect),
 519					     GFP_KERNEL);
 520		vq->log = kmalloc_array(dev->iov_limit, sizeof(*vq->log),
 521					GFP_KERNEL);
 522		vq->heads = kmalloc_array(dev->iov_limit, sizeof(*vq->heads),
 523					  GFP_KERNEL);
 524		vq->nheads = kmalloc_array(dev->iov_limit, sizeof(*vq->nheads),
 525					   GFP_KERNEL);
 526		if (!vq->indirect || !vq->log || !vq->heads || !vq->nheads)
 527			goto err_nomem;
 528	}
 529	return 0;
 530
 531err_nomem:
 532	for (; i >= 0; --i)
 533		vhost_vq_free_iovecs(dev->vqs[i]);
 534	return -ENOMEM;
 535}
 536
 537static void vhost_dev_free_iovecs(struct vhost_dev *dev)
 538{
 539	int i;
 540
 541	for (i = 0; i < dev->nvqs; ++i)
 542		vhost_vq_free_iovecs(dev->vqs[i]);
 543}
 544
 545bool vhost_exceeds_weight(struct vhost_virtqueue *vq,
 546			  int pkts, int total_len)
 547{
 548	struct vhost_dev *dev = vq->dev;
 549
 550	if ((dev->byte_weight && total_len >= dev->byte_weight) ||
 551	    pkts >= dev->weight) {
 552		vhost_poll_queue(&vq->poll);
 553		return true;
 554	}
 555
 556	return false;
 557}
 558EXPORT_SYMBOL_GPL(vhost_exceeds_weight);
 559
 560static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
 561				   unsigned int num)
 562{
 563	size_t event __maybe_unused =
 564	       vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
 565
 566	return size_add(struct_size(vq->avail, ring, num), event);
 567}
 568
 569static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
 570				  unsigned int num)
 571{
 572	size_t event __maybe_unused =
 573	       vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
 574
 575	return size_add(struct_size(vq->used, ring, num), event);
 576}
 577
 578static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
 579				  unsigned int num)
 580{
 581	return sizeof(*vq->desc) * num;
 582}
 583
 584void vhost_dev_init(struct vhost_dev *dev,
 585		    struct vhost_virtqueue **vqs, int nvqs,
 586		    int iov_limit, int weight, int byte_weight,
 587		    bool use_worker,
 588		    int (*msg_handler)(struct vhost_dev *dev, u32 asid,
 589				       struct vhost_iotlb_msg *msg))
 590{
 591	struct vhost_virtqueue *vq;
 592	int i;
 593
 594	dev->vqs = vqs;
 595	dev->nvqs = nvqs;
 596	mutex_init(&dev->mutex);
 597	dev->log_ctx = NULL;
 598	dev->umem = NULL;
 599	dev->iotlb = NULL;
 600	dev->mm = NULL;
 601	dev->iov_limit = iov_limit;
 602	dev->weight = weight;
 603	dev->byte_weight = byte_weight;
 604	dev->use_worker = use_worker;
 605	dev->msg_handler = msg_handler;
 606	dev->fork_owner = fork_from_owner_default;
 607	init_waitqueue_head(&dev->wait);
 608	INIT_LIST_HEAD(&dev->read_list);
 609	INIT_LIST_HEAD(&dev->pending_list);
 610	spin_lock_init(&dev->iotlb_lock);
 611	xa_init_flags(&dev->worker_xa, XA_FLAGS_ALLOC);
 612
 613	for (i = 0; i < dev->nvqs; ++i) {
 614		vq = dev->vqs[i];
 615		vq->log = NULL;
 616		vq->indirect = NULL;
 617		vq->heads = NULL;
 618		vq->nheads = NULL;
 619		vq->dev = dev;
 620		mutex_init(&vq->mutex);
 621		vhost_vq_reset(dev, vq);
 622		if (vq->handle_kick)
 623			vhost_poll_init(&vq->poll, vq->handle_kick,
 624					EPOLLIN, dev, vq);
 625	}
 626}
 627EXPORT_SYMBOL_GPL(vhost_dev_init);
 628
 629/* Caller should have device mutex */
 630long vhost_dev_check_owner(struct vhost_dev *dev)
 631{
 632	/* Are you the owner? If not, I don't think you mean to do that */
 633	return dev->mm == current->mm ? 0 : -EPERM;
 634}
 635EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
 636
 637struct vhost_attach_cgroups_struct {
 638	struct vhost_work work;
 639	struct task_struct *owner;
 640	int ret;
 641};
 642
 643static void vhost_attach_cgroups_work(struct vhost_work *work)
 644{
 645	struct vhost_attach_cgroups_struct *s;
 646
 647	s = container_of(work, struct vhost_attach_cgroups_struct, work);
 648	s->ret = cgroup_attach_task_all(s->owner, current);
 649}
 650
 651static int vhost_attach_task_to_cgroups(struct vhost_worker *worker)
 652{
 653	struct vhost_attach_cgroups_struct attach;
 654	int saved_cnt;
 655
 656	attach.owner = current;
 657
 658	vhost_work_init(&attach.work, vhost_attach_cgroups_work);
 659	vhost_worker_queue(worker, &attach.work);
 660
 661	mutex_lock(&worker->mutex);
 662
 663	/*
 664	 * Bypass attachment_cnt check in __vhost_worker_flush:
 665	 * Temporarily change it to INT_MAX to bypass the check
 666	 */
 667	saved_cnt = worker->attachment_cnt;
 668	worker->attachment_cnt = INT_MAX;
 669	__vhost_worker_flush(worker);
 670	worker->attachment_cnt = saved_cnt;
 671
 672	mutex_unlock(&worker->mutex);
 673
 674	return attach.ret;
 675}
 676
 677/* Caller should have device mutex */
 678bool vhost_dev_has_owner(struct vhost_dev *dev)
 679{
 680	return dev->mm;
 681}
 682EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
 683
 684static void vhost_attach_mm(struct vhost_dev *dev)
 685{
 686	/* No owner, become one */
 687	if (dev->use_worker) {
 688		dev->mm = get_task_mm(current);
 689	} else {
 690		/* vDPA device does not use worker thread, so there's
 691		 * no need to hold the address space for mm. This helps
 692		 * to avoid deadlock in the case of mmap() which may
 693		 * hold the refcnt of the file and depends on release
 694		 * method to remove vma.
 695		 */
 696		dev->mm = current->mm;
 697		mmgrab(dev->mm);
 698	}
 699}
 700
 701static void vhost_detach_mm(struct vhost_dev *dev)
 702{
 703	if (!dev->mm)
 704		return;
 705
 706	if (dev->use_worker)
 707		mmput(dev->mm);
 708	else
 709		mmdrop(dev->mm);
 710
 711	dev->mm = NULL;
 712}
 713
 714static void vhost_worker_destroy(struct vhost_dev *dev,
 715				 struct vhost_worker *worker)
 716{
 717	if (!worker)
 718		return;
 719
 720	WARN_ON(!llist_empty(&worker->work_list));
 721	xa_erase(&dev->worker_xa, worker->id);
 722	worker->ops->stop(worker);
 723	kfree(worker);
 724}
 725
 726static void vhost_workers_free(struct vhost_dev *dev)
 727{
 728	struct vhost_worker *worker;
 729	unsigned long i;
 730
 731	if (!dev->use_worker)
 732		return;
 733
 734	for (i = 0; i < dev->nvqs; i++)
 735		rcu_assign_pointer(dev->vqs[i]->worker, NULL);
 736	/*
 737	 * Free the default worker we created and cleanup workers userspace
 738	 * created but couldn't clean up (it forgot or crashed).
 739	 */
 740	xa_for_each(&dev->worker_xa, i, worker)
 741		vhost_worker_destroy(dev, worker);
 742	xa_destroy(&dev->worker_xa);
 743}
 744
 745static void vhost_task_wakeup(struct vhost_worker *worker)
 746{
 747	return vhost_task_wake(worker->vtsk);
 748}
 749
 750static void vhost_kthread_wakeup(struct vhost_worker *worker)
 751{
 752	wake_up_process(worker->kthread_task);
 753}
 754
 755static void vhost_task_do_stop(struct vhost_worker *worker)
 756{
 757	return vhost_task_stop(worker->vtsk);
 758}
 759
 760static void vhost_kthread_do_stop(struct vhost_worker *worker)
 761{
 762	kthread_stop(worker->kthread_task);
 763}
 764
 765static int vhost_task_worker_create(struct vhost_worker *worker,
 766				    struct vhost_dev *dev, const char *name)
 767{
 768	struct vhost_task *vtsk;
 769	u32 id;
 770	int ret;
 771
 772	vtsk = vhost_task_create(vhost_run_work_list, vhost_worker_killed,
 773				 worker, name);
 774	if (IS_ERR(vtsk))
 775		return PTR_ERR(vtsk);
 776
 777	worker->vtsk = vtsk;
 778	vhost_task_start(vtsk);
 779	ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL);
 780	if (ret < 0) {
 781		vhost_task_do_stop(worker);
 782		return ret;
 783	}
 784	worker->id = id;
 785	return 0;
 786}
 787
 788static int vhost_kthread_worker_create(struct vhost_worker *worker,
 789				       struct vhost_dev *dev, const char *name)
 790{
 791	struct task_struct *task;
 792	u32 id;
 793	int ret;
 794
 795	task = kthread_create(vhost_run_work_kthread_list, worker, "%s", name);
 796	if (IS_ERR(task))
 797		return PTR_ERR(task);
 798
 799	worker->kthread_task = task;
 800	wake_up_process(task);
 801	ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL);
 802	if (ret < 0)
 803		goto stop_worker;
 804
 805	ret = vhost_attach_task_to_cgroups(worker);
 806	if (ret)
 807		goto free_id;
 808
 809	worker->id = id;
 810	return 0;
 811
 812free_id:
 813	xa_erase(&dev->worker_xa, id);
 814stop_worker:
 815	vhost_kthread_do_stop(worker);
 816	return ret;
 817}
 818
 819static const struct vhost_worker_ops kthread_ops = {
 820	.create = vhost_kthread_worker_create,
 821	.stop = vhost_kthread_do_stop,
 822	.wakeup = vhost_kthread_wakeup,
 823};
 824
 825static const struct vhost_worker_ops vhost_task_ops = {
 826	.create = vhost_task_worker_create,
 827	.stop = vhost_task_do_stop,
 828	.wakeup = vhost_task_wakeup,
 829};
 830
 831static struct vhost_worker *vhost_worker_create(struct vhost_dev *dev)
 832{
 833	struct vhost_worker *worker;
 834	char name[TASK_COMM_LEN];
 835	int ret;
 836	const struct vhost_worker_ops *ops = dev->fork_owner ? &vhost_task_ops :
 837							       &kthread_ops;
 838
 839	worker = kzalloc(sizeof(*worker), GFP_KERNEL_ACCOUNT);
 840	if (!worker)
 841		return NULL;
 842
 843	worker->dev = dev;
 844	worker->ops = ops;
 845	snprintf(name, sizeof(name), "vhost-%d", current->pid);
 846
 847	mutex_init(&worker->mutex);
 848	init_llist_head(&worker->work_list);
 849	worker->kcov_handle = kcov_common_handle();
 850	ret = ops->create(worker, dev, name);
 851	if (ret < 0)
 852		goto free_worker;
 853
 854	return worker;
 855
 856free_worker:
 857	kfree(worker);
 858	return NULL;
 859}
 860
 861/* Caller must have device mutex */
 862static void __vhost_vq_attach_worker(struct vhost_virtqueue *vq,
 863				     struct vhost_worker *worker)
 864{
 865	struct vhost_worker *old_worker;
 866
 867	mutex_lock(&worker->mutex);
 868	if (worker->killed) {
 869		mutex_unlock(&worker->mutex);
 870		return;
 871	}
 872
 873	mutex_lock(&vq->mutex);
 874
 875	old_worker = rcu_dereference_check(vq->worker,
 876					   lockdep_is_held(&vq->mutex));
 877	rcu_assign_pointer(vq->worker, worker);
 878	worker->attachment_cnt++;
 879
 880	if (!old_worker) {
 881		mutex_unlock(&vq->mutex);
 882		mutex_unlock(&worker->mutex);
 883		return;
 884	}
 885	mutex_unlock(&vq->mutex);
 886	mutex_unlock(&worker->mutex);
 887
 888	/*
 889	 * Take the worker mutex to make sure we see the work queued from
 890	 * device wide flushes which doesn't use RCU for execution.
 891	 */
 892	mutex_lock(&old_worker->mutex);
 893	if (old_worker->killed) {
 894		mutex_unlock(&old_worker->mutex);
 895		return;
 896	}
 897
 898	/*
 899	 * We don't want to call synchronize_rcu for every vq during setup
 900	 * because it will slow down VM startup. If we haven't done
 901	 * VHOST_SET_VRING_KICK and not done the driver specific
 902	 * SET_ENDPOINT/RUNNING then we can skip the sync since there will
 903	 * not be any works queued for scsi and net.
 904	 */
 905	mutex_lock(&vq->mutex);
 906	if (!vhost_vq_get_backend(vq) && !vq->kick) {
 907		mutex_unlock(&vq->mutex);
 908
 909		old_worker->attachment_cnt--;
 910		mutex_unlock(&old_worker->mutex);
 911		/*
 912		 * vsock can queue anytime after VHOST_VSOCK_SET_GUEST_CID.
 913		 * Warn if it adds support for multiple workers but forgets to
 914		 * handle the early queueing case.
 915		 */
 916		WARN_ON(!old_worker->attachment_cnt &&
 917			!llist_empty(&old_worker->work_list));
 918		return;
 919	}
 920	mutex_unlock(&vq->mutex);
 921
 922	/* Make sure new vq queue/flush/poll calls see the new worker */
 923	synchronize_rcu();
 924	/* Make sure whatever was queued gets run */
 925	__vhost_worker_flush(old_worker);
 926	old_worker->attachment_cnt--;
 927	mutex_unlock(&old_worker->mutex);
 928}
 929
 930 /* Caller must have device mutex */
 931static int vhost_vq_attach_worker(struct vhost_virtqueue *vq,
 932				  struct vhost_vring_worker *info)
 933{
 934	unsigned long index = info->worker_id;
 935	struct vhost_dev *dev = vq->dev;
 936	struct vhost_worker *worker;
 937
 938	if (!dev->use_worker)
 939		return -EINVAL;
 940
 941	worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT);
 942	if (!worker || worker->id != info->worker_id)
 943		return -ENODEV;
 944
 945	__vhost_vq_attach_worker(vq, worker);
 946	return 0;
 947}
 948
 949/* Caller must have device mutex */
 950static int vhost_new_worker(struct vhost_dev *dev,
 951			    struct vhost_worker_state *info)
 952{
 953	struct vhost_worker *worker;
 954
 955	worker = vhost_worker_create(dev);
 956	if (!worker)
 957		return -ENOMEM;
 958
 959	info->worker_id = worker->id;
 960	return 0;
 961}
 962
 963/* Caller must have device mutex */
 964static int vhost_free_worker(struct vhost_dev *dev,
 965			     struct vhost_worker_state *info)
 966{
 967	unsigned long index = info->worker_id;
 968	struct vhost_worker *worker;
 969
 970	worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT);
 971	if (!worker || worker->id != info->worker_id)
 972		return -ENODEV;
 973
 974	mutex_lock(&worker->mutex);
 975	if (worker->attachment_cnt || worker->killed) {
 976		mutex_unlock(&worker->mutex);
 977		return -EBUSY;
 978	}
 979	/*
 980	 * A flush might have raced and snuck in before attachment_cnt was set
 981	 * to zero. Make sure flushes are flushed from the queue before
 982	 * freeing.
 983	 */
 984	__vhost_worker_flush(worker);
 985	mutex_unlock(&worker->mutex);
 986
 987	vhost_worker_destroy(dev, worker);
 988	return 0;
 989}
 990
 991static int vhost_get_vq_from_user(struct vhost_dev *dev, void __user *argp,
 992				  struct vhost_virtqueue **vq, u32 *id)
 993{
 994	u32 __user *idxp = argp;
 995	u32 idx;
 996	long r;
 997
 998	r = get_user(idx, idxp);
 999	if (r < 0)
1000		return r;
1001
1002	if (idx >= dev->nvqs)
1003		return -ENOBUFS;
1004
1005	idx = array_index_nospec(idx, dev->nvqs);
1006
1007	*vq = dev->vqs[idx];
1008	*id = idx;
1009	return 0;
1010}
1011
1012/* Caller must have device mutex */
1013long vhost_worker_ioctl(struct vhost_dev *dev, unsigned int ioctl,
1014			void __user *argp)
1015{
1016	struct vhost_vring_worker ring_worker;
1017	struct vhost_worker_state state;
1018	struct vhost_worker *worker;
1019	struct vhost_virtqueue *vq;
1020	long ret;
1021	u32 idx;
1022
1023	if (!dev->use_worker)
1024		return -EINVAL;
1025
1026	if (!vhost_dev_has_owner(dev))
1027		return -EINVAL;
1028
1029	ret = vhost_dev_check_owner(dev);
1030	if (ret)
1031		return ret;
1032
1033	switch (ioctl) {
1034	/* dev worker ioctls */
1035	case VHOST_NEW_WORKER:
1036		/*
1037		 * vhost_tasks will account for worker threads under the parent's
1038		 * NPROC value but kthreads do not. To avoid userspace overflowing
1039		 * the system with worker threads fork_owner must be true.
1040		 */
1041		if (!dev->fork_owner)
1042			return -EFAULT;
1043
1044		ret = vhost_new_worker(dev, &state);
1045		if (!ret && copy_to_user(argp, &state, sizeof(state)))
1046			ret = -EFAULT;
1047		return ret;
1048	case VHOST_FREE_WORKER:
1049		if (copy_from_user(&state, argp, sizeof(state)))
1050			return -EFAULT;
1051		return vhost_free_worker(dev, &state);
1052	/* vring worker ioctls */
1053	case VHOST_ATTACH_VRING_WORKER:
1054	case VHOST_GET_VRING_WORKER:
1055		break;
1056	default:
1057		return -ENOIOCTLCMD;
1058	}
1059
1060	ret = vhost_get_vq_from_user(dev, argp, &vq, &idx);
1061	if (ret)
1062		return ret;
1063
1064	switch (ioctl) {
1065	case VHOST_ATTACH_VRING_WORKER:
1066		if (copy_from_user(&ring_worker, argp, sizeof(ring_worker))) {
1067			ret = -EFAULT;
1068			break;
1069		}
1070
1071		ret = vhost_vq_attach_worker(vq, &ring_worker);
1072		break;
1073	case VHOST_GET_VRING_WORKER:
1074		worker = rcu_dereference_check(vq->worker,
1075					       lockdep_is_held(&dev->mutex));
1076		if (!worker) {
1077			ret = -EINVAL;
1078			break;
1079		}
1080
1081		ring_worker.index = idx;
1082		ring_worker.worker_id = worker->id;
1083
1084		if (copy_to_user(argp, &ring_worker, sizeof(ring_worker)))
1085			ret = -EFAULT;
1086		break;
1087	default:
1088		ret = -ENOIOCTLCMD;
1089		break;
1090	}
1091
1092	return ret;
1093}
1094EXPORT_SYMBOL_GPL(vhost_worker_ioctl);
1095
1096/* Caller should have device mutex */
1097long vhost_dev_set_owner(struct vhost_dev *dev)
1098{
1099	struct vhost_worker *worker;
1100	int err, i;
1101
1102	/* Is there an owner already? */
1103	if (vhost_dev_has_owner(dev)) {
1104		err = -EBUSY;
1105		goto err_mm;
1106	}
1107
1108	vhost_attach_mm(dev);
1109
1110	err = vhost_dev_alloc_iovecs(dev);
1111	if (err)
1112		goto err_iovecs;
1113
1114	if (dev->use_worker) {
1115		/*
1116		 * This should be done last, because vsock can queue work
1117		 * before VHOST_SET_OWNER so it simplifies the failure path
1118		 * below since we don't have to worry about vsock queueing
1119		 * while we free the worker.
1120		 */
1121		worker = vhost_worker_create(dev);
1122		if (!worker) {
1123			err = -ENOMEM;
1124			goto err_worker;
1125		}
1126
1127		for (i = 0; i < dev->nvqs; i++)
1128			__vhost_vq_attach_worker(dev->vqs[i], worker);
1129	}
1130
1131	return 0;
1132
1133err_worker:
1134	vhost_dev_free_iovecs(dev);
1135err_iovecs:
1136	vhost_detach_mm(dev);
1137err_mm:
1138	return err;
1139}
1140EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
1141
1142static struct vhost_iotlb *iotlb_alloc(void)
1143{
1144	return vhost_iotlb_alloc(max_iotlb_entries,
1145				 VHOST_IOTLB_FLAG_RETIRE);
1146}
1147
1148struct vhost_iotlb *vhost_dev_reset_owner_prepare(void)
1149{
1150	return iotlb_alloc();
1151}
1152EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
1153
1154/* Caller should have device mutex */
1155void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem)
1156{
1157	int i;
1158
1159	vhost_dev_cleanup(dev);
1160
1161	dev->fork_owner = fork_from_owner_default;
1162	dev->umem = umem;
1163	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
1164	 * VQs aren't running.
1165	 */
1166	for (i = 0; i < dev->nvqs; ++i)
1167		dev->vqs[i]->umem = umem;
1168}
1169EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
1170
1171void vhost_dev_stop(struct vhost_dev *dev)
1172{
1173	int i;
1174
1175	for (i = 0; i < dev->nvqs; ++i) {
1176		if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick)
1177			vhost_poll_stop(&dev->vqs[i]->poll);
1178	}
1179
1180	vhost_dev_flush(dev);
1181}
1182EXPORT_SYMBOL_GPL(vhost_dev_stop);
1183
1184void vhost_clear_msg(struct vhost_dev *dev)
1185{
1186	struct vhost_msg_node *node, *n;
1187
1188	spin_lock(&dev->iotlb_lock);
1189
1190	list_for_each_entry_safe(node, n, &dev->read_list, node) {
1191		list_del(&node->node);
1192		kfree(node);
1193	}
1194
1195	list_for_each_entry_safe(node, n, &dev->pending_list, node) {
1196		list_del(&node->node);
1197		kfree(node);
1198	}
1199
1200	spin_unlock(&dev->iotlb_lock);
1201}
1202EXPORT_SYMBOL_GPL(vhost_clear_msg);
1203
1204void vhost_dev_cleanup(struct vhost_dev *dev)
1205{
1206	int i;
1207
1208	for (i = 0; i < dev->nvqs; ++i) {
1209		if (dev->vqs[i]->error_ctx)
1210			eventfd_ctx_put(dev->vqs[i]->error_ctx);
1211		if (dev->vqs[i]->kick)
1212			fput(dev->vqs[i]->kick);
1213		if (dev->vqs[i]->call_ctx.ctx)
1214			eventfd_ctx_put(dev->vqs[i]->call_ctx.ctx);
1215		vhost_vq_reset(dev, dev->vqs[i]);
1216	}
1217	vhost_dev_free_iovecs(dev);
1218	if (dev->log_ctx)
1219		eventfd_ctx_put(dev->log_ctx);
1220	dev->log_ctx = NULL;
1221	/* No one will access memory at this point */
1222	vhost_iotlb_free(dev->umem);
1223	dev->umem = NULL;
1224	vhost_iotlb_free(dev->iotlb);
1225	dev->iotlb = NULL;
1226	vhost_clear_msg(dev);
1227	wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
1228	vhost_workers_free(dev);
1229	vhost_detach_mm(dev);
1230}
1231EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
1232
1233static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
1234{
1235	u64 a = addr / VHOST_PAGE_SIZE / 8;
1236
1237	/* Make sure 64 bit math will not overflow. */
1238	if (a > ULONG_MAX - (unsigned long)log_base ||
1239	    a + (unsigned long)log_base > ULONG_MAX)
1240		return false;
1241
1242	return access_ok(log_base + a,
1243			 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
1244}
1245
1246/* Make sure 64 bit math will not overflow. */
1247static bool vhost_overflow(u64 uaddr, u64 size)
1248{
1249	if (uaddr > ULONG_MAX || size > ULONG_MAX)
1250		return true;
1251
1252	if (!size)
1253		return false;
1254
1255	return uaddr > ULONG_MAX - size + 1;
1256}
1257
1258/* Caller should have vq mutex and device mutex. */
1259static bool vq_memory_access_ok(void __user *log_base, struct vhost_iotlb *umem,
1260				int log_all)
1261{
1262	struct vhost_iotlb_map *map;
1263
1264	if (!umem)
1265		return false;
1266
1267	list_for_each_entry(map, &umem->list, link) {
1268		unsigned long a = map->addr;
1269
1270		if (vhost_overflow(map->addr, map->size))
1271			return false;
1272
1273
1274		if (!access_ok((void __user *)a, map->size))
1275			return false;
1276		else if (log_all && !log_access_ok(log_base,
1277						   map->start,
1278						   map->size))
1279			return false;
1280	}
1281	return true;
1282}
1283
1284static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq,
1285					       u64 addr, unsigned int size,
1286					       int type)
1287{
1288	const struct vhost_iotlb_map *map = vq->meta_iotlb[type];
1289
1290	if (!map)
1291		return NULL;
1292
1293	return (void __user *)(uintptr_t)(map->addr + addr - map->start);
1294}
1295
1296/* Can we switch to this memory table? */
1297/* Caller should have device mutex but not vq mutex */
1298static bool memory_access_ok(struct vhost_dev *d, struct vhost_iotlb *umem,
1299			     int log_all)
1300{
1301	int i;
1302
1303	for (i = 0; i < d->nvqs; ++i) {
1304		bool ok;
1305		bool log;
1306
1307		mutex_lock(&d->vqs[i]->mutex);
1308		log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
1309		/* If ring is inactive, will check when it's enabled. */
1310		if (d->vqs[i]->private_data)
1311			ok = vq_memory_access_ok(d->vqs[i]->log_base,
1312						 umem, log);
1313		else
1314			ok = true;
1315		mutex_unlock(&d->vqs[i]->mutex);
1316		if (!ok)
1317			return false;
1318	}
1319	return true;
1320}
1321
1322static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1323			  struct iovec iov[], int iov_size, int access);
1324
1325static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to,
1326			      const void *from, unsigned size)
1327{
1328	int ret;
1329
1330	if (!vq->iotlb)
1331		return __copy_to_user(to, from, size);
1332	else {
1333		/* This function should be called after iotlb
1334		 * prefetch, which means we're sure that all vq
1335		 * could be access through iotlb. So -EAGAIN should
1336		 * not happen in this case.
1337		 */
1338		struct iov_iter t;
1339		void __user *uaddr = vhost_vq_meta_fetch(vq,
1340				     (u64)(uintptr_t)to, size,
1341				     VHOST_ADDR_USED);
1342
1343		if (uaddr)
1344			return __copy_to_user(uaddr, from, size);
1345
1346		ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
1347				     ARRAY_SIZE(vq->iotlb_iov),
1348				     VHOST_ACCESS_WO);
1349		if (ret < 0)
1350			goto out;
1351		iov_iter_init(&t, ITER_DEST, vq->iotlb_iov, ret, size);
1352		ret = copy_to_iter(from, size, &t);
1353		if (ret == size)
1354			ret = 0;
1355	}
1356out:
1357	return ret;
1358}
1359
1360static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
1361				void __user *from, unsigned size)
1362{
1363	int ret;
1364
1365	if (!vq->iotlb)
1366		return __copy_from_user(to, from, size);
1367	else {
1368		/* This function should be called after iotlb
1369		 * prefetch, which means we're sure that vq
1370		 * could be access through iotlb. So -EAGAIN should
1371		 * not happen in this case.
1372		 */
1373		void __user *uaddr = vhost_vq_meta_fetch(vq,
1374				     (u64)(uintptr_t)from, size,
1375				     VHOST_ADDR_DESC);
1376		struct iov_iter f;
1377
1378		if (uaddr)
1379			return __copy_from_user(to, uaddr, size);
1380
1381		ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
1382				     ARRAY_SIZE(vq->iotlb_iov),
1383				     VHOST_ACCESS_RO);
1384		if (ret < 0) {
1385			vq_err(vq, "IOTLB translation failure: uaddr "
1386			       "%p size 0x%llx\n", from,
1387			       (unsigned long long) size);
1388			goto out;
1389		}
1390		iov_iter_init(&f, ITER_SOURCE, vq->iotlb_iov, ret, size);
1391		ret = copy_from_iter(to, size, &f);
1392		if (ret == size)
1393			ret = 0;
1394	}
1395
1396out:
1397	return ret;
1398}
1399
1400static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq,
1401					  void __user *addr, unsigned int size,
1402					  int type)
1403{
1404	int ret;
1405
1406	ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
1407			     ARRAY_SIZE(vq->iotlb_iov),
1408			     VHOST_ACCESS_RO);
1409	if (ret < 0) {
1410		vq_err(vq, "IOTLB translation failure: uaddr "
1411			"%p size 0x%llx\n", addr,
1412			(unsigned long long) size);
1413		return NULL;
1414	}
1415
1416	if (ret != 1 || vq->iotlb_iov[0].iov_len != size) {
1417		vq_err(vq, "Non atomic userspace memory access: uaddr "
1418			"%p size 0x%llx\n", addr,
1419			(unsigned long long) size);
1420		return NULL;
1421	}
1422
1423	return vq->iotlb_iov[0].iov_base;
1424}
1425
1426/* This function should be called after iotlb
1427 * prefetch, which means we're sure that vq
1428 * could be access through iotlb. So -EAGAIN should
1429 * not happen in this case.
1430 */
1431static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
1432					    void __user *addr, unsigned int size,
1433					    int type)
1434{
1435	void __user *uaddr = vhost_vq_meta_fetch(vq,
1436			     (u64)(uintptr_t)addr, size, type);
1437	if (uaddr)
1438		return uaddr;
1439
1440	return __vhost_get_user_slow(vq, addr, size, type);
1441}
1442
1443#define vhost_put_user(vq, x, ptr)		\
1444({ \
1445	int ret; \
1446	if (!vq->iotlb) { \
1447		ret = __put_user(x, ptr); \
1448	} else { \
1449		__typeof__(ptr) to = \
1450			(__typeof__(ptr)) __vhost_get_user(vq, ptr,	\
1451					  sizeof(*ptr), VHOST_ADDR_USED); \
1452		if (to != NULL) \
1453			ret = __put_user(x, to); \
1454		else \
1455			ret = -EFAULT;	\
1456	} \
1457	ret; \
1458})
1459
1460static inline int vhost_put_avail_event(struct vhost_virtqueue *vq)
1461{
1462	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
1463			      vhost_avail_event(vq));
1464}
1465
1466static inline int vhost_put_used(struct vhost_virtqueue *vq,
1467				 struct vring_used_elem *head, int idx,
1468				 int count)
1469{
1470	return vhost_copy_to_user(vq, vq->used->ring + idx, head,
1471				  count * sizeof(*head));
1472}
1473
1474static inline int vhost_put_used_flags(struct vhost_virtqueue *vq)
1475
1476{
1477	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
1478			      &vq->used->flags);
1479}
1480
1481static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)
1482
1483{
1484	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
1485			      &vq->used->idx);
1486}
1487
1488#define vhost_get_user(vq, x, ptr, type)		\
1489({ \
1490	int ret; \
1491	if (!vq->iotlb) { \
1492		ret = __get_user(x, ptr); \
1493	} else { \
1494		__typeof__(ptr) from = \
1495			(__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1496							   sizeof(*ptr), \
1497							   type); \
1498		if (from != NULL) \
1499			ret = __get_user(x, from); \
1500		else \
1501			ret = -EFAULT; \
1502	} \
1503	ret; \
1504})
1505
1506#define vhost_get_avail(vq, x, ptr) \
1507	vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
1508
1509#define vhost_get_used(vq, x, ptr) \
1510	vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
1511
1512static void vhost_dev_lock_vqs(struct vhost_dev *d)
1513{
1514	int i = 0;
1515	for (i = 0; i < d->nvqs; ++i)
1516		mutex_lock_nested(&d->vqs[i]->mutex, i);
1517}
1518
1519static void vhost_dev_unlock_vqs(struct vhost_dev *d)
1520{
1521	int i = 0;
1522	for (i = 0; i < d->nvqs; ++i)
1523		mutex_unlock(&d->vqs[i]->mutex);
1524}
1525
1526static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq)
1527{
1528	__virtio16 idx;
1529	int r;
1530
1531	r = vhost_get_avail(vq, idx, &vq->avail->idx);
1532	if (unlikely(r < 0)) {
1533		vq_err(vq, "Failed to access available index at %p (%d)\n",
1534		       &vq->avail->idx, r);
1535		return r;
1536	}
1537
1538	/* Check it isn't doing very strange thing with available indexes */
1539	vq->avail_idx = vhost16_to_cpu(vq, idx);
1540	if (unlikely((u16)(vq->avail_idx - vq->last_avail_idx) > vq->num)) {
1541		vq_err(vq, "Invalid available index change from %u to %u",
1542		       vq->last_avail_idx, vq->avail_idx);
1543		return -EINVAL;
1544	}
1545
1546	/* We're done if there is nothing new */
1547	if (vq->avail_idx == vq->last_avail_idx)
1548		return 0;
1549
1550	/*
1551	 * We updated vq->avail_idx so we need a memory barrier between
1552	 * the index read above and the caller reading avail ring entries.
1553	 */
1554	smp_rmb();
1555	return 1;
1556}
1557
1558static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,
1559				       __virtio16 *head, int idx)
1560{
1561	return vhost_get_avail(vq, *head,
1562			       &vq->avail->ring[idx & (vq->num - 1)]);
1563}
1564
1565static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq,
1566					__virtio16 *flags)
1567{
1568	return vhost_get_avail(vq, *flags, &vq->avail->flags);
1569}
1570
1571static inline int vhost_get_used_event(struct vhost_virtqueue *vq,
1572				       __virtio16 *event)
1573{
1574	return vhost_get_avail(vq, *event, vhost_used_event(vq));
1575}
1576
1577static inline int vhost_get_used_idx(struct vhost_virtqueue *vq,
1578				     __virtio16 *idx)
1579{
1580	return vhost_get_used(vq, *idx, &vq->used->idx);
1581}
1582
1583static inline int vhost_get_desc(struct vhost_virtqueue *vq,
1584				 struct vring_desc *desc, int idx)
1585{
1586	return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));
1587}
1588
1589static void vhost_iotlb_notify_vq(struct vhost_dev *d,
1590				  struct vhost_iotlb_msg *msg)
1591{
1592	struct vhost_msg_node *node, *n;
1593
1594	spin_lock(&d->iotlb_lock);
1595
1596	list_for_each_entry_safe(node, n, &d->pending_list, node) {
1597		struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
1598		if (msg->iova <= vq_msg->iova &&
1599		    msg->iova + msg->size - 1 >= vq_msg->iova &&
1600		    vq_msg->type == VHOST_IOTLB_MISS) {
1601			vhost_poll_queue(&node->vq->poll);
1602			list_del(&node->node);
1603			kfree(node);
1604		}
1605	}
1606
1607	spin_unlock(&d->iotlb_lock);
1608}
1609
1610static bool umem_access_ok(u64 uaddr, u64 size, int access)
1611{
1612	unsigned long a = uaddr;
1613
1614	/* Make sure 64 bit math will not overflow. */
1615	if (vhost_overflow(uaddr, size))
1616		return false;
1617
1618	if ((access & VHOST_ACCESS_RO) &&
1619	    !access_ok((void __user *)a, size))
1620		return false;
1621	if ((access & VHOST_ACCESS_WO) &&
1622	    !access_ok((void __user *)a, size))
1623		return false;
1624	return true;
1625}
1626
1627static int vhost_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1628				   struct vhost_iotlb_msg *msg)
1629{
1630	int ret = 0;
1631
1632	if (asid != 0)
1633		return -EINVAL;
1634
1635	mutex_lock(&dev->mutex);
1636	vhost_dev_lock_vqs(dev);
1637	switch (msg->type) {
1638	case VHOST_IOTLB_UPDATE:
1639		if (!dev->iotlb) {
1640			ret = -EFAULT;
1641			break;
1642		}
1643		if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
1644			ret = -EFAULT;
1645			break;
1646		}
1647		vhost_vq_meta_reset(dev);
1648		if (vhost_iotlb_add_range(dev->iotlb, msg->iova,
1649					  msg->iova + msg->size - 1,
1650					  msg->uaddr, msg->perm)) {
1651			ret = -ENOMEM;
1652			break;
1653		}
1654		vhost_iotlb_notify_vq(dev, msg);
1655		break;
1656	case VHOST_IOTLB_INVALIDATE:
1657		if (!dev->iotlb) {
1658			ret = -EFAULT;
1659			break;
1660		}
1661		vhost_vq_meta_reset(dev);
1662		vhost_iotlb_del_range(dev->iotlb, msg->iova,
1663				      msg->iova + msg->size - 1);
1664		break;
1665	default:
1666		ret = -EINVAL;
1667		break;
1668	}
1669
1670	vhost_dev_unlock_vqs(dev);
1671	mutex_unlock(&dev->mutex);
1672
1673	return ret;
1674}
1675ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
1676			     struct iov_iter *from)
1677{
1678	struct vhost_iotlb_msg msg;
1679	size_t offset;
1680	int type, ret;
1681	u32 asid = 0;
1682
1683	ret = copy_from_iter(&type, sizeof(type), from);
1684	if (ret != sizeof(type)) {
1685		ret = -EINVAL;
1686		goto done;
1687	}
1688
1689	switch (type) {
1690	case VHOST_IOTLB_MSG:
1691		/* There maybe a hole after type for V1 message type,
1692		 * so skip it here.
1693		 */
1694		offset = offsetof(struct vhost_msg, iotlb) - sizeof(int);
1695		break;
1696	case VHOST_IOTLB_MSG_V2:
1697		if (vhost_backend_has_feature(dev->vqs[0],
1698					      VHOST_BACKEND_F_IOTLB_ASID)) {
1699			ret = copy_from_iter(&asid, sizeof(asid), from);
1700			if (ret != sizeof(asid)) {
1701				ret = -EINVAL;
1702				goto done;
1703			}
1704			offset = 0;
1705		} else
1706			offset = sizeof(__u32);
1707		break;
1708	default:
1709		ret = -EINVAL;
1710		goto done;
1711	}
1712
1713	iov_iter_advance(from, offset);
1714	ret = copy_from_iter(&msg, sizeof(msg), from);
1715	if (ret != sizeof(msg)) {
1716		ret = -EINVAL;
1717		goto done;
1718	}
1719
1720	if (msg.type == VHOST_IOTLB_UPDATE && msg.size == 0) {
1721		ret = -EINVAL;
1722		goto done;
1723	}
1724
1725	if (dev->msg_handler)
1726		ret = dev->msg_handler(dev, asid, &msg);
1727	else
1728		ret = vhost_process_iotlb_msg(dev, asid, &msg);
1729	if (ret) {
1730		ret = -EFAULT;
1731		goto done;
1732	}
1733
1734	ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) :
1735	      sizeof(struct vhost_msg_v2);
1736done:
1737	return ret;
1738}
1739EXPORT_SYMBOL(vhost_chr_write_iter);
1740
1741__poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev,
1742			    poll_table *wait)
1743{
1744	__poll_t mask = 0;
1745
1746	poll_wait(file, &dev->wait, wait);
1747
1748	if (!list_empty(&dev->read_list))
1749		mask |= EPOLLIN | EPOLLRDNORM;
1750
1751	return mask;
1752}
1753EXPORT_SYMBOL(vhost_chr_poll);
1754
1755ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
1756			    int noblock)
1757{
1758	DEFINE_WAIT(wait);
1759	struct vhost_msg_node *node;
1760	ssize_t ret = 0;
1761	unsigned size = sizeof(struct vhost_msg);
1762
1763	if (iov_iter_count(to) < size)
1764		return 0;
1765
1766	while (1) {
1767		if (!noblock)
1768			prepare_to_wait(&dev->wait, &wait,
1769					TASK_INTERRUPTIBLE);
1770
1771		node = vhost_dequeue_msg(dev, &dev->read_list);
1772		if (node)
1773			break;
1774		if (noblock) {
1775			ret = -EAGAIN;
1776			break;
1777		}
1778		if (signal_pending(current)) {
1779			ret = -ERESTARTSYS;
1780			break;
1781		}
1782		if (!dev->iotlb) {
1783			ret = -EBADFD;
1784			break;
1785		}
1786
1787		schedule();
1788	}
1789
1790	if (!noblock)
1791		finish_wait(&dev->wait, &wait);
1792
1793	if (node) {
1794		struct vhost_iotlb_msg *msg;
1795		void *start = &node->msg;
1796
1797		switch (node->msg.type) {
1798		case VHOST_IOTLB_MSG:
1799			size = sizeof(node->msg);
1800			msg = &node->msg.iotlb;
1801			break;
1802		case VHOST_IOTLB_MSG_V2:
1803			size = sizeof(node->msg_v2);
1804			msg = &node->msg_v2.iotlb;
1805			break;
1806		default:
1807			BUG();
1808			break;
1809		}
1810
1811		ret = copy_to_iter(start, size, to);
1812		if (ret != size || msg->type != VHOST_IOTLB_MISS) {
1813			kfree(node);
1814			return ret;
1815		}
1816		vhost_enqueue_msg(dev, &dev->pending_list, node);
1817	}
1818
1819	return ret;
1820}
1821EXPORT_SYMBOL_GPL(vhost_chr_read_iter);
1822
1823static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
1824{
1825	struct vhost_dev *dev = vq->dev;
1826	struct vhost_msg_node *node;
1827	struct vhost_iotlb_msg *msg;
1828	bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2);
1829
1830	node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG);
1831	if (!node)
1832		return -ENOMEM;
1833
1834	if (v2) {
1835		node->msg_v2.type = VHOST_IOTLB_MSG_V2;
1836		msg = &node->msg_v2.iotlb;
1837	} else {
1838		msg = &node->msg.iotlb;
1839	}
1840
1841	msg->type = VHOST_IOTLB_MISS;
1842	msg->iova = iova;
1843	msg->perm = access;
1844
1845	vhost_enqueue_msg(dev, &dev->read_list, node);
1846
1847	return 0;
1848}
1849
1850static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1851			 vring_desc_t __user *desc,
1852			 vring_avail_t __user *avail,
1853			 vring_used_t __user *used)
1854
1855{
1856	/* If an IOTLB device is present, the vring addresses are
1857	 * GIOVAs. Access validation occurs at prefetch time. */
1858	if (vq->iotlb)
1859		return true;
1860
1861	return access_ok(desc, vhost_get_desc_size(vq, num)) &&
1862	       access_ok(avail, vhost_get_avail_size(vq, num)) &&
1863	       access_ok(used, vhost_get_used_size(vq, num));
1864}
1865
1866static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1867				 const struct vhost_iotlb_map *map,
1868				 int type)
1869{
1870	int access = (type == VHOST_ADDR_USED) ?
1871		     VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1872
1873	if (likely(map->perm & access))
1874		vq->meta_iotlb[type] = map;
1875}
1876
1877static bool iotlb_access_ok(struct vhost_virtqueue *vq,
1878			    int access, u64 addr, u64 len, int type)
1879{
1880	const struct vhost_iotlb_map *map;
1881	struct vhost_iotlb *umem = vq->iotlb;
1882	u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
1883
1884	if (vhost_vq_meta_fetch(vq, addr, len, type))
1885		return true;
1886
1887	while (len > s) {
1888		map = vhost_iotlb_itree_first(umem, addr, last);
1889		if (map == NULL || map->start > addr) {
1890			vhost_iotlb_miss(vq, addr, access);
1891			return false;
1892		} else if (!(map->perm & access)) {
1893			/* Report the possible access violation by
1894			 * request another translation from userspace.
1895			 */
1896			return false;
1897		}
1898
1899		size = map->size - addr + map->start;
1900
1901		if (orig_addr == addr && size >= len)
1902			vhost_vq_meta_update(vq, map, type);
1903
1904		s += size;
1905		addr += size;
1906	}
1907
1908	return true;
1909}
1910
1911int vq_meta_prefetch(struct vhost_virtqueue *vq)
1912{
1913	unsigned int num = vq->num;
1914
1915	if (!vq->iotlb)
1916		return 1;
1917
1918	return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc,
1919			       vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&
1920	       iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail,
1921			       vhost_get_avail_size(vq, num),
1922			       VHOST_ADDR_AVAIL) &&
1923	       iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used,
1924			       vhost_get_used_size(vq, num), VHOST_ADDR_USED);
1925}
1926EXPORT_SYMBOL_GPL(vq_meta_prefetch);
1927
1928/* Can we log writes? */
1929/* Caller should have device mutex but not vq mutex */
1930bool vhost_log_access_ok(struct vhost_dev *dev)
1931{
1932	return memory_access_ok(dev, dev->umem, 1);
1933}
1934EXPORT_SYMBOL_GPL(vhost_log_access_ok);
1935
1936static bool vq_log_used_access_ok(struct vhost_virtqueue *vq,
1937				  void __user *log_base,
1938				  bool log_used,
1939				  u64 log_addr)
1940{
1941	/* If an IOTLB device is present, log_addr is a GIOVA that
1942	 * will never be logged by log_used(). */
1943	if (vq->iotlb)
1944		return true;
1945
1946	return !log_used || log_access_ok(log_base, log_addr,
1947					  vhost_get_used_size(vq, vq->num));
1948}
1949
1950/* Verify access for write logging. */
1951/* Caller should have vq mutex and device mutex */
1952static bool vq_log_access_ok(struct vhost_virtqueue *vq,
1953			     void __user *log_base)
1954{
1955	return vq_memory_access_ok(log_base, vq->umem,
1956				   vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
1957		vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr);
1958}
1959
1960/* Can we start vq? */
1961/* Caller should have vq mutex and device mutex */
1962bool vhost_vq_access_ok(struct vhost_virtqueue *vq)
1963{
1964	if (!vq_log_access_ok(vq, vq->log_base))
1965		return false;
1966
1967	return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);
1968}
1969EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1970
1971static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
1972{
1973	struct vhost_memory mem, *newmem;
1974	struct vhost_memory_region *region;
1975	struct vhost_iotlb *newumem, *oldumem;
1976	unsigned long size = offsetof(struct vhost_memory, regions);
1977	int i;
1978
1979	if (copy_from_user(&mem, m, size))
1980		return -EFAULT;
1981	if (mem.padding)
1982		return -EOPNOTSUPP;
1983	if (mem.nregions > max_mem_regions)
1984		return -E2BIG;
1985	newmem = kvzalloc(struct_size(newmem, regions, mem.nregions),
1986			GFP_KERNEL);
1987	if (!newmem)
1988		return -ENOMEM;
1989
1990	memcpy(newmem, &mem, size);
1991	if (copy_from_user(newmem->regions, m->regions,
1992			   flex_array_size(newmem, regions, mem.nregions))) {
1993		kvfree(newmem);
1994		return -EFAULT;
1995	}
1996
1997	newumem = iotlb_alloc();
1998	if (!newumem) {
1999		kvfree(newmem);
2000		return -ENOMEM;
2001	}
2002
2003	for (region = newmem->regions;
2004	     region < newmem->regions + mem.nregions;
2005	     region++) {
2006		if (vhost_iotlb_add_range(newumem,
2007					  region->guest_phys_addr,
2008					  region->guest_phys_addr +
2009					  region->memory_size - 1,
2010					  region->userspace_addr,
2011					  VHOST_MAP_RW))
2012			goto err;
2013	}
2014
2015	if (!memory_access_ok(d, newumem, 0))
2016		goto err;
2017
2018	oldumem = d->umem;
2019	d->umem = newumem;
2020
2021	/* All memory accesses are done under some VQ mutex. */
2022	for (i = 0; i < d->nvqs; ++i) {
2023		mutex_lock(&d->vqs[i]->mutex);
2024		d->vqs[i]->umem = newumem;
2025		mutex_unlock(&d->vqs[i]->mutex);
2026	}
2027
2028	kvfree(newmem);
2029	vhost_iotlb_free(oldumem);
2030	return 0;
2031
2032err:
2033	vhost_iotlb_free(newumem);
2034	kvfree(newmem);
2035	return -EFAULT;
2036}
2037
2038static long vhost_vring_set_num(struct vhost_dev *d,
2039				struct vhost_virtqueue *vq,
2040				void __user *argp)
2041{
2042	struct vhost_vring_state s;
2043
2044	/* Resizing ring with an active backend?
2045	 * You don't want to do that. */
2046	if (vq->private_data)
2047		return -EBUSY;
2048
2049	if (copy_from_user(&s, argp, sizeof s))
2050		return -EFAULT;
2051
2052	if (!s.num || s.num > 0xffff || (s.num & (s.num - 1)))
2053		return -EINVAL;
2054	vq->num = s.num;
2055
2056	return 0;
2057}
2058
2059static long vhost_vring_set_addr(struct vhost_dev *d,
2060				 struct vhost_virtqueue *vq,
2061				 void __user *argp)
2062{
2063	struct vhost_vring_addr a;
2064
2065	if (copy_from_user(&a, argp, sizeof a))
2066		return -EFAULT;
2067	if (a.flags & ~(0x1 << VHOST_VRING_F_LOG))
2068		return -EOPNOTSUPP;
2069
2070	/* For 32bit, verify that the top 32bits of the user
2071	   data are set to zero. */
2072	if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
2073	    (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
2074	    (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr)
2075		return -EFAULT;
2076
2077	/* Make sure it's safe to cast pointers to vring types. */
2078	BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
2079	BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
2080	if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
2081	    (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
2082	    (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1)))
2083		return -EINVAL;
2084
2085	/* We only verify access here if backend is configured.
2086	 * If it is not, we don't as size might not have been setup.
2087	 * We will verify when backend is configured. */
2088	if (vq->private_data) {
2089		if (!vq_access_ok(vq, vq->num,
2090			(void __user *)(unsigned long)a.desc_user_addr,
2091			(void __user *)(unsigned long)a.avail_user_addr,
2092			(void __user *)(unsigned long)a.used_user_addr))
2093			return -EINVAL;
2094
2095		/* Also validate log access for used ring if enabled. */
2096		if (!vq_log_used_access_ok(vq, vq->log_base,
2097				a.flags & (0x1 << VHOST_VRING_F_LOG),
2098				a.log_guest_addr))
2099			return -EINVAL;
2100	}
2101
2102	vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
2103	vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
2104	vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
2105	vq->log_addr = a.log_guest_addr;
2106	vq->used = (void __user *)(unsigned long)a.used_user_addr;
2107
2108	return 0;
2109}
2110
2111static long vhost_vring_set_num_addr(struct vhost_dev *d,
2112				     struct vhost_virtqueue *vq,
2113				     unsigned int ioctl,
2114				     void __user *argp)
2115{
2116	long r;
2117
2118	mutex_lock(&vq->mutex);
2119
2120	switch (ioctl) {
2121	case VHOST_SET_VRING_NUM:
2122		r = vhost_vring_set_num(d, vq, argp);
2123		break;
2124	case VHOST_SET_VRING_ADDR:
2125		r = vhost_vring_set_addr(d, vq, argp);
2126		break;
2127	default:
2128		BUG();
2129	}
2130
2131	mutex_unlock(&vq->mutex);
2132
2133	return r;
2134}
2135long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
2136{
2137	struct file *eventfp, *filep = NULL;
2138	bool pollstart = false, pollstop = false;
2139	struct eventfd_ctx *ctx = NULL;
2140	struct vhost_virtqueue *vq;
2141	struct vhost_vring_state s;
2142	struct vhost_vring_file f;
2143	u32 idx;
2144	long r;
2145
2146	r = vhost_get_vq_from_user(d, argp, &vq, &idx);
2147	if (r < 0)
2148		return r;
2149
2150	if (ioctl == VHOST_SET_VRING_NUM ||
2151	    ioctl == VHOST_SET_VRING_ADDR) {
2152		return vhost_vring_set_num_addr(d, vq, ioctl, argp);
2153	}
2154
2155	mutex_lock(&vq->mutex);
2156
2157	switch (ioctl) {
2158	case VHOST_SET_VRING_BASE:
2159		/* Moving base with an active backend?
2160		 * You don't want to do that. */
2161		if (vq->private_data) {
2162			r = -EBUSY;
2163			break;
2164		}
2165		if (copy_from_user(&s, argp, sizeof s)) {
2166			r = -EFAULT;
2167			break;
2168		}
2169		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
2170			vq->next_avail_head = vq->last_avail_idx =
2171					      s.num & 0xffff;
2172			vq->last_used_idx = (s.num >> 16) & 0xffff;
2173		} else {
2174			if (s.num > 0xffff) {
2175				r = -EINVAL;
2176				break;
2177			}
2178			vq->next_avail_head = vq->last_avail_idx = s.num;
2179		}
2180		/* Forget the cached index value. */
2181		vq->avail_idx = vq->last_avail_idx;
2182		break;
2183	case VHOST_GET_VRING_BASE:
2184		s.index = idx;
2185		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
2186			s.num = (u32)vq->last_avail_idx | ((u32)vq->last_used_idx << 16);
2187		else
2188			s.num = vq->last_avail_idx;
2189		if (copy_to_user(argp, &s, sizeof s))
2190			r = -EFAULT;
2191		break;
2192	case VHOST_SET_VRING_KICK:
2193		if (copy_from_user(&f, argp, sizeof f)) {
2194			r = -EFAULT;
2195			break;
2196		}
2197		eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd);
2198		if (IS_ERR(eventfp)) {
2199			r = PTR_ERR(eventfp);
2200			break;
2201		}
2202		if (eventfp != vq->kick) {
2203			pollstop = (filep = vq->kick) != NULL;
2204			pollstart = (vq->kick = eventfp) != NULL;
2205		} else
2206			filep = eventfp;
2207		break;
2208	case VHOST_SET_VRING_CALL:
2209		if (copy_from_user(&f, argp, sizeof f)) {
2210			r = -EFAULT;
2211			break;
2212		}
2213		ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
2214		if (IS_ERR(ctx)) {
2215			r = PTR_ERR(ctx);
2216			break;
2217		}
2218
2219		swap(ctx, vq->call_ctx.ctx);
2220		break;
2221	case VHOST_SET_VRING_ERR:
2222		if (copy_from_user(&f, argp, sizeof f)) {
2223			r = -EFAULT;
2224			break;
2225		}
2226		ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
2227		if (IS_ERR(ctx)) {
2228			r = PTR_ERR(ctx);
2229			break;
2230		}
2231		swap(ctx, vq->error_ctx);
2232		break;
2233	case VHOST_SET_VRING_ENDIAN:
2234		r = vhost_set_vring_endian(vq, argp);
2235		break;
2236	case VHOST_GET_VRING_ENDIAN:
2237		r = vhost_get_vring_endian(vq, idx, argp);
2238		break;
2239	case VHOST_SET_VRING_BUSYLOOP_TIMEOUT:
2240		if (copy_from_user(&s, argp, sizeof(s))) {
2241			r = -EFAULT;
2242			break;
2243		}
2244		vq->busyloop_timeout = s.num;
2245		break;
2246	case VHOST_GET_VRING_BUSYLOOP_TIMEOUT:
2247		s.index = idx;
2248		s.num = vq->busyloop_timeout;
2249		if (copy_to_user(argp, &s, sizeof(s)))
2250			r = -EFAULT;
2251		break;
2252	default:
2253		r = -ENOIOCTLCMD;
2254	}
2255
2256	if (pollstop && vq->handle_kick)
2257		vhost_poll_stop(&vq->poll);
2258
2259	if (!IS_ERR_OR_NULL(ctx))
2260		eventfd_ctx_put(ctx);
2261	if (filep)
2262		fput(filep);
2263
2264	if (pollstart && vq->handle_kick)
2265		r = vhost_poll_start(&vq->poll, vq->kick);
2266
2267	mutex_unlock(&vq->mutex);
2268
2269	if (pollstop && vq->handle_kick)
2270		vhost_dev_flush(vq->poll.dev);
2271	return r;
2272}
2273EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
2274
2275int vhost_init_device_iotlb(struct vhost_dev *d)
2276{
2277	struct vhost_iotlb *niotlb, *oiotlb;
2278	int i;
2279
2280	niotlb = iotlb_alloc();
2281	if (!niotlb)
2282		return -ENOMEM;
2283
2284	oiotlb = d->iotlb;
2285	d->iotlb = niotlb;
2286
2287	for (i = 0; i < d->nvqs; ++i) {
2288		struct vhost_virtqueue *vq = d->vqs[i];
2289
2290		mutex_lock(&vq->mutex);
2291		vq->iotlb = niotlb;
2292		__vhost_vq_meta_reset(vq);
2293		mutex_unlock(&vq->mutex);
2294	}
2295
2296	vhost_iotlb_free(oiotlb);
2297
2298	return 0;
2299}
2300EXPORT_SYMBOL_GPL(vhost_init_device_iotlb);
2301
2302/* Caller must have device mutex */
2303long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
2304{
2305	struct eventfd_ctx *ctx;
2306	u64 p;
2307	long r;
2308	int i, fd;
2309
2310	/* If you are not the owner, you can become one */
2311	if (ioctl == VHOST_SET_OWNER) {
2312		r = vhost_dev_set_owner(d);
2313		goto done;
2314	}
2315
2316#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL
2317	if (ioctl == VHOST_SET_FORK_FROM_OWNER) {
2318		/* Only allow modification before owner is set */
2319		if (vhost_dev_has_owner(d)) {
2320			r = -EBUSY;
2321			goto done;
2322		}
2323		u8 fork_owner_val;
2324
2325		if (get_user(fork_owner_val, (u8 __user *)argp)) {
2326			r = -EFAULT;
2327			goto done;
2328		}
2329		if (fork_owner_val != VHOST_FORK_OWNER_TASK &&
2330		    fork_owner_val != VHOST_FORK_OWNER_KTHREAD) {
2331			r = -EINVAL;
2332			goto done;
2333		}
2334		d->fork_owner = !!fork_owner_val;
2335		r = 0;
2336		goto done;
2337	}
2338	if (ioctl == VHOST_GET_FORK_FROM_OWNER) {
2339		u8 fork_owner_val = d->fork_owner;
2340
2341		if (fork_owner_val != VHOST_FORK_OWNER_TASK &&
2342		    fork_owner_val != VHOST_FORK_OWNER_KTHREAD) {
2343			r = -EINVAL;
2344			goto done;
2345		}
2346		if (put_user(fork_owner_val, (u8 __user *)argp)) {
2347			r = -EFAULT;
2348			goto done;
2349		}
2350		r = 0;
2351		goto done;
2352	}
2353#endif
2354
2355	/* You must be the owner to do anything else */
2356	r = vhost_dev_check_owner(d);
2357	if (r)
2358		goto done;
2359
2360	switch (ioctl) {
2361	case VHOST_SET_MEM_TABLE:
2362		r = vhost_set_memory(d, argp);
2363		break;
2364	case VHOST_SET_LOG_BASE:
2365		if (copy_from_user(&p, argp, sizeof p)) {
2366			r = -EFAULT;
2367			break;
2368		}
2369		if ((u64)(unsigned long)p != p) {
2370			r = -EFAULT;
2371			break;
2372		}
2373		for (i = 0; i < d->nvqs; ++i) {
2374			struct vhost_virtqueue *vq;
2375			void __user *base = (void __user *)(unsigned long)p;
2376			vq = d->vqs[i];
2377			mutex_lock(&vq->mutex);
2378			/* If ring is inactive, will check when it's enabled. */
2379			if (vq->private_data && !vq_log_access_ok(vq, base))
2380				r = -EFAULT;
2381			else
2382				vq->log_base = base;
2383			mutex_unlock(&vq->mutex);
2384		}
2385		break;
2386	case VHOST_SET_LOG_FD:
2387		r = get_user(fd, (int __user *)argp);
2388		if (r < 0)
2389			break;
2390		ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
2391		if (IS_ERR(ctx)) {
2392			r = PTR_ERR(ctx);
2393			break;
2394		}
2395		swap(ctx, d->log_ctx);
2396		for (i = 0; i < d->nvqs; ++i) {
2397			mutex_lock(&d->vqs[i]->mutex);
2398			d->vqs[i]->log_ctx = d->log_ctx;
2399			mutex_unlock(&d->vqs[i]->mutex);
2400		}
2401		if (ctx)
2402			eventfd_ctx_put(ctx);
2403		break;
2404	default:
2405		r = -ENOIOCTLCMD;
2406		break;
2407	}
2408done:
2409	return r;
2410}
2411EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
2412
2413/* TODO: This is really inefficient.  We need something like get_user()
2414 * (instruction directly accesses the data, with an exception table entry
2415 * returning -EFAULT). See Documentation/arch/x86/exception-tables.rst.
2416 */
2417static int set_bit_to_user(int nr, void __user *addr)
2418{
2419	unsigned long log = (unsigned long)addr;
2420	struct page *page;
2421	void *base;
2422	int bit = nr + (log % PAGE_SIZE) * 8;
2423	int r;
2424
2425	r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page);
2426	if (r < 0)
2427		return r;
2428	BUG_ON(r != 1);
2429	base = kmap_atomic(page);
2430	set_bit(bit, base);
2431	kunmap_atomic(base);
2432	unpin_user_pages_dirty_lock(&page, 1, true);
2433	return 0;
2434}
2435
2436static int log_write(void __user *log_base,
2437		     u64 write_address, u64 write_length)
2438{
2439	u64 write_page = write_address / VHOST_PAGE_SIZE;
2440	int r;
2441
2442	if (!write_length)
2443		return 0;
2444	write_length += write_address % VHOST_PAGE_SIZE;
2445	for (;;) {
2446		u64 base = (u64)(unsigned long)log_base;
2447		u64 log = base + write_page / 8;
2448		int bit = write_page % 8;
2449		if ((u64)(unsigned long)log != log)
2450			return -EFAULT;
2451		r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
2452		if (r < 0)
2453			return r;
2454		if (write_length <= VHOST_PAGE_SIZE)
2455			break;
2456		write_length -= VHOST_PAGE_SIZE;
2457		write_page += 1;
2458	}
2459	return r;
2460}
2461
2462static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)
2463{
2464	struct vhost_iotlb *umem = vq->umem;
2465	struct vhost_iotlb_map *u;
2466	u64 start, end, l, min;
2467	int r;
2468	bool hit = false;
2469
2470	while (len) {
2471		min = len;
2472		/* More than one GPAs can be mapped into a single HVA. So
2473		 * iterate all possible umems here to be safe.
2474		 */
2475		list_for_each_entry(u, &umem->list, link) {
2476			if (u->addr > hva - 1 + len ||
2477			    u->addr - 1 + u->size < hva)
2478				continue;
2479			start = max(u->addr, hva);
2480			end = min(u->addr - 1 + u->size, hva - 1 + len);
2481			l = end - start + 1;
2482			r = log_write(vq->log_base,
2483				      u->start + start - u->addr,
2484				      l);
2485			if (r < 0)
2486				return r;
2487			hit = true;
2488			min = min(l, min);
2489		}
2490
2491		if (!hit)
2492			return -EFAULT;
2493
2494		len -= min;
2495		hva += min;
2496	}
2497
2498	return 0;
2499}
2500
2501static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len)
2502{
2503	struct iovec *iov = vq->log_iov;
2504	int i, ret;
2505
2506	if (!vq->iotlb)
2507		return log_write(vq->log_base, vq->log_addr + used_offset, len);
2508
2509	ret = translate_desc(vq, (uintptr_t)vq->used + used_offset,
2510			     len, iov, 64, VHOST_ACCESS_WO);
2511	if (ret < 0)
2512		return ret;
2513
2514	for (i = 0; i < ret; i++) {
2515		ret = log_write_hva(vq,	(uintptr_t)iov[i].iov_base,
2516				    iov[i].iov_len);
2517		if (ret)
2518			return ret;
2519	}
2520
2521	return 0;
2522}
2523
2524/*
2525 * vhost_log_write() - Log in dirty page bitmap
2526 * @vq:      vhost virtqueue.
2527 * @log:     Array of dirty memory in GPA.
2528 * @log_num: Size of vhost_log arrary.
2529 * @len:     The total length of memory buffer to log in the dirty bitmap.
2530 *	     Some drivers may only partially use pages shared via the last
2531 *	     vring descriptor (i.e. vhost-net RX buffer).
2532 *	     Use (len == U64_MAX) to indicate the driver would log all
2533 *           pages of vring descriptors.
2534 * @iov:     Array of dirty memory in HVA.
2535 * @count:   Size of iovec array.
2536 */
2537int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
2538		    unsigned int log_num, u64 len, struct iovec *iov, int count)
2539{
2540	int i, r;
2541
2542	/* Make sure data written is seen before log. */
2543	smp_wmb();
2544
2545	if (vq->iotlb) {
2546		for (i = 0; i < count; i++) {
2547			r = log_write_hva(vq, (uintptr_t)iov[i].iov_base,
2548					  iov[i].iov_len);
2549			if (r < 0)
2550				return r;
2551		}
2552		return 0;
2553	}
2554
2555	for (i = 0; i < log_num; ++i) {
2556		u64 l = min(log[i].len, len);
2557		r = log_write(vq->log_base, log[i].addr, l);
2558		if (r < 0)
2559			return r;
2560
2561		if (len != U64_MAX)
2562			len -= l;
2563	}
2564
2565	if (vq->log_ctx)
2566		eventfd_signal(vq->log_ctx);
2567
2568	return 0;
2569}
2570EXPORT_SYMBOL_GPL(vhost_log_write);
2571
2572static int vhost_update_used_flags(struct vhost_virtqueue *vq)
2573{
2574	void __user *used;
2575	if (vhost_put_used_flags(vq))
2576		return -EFAULT;
2577	if (unlikely(vq->log_used)) {
2578		/* Make sure the flag is seen before log. */
2579		smp_wmb();
2580		/* Log used flag write. */
2581		used = &vq->used->flags;
2582		log_used(vq, (used - (void __user *)vq->used),
2583			 sizeof vq->used->flags);
2584		if (vq->log_ctx)
2585			eventfd_signal(vq->log_ctx);
2586	}
2587	return 0;
2588}
2589
2590static int vhost_update_avail_event(struct vhost_virtqueue *vq)
2591{
2592	if (vhost_put_avail_event(vq))
2593		return -EFAULT;
2594	if (unlikely(vq->log_used)) {
2595		void __user *used;
2596		/* Make sure the event is seen before log. */
2597		smp_wmb();
2598		/* Log avail event write */
2599		used = vhost_avail_event(vq);
2600		log_used(vq, (used - (void __user *)vq->used),
2601			 sizeof *vhost_avail_event(vq));
2602		if (vq->log_ctx)
2603			eventfd_signal(vq->log_ctx);
2604	}
2605	return 0;
2606}
2607
2608int vhost_vq_init_access(struct vhost_virtqueue *vq)
2609{
2610	__virtio16 last_used_idx;
2611	int r;
2612	bool is_le = vq->is_le;
2613
2614	if (!vq->private_data)
2615		return 0;
2616
2617	vhost_init_is_le(vq);
2618
2619	r = vhost_update_used_flags(vq);
2620	if (r)
2621		goto err;
2622	vq->signalled_used_valid = false;
2623	if (!vq->iotlb &&
2624	    !access_ok(&vq->used->idx, sizeof vq->used->idx)) {
2625		r = -EFAULT;
2626		goto err;
2627	}
2628	r = vhost_get_used_idx(vq, &last_used_idx);
2629	if (r) {
2630		vq_err(vq, "Can't access used idx at %p\n",
2631		       &vq->used->idx);
2632		goto err;
2633	}
2634	vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
2635	return 0;
2636
2637err:
2638	vq->is_le = is_le;
2639	return r;
2640}
2641EXPORT_SYMBOL_GPL(vhost_vq_init_access);
2642
2643static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
2644			  struct iovec iov[], int iov_size, int access)
2645{
2646	const struct vhost_iotlb_map *map;
2647	struct vhost_dev *dev = vq->dev;
2648	struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem;
2649	struct iovec *_iov;
2650	u64 s = 0, last = addr + len - 1;
2651	int ret = 0;
2652
2653	while ((u64)len > s) {
2654		u64 size;
2655		if (unlikely(ret >= iov_size)) {
2656			ret = -ENOBUFS;
2657			break;
2658		}
2659
2660		map = vhost_iotlb_itree_first(umem, addr, last);
2661		if (map == NULL || map->start > addr) {
2662			if (umem != dev->iotlb) {
2663				ret = -EFAULT;
2664				break;
2665			}
2666			ret = -EAGAIN;
2667			break;
2668		} else if (!(map->perm & access)) {
2669			ret = -EPERM;
2670			break;
2671		}
2672
2673		_iov = iov + ret;
2674		size = map->size - addr + map->start;
2675		_iov->iov_len = min((u64)len - s, size);
2676		_iov->iov_base = (void __user *)(unsigned long)
2677				 (map->addr + addr - map->start);
2678		s += size;
2679		addr += size;
2680		++ret;
2681	}
2682
2683	if (ret == -EAGAIN)
2684		vhost_iotlb_miss(vq, addr, access);
2685	return ret;
2686}
2687
2688/* Each buffer in the virtqueues is actually a chain of descriptors.  This
2689 * function returns the next descriptor in the chain,
2690 * or -1U if we're at the end. */
2691static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)
2692{
2693	unsigned int next;
2694
2695	/* If this descriptor says it doesn't chain, we're done. */
2696	if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
2697		return -1U;
2698
2699	/* Check they're not leading us off end of descriptors. */
2700	next = vhost16_to_cpu(vq, READ_ONCE(desc->next));
2701	return next;
2702}
2703
2704static int get_indirect(struct vhost_virtqueue *vq,
2705			struct iovec iov[], unsigned int iov_size,
2706			unsigned int *out_num, unsigned int *in_num,
2707			struct vhost_log *log, unsigned int *log_num,
2708			struct vring_desc *indirect)
2709{
2710	struct vring_desc desc;
2711	unsigned int i = 0, count, found = 0;
2712	u32 len = vhost32_to_cpu(vq, indirect->len);
2713	struct iov_iter from;
2714	int ret, access;
2715
2716	/* Sanity check */
2717	if (unlikely(len % sizeof desc)) {
2718		vq_err(vq, "Invalid length in indirect descriptor: "
2719		       "len 0x%llx not multiple of 0x%zx\n",
2720		       (unsigned long long)len,
2721		       sizeof desc);
2722		return -EINVAL;
2723	}
2724
2725	ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
2726			     UIO_MAXIOV, VHOST_ACCESS_RO);
2727	if (unlikely(ret < 0)) {
2728		if (ret != -EAGAIN)
2729			vq_err(vq, "Translation failure %d in indirect.\n", ret);
2730		return ret;
2731	}
2732	iov_iter_init(&from, ITER_SOURCE, vq->indirect, ret, len);
2733	count = len / sizeof desc;
2734	/* Buffers are chained via a 16 bit next field, so
2735	 * we can have at most 2^16 of these. */
2736	if (unlikely(count > USHRT_MAX + 1)) {
2737		vq_err(vq, "Indirect buffer length too big: %d\n",
2738		       indirect->len);
2739		return -E2BIG;
2740	}
2741
2742	do {
2743		unsigned iov_count = *in_num + *out_num;
2744		if (unlikely(++found > count)) {
2745			vq_err(vq, "Loop detected: last one at %u "
2746			       "indirect size %u\n",
2747			       i, count);
2748			return -EINVAL;
2749		}
2750		if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) {
2751			vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
2752			       i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
2753			return -EINVAL;
2754		}
2755		if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
2756			vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
2757			       i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
2758			return -EINVAL;
2759		}
2760
2761		if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2762			access = VHOST_ACCESS_WO;
2763		else
2764			access = VHOST_ACCESS_RO;
2765
2766		ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2767				     vhost32_to_cpu(vq, desc.len), iov + iov_count,
2768				     iov_size - iov_count, access);
2769		if (unlikely(ret < 0)) {
2770			if (ret != -EAGAIN)
2771				vq_err(vq, "Translation failure %d indirect idx %d\n",
2772					ret, i);
2773			return ret;
2774		}
2775		/* If this is an input descriptor, increment that count. */
2776		if (access == VHOST_ACCESS_WO) {
2777			*in_num += ret;
2778			if (unlikely(log && ret)) {
2779				log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2780				log[*log_num].len = vhost32_to_cpu(vq, desc.len);
2781				++*log_num;
2782			}
2783		} else {
2784			/* If it's an output descriptor, they're all supposed
2785			 * to come before any input descriptors. */
2786			if (unlikely(*in_num)) {
2787				vq_err(vq, "Indirect descriptor "
2788				       "has out after in: idx %d\n", i);
2789				return -EINVAL;
2790			}
2791			*out_num += ret;
2792		}
2793	} while ((i = next_desc(vq, &desc)) != -1);
2794	return 0;
2795}
2796
2797/**
2798 * vhost_get_vq_desc_n - Fetch the next available descriptor chain and build iovecs
2799 * @vq: target virtqueue
2800 * @iov: array that receives the scatter/gather segments
2801 * @iov_size: capacity of @iov in elements
2802 * @out_num: the number of output segments
2803 * @in_num: the number of input segments
2804 * @log: optional array to record addr/len for each writable segment; NULL if unused
2805 * @log_num: optional output; number of entries written to @log when provided
2806 * @ndesc: optional output; number of descriptors consumed from the available ring
2807 *         (useful for rollback via vhost_discard_vq_desc)
2808 *
2809 * Extracts one available descriptor chain from @vq and translates guest addresses
2810 * into host iovecs.
2811 *
2812 * On success, advances @vq->last_avail_idx by 1 and @vq->next_avail_head by the
2813 * number of descriptors consumed (also stored via @ndesc when non-NULL).
2814 *
2815 * Return:
2816 * - head index in [0, @vq->num) on success;
2817 * - @vq->num if no descriptor is currently available;
2818 * - negative errno on failure
2819 */
2820int vhost_get_vq_desc_n(struct vhost_virtqueue *vq,
2821			struct iovec iov[], unsigned int iov_size,
2822			unsigned int *out_num, unsigned int *in_num,
2823			struct vhost_log *log, unsigned int *log_num,
2824			unsigned int *ndesc)
2825{
2826	bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER);
2827	struct vring_desc desc;
2828	unsigned int i, head, found = 0;
2829	u16 last_avail_idx = vq->last_avail_idx;
2830	__virtio16 ring_head;
2831	int ret, access, c = 0;
2832
2833	if (vq->avail_idx == vq->last_avail_idx) {
2834		ret = vhost_get_avail_idx(vq);
2835		if (unlikely(ret < 0))
2836			return ret;
2837
2838		if (!ret)
2839			return vq->num;
2840	}
2841
2842	if (in_order)
2843		head = vq->next_avail_head & (vq->num - 1);
2844	else {
2845		/* Grab the next descriptor number they're
2846		 * advertising, and increment the index we've seen. */
2847		if (unlikely(vhost_get_avail_head(vq, &ring_head,
2848						  last_avail_idx))) {
2849			vq_err(vq, "Failed to read head: idx %d address %p\n",
2850				last_avail_idx,
2851				&vq->avail->ring[last_avail_idx % vq->num]);
2852			return -EFAULT;
2853		}
2854		head = vhost16_to_cpu(vq, ring_head);
2855	}
2856
2857	/* If their number is silly, that's an error. */
2858	if (unlikely(head >= vq->num)) {
2859		vq_err(vq, "Guest says index %u > %u is available",
2860		       head, vq->num);
2861		return -EINVAL;
2862	}
2863
2864	/* When we start there are none of either input nor output. */
2865	*out_num = *in_num = 0;
2866	if (unlikely(log))
2867		*log_num = 0;
2868
2869	i = head;
2870	do {
2871		unsigned iov_count = *in_num + *out_num;
2872		if (unlikely(i >= vq->num)) {
2873			vq_err(vq, "Desc index is %u > %u, head = %u",
2874			       i, vq->num, head);
2875			return -EINVAL;
2876		}
2877		if (unlikely(++found > vq->num)) {
2878			vq_err(vq, "Loop detected: last one at %u "
2879			       "vq size %u head %u\n",
2880			       i, vq->num, head);
2881			return -EINVAL;
2882		}
2883		ret = vhost_get_desc(vq, &desc, i);
2884		if (unlikely(ret)) {
2885			vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
2886			       i, vq->desc + i);
2887			return -EFAULT;
2888		}
2889		if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
2890			ret = get_indirect(vq, iov, iov_size,
2891					   out_num, in_num,
2892					   log, log_num, &desc);
2893			if (unlikely(ret < 0)) {
2894				if (ret != -EAGAIN)
2895					vq_err(vq, "Failure detected "
2896						"in indirect descriptor at idx %d\n", i);
2897				return ret;
2898			}
2899			++c;
2900			continue;
2901		}
2902
2903		if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2904			access = VHOST_ACCESS_WO;
2905		else
2906			access = VHOST_ACCESS_RO;
2907		ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2908				     vhost32_to_cpu(vq, desc.len), iov + iov_count,
2909				     iov_size - iov_count, access);
2910		if (unlikely(ret < 0)) {
2911			if (ret != -EAGAIN)
2912				vq_err(vq, "Translation failure %d descriptor idx %d\n",
2913					ret, i);
2914			return ret;
2915		}
2916		if (access == VHOST_ACCESS_WO) {
2917			/* If this is an input descriptor,
2918			 * increment that count. */
2919			*in_num += ret;
2920			if (unlikely(log && ret)) {
2921				log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2922				log[*log_num].len = vhost32_to_cpu(vq, desc.len);
2923				++*log_num;
2924			}
2925		} else {
2926			/* If it's an output descriptor, they're all supposed
2927			 * to come before any input descriptors. */
2928			if (unlikely(*in_num)) {
2929				vq_err(vq, "Descriptor has out after in: "
2930				       "idx %d\n", i);
2931				return -EINVAL;
2932			}
2933			*out_num += ret;
2934		}
2935		++c;
2936	} while ((i = next_desc(vq, &desc)) != -1);
2937
2938	/* On success, increment avail index. */
2939	vq->last_avail_idx++;
2940	vq->next_avail_head += c;
2941
2942	if (ndesc)
2943		*ndesc = c;
2944
2945	/* Assume notifications from guest are disabled at this point,
2946	 * if they aren't we would need to update avail_event index. */
2947	BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
2948	return head;
2949}
2950EXPORT_SYMBOL_GPL(vhost_get_vq_desc_n);
2951
2952/* This looks in the virtqueue and for the first available buffer, and converts
2953 * it to an iovec for convenient access.  Since descriptors consist of some
2954 * number of output then some number of input descriptors, it's actually two
2955 * iovecs, but we pack them into one and note how many of each there were.
2956 *
2957 * This function returns the descriptor number found, or vq->num (which is
2958 * never a valid descriptor number) if none was found.  A negative code is
2959 * returned on error.
2960 */
2961int vhost_get_vq_desc(struct vhost_virtqueue *vq,
2962		      struct iovec iov[], unsigned int iov_size,
2963		      unsigned int *out_num, unsigned int *in_num,
2964		      struct vhost_log *log, unsigned int *log_num)
2965{
2966	return vhost_get_vq_desc_n(vq, iov, iov_size, out_num, in_num,
2967				   log, log_num, NULL);
2968}
2969EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
2970
2971/**
2972 * vhost_discard_vq_desc - Reverse the effect of vhost_get_vq_desc_n()
2973 * @vq: target virtqueue
2974 * @nbufs: number of buffers to roll back
2975 * @ndesc: number of descriptors to roll back
2976 *
2977 * Rewinds the internal consumer cursors after a failed attempt to use buffers
2978 * returned by vhost_get_vq_desc_n().
2979 */
2980void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int nbufs,
2981			   unsigned int ndesc)
2982{
2983	vq->next_avail_head -= ndesc;
2984	vq->last_avail_idx -= nbufs;
2985}
2986EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
2987
2988/* After we've used one of their buffers, we tell them about it.  We'll then
2989 * want to notify the guest, using eventfd. */
2990int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
2991{
2992	struct vring_used_elem heads = {
2993		cpu_to_vhost32(vq, head),
2994		cpu_to_vhost32(vq, len)
2995	};
2996	u16 nheads = 1;
2997
2998	return vhost_add_used_n(vq, &heads, &nheads, 1);
2999}
3000EXPORT_SYMBOL_GPL(vhost_add_used);
3001
3002static int __vhost_add_used_n(struct vhost_virtqueue *vq,
3003			    struct vring_used_elem *heads,
3004			    unsigned count)
3005{
3006	vring_used_elem_t __user *used;
3007	u16 old, new;
3008	int start;
3009
3010	start = vq->last_used_idx & (vq->num - 1);
3011	used = vq->used->ring + start;
3012	if (vhost_put_used(vq, heads, start, count)) {
3013		vq_err(vq, "Failed to write used");
3014		return -EFAULT;
3015	}
3016	if (unlikely(vq->log_used)) {
3017		/* Make sure data is seen before log. */
3018		smp_wmb();
3019		/* Log used ring entry write. */
3020		log_used(vq, ((void __user *)used - (void __user *)vq->used),
3021			 count * sizeof *used);
3022	}
3023	old = vq->last_used_idx;
3024	new = (vq->last_used_idx += count);
3025	/* If the driver never bothers to signal in a very long while,
3026	 * used index might wrap around. If that happens, invalidate
3027	 * signalled_used index we stored. TODO: make sure driver
3028	 * signals at least once in 2^16 and remove this. */
3029	if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
3030		vq->signalled_used_valid = false;
3031	return 0;
3032}
3033
3034static int vhost_add_used_n_ooo(struct vhost_virtqueue *vq,
3035				struct vring_used_elem *heads,
3036				unsigned count)
3037{
3038	int start, n, r;
3039
3040	start = vq->last_used_idx & (vq->num - 1);
3041	n = vq->num - start;
3042	if (n < count) {
3043		r = __vhost_add_used_n(vq, heads, n);
3044		if (r < 0)
3045			return r;
3046		heads += n;
3047		count -= n;
3048	}
3049	return __vhost_add_used_n(vq, heads, count);
3050}
3051
3052static int vhost_add_used_n_in_order(struct vhost_virtqueue *vq,
3053				     struct vring_used_elem *heads,
3054				     const u16 *nheads,
3055				     unsigned count)
3056{
3057	vring_used_elem_t __user *used;
3058	u16 old, new = vq->last_used_idx;
3059	int start, i;
3060
3061	if (!nheads)
3062		return -EINVAL;
3063
3064	start = vq->last_used_idx & (vq->num - 1);
3065	used = vq->used->ring + start;
3066
3067	for (i = 0; i < count; i++) {
3068		if (vhost_put_used(vq, &heads[i], start, 1)) {
3069			vq_err(vq, "Failed to write used");
3070			return -EFAULT;
3071		}
3072		start += nheads[i];
3073		new += nheads[i];
3074		if (start >= vq->num)
3075			start -= vq->num;
3076	}
3077
3078	if (unlikely(vq->log_used)) {
3079		/* Make sure data is seen before log. */
3080		smp_wmb();
3081		/* Log used ring entry write. */
3082		log_used(vq, ((void __user *)used - (void __user *)vq->used),
3083			 (vq->num - start) * sizeof *used);
3084		if (start + count > vq->num)
3085			log_used(vq, 0,
3086				 (start + count - vq->num) * sizeof *used);
3087	}
3088
3089	old = vq->last_used_idx;
3090	vq->last_used_idx = new;
3091	/* If the driver never bothers to signal in a very long while,
3092	 * used index might wrap around. If that happens, invalidate
3093	 * signalled_used index we stored. TODO: make sure driver
3094	 * signals at least once in 2^16 and remove this. */
3095	if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
3096		vq->signalled_used_valid = false;
3097	return 0;
3098}
3099
3100/* After we've used one of their buffers, we tell them about it.  We'll then
3101 * want to notify the guest, using eventfd. */
3102int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
3103		     u16 *nheads, unsigned count)
3104{
3105	bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER);
3106	int r;
3107
3108	if (!in_order || !nheads)
3109		r = vhost_add_used_n_ooo(vq, heads, count);
3110	else
3111		r = vhost_add_used_n_in_order(vq, heads, nheads, count);
3112
3113	if (r < 0)
3114		return r;
3115
3116	/* Make sure buffer is written before we update index. */
3117	smp_wmb();
3118	if (vhost_put_used_idx(vq)) {
3119		vq_err(vq, "Failed to increment used idx");
3120		return -EFAULT;
3121	}
3122	if (unlikely(vq->log_used)) {
3123		/* Make sure used idx is seen before log. */
3124		smp_wmb();
3125		/* Log used index update. */
3126		log_used(vq, offsetof(struct vring_used, idx),
3127			 sizeof vq->used->idx);
3128		if (vq->log_ctx)
3129			eventfd_signal(vq->log_ctx);
3130	}
3131	return r;
3132}
3133EXPORT_SYMBOL_GPL(vhost_add_used_n);
3134
3135static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
3136{
3137	__u16 old, new;
3138	__virtio16 event;
3139	bool v;
3140	/* Flush out used index updates. This is paired
3141	 * with the barrier that the Guest executes when enabling
3142	 * interrupts. */
3143	smp_mb();
3144
3145	if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
3146	    unlikely(vq->avail_idx == vq->last_avail_idx))
3147		return true;
3148
3149	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
3150		__virtio16 flags;
3151		if (vhost_get_avail_flags(vq, &flags)) {
3152			vq_err(vq, "Failed to get flags");
3153			return true;
3154		}
3155		return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
3156	}
3157	old = vq->signalled_used;
3158	v = vq->signalled_used_valid;
3159	new = vq->signalled_used = vq->last_used_idx;
3160	vq->signalled_used_valid = true;
3161
3162	if (unlikely(!v))
3163		return true;
3164
3165	if (vhost_get_used_event(vq, &event)) {
3166		vq_err(vq, "Failed to get used event idx");
3167		return true;
3168	}
3169	return vring_need_event(vhost16_to_cpu(vq, event), new, old);
3170}
3171
3172/* This actually signals the guest, using eventfd. */
3173void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
3174{
3175	/* Signal the Guest tell them we used something up. */
3176	if (vq->call_ctx.ctx && vhost_notify(dev, vq))
3177		eventfd_signal(vq->call_ctx.ctx);
3178}
3179EXPORT_SYMBOL_GPL(vhost_signal);
3180
3181/* And here's the combo meal deal.  Supersize me! */
3182void vhost_add_used_and_signal(struct vhost_dev *dev,
3183			       struct vhost_virtqueue *vq,
3184			       unsigned int head, int len)
3185{
3186	vhost_add_used(vq, head, len);
3187	vhost_signal(dev, vq);
3188}
3189EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
3190
3191/* multi-buffer version of vhost_add_used_and_signal */
3192void vhost_add_used_and_signal_n(struct vhost_dev *dev,
3193				 struct vhost_virtqueue *vq,
3194				 struct vring_used_elem *heads,
3195				 u16 *nheads,
3196				 unsigned count)
3197{
3198	vhost_add_used_n(vq, heads, nheads, count);
3199	vhost_signal(dev, vq);
3200}
3201EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
3202
3203/* return true if we're sure that available ring is empty */
3204bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
3205{
3206	int r;
3207
3208	if (vq->avail_idx != vq->last_avail_idx)
3209		return false;
3210
3211	r = vhost_get_avail_idx(vq);
3212
3213	/* Note: we treat error as non-empty here */
3214	return r == 0;
3215}
3216EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
3217
3218/* OK, now we need to know about added descriptors. */
3219bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
3220{
3221	int r;
3222
3223	if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
3224		return false;
3225	vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
3226	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
3227		r = vhost_update_used_flags(vq);
3228		if (r) {
3229			vq_err(vq, "Failed to enable notification at %p: %d\n",
3230			       &vq->used->flags, r);
3231			return false;
3232		}
3233	} else {
3234		r = vhost_update_avail_event(vq);
3235		if (r) {
3236			vq_err(vq, "Failed to update avail event index at %p: %d\n",
3237			       vhost_avail_event(vq), r);
3238			return false;
3239		}
3240	}
3241	/* They could have slipped one in as we were doing that: make
3242	 * sure it's written, then check again. */
3243	smp_mb();
3244
3245	r = vhost_get_avail_idx(vq);
3246	/* Note: we treat error as empty here */
3247	if (unlikely(r < 0))
3248		return false;
3249
3250	return r;
3251}
3252EXPORT_SYMBOL_GPL(vhost_enable_notify);
3253
3254/* We don't need to be notified again. */
3255void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
3256{
3257	int r;
3258
3259	if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
3260		return;
3261	vq->used_flags |= VRING_USED_F_NO_NOTIFY;
3262	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
3263		r = vhost_update_used_flags(vq);
3264		if (r)
3265			vq_err(vq, "Failed to disable notification at %p: %d\n",
3266			       &vq->used->flags, r);
3267	}
3268}
3269EXPORT_SYMBOL_GPL(vhost_disable_notify);
3270
3271/* Create a new message. */
3272struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type)
3273{
3274	/* Make sure all padding within the structure is initialized. */
3275	struct vhost_msg_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
3276	if (!node)
3277		return NULL;
3278
3279	node->vq = vq;
3280	node->msg.type = type;
3281	return node;
3282}
3283EXPORT_SYMBOL_GPL(vhost_new_msg);
3284
3285void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head,
3286		       struct vhost_msg_node *node)
3287{
3288	spin_lock(&dev->iotlb_lock);
3289	list_add_tail(&node->node, head);
3290	spin_unlock(&dev->iotlb_lock);
3291
3292	wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
3293}
3294EXPORT_SYMBOL_GPL(vhost_enqueue_msg);
3295
3296struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev,
3297					 struct list_head *head)
3298{
3299	struct vhost_msg_node *node = NULL;
3300
3301	spin_lock(&dev->iotlb_lock);
3302	if (!list_empty(head)) {
3303		node = list_first_entry(head, struct vhost_msg_node,
3304					node);
3305		list_del(&node->node);
3306	}
3307	spin_unlock(&dev->iotlb_lock);
3308
3309	return node;
3310}
3311EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
3312
3313void vhost_set_backend_features(struct vhost_dev *dev, u64 features)
3314{
3315	struct vhost_virtqueue *vq;
3316	int i;
3317
3318	mutex_lock(&dev->mutex);
3319	for (i = 0; i < dev->nvqs; ++i) {
3320		vq = dev->vqs[i];
3321		mutex_lock(&vq->mutex);
3322		vq->acked_backend_features = features;
3323		mutex_unlock(&vq->mutex);
3324	}
3325	mutex_unlock(&dev->mutex);
3326}
3327EXPORT_SYMBOL_GPL(vhost_set_backend_features);
3328
3329static int __init vhost_init(void)
3330{
3331	return 0;
3332}
3333
3334static void __exit vhost_exit(void)
3335{
3336}
3337
3338module_init(vhost_init);
3339module_exit(vhost_exit);
3340
3341MODULE_VERSION("0.0.1");
3342MODULE_LICENSE("GPL v2");
3343MODULE_AUTHOR("Michael S. Tsirkin");
3344MODULE_DESCRIPTION("Host kernel accelerator for virtio");