vringh.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Helpers for the host side of a virtio ring.
   4 *
   5 * Since these may be in userspace, we use (inline) accessors.
   6 */
   7#include <linux/compiler.h>
   8#include <linux/module.h>
   9#include <linux/vringh.h>
  10#include <linux/virtio_ring.h>
  11#include <linux/kernel.h>
  12#include <linux/ratelimit.h>
  13#include <linux/uaccess.h>
  14#include <linux/slab.h>
  15#include <linux/export.h>
  16#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
  17#include <linux/bvec.h>
  18#include <linux/highmem.h>
  19#include <linux/vhost_iotlb.h>
  20#endif
  21#include <uapi/linux/virtio_config.h>
  22
  23static __printf(1,2) __cold void vringh_bad(const char *fmt, ...)
  24{
  25	static DEFINE_RATELIMIT_STATE(vringh_rs,
  26				      DEFAULT_RATELIMIT_INTERVAL,
  27				      DEFAULT_RATELIMIT_BURST);
  28	if (__ratelimit(&vringh_rs)) {
  29		va_list ap;
  30		va_start(ap, fmt);
  31		printk(KERN_NOTICE "vringh:");
  32		vprintk(fmt, ap);
  33		va_end(ap);
  34	}
  35}
  36
  37/* Returns vring->num if empty, -ve on error. */
  38static inline int __vringh_get_head(const struct vringh *vrh,
  39				    int (*getu16)(const struct vringh *vrh,
  40						  u16 *val, const __virtio16 *p),
  41				    u16 *last_avail_idx)
  42{
  43	u16 avail_idx, i, head;
  44	int err;
  45
  46	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);
  47	if (err) {
  48		vringh_bad("Failed to access avail idx at %p",
  49			   &vrh->vring.avail->idx);
  50		return err;
  51	}
  52
  53	if (*last_avail_idx == avail_idx)
  54		return vrh->vring.num;
  55
  56	/* Only get avail ring entries after they have been exposed by guest. */
  57	virtio_rmb(vrh->weak_barriers);
  58
  59	i = *last_avail_idx & (vrh->vring.num - 1);
  60
  61	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);
  62	if (err) {
  63		vringh_bad("Failed to read head: idx %d address %p",
  64			   *last_avail_idx, &vrh->vring.avail->ring[i]);
  65		return err;
  66	}
  67
  68	if (head >= vrh->vring.num) {
  69		vringh_bad("Guest says index %u > %u is available",
  70			   head, vrh->vring.num);
  71		return -EINVAL;
  72	}
  73
  74	(*last_avail_idx)++;
  75	return head;
  76}
  77
  78/**
  79 * vringh_kiov_advance - skip bytes from vring_kiov
  80 * @iov: an iov passed to vringh_getdesc_*() (updated as we consume)
  81 * @len: the maximum length to advance
  82 */
  83void vringh_kiov_advance(struct vringh_kiov *iov, size_t len)
  84{
  85	while (len && iov->i < iov->used) {
  86		size_t partlen = min(iov->iov[iov->i].iov_len, len);
  87
  88		iov->consumed += partlen;
  89		iov->iov[iov->i].iov_len -= partlen;
  90		iov->iov[iov->i].iov_base += partlen;
  91
  92		if (!iov->iov[iov->i].iov_len) {
  93			/* Fix up old iov element then increment. */
  94			iov->iov[iov->i].iov_len = iov->consumed;
  95			iov->iov[iov->i].iov_base -= iov->consumed;
  96
  97			iov->consumed = 0;
  98			iov->i++;
  99		}
 100
 101		len -= partlen;
 102	}
 103}
 104EXPORT_SYMBOL(vringh_kiov_advance);
 105
 106/* Copy some bytes to/from the iovec.  Returns num copied. */
 107static inline ssize_t vringh_iov_xfer(struct vringh *vrh,
 108				      struct vringh_kiov *iov,
 109				      void *ptr, size_t len,
 110				      int (*xfer)(const struct vringh *vrh,
 111						  void *addr, void *ptr,
 112						  size_t len))
 113{
 114	int err, done = 0;
 115
 116	while (len && iov->i < iov->used) {
 117		size_t partlen;
 118
 119		partlen = min(iov->iov[iov->i].iov_len, len);
 120		err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen);
 121		if (err)
 122			return err;
 123		done += partlen;
 124		len -= partlen;
 125		ptr += partlen;
 126		iov->consumed += partlen;
 127		iov->iov[iov->i].iov_len -= partlen;
 128		iov->iov[iov->i].iov_base += partlen;
 129
 130		if (!iov->iov[iov->i].iov_len) {
 131			/* Fix up old iov element then increment. */
 132			iov->iov[iov->i].iov_len = iov->consumed;
 133			iov->iov[iov->i].iov_base -= iov->consumed;
 134
 135			iov->consumed = 0;
 136			iov->i++;
 137		}
 138	}
 139	return done;
 140}
 141
 142/* May reduce *len if range is shorter. */
 143static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,
 144			       struct vringh_range *range,
 145			       bool (*getrange)(struct vringh *,
 146						u64, struct vringh_range *))
 147{
 148	if (addr < range->start || addr > range->end_incl) {
 149		if (!getrange(vrh, addr, range))
 150			return false;
 151	}
 152	BUG_ON(addr < range->start || addr > range->end_incl);
 153
 154	/* To end of memory? */
 155	if (unlikely(addr + *len == 0)) {
 156		if (range->end_incl == -1ULL)
 157			return true;
 158		goto truncate;
 159	}
 160
 161	/* Otherwise, don't wrap. */
 162	if (addr + *len < addr) {
 163		vringh_bad("Wrapping descriptor %zu@0x%llx",
 164			   *len, (unsigned long long)addr);
 165		return false;
 166	}
 167
 168	if (unlikely(addr + *len - 1 > range->end_incl))
 169		goto truncate;
 170	return true;
 171
 172truncate:
 173	*len = range->end_incl + 1 - addr;
 174	return true;
 175}
 176
 177static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,
 178				  struct vringh_range *range,
 179				  bool (*getrange)(struct vringh *,
 180						   u64, struct vringh_range *))
 181{
 182	return true;
 183}
 184
 185/* No reason for this code to be inline. */
 186static int move_to_indirect(const struct vringh *vrh,
 187			    int *up_next, u16 *i, void *addr,
 188			    const struct vring_desc *desc,
 189			    struct vring_desc **descs, int *desc_max)
 190{
 191	u32 len;
 192
 193	/* Indirect tables can't have indirect. */
 194	if (*up_next != -1) {
 195		vringh_bad("Multilevel indirect %u->%u", *up_next, *i);
 196		return -EINVAL;
 197	}
 198
 199	len = vringh32_to_cpu(vrh, desc->len);
 200	if (unlikely(len % sizeof(struct vring_desc))) {
 201		vringh_bad("Strange indirect len %u", desc->len);
 202		return -EINVAL;
 203	}
 204
 205	/* We will check this when we follow it! */
 206	if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))
 207		*up_next = vringh16_to_cpu(vrh, desc->next);
 208	else
 209		*up_next = -2;
 210	*descs = addr;
 211	*desc_max = len / sizeof(struct vring_desc);
 212
 213	/* Now, start at the first indirect. */
 214	*i = 0;
 215	return 0;
 216}
 217
 218static int resize_iovec(struct vringh_kiov *iov, gfp_t gfp)
 219{
 220	struct kvec *new;
 221	unsigned int flag, new_num = (iov->max_num & ~VRINGH_IOV_ALLOCATED) * 2;
 222
 223	if (new_num < 8)
 224		new_num = 8;
 225
 226	flag = (iov->max_num & VRINGH_IOV_ALLOCATED);
 227	if (flag)
 228		new = krealloc_array(iov->iov, new_num, sizeof(*new), gfp);
 229	else {
 230		new = kmalloc_array(new_num, sizeof(*new), gfp);
 231		if (new) {
 232			memcpy(new, iov->iov,
 233			       iov->max_num * sizeof(struct iovec));
 234			flag = VRINGH_IOV_ALLOCATED;
 235		}
 236	}
 237	if (!new)
 238		return -ENOMEM;
 239	iov->iov = new;
 240	iov->max_num = (new_num | flag);
 241	return 0;
 242}
 243
 244static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next,
 245				       struct vring_desc **descs, int *desc_max)
 246{
 247	u16 i = *up_next;
 248
 249	*up_next = -1;
 250	*descs = vrh->vring.desc;
 251	*desc_max = vrh->vring.num;
 252	return i;
 253}
 254
 255static int slow_copy(struct vringh *vrh, void *dst, const void *src,
 256		     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
 257				    struct vringh_range *range,
 258				    bool (*getrange)(struct vringh *vrh,
 259						     u64,
 260						     struct vringh_range *)),
 261		     bool (*getrange)(struct vringh *vrh,
 262				      u64 addr,
 263				      struct vringh_range *r),
 264		     struct vringh_range *range,
 265		     int (*copy)(const struct vringh *vrh,
 266				 void *dst, const void *src, size_t len))
 267{
 268	size_t part, len = sizeof(struct vring_desc);
 269
 270	do {
 271		u64 addr;
 272		int err;
 273
 274		part = len;
 275		addr = (u64)(unsigned long)src - range->offset;
 276
 277		if (!rcheck(vrh, addr, &part, range, getrange))
 278			return -EINVAL;
 279
 280		err = copy(vrh, dst, src, part);
 281		if (err)
 282			return err;
 283
 284		dst += part;
 285		src += part;
 286		len -= part;
 287	} while (len);
 288	return 0;
 289}
 290
 291static inline int
 292__vringh_iov(struct vringh *vrh, u16 i,
 293	     struct vringh_kiov *riov,
 294	     struct vringh_kiov *wiov,
 295	     bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
 296			    struct vringh_range *range,
 297			    bool (*getrange)(struct vringh *, u64,
 298					     struct vringh_range *)),
 299	     bool (*getrange)(struct vringh *, u64, struct vringh_range *),
 300	     gfp_t gfp,
 301	     int (*copy)(const struct vringh *vrh,
 302			 void *dst, const void *src, size_t len))
 303{
 304	int err, count = 0, indirect_count = 0, up_next, desc_max;
 305	struct vring_desc desc, *descs;
 306	struct vringh_range range = { -1ULL, 0 }, slowrange;
 307	bool slow = false;
 308
 309	/* We start traversing vring's descriptor table. */
 310	descs = vrh->vring.desc;
 311	desc_max = vrh->vring.num;
 312	up_next = -1;
 313
 314	/* You must want something! */
 315	if (WARN_ON(!riov && !wiov))
 316		return -EINVAL;
 317
 318	if (riov)
 319		riov->i = riov->used = riov->consumed = 0;
 320	if (wiov)
 321		wiov->i = wiov->used = wiov->consumed = 0;
 322
 323	for (;;) {
 324		void *addr;
 325		struct vringh_kiov *iov;
 326		size_t len;
 327
 328		if (unlikely(slow))
 329			err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange,
 330					&slowrange, copy);
 331		else
 332			err = copy(vrh, &desc, &descs[i], sizeof(desc));
 333		if (unlikely(err))
 334			goto fail;
 335
 336		if (unlikely(desc.flags &
 337			     cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {
 338			u64 a = vringh64_to_cpu(vrh, desc.addr);
 339
 340			/* Make sure it's OK, and get offset. */
 341			len = vringh32_to_cpu(vrh, desc.len);
 342			if (!rcheck(vrh, a, &len, &range, getrange)) {
 343				err = -EINVAL;
 344				goto fail;
 345			}
 346
 347			if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
 348				slow = true;
 349				/* We need to save this range to use offset */
 350				slowrange = range;
 351			}
 352
 353			addr = (void *)(long)(a + range.offset);
 354			err = move_to_indirect(vrh, &up_next, &i, addr, &desc,
 355					       &descs, &desc_max);
 356			if (err)
 357				goto fail;
 358			continue;
 359		}
 360
 361		if (up_next == -1)
 362			count++;
 363		else
 364			indirect_count++;
 365
 366		if (count > vrh->vring.num || indirect_count > desc_max) {
 367			vringh_bad("Descriptor loop in %p", descs);
 368			err = -ELOOP;
 369			goto fail;
 370		}
 371
 372		if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))
 373			iov = wiov;
 374		else {
 375			iov = riov;
 376			if (unlikely(wiov && wiov->used)) {
 377				vringh_bad("Readable desc %p after writable",
 378					   &descs[i]);
 379				err = -EINVAL;
 380				goto fail;
 381			}
 382		}
 383
 384		if (!iov) {
 385			vringh_bad("Unexpected %s desc",
 386				   !wiov ? "writable" : "readable");
 387			err = -EPROTO;
 388			goto fail;
 389		}
 390
 391	again:
 392		/* Make sure it's OK, and get offset. */
 393		len = vringh32_to_cpu(vrh, desc.len);
 394		if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,
 395			    getrange)) {
 396			err = -EINVAL;
 397			goto fail;
 398		}
 399		addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +
 400					       range.offset);
 401
 402		if (unlikely(iov->used == (iov->max_num & ~VRINGH_IOV_ALLOCATED))) {
 403			err = resize_iovec(iov, gfp);
 404			if (err)
 405				goto fail;
 406		}
 407
 408		iov->iov[iov->used].iov_base = addr;
 409		iov->iov[iov->used].iov_len = len;
 410		iov->used++;
 411
 412		if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
 413			desc.len = cpu_to_vringh32(vrh,
 414				   vringh32_to_cpu(vrh, desc.len) - len);
 415			desc.addr = cpu_to_vringh64(vrh,
 416				    vringh64_to_cpu(vrh, desc.addr) + len);
 417			goto again;
 418		}
 419
 420		if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {
 421			i = vringh16_to_cpu(vrh, desc.next);
 422		} else {
 423			/* Just in case we need to finish traversing above. */
 424			if (unlikely(up_next > 0)) {
 425				i = return_from_indirect(vrh, &up_next,
 426							 &descs, &desc_max);
 427				slow = false;
 428				indirect_count = 0;
 429			} else
 430				break;
 431		}
 432
 433		if (i >= desc_max) {
 434			vringh_bad("Chained index %u > %u", i, desc_max);
 435			err = -EINVAL;
 436			goto fail;
 437		}
 438	}
 439
 440	return 0;
 441
 442fail:
 443	return err;
 444}
 445
 446static inline int __vringh_complete(struct vringh *vrh,
 447				    const struct vring_used_elem *used,
 448				    unsigned int num_used,
 449				    int (*putu16)(const struct vringh *vrh,
 450						  __virtio16 *p, u16 val),
 451				    int (*putused)(const struct vringh *vrh,
 452						   struct vring_used_elem *dst,
 453						   const struct vring_used_elem
 454						   *src, unsigned num))
 455{
 456	struct vring_used *used_ring;
 457	int err;
 458	u16 used_idx, off;
 459
 460	used_ring = vrh->vring.used;
 461	used_idx = vrh->last_used_idx + vrh->completed;
 462
 463	off = used_idx % vrh->vring.num;
 464
 465	/* Compiler knows num_used == 1 sometimes, hence extra check */
 466	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {
 467		u16 part = vrh->vring.num - off;
 468		err = putused(vrh, &used_ring->ring[off], used, part);
 469		if (!err)
 470			err = putused(vrh, &used_ring->ring[0], used + part,
 471				      num_used - part);
 472	} else
 473		err = putused(vrh, &used_ring->ring[off], used, num_used);
 474
 475	if (err) {
 476		vringh_bad("Failed to write %u used entries %u at %p",
 477			   num_used, off, &used_ring->ring[off]);
 478		return err;
 479	}
 480
 481	/* Make sure buffer is written before we update index. */
 482	virtio_wmb(vrh->weak_barriers);
 483
 484	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);
 485	if (err) {
 486		vringh_bad("Failed to update used index at %p",
 487			   &vrh->vring.used->idx);
 488		return err;
 489	}
 490
 491	vrh->completed += num_used;
 492	return 0;
 493}
 494
 495
 496static inline int __vringh_need_notify(struct vringh *vrh,
 497				       int (*getu16)(const struct vringh *vrh,
 498						     u16 *val,
 499						     const __virtio16 *p))
 500{
 501	bool notify;
 502	u16 used_event;
 503	int err;
 504
 505	/* Flush out used index update. This is paired with the
 506	 * barrier that the Guest executes when enabling
 507	 * interrupts. */
 508	virtio_mb(vrh->weak_barriers);
 509
 510	/* Old-style, without event indices. */
 511	if (!vrh->event_indices) {
 512		u16 flags;
 513		err = getu16(vrh, &flags, &vrh->vring.avail->flags);
 514		if (err) {
 515			vringh_bad("Failed to get flags at %p",
 516				   &vrh->vring.avail->flags);
 517			return err;
 518		}
 519		return (!(flags & VRING_AVAIL_F_NO_INTERRUPT));
 520	}
 521
 522	/* Modern: we know when other side wants to know. */
 523	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));
 524	if (err) {
 525		vringh_bad("Failed to get used event idx at %p",
 526			   &vring_used_event(&vrh->vring));
 527		return err;
 528	}
 529
 530	/* Just in case we added so many that we wrap. */
 531	if (unlikely(vrh->completed > 0xffff))
 532		notify = true;
 533	else
 534		notify = vring_need_event(used_event,
 535					  vrh->last_used_idx + vrh->completed,
 536					  vrh->last_used_idx);
 537
 538	vrh->last_used_idx += vrh->completed;
 539	vrh->completed = 0;
 540	return notify;
 541}
 542
 543static inline bool __vringh_notify_enable(struct vringh *vrh,
 544					  int (*getu16)(const struct vringh *vrh,
 545							u16 *val, const __virtio16 *p),
 546					  int (*putu16)(const struct vringh *vrh,
 547							__virtio16 *p, u16 val))
 548{
 549	u16 avail;
 550
 551	if (!vrh->event_indices) {
 552		/* Old-school; update flags. */
 553		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {
 554			vringh_bad("Clearing used flags %p",
 555				   &vrh->vring.used->flags);
 556			return true;
 557		}
 558	} else {
 559		if (putu16(vrh, &vring_avail_event(&vrh->vring),
 560			   vrh->last_avail_idx) != 0) {
 561			vringh_bad("Updating avail event index %p",
 562				   &vring_avail_event(&vrh->vring));
 563			return true;
 564		}
 565	}
 566
 567	/* They could have slipped one in as we were doing that: make
 568	 * sure it's written, then check again. */
 569	virtio_mb(vrh->weak_barriers);
 570
 571	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {
 572		vringh_bad("Failed to check avail idx at %p",
 573			   &vrh->vring.avail->idx);
 574		return true;
 575	}
 576
 577	/* This is unlikely, so we just leave notifications enabled
 578	 * (if we're using event_indices, we'll only get one
 579	 * notification anyway). */
 580	return avail == vrh->last_avail_idx;
 581}
 582
 583static inline void __vringh_notify_disable(struct vringh *vrh,
 584					   int (*putu16)(const struct vringh *vrh,
 585							 __virtio16 *p, u16 val))
 586{
 587	if (!vrh->event_indices) {
 588		/* Old-school; update flags. */
 589		if (putu16(vrh, &vrh->vring.used->flags,
 590			   VRING_USED_F_NO_NOTIFY)) {
 591			vringh_bad("Setting used flags %p",
 592				   &vrh->vring.used->flags);
 593		}
 594	}
 595}
 596
 597/* Userspace access helpers: in this case, addresses are really userspace. */
 598static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)
 599{
 600	__virtio16 v = 0;
 601	int rc = get_user(v, (__force __virtio16 __user *)p);
 602	*val = vringh16_to_cpu(vrh, v);
 603	return rc;
 604}
 605
 606static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)
 607{
 608	__virtio16 v = cpu_to_vringh16(vrh, val);
 609	return put_user(v, (__force __virtio16 __user *)p);
 610}
 611
 612static inline int copydesc_user(const struct vringh *vrh,
 613				void *dst, const void *src, size_t len)
 614{
 615	return copy_from_user(dst, (__force void __user *)src, len) ?
 616		-EFAULT : 0;
 617}
 618
 619static inline int putused_user(const struct vringh *vrh,
 620			       struct vring_used_elem *dst,
 621			       const struct vring_used_elem *src,
 622			       unsigned int num)
 623{
 624	return copy_to_user((__force void __user *)dst, src,
 625			    sizeof(*dst) * num) ? -EFAULT : 0;
 626}
 627
 628static inline int xfer_from_user(const struct vringh *vrh, void *src,
 629				 void *dst, size_t len)
 630{
 631	return copy_from_user(dst, (__force void __user *)src, len) ?
 632		-EFAULT : 0;
 633}
 634
 635static inline int xfer_to_user(const struct vringh *vrh,
 636			       void *dst, void *src, size_t len)
 637{
 638	return copy_to_user((__force void __user *)dst, src, len) ?
 639		-EFAULT : 0;
 640}
 641
 642/**
 643 * vringh_init_user - initialize a vringh for a userspace vring.
 644 * @vrh: the vringh to initialize.
 645 * @features: the feature bits for this ring.
 646 * @num: the number of elements.
 647 * @weak_barriers: true if we only need memory barriers, not I/O.
 648 * @desc: the userspace descriptor pointer.
 649 * @avail: the userspace avail pointer.
 650 * @used: the userspace used pointer.
 651 *
 652 * Returns an error if num is invalid: you should check pointers
 653 * yourself!
 654 */
 655int vringh_init_user(struct vringh *vrh, u64 features,
 656		     unsigned int num, bool weak_barriers,
 657		     vring_desc_t __user *desc,
 658		     vring_avail_t __user *avail,
 659		     vring_used_t __user *used)
 660{
 661	/* Sane power of 2 please! */
 662	if (!num || num > 0xffff || (num & (num - 1))) {
 663		vringh_bad("Bad ring size %u", num);
 664		return -EINVAL;
 665	}
 666
 667	vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
 668	vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
 669	vrh->weak_barriers = weak_barriers;
 670	vrh->completed = 0;
 671	vrh->last_avail_idx = 0;
 672	vrh->last_used_idx = 0;
 673	vrh->vring.num = num;
 674	/* vring expects kernel addresses, but only used via accessors. */
 675	vrh->vring.desc = (__force struct vring_desc *)desc;
 676	vrh->vring.avail = (__force struct vring_avail *)avail;
 677	vrh->vring.used = (__force struct vring_used *)used;
 678	return 0;
 679}
 680EXPORT_SYMBOL(vringh_init_user);
 681
 682/**
 683 * vringh_getdesc_user - get next available descriptor from userspace ring.
 684 * @vrh: the userspace vring.
 685 * @riov: where to put the readable descriptors (or NULL)
 686 * @wiov: where to put the writable descriptors (or NULL)
 687 * @getrange: function to call to check ranges.
 688 * @head: head index we received, for passing to vringh_complete_user().
 689 *
 690 * Returns 0 if there was no descriptor, 1 if there was, or -errno.
 691 *
 692 * Note that on error return, you can tell the difference between an
 693 * invalid ring and a single invalid descriptor: in the former case,
 694 * *head will be vrh->vring.num.  You may be able to ignore an invalid
 695 * descriptor, but there's not much you can do with an invalid ring.
 696 *
 697 * Note that you can reuse riov and wiov with subsequent calls. Content is
 698 * overwritten and memory reallocated if more space is needed.
 699 * When you don't have to use riov and wiov anymore, you should clean up them
 700 * calling vringh_iov_cleanup() to release the memory, even on error!
 701 */
 702int vringh_getdesc_user(struct vringh *vrh,
 703			struct vringh_iov *riov,
 704			struct vringh_iov *wiov,
 705			bool (*getrange)(struct vringh *vrh,
 706					 u64 addr, struct vringh_range *r),
 707			u16 *head)
 708{
 709	int err;
 710
 711	*head = vrh->vring.num;
 712	err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx);
 713	if (err < 0)
 714		return err;
 715
 716	/* Empty... */
 717	if (err == vrh->vring.num)
 718		return 0;
 719
 720	/* We need the layouts to be the identical for this to work */
 721	BUILD_BUG_ON(sizeof(struct vringh_kiov) != sizeof(struct vringh_iov));
 722	BUILD_BUG_ON(offsetof(struct vringh_kiov, iov) !=
 723		     offsetof(struct vringh_iov, iov));
 724	BUILD_BUG_ON(offsetof(struct vringh_kiov, i) !=
 725		     offsetof(struct vringh_iov, i));
 726	BUILD_BUG_ON(offsetof(struct vringh_kiov, used) !=
 727		     offsetof(struct vringh_iov, used));
 728	BUILD_BUG_ON(offsetof(struct vringh_kiov, max_num) !=
 729		     offsetof(struct vringh_iov, max_num));
 730	BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
 731	BUILD_BUG_ON(offsetof(struct iovec, iov_base) !=
 732		     offsetof(struct kvec, iov_base));
 733	BUILD_BUG_ON(offsetof(struct iovec, iov_len) !=
 734		     offsetof(struct kvec, iov_len));
 735	BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_base)
 736		     != sizeof(((struct kvec *)NULL)->iov_base));
 737	BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_len)
 738		     != sizeof(((struct kvec *)NULL)->iov_len));
 739
 740	*head = err;
 741	err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov,
 742			   (struct vringh_kiov *)wiov,
 743			   range_check, getrange, GFP_KERNEL, copydesc_user);
 744	if (err)
 745		return err;
 746
 747	return 1;
 748}
 749EXPORT_SYMBOL(vringh_getdesc_user);
 750
 751/**
 752 * vringh_iov_pull_user - copy bytes from vring_iov.
 753 * @riov: the riov as passed to vringh_getdesc_user() (updated as we consume)
 754 * @dst: the place to copy.
 755 * @len: the maximum length to copy.
 756 *
 757 * Returns the bytes copied <= len or a negative errno.
 758 */
 759ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len)
 760{
 761	return vringh_iov_xfer(NULL, (struct vringh_kiov *)riov,
 762			       dst, len, xfer_from_user);
 763}
 764EXPORT_SYMBOL(vringh_iov_pull_user);
 765
 766/**
 767 * vringh_iov_push_user - copy bytes into vring_iov.
 768 * @wiov: the wiov as passed to vringh_getdesc_user() (updated as we consume)
 769 * @src: the place to copy from.
 770 * @len: the maximum length to copy.
 771 *
 772 * Returns the bytes copied <= len or a negative errno.
 773 */
 774ssize_t vringh_iov_push_user(struct vringh_iov *wiov,
 775			     const void *src, size_t len)
 776{
 777	return vringh_iov_xfer(NULL, (struct vringh_kiov *)wiov,
 778			       (void *)src, len, xfer_to_user);
 779}
 780EXPORT_SYMBOL(vringh_iov_push_user);
 781
 782/**
 783 * vringh_complete_user - we've finished with descriptor, publish it.
 784 * @vrh: the vring.
 785 * @head: the head as filled in by vringh_getdesc_user.
 786 * @len: the length of data we have written.
 787 *
 788 * You should check vringh_need_notify_user() after one or more calls
 789 * to this function.
 790 */
 791int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)
 792{
 793	struct vring_used_elem used;
 794
 795	used.id = cpu_to_vringh32(vrh, head);
 796	used.len = cpu_to_vringh32(vrh, len);
 797	return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);
 798}
 799EXPORT_SYMBOL(vringh_complete_user);
 800
 801/**
 802 * vringh_complete_multi_user - we've finished with many descriptors.
 803 * @vrh: the vring.
 804 * @used: the head, length pairs.
 805 * @num_used: the number of used elements.
 806 *
 807 * You should check vringh_need_notify_user() after one or more calls
 808 * to this function.
 809 */
 810int vringh_complete_multi_user(struct vringh *vrh,
 811			       const struct vring_used_elem used[],
 812			       unsigned num_used)
 813{
 814	return __vringh_complete(vrh, used, num_used,
 815				 putu16_user, putused_user);
 816}
 817EXPORT_SYMBOL(vringh_complete_multi_user);
 818
 819/**
 820 * vringh_notify_enable_user - we want to know if something changes.
 821 * @vrh: the vring.
 822 *
 823 * This always enables notifications, but returns false if there are
 824 * now more buffers available in the vring.
 825 */
 826bool vringh_notify_enable_user(struct vringh *vrh)
 827{
 828	return __vringh_notify_enable(vrh, getu16_user, putu16_user);
 829}
 830EXPORT_SYMBOL(vringh_notify_enable_user);
 831
 832/**
 833 * vringh_notify_disable_user - don't tell us if something changes.
 834 * @vrh: the vring.
 835 *
 836 * This is our normal running state: we disable and then only enable when
 837 * we're going to sleep.
 838 */
 839void vringh_notify_disable_user(struct vringh *vrh)
 840{
 841	__vringh_notify_disable(vrh, putu16_user);
 842}
 843EXPORT_SYMBOL(vringh_notify_disable_user);
 844
 845/**
 846 * vringh_need_notify_user - must we tell the other side about used buffers?
 847 * @vrh: the vring we've called vringh_complete_user() on.
 848 *
 849 * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
 850 */
 851int vringh_need_notify_user(struct vringh *vrh)
 852{
 853	return __vringh_need_notify(vrh, getu16_user);
 854}
 855EXPORT_SYMBOL(vringh_need_notify_user);
 856
 857/* Kernelspace access helpers. */
 858static inline int getu16_kern(const struct vringh *vrh,
 859			      u16 *val, const __virtio16 *p)
 860{
 861	*val = vringh16_to_cpu(vrh, READ_ONCE(*p));
 862	return 0;
 863}
 864
 865static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
 866{
 867	WRITE_ONCE(*p, cpu_to_vringh16(vrh, val));
 868	return 0;
 869}
 870
 871static inline int copydesc_kern(const struct vringh *vrh,
 872				void *dst, const void *src, size_t len)
 873{
 874	memcpy(dst, src, len);
 875	return 0;
 876}
 877
 878static inline int putused_kern(const struct vringh *vrh,
 879			       struct vring_used_elem *dst,
 880			       const struct vring_used_elem *src,
 881			       unsigned int num)
 882{
 883	memcpy(dst, src, num * sizeof(*dst));
 884	return 0;
 885}
 886
 887/**
 888 * vringh_init_kern - initialize a vringh for a kernelspace vring.
 889 * @vrh: the vringh to initialize.
 890 * @features: the feature bits for this ring.
 891 * @num: the number of elements.
 892 * @weak_barriers: true if we only need memory barriers, not I/O.
 893 * @desc: the userspace descriptor pointer.
 894 * @avail: the userspace avail pointer.
 895 * @used: the userspace used pointer.
 896 *
 897 * Returns an error if num is invalid.
 898 */
 899int vringh_init_kern(struct vringh *vrh, u64 features,
 900		     unsigned int num, bool weak_barriers,
 901		     struct vring_desc *desc,
 902		     struct vring_avail *avail,
 903		     struct vring_used *used)
 904{
 905	/* Sane power of 2 please! */
 906	if (!num || num > 0xffff || (num & (num - 1))) {
 907		vringh_bad("Bad ring size %u", num);
 908		return -EINVAL;
 909	}
 910
 911	vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
 912	vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
 913	vrh->weak_barriers = weak_barriers;
 914	vrh->completed = 0;
 915	vrh->last_avail_idx = 0;
 916	vrh->last_used_idx = 0;
 917	vrh->vring.num = num;
 918	vrh->vring.desc = desc;
 919	vrh->vring.avail = avail;
 920	vrh->vring.used = used;
 921	return 0;
 922}
 923EXPORT_SYMBOL(vringh_init_kern);
 924
 925/**
 926 * vringh_getdesc_kern - get next available descriptor from kernelspace ring.
 927 * @vrh: the kernelspace vring.
 928 * @riov: where to put the readable descriptors (or NULL)
 929 * @wiov: where to put the writable descriptors (or NULL)
 930 * @head: head index we received, for passing to vringh_complete_kern().
 931 * @gfp: flags for allocating larger riov/wiov.
 932 *
 933 * Returns 0 if there was no descriptor, 1 if there was, or -errno.
 934 *
 935 * Note that on error return, you can tell the difference between an
 936 * invalid ring and a single invalid descriptor: in the former case,
 937 * *head will be vrh->vring.num.  You may be able to ignore an invalid
 938 * descriptor, but there's not much you can do with an invalid ring.
 939 *
 940 * Note that you can reuse riov and wiov with subsequent calls. Content is
 941 * overwritten and memory reallocated if more space is needed.
 942 * When you don't have to use riov and wiov anymore, you should clean up them
 943 * calling vringh_kiov_cleanup() to release the memory, even on error!
 944 */
 945int vringh_getdesc_kern(struct vringh *vrh,
 946			struct vringh_kiov *riov,
 947			struct vringh_kiov *wiov,
 948			u16 *head,
 949			gfp_t gfp)
 950{
 951	int err;
 952
 953	err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx);
 954	if (err < 0)
 955		return err;
 956
 957	/* Empty... */
 958	if (err == vrh->vring.num)
 959		return 0;
 960
 961	*head = err;
 962	err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
 963			   gfp, copydesc_kern);
 964	if (err)
 965		return err;
 966
 967	return 1;
 968}
 969EXPORT_SYMBOL(vringh_getdesc_kern);
 970
 971/**
 972 * vringh_complete_kern - we've finished with descriptor, publish it.
 973 * @vrh: the vring.
 974 * @head: the head as filled in by vringh_getdesc_kern.
 975 * @len: the length of data we have written.
 976 *
 977 * You should check vringh_need_notify_kern() after one or more calls
 978 * to this function.
 979 */
 980int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)
 981{
 982	struct vring_used_elem used;
 983
 984	used.id = cpu_to_vringh32(vrh, head);
 985	used.len = cpu_to_vringh32(vrh, len);
 986
 987	return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);
 988}
 989EXPORT_SYMBOL(vringh_complete_kern);
 990
 991/**
 992 * vringh_notify_enable_kern - we want to know if something changes.
 993 * @vrh: the vring.
 994 *
 995 * This always enables notifications, but returns false if there are
 996 * now more buffers available in the vring.
 997 */
 998bool vringh_notify_enable_kern(struct vringh *vrh)
 999{
1000	return __vringh_notify_enable(vrh, getu16_kern, putu16_kern);
1001}
1002EXPORT_SYMBOL(vringh_notify_enable_kern);
1003
1004/**
1005 * vringh_notify_disable_kern - don't tell us if something changes.
1006 * @vrh: the vring.
1007 *
1008 * This is our normal running state: we disable and then only enable when
1009 * we're going to sleep.
1010 */
1011void vringh_notify_disable_kern(struct vringh *vrh)
1012{
1013	__vringh_notify_disable(vrh, putu16_kern);
1014}
1015EXPORT_SYMBOL(vringh_notify_disable_kern);
1016
1017/**
1018 * vringh_need_notify_kern - must we tell the other side about used buffers?
1019 * @vrh: the vring we've called vringh_complete_kern() on.
1020 *
1021 * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
1022 */
1023int vringh_need_notify_kern(struct vringh *vrh)
1024{
1025	return __vringh_need_notify(vrh, getu16_kern);
1026}
1027EXPORT_SYMBOL(vringh_need_notify_kern);
1028
1029#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
1030
1031struct iotlb_vec {
1032	union {
1033		struct iovec *iovec;
1034		struct bio_vec *bvec;
1035	} iov;
1036	size_t count;
1037};
1038
1039static int iotlb_translate(const struct vringh *vrh,
1040			   u64 addr, u64 len, u64 *translated,
1041			   struct iotlb_vec *ivec, u32 perm)
1042{
1043	struct vhost_iotlb_map *map;
1044	struct vhost_iotlb *iotlb = vrh->iotlb;
1045	int ret = 0;
1046	u64 s = 0, last = addr + len - 1;
1047
1048	spin_lock(vrh->iotlb_lock);
1049
1050	while (len > s) {
1051		uintptr_t io_addr;
1052		size_t io_len;
1053		u64 size;
1054
1055		if (unlikely(ret >= ivec->count)) {
1056			ret = -ENOBUFS;
1057			break;
1058		}
1059
1060		map = vhost_iotlb_itree_first(iotlb, addr, last);
1061		if (!map || map->start > addr) {
1062			ret = -EINVAL;
1063			break;
1064		} else if (!(map->perm & perm)) {
1065			ret = -EPERM;
1066			break;
1067		}
1068
1069		size = map->size - addr + map->start;
1070		io_len = min(len - s, size);
1071		io_addr = map->addr - map->start + addr;
1072
1073		if (vrh->use_va) {
1074			struct iovec *iovec = ivec->iov.iovec;
1075
1076			iovec[ret].iov_len = io_len;
1077			iovec[ret].iov_base = (void __user *)io_addr;
1078		} else {
1079			u64 pfn = io_addr >> PAGE_SHIFT;
1080			struct bio_vec *bvec = ivec->iov.bvec;
1081
1082			bvec_set_page(&bvec[ret], pfn_to_page(pfn), io_len,
1083				      io_addr & (PAGE_SIZE - 1));
1084		}
1085
1086		s += size;
1087		addr += size;
1088		++ret;
1089	}
1090
1091	spin_unlock(vrh->iotlb_lock);
1092
1093	if (translated)
1094		*translated = min(len, s);
1095
1096	return ret;
1097}
1098
1099#define IOTLB_IOV_STRIDE 16
1100
1101static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
1102				  void *src, size_t len)
1103{
1104	struct iotlb_vec ivec;
1105	union {
1106		struct iovec iovec[IOTLB_IOV_STRIDE];
1107		struct bio_vec bvec[IOTLB_IOV_STRIDE];
1108	} iov;
1109	u64 total_translated = 0;
1110
1111	ivec.iov.iovec = iov.iovec;
1112	ivec.count = IOTLB_IOV_STRIDE;
1113
1114	while (total_translated < len) {
1115		struct iov_iter iter;
1116		u64 translated;
1117		int ret;
1118		size_t size;
1119
1120		ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
1121				      len - total_translated, &translated,
1122				      &ivec, VHOST_MAP_RO);
1123		if (ret == -ENOBUFS)
1124			ret = IOTLB_IOV_STRIDE;
1125		else if (ret < 0)
1126			return ret;
1127
1128		if (vrh->use_va) {
1129			iov_iter_init(&iter, ITER_SOURCE, ivec.iov.iovec, ret,
1130				      translated);
1131		} else {
1132			iov_iter_bvec(&iter, ITER_SOURCE, ivec.iov.bvec, ret,
1133				      translated);
1134		}
1135
1136		size = copy_from_iter(dst, translated, &iter);
1137		if (size != translated)
1138			return -EFAULT;
1139
1140		src += translated;
1141		dst += translated;
1142		total_translated += translated;
1143	}
1144
1145	return total_translated;
1146}
1147
1148static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
1149				void *src, size_t len)
1150{
1151	struct iotlb_vec ivec;
1152	union {
1153		struct iovec iovec[IOTLB_IOV_STRIDE];
1154		struct bio_vec bvec[IOTLB_IOV_STRIDE];
1155	} iov;
1156	u64 total_translated = 0;
1157
1158	ivec.iov.iovec = iov.iovec;
1159	ivec.count = IOTLB_IOV_STRIDE;
1160
1161	while (total_translated < len) {
1162		struct iov_iter iter;
1163		u64 translated;
1164		int ret;
1165		size_t size;
1166
1167		ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
1168				      len - total_translated, &translated,
1169				      &ivec, VHOST_MAP_WO);
1170		if (ret == -ENOBUFS)
1171			ret = IOTLB_IOV_STRIDE;
1172		else if (ret < 0)
1173			return ret;
1174
1175		if (vrh->use_va) {
1176			iov_iter_init(&iter, ITER_DEST, ivec.iov.iovec, ret,
1177				      translated);
1178		} else {
1179			iov_iter_bvec(&iter, ITER_DEST, ivec.iov.bvec, ret,
1180				      translated);
1181		}
1182
1183		size = copy_to_iter(src, translated, &iter);
1184		if (size != translated)
1185			return -EFAULT;
1186
1187		src += translated;
1188		dst += translated;
1189		total_translated += translated;
1190	}
1191
1192	return total_translated;
1193}
1194
1195static inline int getu16_iotlb(const struct vringh *vrh,
1196			       u16 *val, const __virtio16 *p)
1197{
1198	struct iotlb_vec ivec;
1199	union {
1200		struct iovec iovec[1];
1201		struct bio_vec bvec[1];
1202	} iov;
1203	__virtio16 tmp;
1204	int ret;
1205
1206	ivec.iov.iovec = iov.iovec;
1207	ivec.count = 1;
1208
1209	/* Atomic read is needed for getu16 */
1210	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
1211			      NULL, &ivec, VHOST_MAP_RO);
1212	if (ret < 0)
1213		return ret;
1214
1215	if (vrh->use_va) {
1216		ret = __get_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
1217		if (ret)
1218			return ret;
1219	} else {
1220		__virtio16 *from = bvec_kmap_local(&ivec.iov.bvec[0]);
1221
1222		tmp = READ_ONCE(*from);
1223		kunmap_local(from);
1224	}
1225
1226	*val = vringh16_to_cpu(vrh, tmp);
1227
1228	return 0;
1229}
1230
1231static inline int putu16_iotlb(const struct vringh *vrh,
1232			       __virtio16 *p, u16 val)
1233{
1234	struct iotlb_vec ivec;
1235	union {
1236		struct iovec iovec;
1237		struct bio_vec bvec;
1238	} iov;
1239	__virtio16 tmp;
1240	int ret;
1241
1242	ivec.iov.iovec = &iov.iovec;
1243	ivec.count = 1;
1244
1245	/* Atomic write is needed for putu16 */
1246	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
1247			      NULL, &ivec, VHOST_MAP_RO);
1248	if (ret < 0)
1249		return ret;
1250
1251	tmp = cpu_to_vringh16(vrh, val);
1252
1253	if (vrh->use_va) {
1254		ret = __put_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
1255		if (ret)
1256			return ret;
1257	} else {
1258		__virtio16 *to = bvec_kmap_local(&ivec.iov.bvec[0]);
1259
1260		WRITE_ONCE(*to, tmp);
1261		kunmap_local(to);
1262	}
1263
1264	return 0;
1265}
1266
1267static inline int copydesc_iotlb(const struct vringh *vrh,
1268				 void *dst, const void *src, size_t len)
1269{
1270	int ret;
1271
1272	ret = copy_from_iotlb(vrh, dst, (void *)src, len);
1273	if (ret != len)
1274		return -EFAULT;
1275
1276	return 0;
1277}
1278
1279static inline int xfer_from_iotlb(const struct vringh *vrh, void *src,
1280				  void *dst, size_t len)
1281{
1282	int ret;
1283
1284	ret = copy_from_iotlb(vrh, dst, src, len);
1285	if (ret != len)
1286		return -EFAULT;
1287
1288	return 0;
1289}
1290
1291static inline int xfer_to_iotlb(const struct vringh *vrh,
1292			       void *dst, void *src, size_t len)
1293{
1294	int ret;
1295
1296	ret = copy_to_iotlb(vrh, dst, src, len);
1297	if (ret != len)
1298		return -EFAULT;
1299
1300	return 0;
1301}
1302
1303static inline int putused_iotlb(const struct vringh *vrh,
1304				struct vring_used_elem *dst,
1305				const struct vring_used_elem *src,
1306				unsigned int num)
1307{
1308	int size = num * sizeof(*dst);
1309	int ret;
1310
1311	ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst));
1312	if (ret != size)
1313		return -EFAULT;
1314
1315	return 0;
1316}
1317
1318/**
1319 * vringh_init_iotlb - initialize a vringh for a ring with IOTLB.
1320 * @vrh: the vringh to initialize.
1321 * @features: the feature bits for this ring.
1322 * @num: the number of elements.
1323 * @weak_barriers: true if we only need memory barriers, not I/O.
1324 * @desc: the userspace descriptor pointer.
1325 * @avail: the userspace avail pointer.
1326 * @used: the userspace used pointer.
1327 *
1328 * Returns an error if num is invalid.
1329 */
1330int vringh_init_iotlb(struct vringh *vrh, u64 features,
1331		      unsigned int num, bool weak_barriers,
1332		      struct vring_desc *desc,
1333		      struct vring_avail *avail,
1334		      struct vring_used *used)
1335{
1336	vrh->use_va = false;
1337
1338	return vringh_init_kern(vrh, features, num, weak_barriers,
1339				desc, avail, used);
1340}
1341EXPORT_SYMBOL(vringh_init_iotlb);
1342
1343/**
1344 * vringh_init_iotlb_va - initialize a vringh for a ring with IOTLB containing
1345 *                        user VA.
1346 * @vrh: the vringh to initialize.
1347 * @features: the feature bits for this ring.
1348 * @num: the number of elements.
1349 * @weak_barriers: true if we only need memory barriers, not I/O.
1350 * @desc: the userspace descriptor pointer.
1351 * @avail: the userspace avail pointer.
1352 * @used: the userspace used pointer.
1353 *
1354 * Returns an error if num is invalid.
1355 */
1356int vringh_init_iotlb_va(struct vringh *vrh, u64 features,
1357			 unsigned int num, bool weak_barriers,
1358			 struct vring_desc *desc,
1359			 struct vring_avail *avail,
1360			 struct vring_used *used)
1361{
1362	vrh->use_va = true;
1363
1364	return vringh_init_kern(vrh, features, num, weak_barriers,
1365				desc, avail, used);
1366}
1367EXPORT_SYMBOL(vringh_init_iotlb_va);
1368
1369/**
1370 * vringh_set_iotlb - initialize a vringh for a ring with IOTLB.
1371 * @vrh: the vring
1372 * @iotlb: iotlb associated with this vring
1373 * @iotlb_lock: spinlock to synchronize the iotlb accesses
1374 */
1375void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb,
1376		      spinlock_t *iotlb_lock)
1377{
1378	vrh->iotlb = iotlb;
1379	vrh->iotlb_lock = iotlb_lock;
1380}
1381EXPORT_SYMBOL(vringh_set_iotlb);
1382
1383/**
1384 * vringh_getdesc_iotlb - get next available descriptor from ring with
1385 * IOTLB.
1386 * @vrh: the kernelspace vring.
1387 * @riov: where to put the readable descriptors (or NULL)
1388 * @wiov: where to put the writable descriptors (or NULL)
1389 * @head: head index we received, for passing to vringh_complete_iotlb().
1390 * @gfp: flags for allocating larger riov/wiov.
1391 *
1392 * Returns 0 if there was no descriptor, 1 if there was, or -errno.
1393 *
1394 * Note that on error return, you can tell the difference between an
1395 * invalid ring and a single invalid descriptor: in the former case,
1396 * *head will be vrh->vring.num.  You may be able to ignore an invalid
1397 * descriptor, but there's not much you can do with an invalid ring.
1398 *
1399 * Note that you can reuse riov and wiov with subsequent calls. Content is
1400 * overwritten and memory reallocated if more space is needed.
1401 * When you don't have to use riov and wiov anymore, you should clean up them
1402 * calling vringh_kiov_cleanup() to release the memory, even on error!
1403 */
1404int vringh_getdesc_iotlb(struct vringh *vrh,
1405			 struct vringh_kiov *riov,
1406			 struct vringh_kiov *wiov,
1407			 u16 *head,
1408			 gfp_t gfp)
1409{
1410	int err;
1411
1412	err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx);
1413	if (err < 0)
1414		return err;
1415
1416	/* Empty... */
1417	if (err == vrh->vring.num)
1418		return 0;
1419
1420	*head = err;
1421	err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
1422			   gfp, copydesc_iotlb);
1423	if (err)
1424		return err;
1425
1426	return 1;
1427}
1428EXPORT_SYMBOL(vringh_getdesc_iotlb);
1429
1430/**
1431 * vringh_iov_pull_iotlb - copy bytes from vring_iov.
1432 * @vrh: the vring.
1433 * @riov: the riov as passed to vringh_getdesc_iotlb() (updated as we consume)
1434 * @dst: the place to copy.
1435 * @len: the maximum length to copy.
1436 *
1437 * Returns the bytes copied <= len or a negative errno.
1438 */
1439ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,
1440			      struct vringh_kiov *riov,
1441			      void *dst, size_t len)
1442{
1443	return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb);
1444}
1445EXPORT_SYMBOL(vringh_iov_pull_iotlb);
1446
1447/**
1448 * vringh_iov_push_iotlb - copy bytes into vring_iov.
1449 * @vrh: the vring.
1450 * @wiov: the wiov as passed to vringh_getdesc_iotlb() (updated as we consume)
1451 * @src: the place to copy from.
1452 * @len: the maximum length to copy.
1453 *
1454 * Returns the bytes copied <= len or a negative errno.
1455 */
1456ssize_t vringh_iov_push_iotlb(struct vringh *vrh,
1457			      struct vringh_kiov *wiov,
1458			      const void *src, size_t len)
1459{
1460	return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb);
1461}
1462EXPORT_SYMBOL(vringh_iov_push_iotlb);
1463
1464/**
1465 * vringh_complete_iotlb - we've finished with descriptor, publish it.
1466 * @vrh: the vring.
1467 * @head: the head as filled in by vringh_getdesc_iotlb.
1468 * @len: the length of data we have written.
1469 *
1470 * You should check vringh_need_notify_iotlb() after one or more calls
1471 * to this function.
1472 */
1473int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len)
1474{
1475	struct vring_used_elem used;
1476
1477	used.id = cpu_to_vringh32(vrh, head);
1478	used.len = cpu_to_vringh32(vrh, len);
1479
1480	return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb);
1481}
1482EXPORT_SYMBOL(vringh_complete_iotlb);
1483
1484/**
1485 * vringh_need_notify_iotlb - must we tell the other side about used buffers?
1486 * @vrh: the vring we've called vringh_complete_iotlb() on.
1487 *
1488 * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
1489 */
1490int vringh_need_notify_iotlb(struct vringh *vrh)
1491{
1492	return __vringh_need_notify(vrh, getu16_iotlb);
1493}
1494EXPORT_SYMBOL(vringh_need_notify_iotlb);
1495
1496#endif
1497
1498MODULE_DESCRIPTION("host side of a virtio ring");
1499MODULE_LICENSE("GPL");