vdpa.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2018-2020 Intel Corporation.
   4 * Copyright (C) 2020 Red Hat, Inc.
   5 *
   6 * Author: Tiwei Bie <tiwei.bie@intel.com>
   7 *         Jason Wang <jasowang@redhat.com>
   8 *
   9 * Thanks Michael S. Tsirkin for the valuable comments and
  10 * suggestions.  And thanks to Cunming Liang and Zhihong Wang for all
  11 * their supports.
  12 */
  13
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/cdev.h>
  17#include <linux/device.h>
  18#include <linux/mm.h>
  19#include <linux/slab.h>
  20#include <linux/iommu.h>
  21#include <linux/uuid.h>
  22#include <linux/vdpa.h>
  23#include <linux/nospec.h>
  24#include <linux/vhost.h>
  25
  26#include "vhost.h"
  27
  28enum {
  29	VHOST_VDPA_BACKEND_FEATURES =
  30	(1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
  31	(1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
  32	(1ULL << VHOST_BACKEND_F_IOTLB_ASID),
  33};
  34
  35#define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
  36
  37#define VHOST_VDPA_IOTLB_BUCKETS 16
  38
  39struct vhost_vdpa_as {
  40	struct hlist_node hash_link;
  41	struct vhost_iotlb iotlb;
  42	u32 id;
  43};
  44
  45struct vhost_vdpa {
  46	struct vhost_dev vdev;
  47	struct iommu_domain *domain;
  48	struct vhost_virtqueue *vqs;
  49	struct completion completion;
  50	struct vdpa_device *vdpa;
  51	struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
  52	struct device dev;
  53	struct cdev cdev;
  54	atomic_t opened;
  55	u32 nvqs;
  56	int virtio_id;
  57	int minor;
  58	struct eventfd_ctx *config_ctx;
  59	int in_batch;
  60	struct vdpa_iova_range range;
  61	u32 batch_asid;
  62	bool suspended;
  63};
  64
  65static DEFINE_IDA(vhost_vdpa_ida);
  66
  67static dev_t vhost_vdpa_major;
  68
  69static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
  70				   struct vhost_iotlb *iotlb, u64 start,
  71				   u64 last, u32 asid);
  72
  73static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
  74{
  75	struct vhost_vdpa_as *as = container_of(iotlb, struct
  76						vhost_vdpa_as, iotlb);
  77	return as->id;
  78}
  79
  80static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
  81{
  82	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
  83	struct vhost_vdpa_as *as;
  84
  85	hlist_for_each_entry(as, head, hash_link)
  86		if (as->id == asid)
  87			return as;
  88
  89	return NULL;
  90}
  91
  92static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
  93{
  94	struct vhost_vdpa_as *as = asid_to_as(v, asid);
  95
  96	if (!as)
  97		return NULL;
  98
  99	return &as->iotlb;
 100}
 101
 102static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
 103{
 104	struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
 105	struct vhost_vdpa_as *as;
 106
 107	if (asid_to_as(v, asid))
 108		return NULL;
 109
 110	if (asid >= v->vdpa->nas)
 111		return NULL;
 112
 113	as = kmalloc(sizeof(*as), GFP_KERNEL);
 114	if (!as)
 115		return NULL;
 116
 117	vhost_iotlb_init(&as->iotlb, 0, 0);
 118	as->id = asid;
 119	hlist_add_head(&as->hash_link, head);
 120
 121	return as;
 122}
 123
 124static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
 125						      u32 asid)
 126{
 127	struct vhost_vdpa_as *as = asid_to_as(v, asid);
 128
 129	if (as)
 130		return as;
 131
 132	return vhost_vdpa_alloc_as(v, asid);
 133}
 134
 135static void vhost_vdpa_reset_map(struct vhost_vdpa *v, u32 asid)
 136{
 137	struct vdpa_device *vdpa = v->vdpa;
 138	const struct vdpa_config_ops *ops = vdpa->config;
 139
 140	if (ops->reset_map)
 141		ops->reset_map(vdpa, asid);
 142}
 143
 144static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
 145{
 146	struct vhost_vdpa_as *as = asid_to_as(v, asid);
 147
 148	if (!as)
 149		return -EINVAL;
 150
 151	hlist_del(&as->hash_link);
 152	vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
 153	/*
 154	 * Devices with vendor specific IOMMU may need to restore
 155	 * iotlb to the initial or default state, which cannot be
 156	 * cleaned up in the all range unmap call above. Give them
 157	 * a chance to clean up or reset the map to the desired
 158	 * state.
 159	 */
 160	vhost_vdpa_reset_map(v, asid);
 161	kfree(as);
 162
 163	return 0;
 164}
 165
 166static void handle_vq_kick(struct vhost_work *work)
 167{
 168	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
 169						  poll.work);
 170	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
 171	const struct vdpa_config_ops *ops = v->vdpa->config;
 172
 173	ops->kick_vq(v->vdpa, vq - v->vqs);
 174}
 175
 176static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
 177{
 178	struct vhost_virtqueue *vq = private;
 179	struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
 180
 181	if (call_ctx)
 182		eventfd_signal(call_ctx);
 183
 184	return IRQ_HANDLED;
 185}
 186
 187static irqreturn_t vhost_vdpa_config_cb(void *private)
 188{
 189	struct vhost_vdpa *v = private;
 190	struct eventfd_ctx *config_ctx = v->config_ctx;
 191
 192	if (config_ctx)
 193		eventfd_signal(config_ctx);
 194
 195	return IRQ_HANDLED;
 196}
 197
 198static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
 199{
 200	struct vhost_virtqueue *vq = &v->vqs[qid];
 201	const struct vdpa_config_ops *ops = v->vdpa->config;
 202	struct vdpa_device *vdpa = v->vdpa;
 203	int ret, irq;
 204
 205	if (!ops->get_vq_irq)
 206		return;
 207
 208	irq = ops->get_vq_irq(vdpa, qid);
 209	if (irq < 0)
 210		return;
 211
 212	if (!vq->call_ctx.ctx)
 213		return;
 214
 215	ret = irq_bypass_register_producer(&vq->call_ctx.producer,
 216					   vq->call_ctx.ctx, irq);
 217	if (unlikely(ret))
 218		dev_info(&v->dev, "vq %u, irq bypass producer (eventfd %p) registration fails, ret =  %d\n",
 219			 qid, vq->call_ctx.ctx, ret);
 220}
 221
 222static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
 223{
 224	struct vhost_virtqueue *vq = &v->vqs[qid];
 225
 226	irq_bypass_unregister_producer(&vq->call_ctx.producer);
 227}
 228
 229static int _compat_vdpa_reset(struct vhost_vdpa *v)
 230{
 231	struct vdpa_device *vdpa = v->vdpa;
 232	u32 flags = 0;
 233
 234	v->suspended = false;
 235
 236	if (v->vdev.vqs) {
 237		flags |= !vhost_backend_has_feature(v->vdev.vqs[0],
 238						    VHOST_BACKEND_F_IOTLB_PERSIST) ?
 239			 VDPA_RESET_F_CLEAN_MAP : 0;
 240	}
 241
 242	return vdpa_reset(vdpa, flags);
 243}
 244
 245static int vhost_vdpa_reset(struct vhost_vdpa *v)
 246{
 247	v->in_batch = 0;
 248	return _compat_vdpa_reset(v);
 249}
 250
 251static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
 252{
 253	struct vdpa_device *vdpa = v->vdpa;
 254	const struct vdpa_config_ops *ops = vdpa->config;
 255
 256	if (!vdpa->use_va || !ops->bind_mm)
 257		return 0;
 258
 259	return ops->bind_mm(vdpa, v->vdev.mm);
 260}
 261
 262static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
 263{
 264	struct vdpa_device *vdpa = v->vdpa;
 265	const struct vdpa_config_ops *ops = vdpa->config;
 266
 267	if (!vdpa->use_va || !ops->unbind_mm)
 268		return;
 269
 270	ops->unbind_mm(vdpa);
 271}
 272
 273static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
 274{
 275	struct vdpa_device *vdpa = v->vdpa;
 276	const struct vdpa_config_ops *ops = vdpa->config;
 277	u32 device_id;
 278
 279	device_id = ops->get_device_id(vdpa);
 280
 281	if (copy_to_user(argp, &device_id, sizeof(device_id)))
 282		return -EFAULT;
 283
 284	return 0;
 285}
 286
 287static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
 288{
 289	struct vdpa_device *vdpa = v->vdpa;
 290	const struct vdpa_config_ops *ops = vdpa->config;
 291	u8 status;
 292
 293	status = ops->get_status(vdpa);
 294
 295	if (copy_to_user(statusp, &status, sizeof(status)))
 296		return -EFAULT;
 297
 298	return 0;
 299}
 300
 301static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
 302{
 303	struct vdpa_device *vdpa = v->vdpa;
 304	const struct vdpa_config_ops *ops = vdpa->config;
 305	u8 status, status_old;
 306	u32 nvqs = v->nvqs;
 307	int ret;
 308	u16 i;
 309
 310	if (copy_from_user(&status, statusp, sizeof(status)))
 311		return -EFAULT;
 312
 313	status_old = ops->get_status(vdpa);
 314
 315	/*
 316	 * Userspace shouldn't remove status bits unless reset the
 317	 * status to 0.
 318	 */
 319	if (status != 0 && (status_old & ~status) != 0)
 320		return -EINVAL;
 321
 322	if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
 323		for (i = 0; i < nvqs; i++)
 324			vhost_vdpa_unsetup_vq_irq(v, i);
 325
 326	if (status == 0) {
 327		ret = _compat_vdpa_reset(v);
 328		if (ret)
 329			return ret;
 330	} else
 331		vdpa_set_status(vdpa, status);
 332
 333	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
 334		for (i = 0; i < nvqs; i++)
 335			vhost_vdpa_setup_vq_irq(v, i);
 336
 337	return 0;
 338}
 339
 340static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
 341				      struct vhost_vdpa_config *c)
 342{
 343	struct vdpa_device *vdpa = v->vdpa;
 344	size_t size = vdpa->config->get_config_size(vdpa);
 345
 346	if (c->len == 0 || c->off > size)
 347		return -EINVAL;
 348
 349	if (c->len > size - c->off)
 350		return -E2BIG;
 351
 352	return 0;
 353}
 354
 355static long vhost_vdpa_get_config(struct vhost_vdpa *v,
 356				  struct vhost_vdpa_config __user *c)
 357{
 358	struct vdpa_device *vdpa = v->vdpa;
 359	struct vhost_vdpa_config config;
 360	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
 361	u8 *buf;
 362
 363	if (copy_from_user(&config, c, size))
 364		return -EFAULT;
 365	if (vhost_vdpa_config_validate(v, &config))
 366		return -EINVAL;
 367	buf = kvzalloc(config.len, GFP_KERNEL);
 368	if (!buf)
 369		return -ENOMEM;
 370
 371	vdpa_get_config(vdpa, config.off, buf, config.len);
 372
 373	if (copy_to_user(c->buf, buf, config.len)) {
 374		kvfree(buf);
 375		return -EFAULT;
 376	}
 377
 378	kvfree(buf);
 379	return 0;
 380}
 381
 382static long vhost_vdpa_set_config(struct vhost_vdpa *v,
 383				  struct vhost_vdpa_config __user *c)
 384{
 385	struct vdpa_device *vdpa = v->vdpa;
 386	struct vhost_vdpa_config config;
 387	unsigned long size = offsetof(struct vhost_vdpa_config, buf);
 388	u8 *buf;
 389
 390	if (copy_from_user(&config, c, size))
 391		return -EFAULT;
 392	if (vhost_vdpa_config_validate(v, &config))
 393		return -EINVAL;
 394
 395	buf = vmemdup_user(c->buf, config.len);
 396	if (IS_ERR(buf))
 397		return PTR_ERR(buf);
 398
 399	vdpa_set_config(vdpa, config.off, buf, config.len);
 400
 401	kvfree(buf);
 402	return 0;
 403}
 404
 405static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
 406{
 407	struct vdpa_device *vdpa = v->vdpa;
 408	const struct vdpa_config_ops *ops = vdpa->config;
 409
 410	return ops->suspend;
 411}
 412
 413static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)
 414{
 415	struct vdpa_device *vdpa = v->vdpa;
 416	const struct vdpa_config_ops *ops = vdpa->config;
 417
 418	return ops->resume;
 419}
 420
 421static bool vhost_vdpa_has_desc_group(const struct vhost_vdpa *v)
 422{
 423	struct vdpa_device *vdpa = v->vdpa;
 424	const struct vdpa_config_ops *ops = vdpa->config;
 425
 426	return ops->get_vq_desc_group;
 427}
 428
 429static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
 430{
 431	struct vdpa_device *vdpa = v->vdpa;
 432	const struct vdpa_config_ops *ops = vdpa->config;
 433	u64 features;
 434
 435	features = ops->get_device_features(vdpa);
 436
 437	if (copy_to_user(featurep, &features, sizeof(features)))
 438		return -EFAULT;
 439
 440	return 0;
 441}
 442
 443static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v)
 444{
 445	struct vdpa_device *vdpa = v->vdpa;
 446	const struct vdpa_config_ops *ops = vdpa->config;
 447
 448	if (!ops->get_backend_features)
 449		return 0;
 450	else
 451		return ops->get_backend_features(vdpa);
 452}
 453
 454static bool vhost_vdpa_has_persistent_map(const struct vhost_vdpa *v)
 455{
 456	struct vdpa_device *vdpa = v->vdpa;
 457	const struct vdpa_config_ops *ops = vdpa->config;
 458
 459	return (!ops->set_map && !ops->dma_map) || ops->reset_map ||
 460	       vhost_vdpa_get_backend_features(v) & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
 461}
 462
 463static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
 464{
 465	struct vdpa_device *vdpa = v->vdpa;
 466	const struct vdpa_config_ops *ops = vdpa->config;
 467	struct vhost_dev *d = &v->vdev;
 468	u64 actual_features;
 469	u64 features;
 470	int i;
 471
 472	/*
 473	 * It's not allowed to change the features after they have
 474	 * been negotiated.
 475	 */
 476	if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
 477		return -EBUSY;
 478
 479	if (copy_from_user(&features, featurep, sizeof(features)))
 480		return -EFAULT;
 481
 482	if (vdpa_set_features(vdpa, features))
 483		return -EINVAL;
 484
 485	/* let the vqs know what has been configured */
 486	actual_features = ops->get_driver_features(vdpa);
 487	for (i = 0; i < d->nvqs; ++i) {
 488		struct vhost_virtqueue *vq = d->vqs[i];
 489
 490		mutex_lock(&vq->mutex);
 491		vq->acked_features = actual_features;
 492		mutex_unlock(&vq->mutex);
 493	}
 494
 495	return 0;
 496}
 497
 498static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
 499{
 500	struct vdpa_device *vdpa = v->vdpa;
 501	const struct vdpa_config_ops *ops = vdpa->config;
 502	u16 num;
 503
 504	num = ops->get_vq_num_max(vdpa);
 505
 506	if (copy_to_user(argp, &num, sizeof(num)))
 507		return -EFAULT;
 508
 509	return 0;
 510}
 511
 512static void vhost_vdpa_config_put(struct vhost_vdpa *v)
 513{
 514	if (v->config_ctx) {
 515		eventfd_ctx_put(v->config_ctx);
 516		v->config_ctx = NULL;
 517	}
 518}
 519
 520static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
 521{
 522	struct vdpa_callback cb;
 523	int fd;
 524	struct eventfd_ctx *ctx;
 525
 526	cb.callback = vhost_vdpa_config_cb;
 527	cb.private = v;
 528	if (copy_from_user(&fd, argp, sizeof(fd)))
 529		return  -EFAULT;
 530
 531	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
 532	swap(ctx, v->config_ctx);
 533
 534	if (!IS_ERR_OR_NULL(ctx))
 535		eventfd_ctx_put(ctx);
 536
 537	if (IS_ERR(v->config_ctx)) {
 538		long ret = PTR_ERR(v->config_ctx);
 539
 540		v->config_ctx = NULL;
 541		return ret;
 542	}
 543
 544	v->vdpa->config->set_config_cb(v->vdpa, &cb);
 545
 546	return 0;
 547}
 548
 549static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
 550{
 551	struct vhost_vdpa_iova_range range = {
 552		.first = v->range.first,
 553		.last = v->range.last,
 554	};
 555
 556	if (copy_to_user(argp, &range, sizeof(range)))
 557		return -EFAULT;
 558	return 0;
 559}
 560
 561static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
 562{
 563	struct vdpa_device *vdpa = v->vdpa;
 564	const struct vdpa_config_ops *ops = vdpa->config;
 565	u32 size;
 566
 567	size = ops->get_config_size(vdpa);
 568
 569	if (copy_to_user(argp, &size, sizeof(size)))
 570		return -EFAULT;
 571
 572	return 0;
 573}
 574
 575static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
 576{
 577	struct vdpa_device *vdpa = v->vdpa;
 578
 579	if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
 580		return -EFAULT;
 581
 582	return 0;
 583}
 584
 585/* After a successful return of ioctl the device must not process more
 586 * virtqueue descriptors. The device can answer to read or writes of config
 587 * fields as if it were not suspended. In particular, writing to "queue_enable"
 588 * with a value of 1 will not make the device start processing buffers.
 589 */
 590static long vhost_vdpa_suspend(struct vhost_vdpa *v)
 591{
 592	struct vdpa_device *vdpa = v->vdpa;
 593	const struct vdpa_config_ops *ops = vdpa->config;
 594	int ret;
 595
 596	if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
 597		return 0;
 598
 599	if (!ops->suspend)
 600		return -EOPNOTSUPP;
 601
 602	ret = ops->suspend(vdpa);
 603	if (!ret)
 604		v->suspended = true;
 605
 606	return ret;
 607}
 608
 609/* After a successful return of this ioctl the device resumes processing
 610 * virtqueue descriptors. The device becomes fully operational the same way it
 611 * was before it was suspended.
 612 */
 613static long vhost_vdpa_resume(struct vhost_vdpa *v)
 614{
 615	struct vdpa_device *vdpa = v->vdpa;
 616	const struct vdpa_config_ops *ops = vdpa->config;
 617	int ret;
 618
 619	if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
 620		return 0;
 621
 622	if (!ops->resume)
 623		return -EOPNOTSUPP;
 624
 625	ret = ops->resume(vdpa);
 626	if (!ret)
 627		v->suspended = false;
 628
 629	return ret;
 630}
 631
 632static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 633				   void __user *argp)
 634{
 635	struct vdpa_device *vdpa = v->vdpa;
 636	const struct vdpa_config_ops *ops = vdpa->config;
 637	struct vdpa_vq_state vq_state;
 638	struct vdpa_callback cb;
 639	struct vhost_virtqueue *vq;
 640	struct vhost_vring_state s;
 641	u32 idx;
 642	long r;
 643
 644	r = get_user(idx, (u32 __user *)argp);
 645	if (r < 0)
 646		return r;
 647
 648	if (idx >= v->nvqs)
 649		return -ENOBUFS;
 650
 651	idx = array_index_nospec(idx, v->nvqs);
 652	vq = &v->vqs[idx];
 653
 654	switch (cmd) {
 655	case VHOST_VDPA_SET_VRING_ENABLE:
 656		if (copy_from_user(&s, argp, sizeof(s)))
 657			return -EFAULT;
 658		ops->set_vq_ready(vdpa, idx, s.num);
 659		return 0;
 660	case VHOST_VDPA_GET_VRING_GROUP:
 661		if (!ops->get_vq_group)
 662			return -EOPNOTSUPP;
 663		s.index = idx;
 664		s.num = ops->get_vq_group(vdpa, idx);
 665		if (s.num >= vdpa->ngroups)
 666			return -EIO;
 667		else if (copy_to_user(argp, &s, sizeof(s)))
 668			return -EFAULT;
 669		return 0;
 670	case VHOST_VDPA_GET_VRING_DESC_GROUP:
 671		if (!vhost_vdpa_has_desc_group(v))
 672			return -EOPNOTSUPP;
 673		s.index = idx;
 674		s.num = ops->get_vq_desc_group(vdpa, idx);
 675		if (s.num >= vdpa->ngroups)
 676			return -EIO;
 677		else if (copy_to_user(argp, &s, sizeof(s)))
 678			return -EFAULT;
 679		return 0;
 680	case VHOST_VDPA_SET_GROUP_ASID:
 681		if (copy_from_user(&s, argp, sizeof(s)))
 682			return -EFAULT;
 683		if (idx >= vdpa->ngroups || s.num >= vdpa->nas)
 684			return -EINVAL;
 685		if (!ops->set_group_asid)
 686			return -EOPNOTSUPP;
 687		return ops->set_group_asid(vdpa, idx, s.num);
 688	case VHOST_VDPA_GET_VRING_SIZE:
 689		if (!ops->get_vq_size)
 690			return -EOPNOTSUPP;
 691		s.index = idx;
 692		s.num = ops->get_vq_size(vdpa, idx);
 693		if (copy_to_user(argp, &s, sizeof(s)))
 694			return -EFAULT;
 695		return 0;
 696	case VHOST_GET_VRING_BASE:
 697		r = ops->get_vq_state(v->vdpa, idx, &vq_state);
 698		if (r)
 699			return r;
 700
 701		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
 702			vq->last_avail_idx = vq_state.packed.last_avail_idx |
 703					     (vq_state.packed.last_avail_counter << 15);
 704			vq->last_used_idx = vq_state.packed.last_used_idx |
 705					    (vq_state.packed.last_used_counter << 15);
 706		} else {
 707			vq->last_avail_idx = vq_state.split.avail_index;
 708		}
 709		break;
 710	case VHOST_SET_VRING_CALL:
 711		if (vq->call_ctx.ctx) {
 712			if (ops->get_status(vdpa) &
 713			    VIRTIO_CONFIG_S_DRIVER_OK)
 714				vhost_vdpa_unsetup_vq_irq(v, idx);
 715		}
 716		break;
 717	}
 718
 719	r = vhost_vring_ioctl(&v->vdev, cmd, argp);
 720	if (r)
 721		return r;
 722
 723	switch (cmd) {
 724	case VHOST_SET_VRING_ADDR:
 725		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
 726			return -EINVAL;
 727
 728		if (ops->set_vq_address(vdpa, idx,
 729					(u64)(uintptr_t)vq->desc,
 730					(u64)(uintptr_t)vq->avail,
 731					(u64)(uintptr_t)vq->used))
 732			r = -EINVAL;
 733		break;
 734
 735	case VHOST_SET_VRING_BASE:
 736		if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
 737			return -EINVAL;
 738
 739		if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
 740			vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff;
 741			vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000);
 742			vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff;
 743			vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000);
 744		} else {
 745			vq_state.split.avail_index = vq->last_avail_idx;
 746		}
 747		r = ops->set_vq_state(vdpa, idx, &vq_state);
 748		break;
 749
 750	case VHOST_SET_VRING_CALL:
 751		if (vq->call_ctx.ctx) {
 752			cb.callback = vhost_vdpa_virtqueue_cb;
 753			cb.private = vq;
 754			cb.trigger = vq->call_ctx.ctx;
 755			if (ops->get_status(vdpa) &
 756			    VIRTIO_CONFIG_S_DRIVER_OK)
 757				vhost_vdpa_setup_vq_irq(v, idx);
 758		} else {
 759			cb.callback = NULL;
 760			cb.private = NULL;
 761			cb.trigger = NULL;
 762		}
 763		ops->set_vq_cb(vdpa, idx, &cb);
 764		break;
 765
 766	case VHOST_SET_VRING_NUM:
 767		ops->set_vq_num(vdpa, idx, vq->num);
 768		break;
 769	}
 770
 771	return r;
 772}
 773
 774static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 775				      unsigned int cmd, unsigned long arg)
 776{
 777	struct vhost_vdpa *v = filep->private_data;
 778	struct vhost_dev *d = &v->vdev;
 779	void __user *argp = (void __user *)arg;
 780	u64 __user *featurep = argp;
 781	u64 features;
 782	long r = 0;
 783
 784	if (cmd == VHOST_SET_BACKEND_FEATURES) {
 785		if (copy_from_user(&features, featurep, sizeof(features)))
 786			return -EFAULT;
 787		if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
 788				 BIT_ULL(VHOST_BACKEND_F_DESC_ASID) |
 789				 BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST) |
 790				 BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
 791				 BIT_ULL(VHOST_BACKEND_F_RESUME) |
 792				 BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK)))
 793			return -EOPNOTSUPP;
 794		if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
 795		     !vhost_vdpa_can_suspend(v))
 796			return -EOPNOTSUPP;
 797		if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) &&
 798		     !vhost_vdpa_can_resume(v))
 799			return -EOPNOTSUPP;
 800		if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
 801		    !(features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID)))
 802			return -EINVAL;
 803		if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
 804		     !vhost_vdpa_has_desc_group(v))
 805			return -EOPNOTSUPP;
 806		if ((features & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST)) &&
 807		     !vhost_vdpa_has_persistent_map(v))
 808			return -EOPNOTSUPP;
 809		vhost_set_backend_features(&v->vdev, features);
 810		return 0;
 811	}
 812
 813	mutex_lock(&d->mutex);
 814
 815	switch (cmd) {
 816	case VHOST_VDPA_GET_DEVICE_ID:
 817		r = vhost_vdpa_get_device_id(v, argp);
 818		break;
 819	case VHOST_VDPA_GET_STATUS:
 820		r = vhost_vdpa_get_status(v, argp);
 821		break;
 822	case VHOST_VDPA_SET_STATUS:
 823		r = vhost_vdpa_set_status(v, argp);
 824		break;
 825	case VHOST_VDPA_GET_CONFIG:
 826		r = vhost_vdpa_get_config(v, argp);
 827		break;
 828	case VHOST_VDPA_SET_CONFIG:
 829		r = vhost_vdpa_set_config(v, argp);
 830		break;
 831	case VHOST_GET_FEATURES:
 832		r = vhost_vdpa_get_features(v, argp);
 833		break;
 834	case VHOST_SET_FEATURES:
 835		r = vhost_vdpa_set_features(v, argp);
 836		break;
 837	case VHOST_VDPA_GET_VRING_NUM:
 838		r = vhost_vdpa_get_vring_num(v, argp);
 839		break;
 840	case VHOST_VDPA_GET_GROUP_NUM:
 841		if (copy_to_user(argp, &v->vdpa->ngroups,
 842				 sizeof(v->vdpa->ngroups)))
 843			r = -EFAULT;
 844		break;
 845	case VHOST_VDPA_GET_AS_NUM:
 846		if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
 847			r = -EFAULT;
 848		break;
 849	case VHOST_SET_LOG_BASE:
 850	case VHOST_SET_LOG_FD:
 851		r = -ENOIOCTLCMD;
 852		break;
 853	case VHOST_VDPA_SET_CONFIG_CALL:
 854		r = vhost_vdpa_set_config_call(v, argp);
 855		break;
 856	case VHOST_GET_BACKEND_FEATURES:
 857		features = VHOST_VDPA_BACKEND_FEATURES;
 858		if (vhost_vdpa_can_suspend(v))
 859			features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
 860		if (vhost_vdpa_can_resume(v))
 861			features |= BIT_ULL(VHOST_BACKEND_F_RESUME);
 862		if (vhost_vdpa_has_desc_group(v))
 863			features |= BIT_ULL(VHOST_BACKEND_F_DESC_ASID);
 864		if (vhost_vdpa_has_persistent_map(v))
 865			features |= BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
 866		features |= vhost_vdpa_get_backend_features(v);
 867		if (copy_to_user(featurep, &features, sizeof(features)))
 868			r = -EFAULT;
 869		break;
 870	case VHOST_VDPA_GET_IOVA_RANGE:
 871		r = vhost_vdpa_get_iova_range(v, argp);
 872		break;
 873	case VHOST_VDPA_GET_CONFIG_SIZE:
 874		r = vhost_vdpa_get_config_size(v, argp);
 875		break;
 876	case VHOST_VDPA_GET_VQS_COUNT:
 877		r = vhost_vdpa_get_vqs_count(v, argp);
 878		break;
 879	case VHOST_VDPA_SUSPEND:
 880		r = vhost_vdpa_suspend(v);
 881		break;
 882	case VHOST_VDPA_RESUME:
 883		r = vhost_vdpa_resume(v);
 884		break;
 885	default:
 886		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
 887		if (r == -ENOIOCTLCMD)
 888			r = vhost_vdpa_vring_ioctl(v, cmd, argp);
 889		break;
 890	}
 891
 892	if (r)
 893		goto out;
 894
 895	switch (cmd) {
 896	case VHOST_SET_OWNER:
 897		r = vhost_vdpa_bind_mm(v);
 898		if (r)
 899			vhost_dev_reset_owner(d, NULL);
 900		break;
 901	}
 902out:
 903	mutex_unlock(&d->mutex);
 904	return r;
 905}
 906static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
 907				     struct vhost_iotlb_map *map, u32 asid)
 908{
 909	struct vdpa_device *vdpa = v->vdpa;
 910	const struct vdpa_config_ops *ops = vdpa->config;
 911	if (ops->dma_map) {
 912		ops->dma_unmap(vdpa, asid, map->start, map->size);
 913	} else if (ops->set_map == NULL) {
 914		iommu_unmap(v->domain, map->start, map->size);
 915	}
 916}
 917
 918static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 919				u64 start, u64 last, u32 asid)
 920{
 921	struct vhost_dev *dev = &v->vdev;
 922	struct vhost_iotlb_map *map;
 923	struct page *page;
 924	unsigned long pfn, pinned;
 925
 926	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
 927		pinned = PFN_DOWN(map->size);
 928		for (pfn = PFN_DOWN(map->addr);
 929		     pinned > 0; pfn++, pinned--) {
 930			page = pfn_to_page(pfn);
 931			if (map->perm & VHOST_ACCESS_WO)
 932				set_page_dirty_lock(page);
 933			unpin_user_page(page);
 934		}
 935		atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
 936		vhost_vdpa_general_unmap(v, map, asid);
 937		vhost_iotlb_map_free(iotlb, map);
 938	}
 939}
 940
 941static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 942				u64 start, u64 last, u32 asid)
 943{
 944	struct vhost_iotlb_map *map;
 945	struct vdpa_map_file *map_file;
 946
 947	while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
 948		map_file = (struct vdpa_map_file *)map->opaque;
 949		fput(map_file->file);
 950		kfree(map_file);
 951		vhost_vdpa_general_unmap(v, map, asid);
 952		vhost_iotlb_map_free(iotlb, map);
 953	}
 954}
 955
 956static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
 957				   struct vhost_iotlb *iotlb, u64 start,
 958				   u64 last, u32 asid)
 959{
 960	struct vdpa_device *vdpa = v->vdpa;
 961
 962	if (vdpa->use_va)
 963		return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
 964
 965	return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
 966}
 967
 968static int perm_to_iommu_flags(u32 perm)
 969{
 970	int flags = 0;
 971
 972	switch (perm) {
 973	case VHOST_ACCESS_WO:
 974		flags |= IOMMU_WRITE;
 975		break;
 976	case VHOST_ACCESS_RO:
 977		flags |= IOMMU_READ;
 978		break;
 979	case VHOST_ACCESS_RW:
 980		flags |= (IOMMU_WRITE | IOMMU_READ);
 981		break;
 982	default:
 983		WARN(1, "invalidate vhost IOTLB permission\n");
 984		break;
 985	}
 986
 987	return flags | IOMMU_CACHE;
 988}
 989
 990static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
 991			  u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
 992{
 993	struct vhost_dev *dev = &v->vdev;
 994	struct vdpa_device *vdpa = v->vdpa;
 995	const struct vdpa_config_ops *ops = vdpa->config;
 996	u32 asid = iotlb_to_asid(iotlb);
 997	int r = 0;
 998
 999	r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
1000				      pa, perm, opaque);
1001	if (r)
1002		return r;
1003
1004	if (ops->dma_map) {
1005		r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
1006	} else if (ops->set_map) {
1007		if (!v->in_batch)
1008			r = ops->set_map(vdpa, asid, iotlb);
1009	} else {
1010		r = iommu_map(v->domain, iova, pa, size,
1011			      perm_to_iommu_flags(perm),
1012			      GFP_KERNEL_ACCOUNT);
1013	}
1014	if (r) {
1015		vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
1016		return r;
1017	}
1018
1019	if (!vdpa->use_va)
1020		atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
1021
1022	return 0;
1023}
1024
1025static void vhost_vdpa_unmap(struct vhost_vdpa *v,
1026			     struct vhost_iotlb *iotlb,
1027			     u64 iova, u64 size)
1028{
1029	struct vdpa_device *vdpa = v->vdpa;
1030	const struct vdpa_config_ops *ops = vdpa->config;
1031	u32 asid = iotlb_to_asid(iotlb);
1032
1033	vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
1034
1035	if (ops->set_map) {
1036		if (!v->in_batch)
1037			ops->set_map(vdpa, asid, iotlb);
1038	}
1039
1040}
1041
1042static int vhost_vdpa_va_map(struct vhost_vdpa *v,
1043			     struct vhost_iotlb *iotlb,
1044			     u64 iova, u64 size, u64 uaddr, u32 perm)
1045{
1046	struct vhost_dev *dev = &v->vdev;
1047	u64 offset, map_size, map_iova = iova;
1048	struct vdpa_map_file *map_file;
1049	struct vm_area_struct *vma;
1050	int ret = 0;
1051
1052	mmap_read_lock(dev->mm);
1053
1054	while (size) {
1055		vma = find_vma(dev->mm, uaddr);
1056		if (!vma) {
1057			ret = -EINVAL;
1058			break;
1059		}
1060		map_size = min(size, vma->vm_end - uaddr);
1061		if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
1062			!(vma->vm_flags & (VM_IO | VM_PFNMAP))))
1063			goto next;
1064
1065		map_file = kzalloc(sizeof(*map_file), GFP_KERNEL);
1066		if (!map_file) {
1067			ret = -ENOMEM;
1068			break;
1069		}
1070		offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
1071		map_file->offset = offset;
1072		map_file->file = get_file(vma->vm_file);
1073		ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
1074				     perm, map_file);
1075		if (ret) {
1076			fput(map_file->file);
1077			kfree(map_file);
1078			break;
1079		}
1080next:
1081		size -= map_size;
1082		uaddr += map_size;
1083		map_iova += map_size;
1084	}
1085	if (ret)
1086		vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
1087
1088	mmap_read_unlock(dev->mm);
1089
1090	return ret;
1091}
1092
1093static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
1094			     struct vhost_iotlb *iotlb,
1095			     u64 iova, u64 size, u64 uaddr, u32 perm)
1096{
1097	struct vhost_dev *dev = &v->vdev;
1098	struct page **page_list;
1099	unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
1100	unsigned int gup_flags = FOLL_LONGTERM;
1101	unsigned long npages, cur_base, map_pfn, last_pfn = 0;
1102	unsigned long lock_limit, sz2pin, nchunks, i;
1103	u64 start = iova;
1104	long pinned;
1105	int ret = 0;
1106
1107	/* Limit the use of memory for bookkeeping */
1108	page_list = (struct page **) __get_free_page(GFP_KERNEL);
1109	if (!page_list)
1110		return -ENOMEM;
1111
1112	if (perm & VHOST_ACCESS_WO)
1113		gup_flags |= FOLL_WRITE;
1114
1115	npages = PFN_UP(size + (iova & ~PAGE_MASK));
1116	if (!npages) {
1117		ret = -EINVAL;
1118		goto free;
1119	}
1120
1121	mmap_read_lock(dev->mm);
1122
1123	lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1124	if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
1125		ret = -ENOMEM;
1126		goto unlock;
1127	}
1128
1129	cur_base = uaddr & PAGE_MASK;
1130	iova &= PAGE_MASK;
1131	nchunks = 0;
1132
1133	while (npages) {
1134		sz2pin = min_t(unsigned long, npages, list_size);
1135		pinned = pin_user_pages(cur_base, sz2pin,
1136					gup_flags, page_list);
1137		if (sz2pin != pinned) {
1138			if (pinned < 0) {
1139				ret = pinned;
1140			} else {
1141				unpin_user_pages(page_list, pinned);
1142				ret = -ENOMEM;
1143			}
1144			goto out;
1145		}
1146		nchunks++;
1147
1148		if (!last_pfn)
1149			map_pfn = page_to_pfn(page_list[0]);
1150
1151		for (i = 0; i < pinned; i++) {
1152			unsigned long this_pfn = page_to_pfn(page_list[i]);
1153			u64 csize;
1154
1155			if (last_pfn && (this_pfn != last_pfn + 1)) {
1156				/* Pin a contiguous chunk of memory */
1157				csize = PFN_PHYS(last_pfn - map_pfn + 1);
1158				ret = vhost_vdpa_map(v, iotlb, iova, csize,
1159						     PFN_PHYS(map_pfn),
1160						     perm, NULL);
1161				if (ret) {
1162					/*
1163					 * Unpin the pages that are left unmapped
1164					 * from this point on in the current
1165					 * page_list. The remaining outstanding
1166					 * ones which may stride across several
1167					 * chunks will be covered in the common
1168					 * error path subsequently.
1169					 */
1170					unpin_user_pages(&page_list[i],
1171							 pinned - i);
1172					goto out;
1173				}
1174
1175				map_pfn = this_pfn;
1176				iova += csize;
1177				nchunks = 0;
1178			}
1179
1180			last_pfn = this_pfn;
1181		}
1182
1183		cur_base += PFN_PHYS(pinned);
1184		npages -= pinned;
1185	}
1186
1187	/* Pin the rest chunk */
1188	ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
1189			     PFN_PHYS(map_pfn), perm, NULL);
1190out:
1191	if (ret) {
1192		if (nchunks) {
1193			unsigned long pfn;
1194
1195			/*
1196			 * Unpin the outstanding pages which are yet to be
1197			 * mapped but haven't due to vdpa_map() or
1198			 * pin_user_pages() failure.
1199			 *
1200			 * Mapped pages are accounted in vdpa_map(), hence
1201			 * the corresponding unpinning will be handled by
1202			 * vdpa_unmap().
1203			 */
1204			WARN_ON(!last_pfn);
1205			for (pfn = map_pfn; pfn <= last_pfn; pfn++)
1206				unpin_user_page(pfn_to_page(pfn));
1207		}
1208		vhost_vdpa_unmap(v, iotlb, start, size);
1209	}
1210unlock:
1211	mmap_read_unlock(dev->mm);
1212free:
1213	free_page((unsigned long)page_list);
1214	return ret;
1215
1216}
1217
1218static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1219					   struct vhost_iotlb *iotlb,
1220					   struct vhost_iotlb_msg *msg)
1221{
1222	struct vdpa_device *vdpa = v->vdpa;
1223
1224	if (msg->iova < v->range.first || !msg->size ||
1225	    msg->iova > U64_MAX - msg->size + 1 ||
1226	    msg->iova + msg->size - 1 > v->range.last)
1227		return -EINVAL;
1228
1229	if (vhost_iotlb_itree_first(iotlb, msg->iova,
1230				    msg->iova + msg->size - 1))
1231		return -EEXIST;
1232
1233	if (vdpa->use_va)
1234		return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1235					 msg->uaddr, msg->perm);
1236
1237	return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1238				 msg->perm);
1239}
1240
1241static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1242					struct vhost_iotlb_msg *msg)
1243{
1244	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1245	struct vdpa_device *vdpa = v->vdpa;
1246	const struct vdpa_config_ops *ops = vdpa->config;
1247	struct vhost_iotlb *iotlb = NULL;
1248	struct vhost_vdpa_as *as = NULL;
1249	int r = 0;
1250
1251	mutex_lock(&dev->mutex);
1252
1253	r = vhost_dev_check_owner(dev);
1254	if (r)
1255		goto unlock;
1256
1257	if (msg->type == VHOST_IOTLB_UPDATE ||
1258	    msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1259		as = vhost_vdpa_find_alloc_as(v, asid);
1260		if (!as) {
1261			dev_err(&v->dev, "can't find and alloc asid %d\n",
1262				asid);
1263			r = -EINVAL;
1264			goto unlock;
1265		}
1266		iotlb = &as->iotlb;
1267	} else
1268		iotlb = asid_to_iotlb(v, asid);
1269
1270	if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1271		if (v->in_batch && v->batch_asid != asid) {
1272			dev_info(&v->dev, "batch id %d asid %d\n",
1273				 v->batch_asid, asid);
1274		}
1275		if (!iotlb)
1276			dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1277		r = -EINVAL;
1278		goto unlock;
1279	}
1280
1281	switch (msg->type) {
1282	case VHOST_IOTLB_UPDATE:
1283		r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1284		break;
1285	case VHOST_IOTLB_INVALIDATE:
1286		vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1287		break;
1288	case VHOST_IOTLB_BATCH_BEGIN:
1289		v->batch_asid = asid;
1290		v->in_batch = true;
1291		break;
1292	case VHOST_IOTLB_BATCH_END:
1293		if (v->in_batch && ops->set_map)
1294			ops->set_map(vdpa, asid, iotlb);
1295		v->in_batch = false;
1296		break;
1297	default:
1298		r = -EINVAL;
1299		break;
1300	}
1301unlock:
1302	mutex_unlock(&dev->mutex);
1303
1304	return r;
1305}
1306
1307static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1308					 struct iov_iter *from)
1309{
1310	struct file *file = iocb->ki_filp;
1311	struct vhost_vdpa *v = file->private_data;
1312	struct vhost_dev *dev = &v->vdev;
1313
1314	return vhost_chr_write_iter(dev, from);
1315}
1316
1317static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1318{
1319	struct vdpa_device *vdpa = v->vdpa;
1320	const struct vdpa_config_ops *ops = vdpa->config;
1321	union virtio_map map = vdpa_get_map(vdpa);
1322	struct device *dma_dev = map.dma_dev;
1323	int ret;
1324
1325	/* Device want to do DMA by itself */
1326	if (ops->set_map || ops->dma_map)
1327		return 0;
1328
1329	if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) {
1330		dev_warn_once(&v->dev,
1331			      "Failed to allocate domain, device is not IOMMU cache coherent capable\n");
1332		return -ENOTSUPP;
1333	}
1334
1335	v->domain = iommu_paging_domain_alloc(dma_dev);
1336	if (IS_ERR(v->domain)) {
1337		ret = PTR_ERR(v->domain);
1338		v->domain = NULL;
1339		return ret;
1340	}
1341
1342	ret = iommu_attach_device(v->domain, dma_dev);
1343	if (ret)
1344		goto err_attach;
1345
1346	return 0;
1347
1348err_attach:
1349	iommu_domain_free(v->domain);
1350	v->domain = NULL;
1351	return ret;
1352}
1353
1354static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1355{
1356	struct vdpa_device *vdpa = v->vdpa;
1357	union virtio_map map = vdpa_get_map(vdpa);
1358	struct device *dma_dev = map.dma_dev;
1359
1360	if (v->domain) {
1361		iommu_detach_device(v->domain, dma_dev);
1362		iommu_domain_free(v->domain);
1363	}
1364
1365	v->domain = NULL;
1366}
1367
1368static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1369{
1370	struct vdpa_iova_range *range = &v->range;
1371	struct vdpa_device *vdpa = v->vdpa;
1372	const struct vdpa_config_ops *ops = vdpa->config;
1373
1374	if (ops->get_iova_range) {
1375		*range = ops->get_iova_range(vdpa);
1376	} else if (v->domain && v->domain->geometry.force_aperture) {
1377		range->first = v->domain->geometry.aperture_start;
1378		range->last = v->domain->geometry.aperture_end;
1379	} else {
1380		range->first = 0;
1381		range->last = ULLONG_MAX;
1382	}
1383}
1384
1385static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1386{
1387	struct vhost_vdpa_as *as;
1388	u32 asid;
1389
1390	for (asid = 0; asid < v->vdpa->nas; asid++) {
1391		as = asid_to_as(v, asid);
1392		if (as)
1393			vhost_vdpa_remove_as(v, asid);
1394	}
1395
1396	vhost_vdpa_free_domain(v);
1397	vhost_dev_cleanup(&v->vdev);
1398	kfree(v->vdev.vqs);
1399	v->vdev.vqs = NULL;
1400}
1401
1402static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1403{
1404	struct vhost_vdpa *v;
1405	struct vhost_dev *dev;
1406	struct vhost_virtqueue **vqs;
1407	int r, opened;
1408	u32 i, nvqs;
1409
1410	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1411
1412	opened = atomic_cmpxchg(&v->opened, 0, 1);
1413	if (opened)
1414		return -EBUSY;
1415
1416	nvqs = v->nvqs;
1417	r = vhost_vdpa_reset(v);
1418	if (r)
1419		goto err;
1420
1421	vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1422	if (!vqs) {
1423		r = -ENOMEM;
1424		goto err;
1425	}
1426
1427	dev = &v->vdev;
1428	for (i = 0; i < nvqs; i++) {
1429		vqs[i] = &v->vqs[i];
1430		vqs[i]->handle_kick = handle_vq_kick;
1431		vqs[i]->call_ctx.ctx = NULL;
1432	}
1433	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1434		       vhost_vdpa_process_iotlb_msg);
1435
1436	r = vhost_vdpa_alloc_domain(v);
1437	if (r)
1438		goto err_alloc_domain;
1439
1440	vhost_vdpa_set_iova_range(v);
1441
1442	filep->private_data = v;
1443
1444	return 0;
1445
1446err_alloc_domain:
1447	vhost_vdpa_cleanup(v);
1448err:
1449	atomic_dec(&v->opened);
1450	return r;
1451}
1452
1453static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1454{
1455	u32 i;
1456
1457	for (i = 0; i < v->nvqs; i++)
1458		vhost_vdpa_unsetup_vq_irq(v, i);
1459}
1460
1461static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1462{
1463	struct vhost_vdpa *v = filep->private_data;
1464	struct vhost_dev *d = &v->vdev;
1465
1466	mutex_lock(&d->mutex);
1467	filep->private_data = NULL;
1468	vhost_vdpa_clean_irq(v);
1469	vhost_vdpa_reset(v);
1470	vhost_dev_stop(&v->vdev);
1471	vhost_vdpa_unbind_mm(v);
1472	vhost_vdpa_config_put(v);
1473	vhost_vdpa_cleanup(v);
1474	mutex_unlock(&d->mutex);
1475
1476	atomic_dec(&v->opened);
1477	complete(&v->completion);
1478
1479	return 0;
1480}
1481
1482#ifdef CONFIG_MMU
1483static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1484{
1485	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1486	struct vdpa_device *vdpa = v->vdpa;
1487	const struct vdpa_config_ops *ops = vdpa->config;
1488	struct vdpa_notification_area notify;
1489	struct vm_area_struct *vma = vmf->vma;
1490	u16 index = vma->vm_pgoff;
1491
1492	notify = ops->get_vq_notification(vdpa, index);
1493
1494	return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr));
1495}
1496
1497static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1498	.fault = vhost_vdpa_fault,
1499};
1500
1501static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1502{
1503	struct vhost_vdpa *v = vma->vm_file->private_data;
1504	struct vdpa_device *vdpa = v->vdpa;
1505	const struct vdpa_config_ops *ops = vdpa->config;
1506	struct vdpa_notification_area notify;
1507	unsigned long index = vma->vm_pgoff;
1508
1509	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1510		return -EINVAL;
1511	if ((vma->vm_flags & VM_SHARED) == 0)
1512		return -EINVAL;
1513	if (vma->vm_flags & VM_READ)
1514		return -EINVAL;
1515	if (index > 65535)
1516		return -EINVAL;
1517	if (!ops->get_vq_notification)
1518		return -ENOTSUPP;
1519
1520	/* To be safe and easily modelled by userspace, We only
1521	 * support the doorbell which sits on the page boundary and
1522	 * does not share the page with other registers.
1523	 */
1524	notify = ops->get_vq_notification(vdpa, index);
1525	if (notify.addr & (PAGE_SIZE - 1))
1526		return -EINVAL;
1527	if (vma->vm_end - vma->vm_start != notify.size)
1528		return -ENOTSUPP;
1529
1530	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1531	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1532	vma->vm_ops = &vhost_vdpa_vm_ops;
1533	return 0;
1534}
1535#endif /* CONFIG_MMU */
1536
1537static const struct file_operations vhost_vdpa_fops = {
1538	.owner		= THIS_MODULE,
1539	.open		= vhost_vdpa_open,
1540	.release	= vhost_vdpa_release,
1541	.write_iter	= vhost_vdpa_chr_write_iter,
1542	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
1543#ifdef CONFIG_MMU
1544	.mmap		= vhost_vdpa_mmap,
1545#endif /* CONFIG_MMU */
1546	.compat_ioctl	= compat_ptr_ioctl,
1547};
1548
1549static void vhost_vdpa_release_dev(struct device *device)
1550{
1551	struct vhost_vdpa *v =
1552	       container_of(device, struct vhost_vdpa, dev);
1553
1554	ida_free(&vhost_vdpa_ida, v->minor);
1555	kfree(v->vqs);
1556	kfree(v);
1557}
1558
1559static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1560{
1561	const struct vdpa_config_ops *ops = vdpa->config;
1562	struct vhost_vdpa *v;
1563	int minor;
1564	int i, r;
1565
1566	/* We can't support platform IOMMU device with more than 1
1567	 * group or as
1568	 */
1569	if (!ops->set_map && !ops->dma_map &&
1570	    (vdpa->ngroups > 1 || vdpa->nas > 1))
1571		return -EOPNOTSUPP;
1572
1573	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1574	if (!v)
1575		return -ENOMEM;
1576
1577	minor = ida_alloc_max(&vhost_vdpa_ida, VHOST_VDPA_DEV_MAX - 1,
1578			      GFP_KERNEL);
1579	if (minor < 0) {
1580		kfree(v);
1581		return minor;
1582	}
1583
1584	atomic_set(&v->opened, 0);
1585	v->minor = minor;
1586	v->vdpa = vdpa;
1587	v->nvqs = vdpa->nvqs;
1588	v->virtio_id = ops->get_device_id(vdpa);
1589
1590	device_initialize(&v->dev);
1591	v->dev.release = vhost_vdpa_release_dev;
1592	v->dev.parent = &vdpa->dev;
1593	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1594	v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),
1595			       GFP_KERNEL);
1596	if (!v->vqs) {
1597		r = -ENOMEM;
1598		goto err;
1599	}
1600
1601	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1602	if (r)
1603		goto err;
1604
1605	cdev_init(&v->cdev, &vhost_vdpa_fops);
1606	v->cdev.owner = THIS_MODULE;
1607
1608	r = cdev_device_add(&v->cdev, &v->dev);
1609	if (r)
1610		goto err;
1611
1612	init_completion(&v->completion);
1613	vdpa_set_drvdata(vdpa, v);
1614
1615	for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1616		INIT_HLIST_HEAD(&v->as[i]);
1617
1618	return 0;
1619
1620err:
1621	put_device(&v->dev);
1622	return r;
1623}
1624
1625static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1626{
1627	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1628	int opened;
1629
1630	cdev_device_del(&v->cdev, &v->dev);
1631
1632	do {
1633		opened = atomic_cmpxchg(&v->opened, 0, 1);
1634		if (!opened)
1635			break;
1636		wait_for_completion(&v->completion);
1637	} while (1);
1638
1639	put_device(&v->dev);
1640}
1641
1642static struct vdpa_driver vhost_vdpa_driver = {
1643	.driver = {
1644		.name	= "vhost_vdpa",
1645	},
1646	.probe	= vhost_vdpa_probe,
1647	.remove	= vhost_vdpa_remove,
1648};
1649
1650static int __init vhost_vdpa_init(void)
1651{
1652	int r;
1653
1654	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1655				"vhost-vdpa");
1656	if (r)
1657		goto err_alloc_chrdev;
1658
1659	r = vdpa_register_driver(&vhost_vdpa_driver);
1660	if (r)
1661		goto err_vdpa_register_driver;
1662
1663	return 0;
1664
1665err_vdpa_register_driver:
1666	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1667err_alloc_chrdev:
1668	return r;
1669}
1670module_init(vhost_vdpa_init);
1671
1672static void __exit vhost_vdpa_exit(void)
1673{
1674	vdpa_unregister_driver(&vhost_vdpa_driver);
1675	unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1676}
1677module_exit(vhost_vdpa_exit);
1678
1679MODULE_VERSION("0.0.1");
1680MODULE_LICENSE("GPL v2");
1681MODULE_AUTHOR("Intel Corporation");
1682MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");