vsock.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * vhost transport for vsock
  4 *
  5 * Copyright (C) 2013-2015 Red Hat, Inc.
  6 * Author: Asias He <asias@redhat.com>
  7 *         Stefan Hajnoczi <stefanha@redhat.com>
  8 */
  9#include <linux/miscdevice.h>
 10#include <linux/atomic.h>
 11#include <linux/module.h>
 12#include <linux/mutex.h>
 13#include <linux/vmalloc.h>
 14#include <net/sock.h>
 15#include <linux/virtio_vsock.h>
 16#include <linux/vhost.h>
 17#include <linux/hashtable.h>
 18
 19#include <net/af_vsock.h>
 20#include "vhost.h"
 21
 22#define VHOST_VSOCK_DEFAULT_HOST_CID	2
 23/* Max number of bytes transferred before requeueing the job.
 24 * Using this limit prevents one virtqueue from starving others. */
 25#define VHOST_VSOCK_WEIGHT 0x80000
 26/* Max number of packets transferred before requeueing the job.
 27 * Using this limit prevents one virtqueue from starving others with
 28 * small pkts.
 29 */
 30#define VHOST_VSOCK_PKT_WEIGHT 256
 31
 32static const int vhost_vsock_bits[] = {
 33	VHOST_FEATURES,
 34	VIRTIO_F_ACCESS_PLATFORM,
 35	VIRTIO_VSOCK_F_SEQPACKET
 36};
 37
 38#define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0)
 39
 40enum {
 41	VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
 42};
 43
 44/* Used to track all the vhost_vsock instances on the system. */
 45static DEFINE_MUTEX(vhost_vsock_mutex);
 46static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8);
 47
 48struct vhost_vsock {
 49	struct vhost_dev dev;
 50	struct vhost_virtqueue vqs[2];
 51
 52	/* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */
 53	struct hlist_node hash;
 54
 55	struct vhost_work send_pkt_work;
 56	struct sk_buff_head send_pkt_queue; /* host->guest pending packets */
 57
 58	atomic_t queued_replies;
 59
 60	u32 guest_cid;
 61	bool seqpacket_allow;
 62};
 63
 64static u32 vhost_transport_get_local_cid(void)
 65{
 66	return VHOST_VSOCK_DEFAULT_HOST_CID;
 67}
 68
 69/* Callers must be in an RCU read section or hold the vhost_vsock_mutex.
 70 * The return value can only be dereferenced while within the section.
 71 */
 72static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
 73{
 74	struct vhost_vsock *vsock;
 75
 76	hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid,
 77				   lockdep_is_held(&vhost_vsock_mutex)) {
 78		u32 other_cid = vsock->guest_cid;
 79
 80		/* Skip instances that have no CID yet */
 81		if (other_cid == 0)
 82			continue;
 83
 84		if (other_cid == guest_cid)
 85			return vsock;
 86
 87	}
 88
 89	return NULL;
 90}
 91
 92static void
 93vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
 94			    struct vhost_virtqueue *vq)
 95{
 96	struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
 97	int pkts = 0, total_len = 0;
 98	bool added = false;
 99	bool restart_tx = false;
100
101	mutex_lock(&vq->mutex);
102
103	if (!vhost_vq_get_backend(vq))
104		goto out;
105
106	if (!vq_meta_prefetch(vq))
107		goto out;
108
109	/* Avoid further vmexits, we're already processing the virtqueue */
110	vhost_disable_notify(&vsock->dev, vq);
111
112	do {
113		struct virtio_vsock_hdr *hdr;
114		size_t iov_len, payload_len;
115		struct iov_iter iov_iter;
116		u32 flags_to_restore = 0;
117		struct sk_buff *skb;
118		unsigned out, in;
119		size_t nbytes;
120		u32 offset;
121		int head;
122
123		skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);
124
125		if (!skb) {
126			vhost_enable_notify(&vsock->dev, vq);
127			break;
128		}
129
130		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
131					 &out, &in, NULL, NULL);
132		if (head < 0) {
133			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
134			break;
135		}
136
137		if (head == vq->num) {
138			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
139			/* We cannot finish yet if more buffers snuck in while
140			 * re-enabling notify.
141			 */
142			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
143				vhost_disable_notify(&vsock->dev, vq);
144				continue;
145			}
146			break;
147		}
148
149		if (out) {
150			kfree_skb(skb);
151			vq_err(vq, "Expected 0 output buffers, got %u\n", out);
152			break;
153		}
154
155		iov_len = iov_length(&vq->iov[out], in);
156		if (iov_len < sizeof(*hdr)) {
157			kfree_skb(skb);
158			vq_err(vq, "Buffer len [%zu] too small\n", iov_len);
159			break;
160		}
161
162		iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len);
163		offset = VIRTIO_VSOCK_SKB_CB(skb)->offset;
164		payload_len = skb->len - offset;
165		hdr = virtio_vsock_hdr(skb);
166
167		/* If the packet is greater than the space available in the
168		 * buffer, we split it using multiple buffers.
169		 */
170		if (payload_len > iov_len - sizeof(*hdr)) {
171			payload_len = iov_len - sizeof(*hdr);
172
173			/* As we are copying pieces of large packet's buffer to
174			 * small rx buffers, headers of packets in rx queue are
175			 * created dynamically and are initialized with header
176			 * of current packet(except length). But in case of
177			 * SOCK_SEQPACKET, we also must clear message delimeter
178			 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit
179			 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise,
180			 * there will be sequence of packets with these
181			 * bits set. After initialized header will be copied to
182			 * rx buffer, these required bits will be restored.
183			 */
184			if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
185				hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);
186				flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM;
187
188				if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) {
189					hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
190					flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR;
191				}
192			}
193		}
194
195		/* Set the correct length in the header */
196		hdr->len = cpu_to_le32(payload_len);
197
198		nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter);
199		if (nbytes != sizeof(*hdr)) {
200			kfree_skb(skb);
201			vq_err(vq, "Faulted on copying pkt hdr\n");
202			break;
203		}
204
205		if (skb_copy_datagram_iter(skb,
206					   offset,
207					   &iov_iter,
208					   payload_len)) {
209			kfree_skb(skb);
210			vq_err(vq, "Faulted on copying pkt buf\n");
211			break;
212		}
213
214		/* Deliver to monitoring devices all packets that we
215		 * will transmit.
216		 */
217		virtio_transport_deliver_tap_pkt(skb);
218
219		vhost_add_used(vq, head, sizeof(*hdr) + payload_len);
220		added = true;
221
222		VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len;
223		total_len += payload_len;
224
225		/* If we didn't send all the payload we can requeue the packet
226		 * to send it with the next available buffer.
227		 */
228		if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) {
229			hdr->flags |= cpu_to_le32(flags_to_restore);
230
231			/* We are queueing the same skb to handle
232			 * the remaining bytes, and we want to deliver it
233			 * to monitoring devices in the next iteration.
234			 */
235			virtio_vsock_skb_clear_tap_delivered(skb);
236			virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
237		} else {
238			if (virtio_vsock_skb_reply(skb)) {
239				int val;
240
241				val = atomic_dec_return(&vsock->queued_replies);
242
243				/* Do we have resources to resume tx
244				 * processing?
245				 */
246				if (val + 1 == tx_vq->num)
247					restart_tx = true;
248			}
249
250			virtio_transport_consume_skb_sent(skb, true);
251		}
252	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
253	if (added)
254		vhost_signal(&vsock->dev, vq);
255
256out:
257	mutex_unlock(&vq->mutex);
258
259	if (restart_tx)
260		vhost_poll_queue(&tx_vq->poll);
261}
262
263static void vhost_transport_send_pkt_work(struct vhost_work *work)
264{
265	struct vhost_virtqueue *vq;
266	struct vhost_vsock *vsock;
267
268	vsock = container_of(work, struct vhost_vsock, send_pkt_work);
269	vq = &vsock->vqs[VSOCK_VQ_RX];
270
271	vhost_transport_do_send_pkt(vsock, vq);
272}
273
274static int
275vhost_transport_send_pkt(struct sk_buff *skb)
276{
277	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
278	struct vhost_vsock *vsock;
279	int len = skb->len;
280
281	rcu_read_lock();
282
283	/* Find the vhost_vsock according to guest context id  */
284	vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid));
285	if (!vsock) {
286		rcu_read_unlock();
287		kfree_skb(skb);
288		return -ENODEV;
289	}
290
291	if (virtio_vsock_skb_reply(skb))
292		atomic_inc(&vsock->queued_replies);
293
294	virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
295	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
296
297	rcu_read_unlock();
298	return len;
299}
300
301static int
302vhost_transport_cancel_pkt(struct vsock_sock *vsk)
303{
304	struct vhost_vsock *vsock;
305	int cnt = 0;
306	int ret = -ENODEV;
307
308	rcu_read_lock();
309
310	/* Find the vhost_vsock according to guest context id  */
311	vsock = vhost_vsock_get(vsk->remote_addr.svm_cid);
312	if (!vsock)
313		goto out;
314
315	cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);
316
317	if (cnt) {
318		struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
319		int new_cnt;
320
321		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
322		if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num)
323			vhost_poll_queue(&tx_vq->poll);
324	}
325
326	ret = 0;
327out:
328	rcu_read_unlock();
329	return ret;
330}
331
332static struct sk_buff *
333vhost_vsock_alloc_skb(struct vhost_virtqueue *vq,
334		      unsigned int out, unsigned int in)
335{
336	struct virtio_vsock_hdr *hdr;
337	struct iov_iter iov_iter;
338	struct sk_buff *skb;
339	size_t payload_len;
340	size_t nbytes;
341	size_t len;
342
343	if (in != 0) {
344		vq_err(vq, "Expected 0 input buffers, got %u\n", in);
345		return NULL;
346	}
347
348	len = iov_length(vq->iov, out);
349
350	if (len < VIRTIO_VSOCK_SKB_HEADROOM ||
351	    len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM)
352		return NULL;
353
354	/* len contains both payload and hdr */
355	skb = virtio_vsock_alloc_skb(len, GFP_KERNEL);
356	if (!skb)
357		return NULL;
358
359	iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len);
360
361	hdr = virtio_vsock_hdr(skb);
362	nbytes = copy_from_iter(hdr, sizeof(*hdr), &iov_iter);
363	if (nbytes != sizeof(*hdr)) {
364		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
365		       sizeof(*hdr), nbytes);
366		kfree_skb(skb);
367		return NULL;
368	}
369
370	payload_len = le32_to_cpu(hdr->len);
371
372	/* No payload */
373	if (!payload_len)
374		return skb;
375
376	/* The pkt is too big or the length in the header is invalid */
377	if (payload_len + sizeof(*hdr) > len) {
378		kfree_skb(skb);
379		return NULL;
380	}
381
382	virtio_vsock_skb_put(skb, payload_len);
383
384	if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) {
385		vq_err(vq, "Failed to copy %zu byte payload\n", payload_len);
386		kfree_skb(skb);
387		return NULL;
388	}
389
390	return skb;
391}
392
393/* Is there space left for replies to rx packets? */
394static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)
395{
396	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];
397	int val;
398
399	smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
400	val = atomic_read(&vsock->queued_replies);
401
402	return val < vq->num;
403}
404
405static bool vhost_transport_msgzerocopy_allow(void)
406{
407	return true;
408}
409
410static bool vhost_transport_seqpacket_allow(u32 remote_cid);
411
412static struct virtio_transport vhost_transport = {
413	.transport = {
414		.module                   = THIS_MODULE,
415
416		.get_local_cid            = vhost_transport_get_local_cid,
417
418		.init                     = virtio_transport_do_socket_init,
419		.destruct                 = virtio_transport_destruct,
420		.release                  = virtio_transport_release,
421		.connect                  = virtio_transport_connect,
422		.shutdown                 = virtio_transport_shutdown,
423		.cancel_pkt               = vhost_transport_cancel_pkt,
424
425		.dgram_enqueue            = virtio_transport_dgram_enqueue,
426		.dgram_dequeue            = virtio_transport_dgram_dequeue,
427		.dgram_bind               = virtio_transport_dgram_bind,
428		.dgram_allow              = virtio_transport_dgram_allow,
429
430		.stream_enqueue           = virtio_transport_stream_enqueue,
431		.stream_dequeue           = virtio_transport_stream_dequeue,
432		.stream_has_data          = virtio_transport_stream_has_data,
433		.stream_has_space         = virtio_transport_stream_has_space,
434		.stream_rcvhiwat          = virtio_transport_stream_rcvhiwat,
435		.stream_is_active         = virtio_transport_stream_is_active,
436		.stream_allow             = virtio_transport_stream_allow,
437
438		.seqpacket_dequeue        = virtio_transport_seqpacket_dequeue,
439		.seqpacket_enqueue        = virtio_transport_seqpacket_enqueue,
440		.seqpacket_allow          = vhost_transport_seqpacket_allow,
441		.seqpacket_has_data       = virtio_transport_seqpacket_has_data,
442
443		.msgzerocopy_allow        = vhost_transport_msgzerocopy_allow,
444
445		.notify_poll_in           = virtio_transport_notify_poll_in,
446		.notify_poll_out          = virtio_transport_notify_poll_out,
447		.notify_recv_init         = virtio_transport_notify_recv_init,
448		.notify_recv_pre_block    = virtio_transport_notify_recv_pre_block,
449		.notify_recv_pre_dequeue  = virtio_transport_notify_recv_pre_dequeue,
450		.notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
451		.notify_send_init         = virtio_transport_notify_send_init,
452		.notify_send_pre_block    = virtio_transport_notify_send_pre_block,
453		.notify_send_pre_enqueue  = virtio_transport_notify_send_pre_enqueue,
454		.notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
455		.notify_buffer_size       = virtio_transport_notify_buffer_size,
456		.notify_set_rcvlowat      = virtio_transport_notify_set_rcvlowat,
457
458		.unsent_bytes             = virtio_transport_unsent_bytes,
459
460		.read_skb = virtio_transport_read_skb,
461	},
462
463	.send_pkt = vhost_transport_send_pkt,
464};
465
466static bool vhost_transport_seqpacket_allow(u32 remote_cid)
467{
468	struct vhost_vsock *vsock;
469	bool seqpacket_allow = false;
470
471	rcu_read_lock();
472	vsock = vhost_vsock_get(remote_cid);
473
474	if (vsock)
475		seqpacket_allow = vsock->seqpacket_allow;
476
477	rcu_read_unlock();
478
479	return seqpacket_allow;
480}
481
482static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
483{
484	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
485						  poll.work);
486	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
487						 dev);
488	int head, pkts = 0, total_len = 0;
489	unsigned int out, in;
490	struct sk_buff *skb;
491	bool added = false;
492
493	mutex_lock(&vq->mutex);
494
495	if (!vhost_vq_get_backend(vq))
496		goto out;
497
498	if (!vq_meta_prefetch(vq))
499		goto out;
500
501	vhost_disable_notify(&vsock->dev, vq);
502	do {
503		struct virtio_vsock_hdr *hdr;
504
505		if (!vhost_vsock_more_replies(vsock)) {
506			/* Stop tx until the device processes already
507			 * pending replies.  Leave tx virtqueue
508			 * callbacks disabled.
509			 */
510			goto no_more_replies;
511		}
512
513		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
514					 &out, &in, NULL, NULL);
515		if (head < 0)
516			break;
517
518		if (head == vq->num) {
519			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
520				vhost_disable_notify(&vsock->dev, vq);
521				continue;
522			}
523			break;
524		}
525
526		skb = vhost_vsock_alloc_skb(vq, out, in);
527		if (!skb) {
528			vq_err(vq, "Faulted on pkt\n");
529			continue;
530		}
531
532		total_len += sizeof(*hdr) + skb->len;
533
534		/* Deliver to monitoring devices all received packets */
535		virtio_transport_deliver_tap_pkt(skb);
536
537		hdr = virtio_vsock_hdr(skb);
538
539		/* Only accept correctly addressed packets */
540		if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid &&
541		    le64_to_cpu(hdr->dst_cid) ==
542		    vhost_transport_get_local_cid())
543			virtio_transport_recv_pkt(&vhost_transport, skb);
544		else
545			kfree_skb(skb);
546
547		vhost_add_used(vq, head, 0);
548		added = true;
549	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
550
551no_more_replies:
552	if (added)
553		vhost_signal(&vsock->dev, vq);
554
555out:
556	mutex_unlock(&vq->mutex);
557}
558
559static void vhost_vsock_handle_rx_kick(struct vhost_work *work)
560{
561	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
562						poll.work);
563	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
564						 dev);
565
566	vhost_transport_do_send_pkt(vsock, vq);
567}
568
569static int vhost_vsock_start(struct vhost_vsock *vsock)
570{
571	struct vhost_virtqueue *vq;
572	size_t i;
573	int ret;
574
575	mutex_lock(&vsock->dev.mutex);
576
577	ret = vhost_dev_check_owner(&vsock->dev);
578	if (ret)
579		goto err;
580
581	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
582		vq = &vsock->vqs[i];
583
584		mutex_lock(&vq->mutex);
585
586		if (!vhost_vq_access_ok(vq)) {
587			ret = -EFAULT;
588			goto err_vq;
589		}
590
591		if (!vhost_vq_get_backend(vq)) {
592			vhost_vq_set_backend(vq, vsock);
593			ret = vhost_vq_init_access(vq);
594			if (ret)
595				goto err_vq;
596		}
597
598		mutex_unlock(&vq->mutex);
599	}
600
601	/* Some packets may have been queued before the device was started,
602	 * let's kick the send worker to send them.
603	 */
604	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
605
606	mutex_unlock(&vsock->dev.mutex);
607	return 0;
608
609err_vq:
610	vhost_vq_set_backend(vq, NULL);
611	mutex_unlock(&vq->mutex);
612
613	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
614		vq = &vsock->vqs[i];
615
616		mutex_lock(&vq->mutex);
617		vhost_vq_set_backend(vq, NULL);
618		mutex_unlock(&vq->mutex);
619	}
620err:
621	mutex_unlock(&vsock->dev.mutex);
622	return ret;
623}
624
625static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
626{
627	size_t i;
628	int ret = 0;
629
630	mutex_lock(&vsock->dev.mutex);
631
632	if (check_owner) {
633		ret = vhost_dev_check_owner(&vsock->dev);
634		if (ret)
635			goto err;
636	}
637
638	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
639		struct vhost_virtqueue *vq = &vsock->vqs[i];
640
641		mutex_lock(&vq->mutex);
642		vhost_vq_set_backend(vq, NULL);
643		mutex_unlock(&vq->mutex);
644	}
645
646err:
647	mutex_unlock(&vsock->dev.mutex);
648	return ret;
649}
650
651static void vhost_vsock_free(struct vhost_vsock *vsock)
652{
653	kvfree(vsock);
654}
655
656static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
657{
658	struct vhost_virtqueue **vqs;
659	struct vhost_vsock *vsock;
660	int ret;
661
662	/* This struct is large and allocation could fail, fall back to vmalloc
663	 * if there is no other way.
664	 */
665	vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL);
666	if (!vsock)
667		return -ENOMEM;
668
669	vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL);
670	if (!vqs) {
671		ret = -ENOMEM;
672		goto out;
673	}
674
675	vsock->guest_cid = 0; /* no CID assigned yet */
676	vsock->seqpacket_allow = false;
677
678	atomic_set(&vsock->queued_replies, 0);
679
680	vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];
681	vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];
682	vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;
683	vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;
684
685	vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
686		       UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
687		       VHOST_VSOCK_WEIGHT, true, NULL);
688
689	file->private_data = vsock;
690	skb_queue_head_init(&vsock->send_pkt_queue);
691	vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);
692	return 0;
693
694out:
695	vhost_vsock_free(vsock);
696	return ret;
697}
698
699static void vhost_vsock_flush(struct vhost_vsock *vsock)
700{
701	vhost_dev_flush(&vsock->dev);
702}
703
704static void vhost_vsock_reset_orphans(struct sock *sk)
705{
706	struct vsock_sock *vsk = vsock_sk(sk);
707
708	/* vmci_transport.c doesn't take sk_lock here either.  At least we're
709	 * under vsock_table_lock so the sock cannot disappear while we're
710	 * executing.
711	 */
712
713	rcu_read_lock();
714
715	/* If the peer is still valid, no need to reset connection */
716	if (vhost_vsock_get(vsk->remote_addr.svm_cid)) {
717		rcu_read_unlock();
718		return;
719	}
720
721	rcu_read_unlock();
722
723	/* If the close timeout is pending, let it expire.  This avoids races
724	 * with the timeout callback.
725	 */
726	if (vsk->close_work_scheduled)
727		return;
728
729	sock_set_flag(sk, SOCK_DONE);
730	vsk->peer_shutdown = SHUTDOWN_MASK;
731	sk->sk_state = SS_UNCONNECTED;
732	sk->sk_err = ECONNRESET;
733	sk_error_report(sk);
734}
735
736static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
737{
738	struct vhost_vsock *vsock = file->private_data;
739
740	mutex_lock(&vhost_vsock_mutex);
741	if (vsock->guest_cid)
742		hash_del_rcu(&vsock->hash);
743	mutex_unlock(&vhost_vsock_mutex);
744
745	/* Wait for other CPUs to finish using vsock */
746	synchronize_rcu();
747
748	/* Iterating over all connections for all CIDs to find orphans is
749	 * inefficient.  Room for improvement here. */
750	vsock_for_each_connected_socket(&vhost_transport.transport,
751					vhost_vsock_reset_orphans);
752
753	/* Don't check the owner, because we are in the release path, so we
754	 * need to stop the vsock device in any case.
755	 * vhost_vsock_stop() can not fail in this case, so we don't need to
756	 * check the return code.
757	 */
758	vhost_vsock_stop(vsock, false);
759	vhost_vsock_flush(vsock);
760	vhost_dev_stop(&vsock->dev);
761
762	virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);
763
764	vhost_dev_cleanup(&vsock->dev);
765	kfree(vsock->dev.vqs);
766	vhost_vsock_free(vsock);
767	return 0;
768}
769
770static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
771{
772	struct vhost_vsock *other;
773
774	/* Refuse reserved CIDs */
775	if (guest_cid <= VMADDR_CID_HOST ||
776	    guest_cid == U32_MAX)
777		return -EINVAL;
778
779	/* 64-bit CIDs are not yet supported */
780	if (guest_cid > U32_MAX)
781		return -EINVAL;
782
783	/* Refuse if CID is assigned to the guest->host transport (i.e. nested
784	 * VM), to make the loopback work.
785	 */
786	if (vsock_find_cid(guest_cid))
787		return -EADDRINUSE;
788
789	/* Refuse if CID is already in use */
790	mutex_lock(&vhost_vsock_mutex);
791	other = vhost_vsock_get(guest_cid);
792	if (other && other != vsock) {
793		mutex_unlock(&vhost_vsock_mutex);
794		return -EADDRINUSE;
795	}
796
797	if (vsock->guest_cid)
798		hash_del_rcu(&vsock->hash);
799
800	vsock->guest_cid = guest_cid;
801	hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);
802	mutex_unlock(&vhost_vsock_mutex);
803
804	return 0;
805}
806
807static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
808{
809	struct vhost_virtqueue *vq;
810	int i;
811
812	if (features & ~VHOST_VSOCK_FEATURES)
813		return -EOPNOTSUPP;
814
815	mutex_lock(&vsock->dev.mutex);
816	if ((features & (1 << VHOST_F_LOG_ALL)) &&
817	    !vhost_log_access_ok(&vsock->dev)) {
818		goto err;
819	}
820
821	if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
822		if (vhost_init_device_iotlb(&vsock->dev))
823			goto err;
824	}
825
826	vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
827
828	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
829		vq = &vsock->vqs[i];
830		mutex_lock(&vq->mutex);
831		vq->acked_features = features;
832		mutex_unlock(&vq->mutex);
833	}
834	mutex_unlock(&vsock->dev.mutex);
835	return 0;
836
837err:
838	mutex_unlock(&vsock->dev.mutex);
839	return -EFAULT;
840}
841
842static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
843				  unsigned long arg)
844{
845	struct vhost_vsock *vsock = f->private_data;
846	void __user *argp = (void __user *)arg;
847	u64 guest_cid;
848	u64 features;
849	int start;
850	int r;
851
852	switch (ioctl) {
853	case VHOST_VSOCK_SET_GUEST_CID:
854		if (copy_from_user(&guest_cid, argp, sizeof(guest_cid)))
855			return -EFAULT;
856		return vhost_vsock_set_cid(vsock, guest_cid);
857	case VHOST_VSOCK_SET_RUNNING:
858		if (copy_from_user(&start, argp, sizeof(start)))
859			return -EFAULT;
860		if (start)
861			return vhost_vsock_start(vsock);
862		else
863			return vhost_vsock_stop(vsock, true);
864	case VHOST_GET_FEATURES:
865		features = VHOST_VSOCK_FEATURES;
866		if (copy_to_user(argp, &features, sizeof(features)))
867			return -EFAULT;
868		return 0;
869	case VHOST_SET_FEATURES:
870		if (copy_from_user(&features, argp, sizeof(features)))
871			return -EFAULT;
872		return vhost_vsock_set_features(vsock, features);
873	case VHOST_GET_BACKEND_FEATURES:
874		features = VHOST_VSOCK_BACKEND_FEATURES;
875		if (copy_to_user(argp, &features, sizeof(features)))
876			return -EFAULT;
877		return 0;
878	case VHOST_SET_BACKEND_FEATURES:
879		if (copy_from_user(&features, argp, sizeof(features)))
880			return -EFAULT;
881		if (features & ~VHOST_VSOCK_BACKEND_FEATURES)
882			return -EOPNOTSUPP;
883		vhost_set_backend_features(&vsock->dev, features);
884		return 0;
885	default:
886		mutex_lock(&vsock->dev.mutex);
887		r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
888		if (r == -ENOIOCTLCMD)
889			r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);
890		else
891			vhost_vsock_flush(vsock);
892		mutex_unlock(&vsock->dev.mutex);
893		return r;
894	}
895}
896
897static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
898{
899	struct file *file = iocb->ki_filp;
900	struct vhost_vsock *vsock = file->private_data;
901	struct vhost_dev *dev = &vsock->dev;
902	int noblock = file->f_flags & O_NONBLOCK;
903
904	return vhost_chr_read_iter(dev, to, noblock);
905}
906
907static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb,
908					struct iov_iter *from)
909{
910	struct file *file = iocb->ki_filp;
911	struct vhost_vsock *vsock = file->private_data;
912	struct vhost_dev *dev = &vsock->dev;
913
914	return vhost_chr_write_iter(dev, from);
915}
916
917static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait)
918{
919	struct vhost_vsock *vsock = file->private_data;
920	struct vhost_dev *dev = &vsock->dev;
921
922	return vhost_chr_poll(file, dev, wait);
923}
924
925static const struct file_operations vhost_vsock_fops = {
926	.owner          = THIS_MODULE,
927	.open           = vhost_vsock_dev_open,
928	.release        = vhost_vsock_dev_release,
929	.llseek		= noop_llseek,
930	.unlocked_ioctl = vhost_vsock_dev_ioctl,
931	.compat_ioctl   = compat_ptr_ioctl,
932	.read_iter      = vhost_vsock_chr_read_iter,
933	.write_iter     = vhost_vsock_chr_write_iter,
934	.poll           = vhost_vsock_chr_poll,
935};
936
937static struct miscdevice vhost_vsock_misc = {
938	.minor = VHOST_VSOCK_MINOR,
939	.name = "vhost-vsock",
940	.fops = &vhost_vsock_fops,
941};
942
943static int __init vhost_vsock_init(void)
944{
945	int ret;
946
947	ret = vsock_core_register(&vhost_transport.transport,
948				  VSOCK_TRANSPORT_F_H2G);
949	if (ret < 0)
950		return ret;
951
952	ret = misc_register(&vhost_vsock_misc);
953	if (ret) {
954		vsock_core_unregister(&vhost_transport.transport);
955		return ret;
956	}
957
958	return 0;
959};
960
961static void __exit vhost_vsock_exit(void)
962{
963	misc_deregister(&vhost_vsock_misc);
964	vsock_core_unregister(&vhost_transport.transport);
965};
966
967module_init(vhost_vsock_init);
968module_exit(vhost_vsock_exit);
969MODULE_LICENSE("GPL v2");
970MODULE_AUTHOR("Asias He");
971MODULE_DESCRIPTION("vhost transport for vsock ");
972MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR);
973MODULE_ALIAS("devname:vhost-vsock");