virtio_transport_common.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * common code for virtio vsock
   4 *
   5 * Copyright (C) 2013-2015 Red Hat, Inc.
   6 * Author: Asias He <asias@redhat.com>
   7 *         Stefan Hajnoczi <stefanha@redhat.com>
   8 */
   9#include <linux/spinlock.h>
  10#include <linux/module.h>
  11#include <linux/sched/signal.h>
  12#include <linux/ctype.h>
  13#include <linux/list.h>
  14#include <linux/virtio_vsock.h>
  15#include <uapi/linux/vsockmon.h>
  16
  17#include <net/sock.h>
  18#include <net/af_vsock.h>
  19
  20#define CREATE_TRACE_POINTS
  21#include <trace/events/vsock_virtio_transport_common.h>
  22
  23/* How long to wait for graceful shutdown of a connection */
  24#define VSOCK_CLOSE_TIMEOUT (8 * HZ)
  25
  26/* Threshold for detecting small packets to copy */
  27#define GOOD_COPY_LEN  128
  28
  29static void virtio_transport_cancel_close_work(struct vsock_sock *vsk,
  30					       bool cancel_timeout);
  31static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs);
  32
  33static const struct virtio_transport *
  34virtio_transport_get_ops(struct vsock_sock *vsk)
  35{
  36	const struct vsock_transport *t = vsock_core_get_transport(vsk);
  37
  38	if (WARN_ON(!t))
  39		return NULL;
  40
  41	return container_of(t, struct virtio_transport, transport);
  42}
  43
  44static bool virtio_transport_can_zcopy(const struct virtio_transport *t_ops,
  45				       struct virtio_vsock_pkt_info *info,
  46				       size_t pkt_len)
  47{
  48	struct iov_iter *iov_iter;
  49
  50	if (!info->msg)
  51		return false;
  52
  53	iov_iter = &info->msg->msg_iter;
  54
  55	if (iov_iter->iov_offset)
  56		return false;
  57
  58	/* We can't send whole iov. */
  59	if (iov_iter->count > pkt_len)
  60		return false;
  61
  62	/* Check that transport can send data in zerocopy mode. */
  63	t_ops = virtio_transport_get_ops(info->vsk);
  64
  65	if (t_ops->can_msgzerocopy) {
  66		int pages_to_send = iov_iter_npages(iov_iter, MAX_SKB_FRAGS);
  67
  68		/* +1 is for packet header. */
  69		return t_ops->can_msgzerocopy(pages_to_send + 1);
  70	}
  71
  72	return true;
  73}
  74
  75static int virtio_transport_init_zcopy_skb(struct vsock_sock *vsk,
  76					   struct sk_buff *skb,
  77					   struct msghdr *msg,
  78					   bool zerocopy)
  79{
  80	struct ubuf_info *uarg;
  81
  82	if (msg->msg_ubuf) {
  83		uarg = msg->msg_ubuf;
  84		net_zcopy_get(uarg);
  85	} else {
  86		struct iov_iter *iter = &msg->msg_iter;
  87		struct ubuf_info_msgzc *uarg_zc;
  88
  89		uarg = msg_zerocopy_realloc(sk_vsock(vsk),
  90					    iter->count,
  91					    NULL, false);
  92		if (!uarg)
  93			return -1;
  94
  95		uarg_zc = uarg_to_msgzc(uarg);
  96		uarg_zc->zerocopy = zerocopy ? 1 : 0;
  97	}
  98
  99	skb_zcopy_init(skb, uarg);
 100
 101	return 0;
 102}
 103
 104static int virtio_transport_fill_skb(struct sk_buff *skb,
 105				     struct virtio_vsock_pkt_info *info,
 106				     size_t len,
 107				     bool zcopy)
 108{
 109	struct msghdr *msg = info->msg;
 110
 111	if (zcopy)
 112		return __zerocopy_sg_from_iter(msg, NULL, skb,
 113					       &msg->msg_iter, len, NULL);
 114
 115	virtio_vsock_skb_put(skb, len);
 116	return skb_copy_datagram_from_iter_full(skb, 0, &msg->msg_iter, len);
 117}
 118
 119static void virtio_transport_init_hdr(struct sk_buff *skb,
 120				      struct virtio_vsock_pkt_info *info,
 121				      size_t payload_len,
 122				      u32 src_cid,
 123				      u32 src_port,
 124				      u32 dst_cid,
 125				      u32 dst_port)
 126{
 127	struct virtio_vsock_hdr *hdr;
 128
 129	hdr = virtio_vsock_hdr(skb);
 130	hdr->type	= cpu_to_le16(info->type);
 131	hdr->op		= cpu_to_le16(info->op);
 132	hdr->src_cid	= cpu_to_le64(src_cid);
 133	hdr->dst_cid	= cpu_to_le64(dst_cid);
 134	hdr->src_port	= cpu_to_le32(src_port);
 135	hdr->dst_port	= cpu_to_le32(dst_port);
 136	hdr->flags	= cpu_to_le32(info->flags);
 137	hdr->len	= cpu_to_le32(payload_len);
 138	hdr->buf_alloc	= cpu_to_le32(0);
 139	hdr->fwd_cnt	= cpu_to_le32(0);
 140}
 141
 142static void virtio_transport_copy_nonlinear_skb(const struct sk_buff *skb,
 143						void *dst,
 144						size_t len)
 145{
 146	struct iov_iter iov_iter = { 0 };
 147	struct kvec kvec;
 148	size_t to_copy;
 149
 150	kvec.iov_base = dst;
 151	kvec.iov_len = len;
 152
 153	iov_iter.iter_type = ITER_KVEC;
 154	iov_iter.kvec = &kvec;
 155	iov_iter.nr_segs = 1;
 156
 157	to_copy = min_t(size_t, len, skb->len);
 158
 159	skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset,
 160			       &iov_iter, to_copy);
 161}
 162
 163/* Packet capture */
 164static struct sk_buff *virtio_transport_build_skb(void *opaque)
 165{
 166	struct virtio_vsock_hdr *pkt_hdr;
 167	struct sk_buff *pkt = opaque;
 168	struct af_vsockmon_hdr *hdr;
 169	struct sk_buff *skb;
 170	size_t payload_len;
 171
 172	/* A packet could be split to fit the RX buffer, so we can retrieve
 173	 * the payload length from the header and the buffer pointer taking
 174	 * care of the offset in the original packet.
 175	 */
 176	pkt_hdr = virtio_vsock_hdr(pkt);
 177	payload_len = pkt->len;
 178
 179	skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len,
 180			GFP_ATOMIC);
 181	if (!skb)
 182		return NULL;
 183
 184	hdr = skb_put(skb, sizeof(*hdr));
 185
 186	/* pkt->hdr is little-endian so no need to byteswap here */
 187	hdr->src_cid = pkt_hdr->src_cid;
 188	hdr->src_port = pkt_hdr->src_port;
 189	hdr->dst_cid = pkt_hdr->dst_cid;
 190	hdr->dst_port = pkt_hdr->dst_port;
 191
 192	hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO);
 193	hdr->len = cpu_to_le16(sizeof(*pkt_hdr));
 194	memset(hdr->reserved, 0, sizeof(hdr->reserved));
 195
 196	switch (le16_to_cpu(pkt_hdr->op)) {
 197	case VIRTIO_VSOCK_OP_REQUEST:
 198	case VIRTIO_VSOCK_OP_RESPONSE:
 199		hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT);
 200		break;
 201	case VIRTIO_VSOCK_OP_RST:
 202	case VIRTIO_VSOCK_OP_SHUTDOWN:
 203		hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT);
 204		break;
 205	case VIRTIO_VSOCK_OP_RW:
 206		hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD);
 207		break;
 208	case VIRTIO_VSOCK_OP_CREDIT_UPDATE:
 209	case VIRTIO_VSOCK_OP_CREDIT_REQUEST:
 210		hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL);
 211		break;
 212	default:
 213		hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN);
 214		break;
 215	}
 216
 217	skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr));
 218
 219	if (payload_len) {
 220		if (skb_is_nonlinear(pkt)) {
 221			void *data = skb_put(skb, payload_len);
 222
 223			virtio_transport_copy_nonlinear_skb(pkt, data, payload_len);
 224		} else {
 225			skb_put_data(skb, pkt->data, payload_len);
 226		}
 227	}
 228
 229	return skb;
 230}
 231
 232void virtio_transport_deliver_tap_pkt(struct sk_buff *skb)
 233{
 234	if (virtio_vsock_skb_tap_delivered(skb))
 235		return;
 236
 237	vsock_deliver_tap(virtio_transport_build_skb, skb);
 238	virtio_vsock_skb_set_tap_delivered(skb);
 239}
 240EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt);
 241
 242static u16 virtio_transport_get_type(struct sock *sk)
 243{
 244	if (sk->sk_type == SOCK_STREAM)
 245		return VIRTIO_VSOCK_TYPE_STREAM;
 246	else
 247		return VIRTIO_VSOCK_TYPE_SEQPACKET;
 248}
 249
 250/* Returns new sk_buff on success, otherwise returns NULL. */
 251static struct sk_buff *virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info,
 252						  size_t payload_len,
 253						  bool zcopy,
 254						  u32 src_cid,
 255						  u32 src_port,
 256						  u32 dst_cid,
 257						  u32 dst_port)
 258{
 259	struct vsock_sock *vsk;
 260	struct sk_buff *skb;
 261	size_t skb_len;
 262
 263	skb_len = VIRTIO_VSOCK_SKB_HEADROOM;
 264
 265	if (!zcopy)
 266		skb_len += payload_len;
 267
 268	skb = virtio_vsock_alloc_skb(skb_len, GFP_KERNEL);
 269	if (!skb)
 270		return NULL;
 271
 272	virtio_transport_init_hdr(skb, info, payload_len, src_cid, src_port,
 273				  dst_cid, dst_port);
 274
 275	vsk = info->vsk;
 276
 277	/* If 'vsk' != NULL then payload is always present, so we
 278	 * will never call '__zerocopy_sg_from_iter()' below without
 279	 * setting skb owner in 'skb_set_owner_w()'. The only case
 280	 * when 'vsk' == NULL is VIRTIO_VSOCK_OP_RST control message
 281	 * without payload.
 282	 */
 283	WARN_ON_ONCE(!(vsk && (info->msg && payload_len)) && zcopy);
 284
 285	/* Set owner here, because '__zerocopy_sg_from_iter()' uses
 286	 * owner of skb without check to update 'sk_wmem_alloc'.
 287	 */
 288	if (vsk)
 289		skb_set_owner_w(skb, sk_vsock(vsk));
 290
 291	if (info->msg && payload_len > 0) {
 292		int err;
 293
 294		err = virtio_transport_fill_skb(skb, info, payload_len, zcopy);
 295		if (err)
 296			goto out;
 297
 298		if (msg_data_left(info->msg) == 0 &&
 299		    info->type == VIRTIO_VSOCK_TYPE_SEQPACKET) {
 300			struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
 301
 302			hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);
 303
 304			if (info->msg->msg_flags & MSG_EOR)
 305				hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
 306		}
 307	}
 308
 309	if (info->reply)
 310		virtio_vsock_skb_set_reply(skb);
 311
 312	trace_virtio_transport_alloc_pkt(src_cid, src_port,
 313					 dst_cid, dst_port,
 314					 payload_len,
 315					 info->type,
 316					 info->op,
 317					 info->flags,
 318					 zcopy);
 319
 320	return skb;
 321out:
 322	kfree_skb(skb);
 323	return NULL;
 324}
 325
 326/* This function can only be used on connecting/connected sockets,
 327 * since a socket assigned to a transport is required.
 328 *
 329 * Do not use on listener sockets!
 330 */
 331static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 332					  struct virtio_vsock_pkt_info *info)
 333{
 334	u32 max_skb_len = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE;
 335	u32 src_cid, src_port, dst_cid, dst_port;
 336	const struct virtio_transport *t_ops;
 337	struct virtio_vsock_sock *vvs;
 338	u32 pkt_len = info->pkt_len;
 339	bool can_zcopy = false;
 340	u32 rest_len;
 341	int ret;
 342
 343	info->type = virtio_transport_get_type(sk_vsock(vsk));
 344
 345	t_ops = virtio_transport_get_ops(vsk);
 346	if (unlikely(!t_ops))
 347		return -EFAULT;
 348
 349	src_cid = t_ops->transport.get_local_cid();
 350	src_port = vsk->local_addr.svm_port;
 351	if (!info->remote_cid) {
 352		dst_cid	= vsk->remote_addr.svm_cid;
 353		dst_port = vsk->remote_addr.svm_port;
 354	} else {
 355		dst_cid = info->remote_cid;
 356		dst_port = info->remote_port;
 357	}
 358
 359	vvs = vsk->trans;
 360
 361	/* virtio_transport_get_credit might return less than pkt_len credit */
 362	pkt_len = virtio_transport_get_credit(vvs, pkt_len);
 363
 364	/* Do not send zero length OP_RW pkt */
 365	if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW)
 366		return pkt_len;
 367
 368	if (info->msg) {
 369		/* If zerocopy is not enabled by 'setsockopt()', we behave as
 370		 * there is no MSG_ZEROCOPY flag set.
 371		 */
 372		if (!sock_flag(sk_vsock(vsk), SOCK_ZEROCOPY))
 373			info->msg->msg_flags &= ~MSG_ZEROCOPY;
 374
 375		if (info->msg->msg_flags & MSG_ZEROCOPY)
 376			can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len);
 377
 378		if (can_zcopy)
 379			max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE,
 380					    (MAX_SKB_FRAGS * PAGE_SIZE));
 381	}
 382
 383	rest_len = pkt_len;
 384
 385	do {
 386		struct sk_buff *skb;
 387		size_t skb_len;
 388
 389		skb_len = min(max_skb_len, rest_len);
 390
 391		skb = virtio_transport_alloc_skb(info, skb_len, can_zcopy,
 392						 src_cid, src_port,
 393						 dst_cid, dst_port);
 394		if (!skb) {
 395			ret = -ENOMEM;
 396			break;
 397		}
 398
 399		/* We process buffer part by part, allocating skb on
 400		 * each iteration. If this is last skb for this buffer
 401		 * and MSG_ZEROCOPY mode is in use - we must allocate
 402		 * completion for the current syscall.
 403		 */
 404		if (info->msg && info->msg->msg_flags & MSG_ZEROCOPY &&
 405		    skb_len == rest_len && info->op == VIRTIO_VSOCK_OP_RW) {
 406			if (virtio_transport_init_zcopy_skb(vsk, skb,
 407							    info->msg,
 408							    can_zcopy)) {
 409				kfree_skb(skb);
 410				ret = -ENOMEM;
 411				break;
 412			}
 413		}
 414
 415		virtio_transport_inc_tx_pkt(vvs, skb);
 416
 417		ret = t_ops->send_pkt(skb);
 418		if (ret < 0)
 419			break;
 420
 421		/* Both virtio and vhost 'send_pkt()' returns 'skb_len',
 422		 * but for reliability use 'ret' instead of 'skb_len'.
 423		 * Also if partial send happens (e.g. 'ret' != 'skb_len')
 424		 * somehow, we break this loop, but account such returned
 425		 * value in 'virtio_transport_put_credit()'.
 426		 */
 427		rest_len -= ret;
 428
 429		if (WARN_ONCE(ret != skb_len,
 430			      "'send_pkt()' returns %i, but %zu expected\n",
 431			      ret, skb_len))
 432			break;
 433	} while (rest_len);
 434
 435	virtio_transport_put_credit(vvs, rest_len);
 436
 437	/* Return number of bytes, if any data has been sent. */
 438	if (rest_len != pkt_len)
 439		ret = pkt_len - rest_len;
 440
 441	return ret;
 442}
 443
 444static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs,
 445					u32 len)
 446{
 447	if (vvs->buf_used + len > vvs->buf_alloc)
 448		return false;
 449
 450	vvs->rx_bytes += len;
 451	vvs->buf_used += len;
 452	return true;
 453}
 454
 455static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs,
 456					u32 bytes_read, u32 bytes_dequeued)
 457{
 458	vvs->rx_bytes -= bytes_read;
 459	vvs->buf_used -= bytes_dequeued;
 460	vvs->fwd_cnt += bytes_dequeued;
 461}
 462
 463void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb)
 464{
 465	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
 466
 467	spin_lock_bh(&vvs->rx_lock);
 468	vvs->last_fwd_cnt = vvs->fwd_cnt;
 469	hdr->fwd_cnt = cpu_to_le32(vvs->fwd_cnt);
 470	hdr->buf_alloc = cpu_to_le32(vvs->buf_alloc);
 471	spin_unlock_bh(&vvs->rx_lock);
 472}
 473EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt);
 474
 475void virtio_transport_consume_skb_sent(struct sk_buff *skb, bool consume)
 476{
 477	struct sock *s = skb->sk;
 478
 479	if (s && skb->len) {
 480		struct vsock_sock *vs = vsock_sk(s);
 481		struct virtio_vsock_sock *vvs;
 482
 483		vvs = vs->trans;
 484
 485		spin_lock_bh(&vvs->tx_lock);
 486		vvs->bytes_unsent -= skb->len;
 487		spin_unlock_bh(&vvs->tx_lock);
 488	}
 489
 490	if (consume)
 491		consume_skb(skb);
 492}
 493EXPORT_SYMBOL_GPL(virtio_transport_consume_skb_sent);
 494
 495u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit)
 496{
 497	u32 ret;
 498
 499	if (!credit)
 500		return 0;
 501
 502	spin_lock_bh(&vvs->tx_lock);
 503	ret = min_t(u32, credit, virtio_transport_has_space(vvs));
 504	vvs->tx_cnt += ret;
 505	vvs->bytes_unsent += ret;
 506	spin_unlock_bh(&vvs->tx_lock);
 507
 508	return ret;
 509}
 510EXPORT_SYMBOL_GPL(virtio_transport_get_credit);
 511
 512void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit)
 513{
 514	if (!credit)
 515		return;
 516
 517	spin_lock_bh(&vvs->tx_lock);
 518	vvs->tx_cnt -= credit;
 519	vvs->bytes_unsent -= credit;
 520	spin_unlock_bh(&vvs->tx_lock);
 521}
 522EXPORT_SYMBOL_GPL(virtio_transport_put_credit);
 523
 524static int virtio_transport_send_credit_update(struct vsock_sock *vsk)
 525{
 526	struct virtio_vsock_pkt_info info = {
 527		.op = VIRTIO_VSOCK_OP_CREDIT_UPDATE,
 528		.vsk = vsk,
 529	};
 530
 531	return virtio_transport_send_pkt_info(vsk, &info);
 532}
 533
 534static ssize_t
 535virtio_transport_stream_do_peek(struct vsock_sock *vsk,
 536				struct msghdr *msg,
 537				size_t len)
 538{
 539	struct virtio_vsock_sock *vvs = vsk->trans;
 540	struct sk_buff *skb;
 541	size_t total = 0;
 542	int err;
 543
 544	spin_lock_bh(&vvs->rx_lock);
 545
 546	skb_queue_walk(&vvs->rx_queue, skb) {
 547		size_t bytes;
 548
 549		bytes = len - total;
 550		if (bytes > skb->len)
 551			bytes = skb->len;
 552
 553		spin_unlock_bh(&vvs->rx_lock);
 554
 555		/* sk_lock is held by caller so no one else can dequeue.
 556		 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
 557		 */
 558		err = skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset,
 559					     &msg->msg_iter, bytes);
 560		if (err)
 561			goto out;
 562
 563		total += bytes;
 564
 565		spin_lock_bh(&vvs->rx_lock);
 566
 567		if (total == len)
 568			break;
 569	}
 570
 571	spin_unlock_bh(&vvs->rx_lock);
 572
 573	return total;
 574
 575out:
 576	if (total)
 577		err = total;
 578	return err;
 579}
 580
 581static ssize_t
 582virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
 583				   struct msghdr *msg,
 584				   size_t len)
 585{
 586	struct virtio_vsock_sock *vvs = vsk->trans;
 587	struct sk_buff *skb;
 588	u32 fwd_cnt_delta;
 589	bool low_rx_bytes;
 590	int err = -EFAULT;
 591	size_t total = 0;
 592	u32 free_space;
 593
 594	spin_lock_bh(&vvs->rx_lock);
 595
 596	if (WARN_ONCE(skb_queue_empty(&vvs->rx_queue) && vvs->rx_bytes,
 597		      "rx_queue is empty, but rx_bytes is non-zero\n")) {
 598		spin_unlock_bh(&vvs->rx_lock);
 599		return err;
 600	}
 601
 602	while (total < len && !skb_queue_empty(&vvs->rx_queue)) {
 603		size_t bytes, dequeued = 0;
 604
 605		skb = skb_peek(&vvs->rx_queue);
 606
 607		bytes = min_t(size_t, len - total,
 608			      skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset);
 609
 610		/* sk_lock is held by caller so no one else can dequeue.
 611		 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
 612		 */
 613		spin_unlock_bh(&vvs->rx_lock);
 614
 615		err = skb_copy_datagram_iter(skb,
 616					     VIRTIO_VSOCK_SKB_CB(skb)->offset,
 617					     &msg->msg_iter, bytes);
 618		if (err)
 619			goto out;
 620
 621		spin_lock_bh(&vvs->rx_lock);
 622
 623		total += bytes;
 624
 625		VIRTIO_VSOCK_SKB_CB(skb)->offset += bytes;
 626
 627		if (skb->len == VIRTIO_VSOCK_SKB_CB(skb)->offset) {
 628			dequeued = le32_to_cpu(virtio_vsock_hdr(skb)->len);
 629			__skb_unlink(skb, &vvs->rx_queue);
 630			consume_skb(skb);
 631		}
 632
 633		virtio_transport_dec_rx_pkt(vvs, bytes, dequeued);
 634	}
 635
 636	fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
 637	free_space = vvs->buf_alloc - fwd_cnt_delta;
 638	low_rx_bytes = (vvs->rx_bytes <
 639			sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
 640
 641	spin_unlock_bh(&vvs->rx_lock);
 642
 643	/* To reduce the number of credit update messages,
 644	 * don't update credits as long as lots of space is available.
 645	 * Note: the limit chosen here is arbitrary. Setting the limit
 646	 * too high causes extra messages. Too low causes transmitter
 647	 * stalls. As stalls are in theory more expensive than extra
 648	 * messages, we set the limit to a high value. TODO: experiment
 649	 * with different values. Also send credit update message when
 650	 * number of bytes in rx queue is not enough to wake up reader.
 651	 */
 652	if (fwd_cnt_delta &&
 653	    (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
 654		virtio_transport_send_credit_update(vsk);
 655
 656	return total;
 657
 658out:
 659	if (total)
 660		err = total;
 661	return err;
 662}
 663
 664static ssize_t
 665virtio_transport_seqpacket_do_peek(struct vsock_sock *vsk,
 666				   struct msghdr *msg)
 667{
 668	struct virtio_vsock_sock *vvs = vsk->trans;
 669	struct sk_buff *skb;
 670	size_t total, len;
 671
 672	spin_lock_bh(&vvs->rx_lock);
 673
 674	if (!vvs->msg_count) {
 675		spin_unlock_bh(&vvs->rx_lock);
 676		return 0;
 677	}
 678
 679	total = 0;
 680	len = msg_data_left(msg);
 681
 682	skb_queue_walk(&vvs->rx_queue, skb) {
 683		struct virtio_vsock_hdr *hdr;
 684
 685		if (total < len) {
 686			size_t bytes;
 687			int err;
 688
 689			bytes = len - total;
 690			if (bytes > skb->len)
 691				bytes = skb->len;
 692
 693			spin_unlock_bh(&vvs->rx_lock);
 694
 695			/* sk_lock is held by caller so no one else can dequeue.
 696			 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
 697			 */
 698			err = skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset,
 699						     &msg->msg_iter, bytes);
 700			if (err)
 701				return err;
 702
 703			spin_lock_bh(&vvs->rx_lock);
 704		}
 705
 706		total += skb->len;
 707		hdr = virtio_vsock_hdr(skb);
 708
 709		if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
 710			if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR)
 711				msg->msg_flags |= MSG_EOR;
 712
 713			break;
 714		}
 715	}
 716
 717	spin_unlock_bh(&vvs->rx_lock);
 718
 719	return total;
 720}
 721
 722static int virtio_transport_seqpacket_do_dequeue(struct vsock_sock *vsk,
 723						 struct msghdr *msg,
 724						 int flags)
 725{
 726	struct virtio_vsock_sock *vvs = vsk->trans;
 727	int dequeued_len = 0;
 728	size_t user_buf_len = msg_data_left(msg);
 729	bool msg_ready = false;
 730	struct sk_buff *skb;
 731
 732	spin_lock_bh(&vvs->rx_lock);
 733
 734	if (vvs->msg_count == 0) {
 735		spin_unlock_bh(&vvs->rx_lock);
 736		return 0;
 737	}
 738
 739	while (!msg_ready) {
 740		struct virtio_vsock_hdr *hdr;
 741		size_t pkt_len;
 742
 743		skb = __skb_dequeue(&vvs->rx_queue);
 744		if (!skb)
 745			break;
 746		hdr = virtio_vsock_hdr(skb);
 747		pkt_len = (size_t)le32_to_cpu(hdr->len);
 748
 749		if (dequeued_len >= 0) {
 750			size_t bytes_to_copy;
 751
 752			bytes_to_copy = min(user_buf_len, pkt_len);
 753
 754			if (bytes_to_copy) {
 755				int err;
 756
 757				/* sk_lock is held by caller so no one else can dequeue.
 758				 * Unlock rx_lock since skb_copy_datagram_iter() may sleep.
 759				 */
 760				spin_unlock_bh(&vvs->rx_lock);
 761
 762				err = skb_copy_datagram_iter(skb, 0,
 763							     &msg->msg_iter,
 764							     bytes_to_copy);
 765				if (err) {
 766					/* Copy of message failed. Rest of
 767					 * fragments will be freed without copy.
 768					 */
 769					dequeued_len = err;
 770				} else {
 771					user_buf_len -= bytes_to_copy;
 772				}
 773
 774				spin_lock_bh(&vvs->rx_lock);
 775			}
 776
 777			if (dequeued_len >= 0)
 778				dequeued_len += pkt_len;
 779		}
 780
 781		if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
 782			msg_ready = true;
 783			vvs->msg_count--;
 784
 785			if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR)
 786				msg->msg_flags |= MSG_EOR;
 787		}
 788
 789		virtio_transport_dec_rx_pkt(vvs, pkt_len, pkt_len);
 790		kfree_skb(skb);
 791	}
 792
 793	spin_unlock_bh(&vvs->rx_lock);
 794
 795	virtio_transport_send_credit_update(vsk);
 796
 797	return dequeued_len;
 798}
 799
 800ssize_t
 801virtio_transport_stream_dequeue(struct vsock_sock *vsk,
 802				struct msghdr *msg,
 803				size_t len, int flags)
 804{
 805	if (flags & MSG_PEEK)
 806		return virtio_transport_stream_do_peek(vsk, msg, len);
 807	else
 808		return virtio_transport_stream_do_dequeue(vsk, msg, len);
 809}
 810EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue);
 811
 812ssize_t
 813virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk,
 814				   struct msghdr *msg,
 815				   int flags)
 816{
 817	if (flags & MSG_PEEK)
 818		return virtio_transport_seqpacket_do_peek(vsk, msg);
 819	else
 820		return virtio_transport_seqpacket_do_dequeue(vsk, msg, flags);
 821}
 822EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_dequeue);
 823
 824static u32 virtio_transport_tx_buf_size(struct virtio_vsock_sock *vvs)
 825{
 826	/* The peer advertises its receive buffer via peer_buf_alloc, but we
 827	 * cap it to our local buf_alloc so a remote peer cannot force us to
 828	 * queue more data than our own buffer configuration allows.
 829	 */
 830	return min(vvs->peer_buf_alloc, vvs->buf_alloc);
 831}
 832
 833int
 834virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk,
 835				   struct msghdr *msg,
 836				   size_t len)
 837{
 838	struct virtio_vsock_sock *vvs = vsk->trans;
 839
 840	spin_lock_bh(&vvs->tx_lock);
 841
 842	if (len > virtio_transport_tx_buf_size(vvs)) {
 843		spin_unlock_bh(&vvs->tx_lock);
 844		return -EMSGSIZE;
 845	}
 846
 847	spin_unlock_bh(&vvs->tx_lock);
 848
 849	return virtio_transport_stream_enqueue(vsk, msg, len);
 850}
 851EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_enqueue);
 852
 853int
 854virtio_transport_dgram_dequeue(struct vsock_sock *vsk,
 855			       struct msghdr *msg,
 856			       size_t len, int flags)
 857{
 858	return -EOPNOTSUPP;
 859}
 860EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue);
 861
 862s64 virtio_transport_stream_has_data(struct vsock_sock *vsk)
 863{
 864	struct virtio_vsock_sock *vvs = vsk->trans;
 865	s64 bytes;
 866
 867	spin_lock_bh(&vvs->rx_lock);
 868	bytes = vvs->rx_bytes;
 869	spin_unlock_bh(&vvs->rx_lock);
 870
 871	return bytes;
 872}
 873EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data);
 874
 875u32 virtio_transport_seqpacket_has_data(struct vsock_sock *vsk)
 876{
 877	struct virtio_vsock_sock *vvs = vsk->trans;
 878	u32 msg_count;
 879
 880	spin_lock_bh(&vvs->rx_lock);
 881	msg_count = vvs->msg_count;
 882	spin_unlock_bh(&vvs->rx_lock);
 883
 884	return msg_count;
 885}
 886EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_has_data);
 887
 888static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs)
 889{
 890	s64 bytes;
 891
 892	/* Use s64 arithmetic so if the peer shrinks peer_buf_alloc while
 893	 * we have bytes in flight (tx_cnt - peer_fwd_cnt), the subtraction
 894	 * does not underflow.
 895	 */
 896	bytes = (s64)virtio_transport_tx_buf_size(vvs) -
 897		(vvs->tx_cnt - vvs->peer_fwd_cnt);
 898	if (bytes < 0)
 899		bytes = 0;
 900
 901	return bytes;
 902}
 903
 904s64 virtio_transport_stream_has_space(struct vsock_sock *vsk)
 905{
 906	struct virtio_vsock_sock *vvs = vsk->trans;
 907	s64 bytes;
 908
 909	spin_lock_bh(&vvs->tx_lock);
 910	bytes = virtio_transport_has_space(vvs);
 911	spin_unlock_bh(&vvs->tx_lock);
 912
 913	return bytes;
 914}
 915EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space);
 916
 917int virtio_transport_do_socket_init(struct vsock_sock *vsk,
 918				    struct vsock_sock *psk)
 919{
 920	struct virtio_vsock_sock *vvs;
 921
 922	vvs = kzalloc(sizeof(*vvs), GFP_KERNEL);
 923	if (!vvs)
 924		return -ENOMEM;
 925
 926	vsk->trans = vvs;
 927	vvs->vsk = vsk;
 928	if (psk && psk->trans) {
 929		struct virtio_vsock_sock *ptrans = psk->trans;
 930
 931		vvs->peer_buf_alloc = ptrans->peer_buf_alloc;
 932	}
 933
 934	if (vsk->buffer_size > VIRTIO_VSOCK_MAX_BUF_SIZE)
 935		vsk->buffer_size = VIRTIO_VSOCK_MAX_BUF_SIZE;
 936
 937	vvs->buf_alloc = vsk->buffer_size;
 938
 939	spin_lock_init(&vvs->rx_lock);
 940	spin_lock_init(&vvs->tx_lock);
 941	skb_queue_head_init(&vvs->rx_queue);
 942
 943	return 0;
 944}
 945EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init);
 946
 947/* sk_lock held by the caller */
 948void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val)
 949{
 950	struct virtio_vsock_sock *vvs = vsk->trans;
 951
 952	if (*val > VIRTIO_VSOCK_MAX_BUF_SIZE)
 953		*val = VIRTIO_VSOCK_MAX_BUF_SIZE;
 954
 955	vvs->buf_alloc = *val;
 956
 957	virtio_transport_send_credit_update(vsk);
 958}
 959EXPORT_SYMBOL_GPL(virtio_transport_notify_buffer_size);
 960
 961int
 962virtio_transport_notify_poll_in(struct vsock_sock *vsk,
 963				size_t target,
 964				bool *data_ready_now)
 965{
 966	*data_ready_now = vsock_stream_has_data(vsk) >= target;
 967
 968	return 0;
 969}
 970EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in);
 971
 972int
 973virtio_transport_notify_poll_out(struct vsock_sock *vsk,
 974				 size_t target,
 975				 bool *space_avail_now)
 976{
 977	s64 free_space;
 978
 979	free_space = vsock_stream_has_space(vsk);
 980	if (free_space > 0)
 981		*space_avail_now = true;
 982	else if (free_space == 0)
 983		*space_avail_now = false;
 984
 985	return 0;
 986}
 987EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out);
 988
 989int virtio_transport_notify_recv_init(struct vsock_sock *vsk,
 990	size_t target, struct vsock_transport_recv_notify_data *data)
 991{
 992	return 0;
 993}
 994EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init);
 995
 996int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk,
 997	size_t target, struct vsock_transport_recv_notify_data *data)
 998{
 999	return 0;
1000}
1001EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block);
1002
1003int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk,
1004	size_t target, struct vsock_transport_recv_notify_data *data)
1005{
1006	return 0;
1007}
1008EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue);
1009
1010int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk,
1011	size_t target, ssize_t copied, bool data_read,
1012	struct vsock_transport_recv_notify_data *data)
1013{
1014	return 0;
1015}
1016EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue);
1017
1018int virtio_transport_notify_send_init(struct vsock_sock *vsk,
1019	struct vsock_transport_send_notify_data *data)
1020{
1021	return 0;
1022}
1023EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init);
1024
1025int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk,
1026	struct vsock_transport_send_notify_data *data)
1027{
1028	return 0;
1029}
1030EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block);
1031
1032int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk,
1033	struct vsock_transport_send_notify_data *data)
1034{
1035	return 0;
1036}
1037EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue);
1038
1039int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk,
1040	ssize_t written, struct vsock_transport_send_notify_data *data)
1041{
1042	return 0;
1043}
1044EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue);
1045
1046u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk)
1047{
1048	return vsk->buffer_size;
1049}
1050EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat);
1051
1052bool virtio_transport_stream_is_active(struct vsock_sock *vsk)
1053{
1054	return true;
1055}
1056EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active);
1057
1058bool virtio_transport_stream_allow(u32 cid, u32 port)
1059{
1060	return true;
1061}
1062EXPORT_SYMBOL_GPL(virtio_transport_stream_allow);
1063
1064int virtio_transport_dgram_bind(struct vsock_sock *vsk,
1065				struct sockaddr_vm *addr)
1066{
1067	return -EOPNOTSUPP;
1068}
1069EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind);
1070
1071bool virtio_transport_dgram_allow(u32 cid, u32 port)
1072{
1073	return false;
1074}
1075EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow);
1076
1077int virtio_transport_connect(struct vsock_sock *vsk)
1078{
1079	struct virtio_vsock_pkt_info info = {
1080		.op = VIRTIO_VSOCK_OP_REQUEST,
1081		.vsk = vsk,
1082	};
1083
1084	return virtio_transport_send_pkt_info(vsk, &info);
1085}
1086EXPORT_SYMBOL_GPL(virtio_transport_connect);
1087
1088int virtio_transport_shutdown(struct vsock_sock *vsk, int mode)
1089{
1090	struct virtio_vsock_pkt_info info = {
1091		.op = VIRTIO_VSOCK_OP_SHUTDOWN,
1092		.flags = (mode & RCV_SHUTDOWN ?
1093			  VIRTIO_VSOCK_SHUTDOWN_RCV : 0) |
1094			 (mode & SEND_SHUTDOWN ?
1095			  VIRTIO_VSOCK_SHUTDOWN_SEND : 0),
1096		.vsk = vsk,
1097	};
1098
1099	return virtio_transport_send_pkt_info(vsk, &info);
1100}
1101EXPORT_SYMBOL_GPL(virtio_transport_shutdown);
1102
1103int
1104virtio_transport_dgram_enqueue(struct vsock_sock *vsk,
1105			       struct sockaddr_vm *remote_addr,
1106			       struct msghdr *msg,
1107			       size_t dgram_len)
1108{
1109	return -EOPNOTSUPP;
1110}
1111EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue);
1112
1113ssize_t
1114virtio_transport_stream_enqueue(struct vsock_sock *vsk,
1115				struct msghdr *msg,
1116				size_t len)
1117{
1118	struct virtio_vsock_pkt_info info = {
1119		.op = VIRTIO_VSOCK_OP_RW,
1120		.msg = msg,
1121		.pkt_len = len,
1122		.vsk = vsk,
1123	};
1124
1125	return virtio_transport_send_pkt_info(vsk, &info);
1126}
1127EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue);
1128
1129void virtio_transport_destruct(struct vsock_sock *vsk)
1130{
1131	struct virtio_vsock_sock *vvs = vsk->trans;
1132
1133	virtio_transport_cancel_close_work(vsk, true);
1134
1135	kfree(vvs);
1136	vsk->trans = NULL;
1137}
1138EXPORT_SYMBOL_GPL(virtio_transport_destruct);
1139
1140ssize_t virtio_transport_unsent_bytes(struct vsock_sock *vsk)
1141{
1142	struct virtio_vsock_sock *vvs = vsk->trans;
1143	size_t ret;
1144
1145	spin_lock_bh(&vvs->tx_lock);
1146	ret = vvs->bytes_unsent;
1147	spin_unlock_bh(&vvs->tx_lock);
1148
1149	return ret;
1150}
1151EXPORT_SYMBOL_GPL(virtio_transport_unsent_bytes);
1152
1153static int virtio_transport_reset(struct vsock_sock *vsk,
1154				  struct sk_buff *skb)
1155{
1156	struct virtio_vsock_pkt_info info = {
1157		.op = VIRTIO_VSOCK_OP_RST,
1158		.reply = !!skb,
1159		.vsk = vsk,
1160	};
1161
1162	/* Send RST only if the original pkt is not a RST pkt */
1163	if (skb && le16_to_cpu(virtio_vsock_hdr(skb)->op) == VIRTIO_VSOCK_OP_RST)
1164		return 0;
1165
1166	return virtio_transport_send_pkt_info(vsk, &info);
1167}
1168
1169/* Normally packets are associated with a socket.  There may be no socket if an
1170 * attempt was made to connect to a socket that does not exist.
1171 */
1172static int virtio_transport_reset_no_sock(const struct virtio_transport *t,
1173					  struct sk_buff *skb)
1174{
1175	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1176	struct virtio_vsock_pkt_info info = {
1177		.op = VIRTIO_VSOCK_OP_RST,
1178		.type = le16_to_cpu(hdr->type),
1179		.reply = true,
1180	};
1181	struct sk_buff *reply;
1182
1183	/* Send RST only if the original pkt is not a RST pkt */
1184	if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST)
1185		return 0;
1186
1187	if (!t)
1188		return -ENOTCONN;
1189
1190	reply = virtio_transport_alloc_skb(&info, 0, false,
1191					   le64_to_cpu(hdr->dst_cid),
1192					   le32_to_cpu(hdr->dst_port),
1193					   le64_to_cpu(hdr->src_cid),
1194					   le32_to_cpu(hdr->src_port));
1195	if (!reply)
1196		return -ENOMEM;
1197
1198	return t->send_pkt(reply);
1199}
1200
1201/* This function should be called with sk_lock held and SOCK_DONE set */
1202static void virtio_transport_remove_sock(struct vsock_sock *vsk)
1203{
1204	struct virtio_vsock_sock *vvs = vsk->trans;
1205
1206	/* We don't need to take rx_lock, as the socket is closing and we are
1207	 * removing it.
1208	 */
1209	__skb_queue_purge(&vvs->rx_queue);
1210	vsock_remove_sock(vsk);
1211}
1212
1213static void virtio_transport_cancel_close_work(struct vsock_sock *vsk,
1214					       bool cancel_timeout)
1215{
1216	struct sock *sk = sk_vsock(vsk);
1217
1218	if (vsk->close_work_scheduled &&
1219	    (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) {
1220		vsk->close_work_scheduled = false;
1221
1222		virtio_transport_remove_sock(vsk);
1223
1224		/* Release refcnt obtained when we scheduled the timeout */
1225		sock_put(sk);
1226	}
1227}
1228
1229static void virtio_transport_do_close(struct vsock_sock *vsk,
1230				      bool cancel_timeout)
1231{
1232	struct sock *sk = sk_vsock(vsk);
1233
1234	sock_set_flag(sk, SOCK_DONE);
1235	vsk->peer_shutdown = SHUTDOWN_MASK;
1236	if (vsock_stream_has_data(vsk) <= 0)
1237		sk->sk_state = TCP_CLOSING;
1238	sk->sk_state_change(sk);
1239
1240	virtio_transport_cancel_close_work(vsk, cancel_timeout);
1241}
1242
1243static void virtio_transport_close_timeout(struct work_struct *work)
1244{
1245	struct vsock_sock *vsk =
1246		container_of(work, struct vsock_sock, close_work.work);
1247	struct sock *sk = sk_vsock(vsk);
1248
1249	sock_hold(sk);
1250	lock_sock(sk);
1251
1252	if (!sock_flag(sk, SOCK_DONE)) {
1253		(void)virtio_transport_reset(vsk, NULL);
1254
1255		virtio_transport_do_close(vsk, false);
1256	}
1257
1258	vsk->close_work_scheduled = false;
1259
1260	release_sock(sk);
1261	sock_put(sk);
1262}
1263
1264/* User context, vsk->sk is locked */
1265static bool virtio_transport_close(struct vsock_sock *vsk)
1266{
1267	struct sock *sk = &vsk->sk;
1268
1269	if (!(sk->sk_state == TCP_ESTABLISHED ||
1270	      sk->sk_state == TCP_CLOSING))
1271		return true;
1272
1273	/* Already received SHUTDOWN from peer, reply with RST */
1274	if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) {
1275		(void)virtio_transport_reset(vsk, NULL);
1276		return true;
1277	}
1278
1279	if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK)
1280		(void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK);
1281
1282	if (!(current->flags & PF_EXITING))
1283		vsock_linger(sk);
1284
1285	if (sock_flag(sk, SOCK_DONE)) {
1286		return true;
1287	}
1288
1289	sock_hold(sk);
1290	INIT_DELAYED_WORK(&vsk->close_work,
1291			  virtio_transport_close_timeout);
1292	vsk->close_work_scheduled = true;
1293	schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT);
1294	return false;
1295}
1296
1297void virtio_transport_release(struct vsock_sock *vsk)
1298{
1299	struct sock *sk = &vsk->sk;
1300	bool remove_sock = true;
1301
1302	if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET)
1303		remove_sock = virtio_transport_close(vsk);
1304
1305	if (remove_sock) {
1306		sock_set_flag(sk, SOCK_DONE);
1307		virtio_transport_remove_sock(vsk);
1308	}
1309}
1310EXPORT_SYMBOL_GPL(virtio_transport_release);
1311
1312static int
1313virtio_transport_recv_connecting(struct sock *sk,
1314				 struct sk_buff *skb)
1315{
1316	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1317	struct vsock_sock *vsk = vsock_sk(sk);
1318	int skerr;
1319	int err;
1320
1321	switch (le16_to_cpu(hdr->op)) {
1322	case VIRTIO_VSOCK_OP_RESPONSE:
1323		sk->sk_state = TCP_ESTABLISHED;
1324		sk->sk_socket->state = SS_CONNECTED;
1325		vsock_insert_connected(vsk);
1326		sk->sk_state_change(sk);
1327		break;
1328	case VIRTIO_VSOCK_OP_INVALID:
1329		break;
1330	case VIRTIO_VSOCK_OP_RST:
1331		skerr = ECONNRESET;
1332		err = 0;
1333		goto destroy;
1334	default:
1335		skerr = EPROTO;
1336		err = -EINVAL;
1337		goto destroy;
1338	}
1339	return 0;
1340
1341destroy:
1342	virtio_transport_reset(vsk, skb);
1343	sk->sk_state = TCP_CLOSE;
1344	sk->sk_err = skerr;
1345	sk_error_report(sk);
1346	return err;
1347}
1348
1349static void
1350virtio_transport_recv_enqueue(struct vsock_sock *vsk,
1351			      struct sk_buff *skb)
1352{
1353	struct virtio_vsock_sock *vvs = vsk->trans;
1354	bool can_enqueue, free_pkt = false;
1355	struct virtio_vsock_hdr *hdr;
1356	u32 len;
1357
1358	hdr = virtio_vsock_hdr(skb);
1359	len = le32_to_cpu(hdr->len);
1360
1361	spin_lock_bh(&vvs->rx_lock);
1362
1363	can_enqueue = virtio_transport_inc_rx_pkt(vvs, len);
1364	if (!can_enqueue) {
1365		free_pkt = true;
1366		goto out;
1367	}
1368
1369	if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)
1370		vvs->msg_count++;
1371
1372	/* Try to copy small packets into the buffer of last packet queued,
1373	 * to avoid wasting memory queueing the entire buffer with a small
1374	 * payload. Skip non-linear (e.g. zerocopy) skbs; these carry payload
1375	 * in skb_shinfo.
1376	 */
1377	if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue) &&
1378	    !skb_is_nonlinear(skb)) {
1379		struct virtio_vsock_hdr *last_hdr;
1380		struct sk_buff *last_skb;
1381
1382		last_skb = skb_peek_tail(&vvs->rx_queue);
1383		last_hdr = virtio_vsock_hdr(last_skb);
1384
1385		/* If there is space in the last packet queued, we copy the
1386		 * new packet in its buffer. We avoid this if the last packet
1387		 * queued has VIRTIO_VSOCK_SEQ_EOM set, because this is
1388		 * delimiter of SEQPACKET message, so 'pkt' is the first packet
1389		 * of a new message.
1390		 */
1391		if (skb->len < skb_tailroom(last_skb) &&
1392		    !(le32_to_cpu(last_hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)) {
1393			memcpy(skb_put(last_skb, skb->len), skb->data, skb->len);
1394			free_pkt = true;
1395			last_hdr->flags |= hdr->flags;
1396			le32_add_cpu(&last_hdr->len, len);
1397			goto out;
1398		}
1399	}
1400
1401	__skb_queue_tail(&vvs->rx_queue, skb);
1402
1403out:
1404	spin_unlock_bh(&vvs->rx_lock);
1405	if (free_pkt)
1406		kfree_skb(skb);
1407}
1408
1409static int
1410virtio_transport_recv_connected(struct sock *sk,
1411				struct sk_buff *skb)
1412{
1413	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1414	struct vsock_sock *vsk = vsock_sk(sk);
1415	int err = 0;
1416
1417	switch (le16_to_cpu(hdr->op)) {
1418	case VIRTIO_VSOCK_OP_RW:
1419		virtio_transport_recv_enqueue(vsk, skb);
1420		vsock_data_ready(sk);
1421		return err;
1422	case VIRTIO_VSOCK_OP_CREDIT_REQUEST:
1423		virtio_transport_send_credit_update(vsk);
1424		break;
1425	case VIRTIO_VSOCK_OP_CREDIT_UPDATE:
1426		sk->sk_write_space(sk);
1427		break;
1428	case VIRTIO_VSOCK_OP_SHUTDOWN:
1429		if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_RCV)
1430			vsk->peer_shutdown |= RCV_SHUTDOWN;
1431		if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_SEND)
1432			vsk->peer_shutdown |= SEND_SHUTDOWN;
1433		if (vsk->peer_shutdown == SHUTDOWN_MASK) {
1434			if (vsock_stream_has_data(vsk) <= 0 && !sock_flag(sk, SOCK_DONE)) {
1435				(void)virtio_transport_reset(vsk, NULL);
1436				virtio_transport_do_close(vsk, true);
1437			}
1438			/* Remove this socket anyway because the remote peer sent
1439			 * the shutdown. This way a new connection will succeed
1440			 * if the remote peer uses the same source port,
1441			 * even if the old socket is still unreleased, but now disconnected.
1442			 */
1443			vsock_remove_sock(vsk);
1444		}
1445		if (le32_to_cpu(virtio_vsock_hdr(skb)->flags))
1446			sk->sk_state_change(sk);
1447		break;
1448	case VIRTIO_VSOCK_OP_RST:
1449		virtio_transport_do_close(vsk, true);
1450		break;
1451	default:
1452		err = -EINVAL;
1453		break;
1454	}
1455
1456	kfree_skb(skb);
1457	return err;
1458}
1459
1460static void
1461virtio_transport_recv_disconnecting(struct sock *sk,
1462				    struct sk_buff *skb)
1463{
1464	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1465	struct vsock_sock *vsk = vsock_sk(sk);
1466
1467	if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST)
1468		virtio_transport_do_close(vsk, true);
1469}
1470
1471static int
1472virtio_transport_send_response(struct vsock_sock *vsk,
1473			       struct sk_buff *skb)
1474{
1475	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1476	struct virtio_vsock_pkt_info info = {
1477		.op = VIRTIO_VSOCK_OP_RESPONSE,
1478		.remote_cid = le64_to_cpu(hdr->src_cid),
1479		.remote_port = le32_to_cpu(hdr->src_port),
1480		.reply = true,
1481		.vsk = vsk,
1482	};
1483
1484	return virtio_transport_send_pkt_info(vsk, &info);
1485}
1486
1487static bool virtio_transport_space_update(struct sock *sk,
1488					  struct sk_buff *skb)
1489{
1490	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1491	struct vsock_sock *vsk = vsock_sk(sk);
1492	struct virtio_vsock_sock *vvs = vsk->trans;
1493	bool space_available;
1494
1495	/* Listener sockets are not associated with any transport, so we are
1496	 * not able to take the state to see if there is space available in the
1497	 * remote peer, but since they are only used to receive requests, we
1498	 * can assume that there is always space available in the other peer.
1499	 */
1500	if (!vvs)
1501		return true;
1502
1503	/* buf_alloc and fwd_cnt is always included in the hdr */
1504	spin_lock_bh(&vvs->tx_lock);
1505	vvs->peer_buf_alloc = le32_to_cpu(hdr->buf_alloc);
1506	vvs->peer_fwd_cnt = le32_to_cpu(hdr->fwd_cnt);
1507	space_available = virtio_transport_has_space(vvs);
1508	spin_unlock_bh(&vvs->tx_lock);
1509	return space_available;
1510}
1511
1512/* Handle server socket */
1513static int
1514virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb,
1515			     struct virtio_transport *t)
1516{
1517	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1518	struct vsock_sock *vsk = vsock_sk(sk);
1519	struct vsock_sock *vchild;
1520	struct sock *child;
1521	int ret;
1522
1523	if (le16_to_cpu(hdr->op) != VIRTIO_VSOCK_OP_REQUEST) {
1524		virtio_transport_reset_no_sock(t, skb);
1525		return -EINVAL;
1526	}
1527
1528	if (sk_acceptq_is_full(sk)) {
1529		virtio_transport_reset_no_sock(t, skb);
1530		return -ENOMEM;
1531	}
1532
1533	/* __vsock_release() might have already flushed accept_queue.
1534	 * Subsequent enqueues would lead to a memory leak.
1535	 */
1536	if (sk->sk_shutdown == SHUTDOWN_MASK) {
1537		virtio_transport_reset_no_sock(t, skb);
1538		return -ESHUTDOWN;
1539	}
1540
1541	child = vsock_create_connected(sk);
1542	if (!child) {
1543		virtio_transport_reset_no_sock(t, skb);
1544		return -ENOMEM;
1545	}
1546
1547	sk_acceptq_added(sk);
1548
1549	lock_sock_nested(child, SINGLE_DEPTH_NESTING);
1550
1551	child->sk_state = TCP_ESTABLISHED;
1552
1553	vchild = vsock_sk(child);
1554	vsock_addr_init(&vchild->local_addr, le64_to_cpu(hdr->dst_cid),
1555			le32_to_cpu(hdr->dst_port));
1556	vsock_addr_init(&vchild->remote_addr, le64_to_cpu(hdr->src_cid),
1557			le32_to_cpu(hdr->src_port));
1558
1559	ret = vsock_assign_transport(vchild, vsk);
1560	/* Transport assigned (looking at remote_addr) must be the same
1561	 * where we received the request.
1562	 */
1563	if (ret || vchild->transport != &t->transport) {
1564		release_sock(child);
1565		virtio_transport_reset_no_sock(t, skb);
1566		sock_put(child);
1567		return ret;
1568	}
1569
1570	if (virtio_transport_space_update(child, skb))
1571		child->sk_write_space(child);
1572
1573	vsock_insert_connected(vchild);
1574	vsock_enqueue_accept(sk, child);
1575	virtio_transport_send_response(vchild, skb);
1576
1577	release_sock(child);
1578
1579	sk->sk_data_ready(sk);
1580	return 0;
1581}
1582
1583static bool virtio_transport_valid_type(u16 type)
1584{
1585	return (type == VIRTIO_VSOCK_TYPE_STREAM) ||
1586	       (type == VIRTIO_VSOCK_TYPE_SEQPACKET);
1587}
1588
1589/* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex
1590 * lock.
1591 */
1592void virtio_transport_recv_pkt(struct virtio_transport *t,
1593			       struct sk_buff *skb)
1594{
1595	struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
1596	struct sockaddr_vm src, dst;
1597	struct vsock_sock *vsk;
1598	struct sock *sk;
1599	bool space_available;
1600
1601	vsock_addr_init(&src, le64_to_cpu(hdr->src_cid),
1602			le32_to_cpu(hdr->src_port));
1603	vsock_addr_init(&dst, le64_to_cpu(hdr->dst_cid),
1604			le32_to_cpu(hdr->dst_port));
1605
1606	trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port,
1607					dst.svm_cid, dst.svm_port,
1608					le32_to_cpu(hdr->len),
1609					le16_to_cpu(hdr->type),
1610					le16_to_cpu(hdr->op),
1611					le32_to_cpu(hdr->flags),
1612					le32_to_cpu(hdr->buf_alloc),
1613					le32_to_cpu(hdr->fwd_cnt));
1614
1615	if (!virtio_transport_valid_type(le16_to_cpu(hdr->type))) {
1616		(void)virtio_transport_reset_no_sock(t, skb);
1617		goto free_pkt;
1618	}
1619
1620	/* The socket must be in connected or bound table
1621	 * otherwise send reset back
1622	 */
1623	sk = vsock_find_connected_socket(&src, &dst);
1624	if (!sk) {
1625		sk = vsock_find_bound_socket(&dst);
1626		if (!sk) {
1627			(void)virtio_transport_reset_no_sock(t, skb);
1628			goto free_pkt;
1629		}
1630	}
1631
1632	if (virtio_transport_get_type(sk) != le16_to_cpu(hdr->type)) {
1633		(void)virtio_transport_reset_no_sock(t, skb);
1634		sock_put(sk);
1635		goto free_pkt;
1636	}
1637
1638	if (!skb_set_owner_sk_safe(skb, sk)) {
1639		WARN_ONCE(1, "receiving vsock socket has sk_refcnt == 0\n");
1640		goto free_pkt;
1641	}
1642
1643	vsk = vsock_sk(sk);
1644
1645	lock_sock(sk);
1646
1647	/* Check if sk has been closed or assigned to another transport before
1648	 * lock_sock (note: listener sockets are not assigned to any transport)
1649	 */
1650	if (sock_flag(sk, SOCK_DONE) ||
1651	    (sk->sk_state != TCP_LISTEN && vsk->transport != &t->transport)) {
1652		(void)virtio_transport_reset_no_sock(t, skb);
1653		release_sock(sk);
1654		sock_put(sk);
1655		goto free_pkt;
1656	}
1657
1658	space_available = virtio_transport_space_update(sk, skb);
1659
1660	/* Update CID in case it has changed after a transport reset event */
1661	if (vsk->local_addr.svm_cid != VMADDR_CID_ANY)
1662		vsk->local_addr.svm_cid = dst.svm_cid;
1663
1664	if (space_available)
1665		sk->sk_write_space(sk);
1666
1667	switch (sk->sk_state) {
1668	case TCP_LISTEN:
1669		virtio_transport_recv_listen(sk, skb, t);
1670		kfree_skb(skb);
1671		break;
1672	case TCP_SYN_SENT:
1673		virtio_transport_recv_connecting(sk, skb);
1674		kfree_skb(skb);
1675		break;
1676	case TCP_ESTABLISHED:
1677		virtio_transport_recv_connected(sk, skb);
1678		break;
1679	case TCP_CLOSING:
1680		virtio_transport_recv_disconnecting(sk, skb);
1681		kfree_skb(skb);
1682		break;
1683	default:
1684		(void)virtio_transport_reset_no_sock(t, skb);
1685		kfree_skb(skb);
1686		break;
1687	}
1688
1689	release_sock(sk);
1690
1691	/* Release refcnt obtained when we fetched this socket out of the
1692	 * bound or connected list.
1693	 */
1694	sock_put(sk);
1695	return;
1696
1697free_pkt:
1698	kfree_skb(skb);
1699}
1700EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt);
1701
1702/* Remove skbs found in a queue that have a vsk that matches.
1703 *
1704 * Each skb is freed.
1705 *
1706 * Returns the count of skbs that were reply packets.
1707 */
1708int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *queue)
1709{
1710	struct sk_buff_head freeme;
1711	struct sk_buff *skb, *tmp;
1712	int cnt = 0;
1713
1714	skb_queue_head_init(&freeme);
1715
1716	spin_lock_bh(&queue->lock);
1717	skb_queue_walk_safe(queue, skb, tmp) {
1718		if (vsock_sk(skb->sk) != vsk)
1719			continue;
1720
1721		__skb_unlink(skb, queue);
1722		__skb_queue_tail(&freeme, skb);
1723
1724		if (virtio_vsock_skb_reply(skb))
1725			cnt++;
1726	}
1727	spin_unlock_bh(&queue->lock);
1728
1729	__skb_queue_purge(&freeme);
1730
1731	return cnt;
1732}
1733EXPORT_SYMBOL_GPL(virtio_transport_purge_skbs);
1734
1735int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_actor)
1736{
1737	struct virtio_vsock_sock *vvs = vsk->trans;
1738	struct sock *sk = sk_vsock(vsk);
1739	struct virtio_vsock_hdr *hdr;
1740	struct sk_buff *skb;
1741	u32 pkt_len;
1742	int off = 0;
1743	int err;
1744
1745	spin_lock_bh(&vvs->rx_lock);
1746	/* Use __skb_recv_datagram() for race-free handling of the receive. It
1747	 * works for types other than dgrams.
1748	 */
1749	skb = __skb_recv_datagram(sk, &vvs->rx_queue, MSG_DONTWAIT, &off, &err);
1750	if (!skb) {
1751		spin_unlock_bh(&vvs->rx_lock);
1752		return err;
1753	}
1754
1755	hdr = virtio_vsock_hdr(skb);
1756	if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)
1757		vvs->msg_count--;
1758
1759	pkt_len = le32_to_cpu(hdr->len);
1760	virtio_transport_dec_rx_pkt(vvs, pkt_len, pkt_len);
1761	spin_unlock_bh(&vvs->rx_lock);
1762
1763	virtio_transport_send_credit_update(vsk);
1764
1765	return recv_actor(sk, skb);
1766}
1767EXPORT_SYMBOL_GPL(virtio_transport_read_skb);
1768
1769int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val)
1770{
1771	struct virtio_vsock_sock *vvs = vsk->trans;
1772	bool send_update;
1773
1774	spin_lock_bh(&vvs->rx_lock);
1775
1776	/* If number of available bytes is less than new SO_RCVLOWAT value,
1777	 * kick sender to send more data, because sender may sleep in its
1778	 * 'send()' syscall waiting for enough space at our side. Also
1779	 * don't send credit update when peer already knows actual value -
1780	 * such transmission will be useless.
1781	 */
1782	send_update = (vvs->rx_bytes < val) &&
1783		      (vvs->fwd_cnt != vvs->last_fwd_cnt);
1784
1785	spin_unlock_bh(&vvs->rx_lock);
1786
1787	if (send_update) {
1788		int err;
1789
1790		err = virtio_transport_send_credit_update(vsk);
1791		if (err < 0)
1792			return err;
1793	}
1794
1795	return 0;
1796}
1797EXPORT_SYMBOL_GPL(virtio_transport_notify_set_rcvlowat);
1798
1799MODULE_LICENSE("GPL v2");
1800MODULE_AUTHOR("Asias He");
1801MODULE_DESCRIPTION("common code for virtio vsock");