virtio_uml.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Virtio vhost-user driver
   4 *
   5 * Copyright(c) 2019 Intel Corporation
   6 *
   7 * This driver allows virtio devices to be used over a vhost-user socket.
   8 *
   9 * Guest devices can be instantiated by kernel module or command line
  10 * parameters. One device will be created for each parameter. Syntax:
  11 *
  12 *		virtio_uml.device=<socket>:<virtio_id>[:<platform_id>]
  13 * where:
  14 *		<socket>	:= vhost-user socket path to connect
  15 *		<virtio_id>	:= virtio device id (as in virtio_ids.h)
  16 *		<platform_id>	:= (optional) platform device id
  17 *
  18 * example:
  19 *		virtio_uml.device=/var/uml.socket:1
  20 *
  21 * Based on Virtio MMIO driver by Pawel Moll, copyright 2011-2014, ARM Ltd.
  22 */
  23#include <linux/module.h>
  24#include <linux/of.h>
  25#include <linux/platform_device.h>
  26#include <linux/slab.h>
  27#include <linux/string_choices.h>
  28#include <linux/virtio.h>
  29#include <linux/virtio_config.h>
  30#include <linux/virtio_ring.h>
  31#include <linux/time-internal.h>
  32#include <linux/virtio-uml.h>
  33#include <shared/as-layout.h>
  34#include <irq_kern.h>
  35#include <init.h>
  36#include <os.h>
  37#include "vhost_user.h"
  38
  39#define MAX_SUPPORTED_QUEUE_SIZE	256
  40
  41#define to_virtio_uml_device(_vdev) \
  42	container_of(_vdev, struct virtio_uml_device, vdev)
  43
  44struct virtio_uml_platform_data {
  45	u32 virtio_device_id;
  46	const char *socket_path;
  47	struct work_struct conn_broken_wk;
  48	struct platform_device *pdev;
  49};
  50
  51struct virtio_uml_device {
  52	struct virtio_device vdev;
  53	struct platform_device *pdev;
  54	struct virtio_uml_platform_data *pdata;
  55
  56	raw_spinlock_t sock_lock;
  57	int sock, req_fd, irq;
  58	u64 features;
  59	u64 protocol_features;
  60	u64 max_vqs;
  61	u8 status;
  62	u8 registered:1;
  63	u8 suspended:1;
  64	u8 no_vq_suspend:1;
  65
  66	u8 config_changed_irq:1;
  67	uint64_t vq_irq_vq_map;
  68	int recv_rc;
  69};
  70
  71struct virtio_uml_vq_info {
  72	int kick_fd, call_fd;
  73	char name[32];
  74	bool suspended;
  75};
  76
  77#define vu_err(vu_dev, ...)	dev_err(&(vu_dev)->pdev->dev, ##__VA_ARGS__)
  78
  79/* Vhost-user protocol */
  80
  81static int full_sendmsg_fds(int fd, const void *buf, unsigned int len,
  82			    const int *fds, unsigned int fds_num)
  83{
  84	int rc;
  85
  86	do {
  87		rc = os_sendmsg_fds(fd, buf, len, fds, fds_num);
  88		if (rc > 0) {
  89			buf += rc;
  90			len -= rc;
  91			fds = NULL;
  92			fds_num = 0;
  93		}
  94	} while (len && (rc >= 0 || rc == -EINTR));
  95
  96	if (rc < 0)
  97		return rc;
  98	return 0;
  99}
 100
 101static int full_read(int fd, void *buf, int len, bool abortable)
 102{
 103	int rc;
 104
 105	if (!len)
 106		return 0;
 107
 108	do {
 109		rc = os_read_file(fd, buf, len);
 110		if (rc > 0) {
 111			buf += rc;
 112			len -= rc;
 113		}
 114	} while (len && (rc > 0 || rc == -EINTR || (!abortable && rc == -EAGAIN)));
 115
 116	if (rc < 0)
 117		return rc;
 118	if (rc == 0)
 119		return -ECONNRESET;
 120	return 0;
 121}
 122
 123static int vhost_user_recv_header(int fd, struct vhost_user_msg *msg)
 124{
 125	return full_read(fd, msg, sizeof(msg->header), true);
 126}
 127
 128static int vhost_user_recv(struct virtio_uml_device *vu_dev,
 129			   int fd, struct vhost_user_msg *msg,
 130			   size_t max_payload_size, bool wait)
 131{
 132	size_t size;
 133	int rc;
 134
 135	/*
 136	 * In virtio time-travel mode, we're handling all the vhost-user
 137	 * FDs by polling them whenever appropriate. However, we may get
 138	 * into a situation where we're sending out an interrupt message
 139	 * to a device (e.g. a net device) and need to handle a simulation
 140	 * time message while doing so, e.g. one that tells us to update
 141	 * our idea of how long we can run without scheduling.
 142	 *
 143	 * Thus, we need to not just read() from the given fd, but need
 144	 * to also handle messages for the simulation time - this function
 145	 * does that for us while waiting for the given fd to be readable.
 146	 */
 147	if (wait)
 148		time_travel_wait_readable(fd);
 149
 150	rc = vhost_user_recv_header(fd, msg);
 151
 152	if (rc)
 153		return rc;
 154	size = msg->header.size;
 155	if (size > max_payload_size)
 156		return -EPROTO;
 157	return full_read(fd, &msg->payload, size, false);
 158}
 159
 160static void vhost_user_check_reset(struct virtio_uml_device *vu_dev,
 161				   int rc)
 162{
 163	struct virtio_uml_platform_data *pdata = vu_dev->pdata;
 164
 165	if (rc != -ECONNRESET)
 166		return;
 167
 168	if (!vu_dev->registered)
 169		return;
 170
 171	vu_dev->registered = 0;
 172
 173	schedule_work(&pdata->conn_broken_wk);
 174}
 175
 176static int vhost_user_recv_resp(struct virtio_uml_device *vu_dev,
 177				struct vhost_user_msg *msg,
 178				size_t max_payload_size)
 179{
 180	int rc = vhost_user_recv(vu_dev, vu_dev->sock, msg,
 181				 max_payload_size, true);
 182
 183	if (rc) {
 184		vhost_user_check_reset(vu_dev, rc);
 185		return rc;
 186	}
 187
 188	if (msg->header.flags != (VHOST_USER_FLAG_REPLY | VHOST_USER_VERSION))
 189		return -EPROTO;
 190
 191	return 0;
 192}
 193
 194static int vhost_user_recv_u64(struct virtio_uml_device *vu_dev,
 195			       u64 *value)
 196{
 197	struct vhost_user_msg msg;
 198	int rc = vhost_user_recv_resp(vu_dev, &msg,
 199				      sizeof(msg.payload.integer));
 200
 201	if (rc)
 202		return rc;
 203	if (msg.header.size != sizeof(msg.payload.integer))
 204		return -EPROTO;
 205	*value = msg.payload.integer;
 206	return 0;
 207}
 208
 209static int vhost_user_recv_req(struct virtio_uml_device *vu_dev,
 210			       struct vhost_user_msg *msg,
 211			       size_t max_payload_size)
 212{
 213	int rc = vhost_user_recv(vu_dev, vu_dev->req_fd, msg,
 214				 max_payload_size, false);
 215
 216	if (rc)
 217		return rc;
 218
 219	if ((msg->header.flags & ~VHOST_USER_FLAG_NEED_REPLY) !=
 220			VHOST_USER_VERSION)
 221		return -EPROTO;
 222
 223	return 0;
 224}
 225
 226static int vhost_user_send(struct virtio_uml_device *vu_dev,
 227			   bool need_response, struct vhost_user_msg *msg,
 228			   int *fds, size_t num_fds)
 229{
 230	size_t size = sizeof(msg->header) + msg->header.size;
 231	unsigned long flags;
 232	bool request_ack;
 233	int rc;
 234
 235	msg->header.flags |= VHOST_USER_VERSION;
 236
 237	/*
 238	 * The need_response flag indicates that we already need a response,
 239	 * e.g. to read the features. In these cases, don't request an ACK as
 240	 * it is meaningless. Also request an ACK only if supported.
 241	 */
 242	request_ack = !need_response;
 243	if (!(vu_dev->protocol_features &
 244			BIT_ULL(VHOST_USER_PROTOCOL_F_REPLY_ACK)))
 245		request_ack = false;
 246
 247	if (request_ack)
 248		msg->header.flags |= VHOST_USER_FLAG_NEED_REPLY;
 249
 250	raw_spin_lock_irqsave(&vu_dev->sock_lock, flags);
 251	rc = full_sendmsg_fds(vu_dev->sock, msg, size, fds, num_fds);
 252	if (rc < 0)
 253		goto out;
 254
 255	if (request_ack) {
 256		uint64_t status;
 257
 258		rc = vhost_user_recv_u64(vu_dev, &status);
 259		if (rc)
 260			goto out;
 261
 262		if (status) {
 263			vu_err(vu_dev, "slave reports error: %llu\n", status);
 264			rc = -EIO;
 265			goto out;
 266		}
 267	}
 268
 269out:
 270	raw_spin_unlock_irqrestore(&vu_dev->sock_lock, flags);
 271	return rc;
 272}
 273
 274static int vhost_user_send_no_payload(struct virtio_uml_device *vu_dev,
 275				      bool need_response, u32 request)
 276{
 277	struct vhost_user_msg msg = {
 278		.header.request = request,
 279	};
 280
 281	return vhost_user_send(vu_dev, need_response, &msg, NULL, 0);
 282}
 283
 284static int vhost_user_send_no_payload_fd(struct virtio_uml_device *vu_dev,
 285					 u32 request, int fd)
 286{
 287	struct vhost_user_msg msg = {
 288		.header.request = request,
 289	};
 290
 291	return vhost_user_send(vu_dev, false, &msg, &fd, 1);
 292}
 293
 294static int vhost_user_send_u64(struct virtio_uml_device *vu_dev,
 295			       u32 request, u64 value)
 296{
 297	struct vhost_user_msg msg = {
 298		.header.request = request,
 299		.header.size = sizeof(msg.payload.integer),
 300		.payload.integer = value,
 301	};
 302
 303	return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 304}
 305
 306static int vhost_user_set_owner(struct virtio_uml_device *vu_dev)
 307{
 308	return vhost_user_send_no_payload(vu_dev, false, VHOST_USER_SET_OWNER);
 309}
 310
 311static int vhost_user_get_features(struct virtio_uml_device *vu_dev,
 312				   u64 *features)
 313{
 314	int rc = vhost_user_send_no_payload(vu_dev, true,
 315					    VHOST_USER_GET_FEATURES);
 316
 317	if (rc)
 318		return rc;
 319	return vhost_user_recv_u64(vu_dev, features);
 320}
 321
 322static int vhost_user_set_features(struct virtio_uml_device *vu_dev,
 323				   u64 features)
 324{
 325	return vhost_user_send_u64(vu_dev, VHOST_USER_SET_FEATURES, features);
 326}
 327
 328static int vhost_user_get_protocol_features(struct virtio_uml_device *vu_dev,
 329					    u64 *protocol_features)
 330{
 331	int rc = vhost_user_send_no_payload(vu_dev, true,
 332			VHOST_USER_GET_PROTOCOL_FEATURES);
 333
 334	if (rc)
 335		return rc;
 336	return vhost_user_recv_u64(vu_dev, protocol_features);
 337}
 338
 339static int vhost_user_set_protocol_features(struct virtio_uml_device *vu_dev,
 340					    u64 protocol_features)
 341{
 342	return vhost_user_send_u64(vu_dev, VHOST_USER_SET_PROTOCOL_FEATURES,
 343				   protocol_features);
 344}
 345
 346static int vhost_user_get_queue_num(struct virtio_uml_device *vu_dev,
 347				    u64 *queue_num)
 348{
 349	int rc = vhost_user_send_no_payload(vu_dev, true,
 350			VHOST_USER_GET_QUEUE_NUM);
 351
 352	if (rc)
 353		return rc;
 354	return vhost_user_recv_u64(vu_dev, queue_num);
 355}
 356
 357static void vhost_user_reply(struct virtio_uml_device *vu_dev,
 358			     struct vhost_user_msg *msg, int response)
 359{
 360	struct vhost_user_msg reply = {
 361		.payload.integer = response,
 362	};
 363	size_t size = sizeof(reply.header) + sizeof(reply.payload.integer);
 364	int rc;
 365
 366	reply.header = msg->header;
 367	reply.header.flags &= ~VHOST_USER_FLAG_NEED_REPLY;
 368	reply.header.flags |= VHOST_USER_FLAG_REPLY;
 369	reply.header.size = sizeof(reply.payload.integer);
 370
 371	rc = full_sendmsg_fds(vu_dev->req_fd, &reply, size, NULL, 0);
 372
 373	if (rc)
 374		vu_err(vu_dev,
 375		       "sending reply to slave request failed: %d (size %zu)\n",
 376		       rc, size);
 377}
 378
 379static irqreturn_t vu_req_read_message(struct virtio_uml_device *vu_dev,
 380				       struct time_travel_event *ev)
 381{
 382	struct virtqueue *vq;
 383	int response = 1;
 384	struct {
 385		struct vhost_user_msg msg;
 386		u8 extra_payload[512];
 387	} msg;
 388	int rc;
 389	irqreturn_t irq_rc = IRQ_NONE;
 390
 391	while (1) {
 392		rc = vhost_user_recv_req(vu_dev, &msg.msg,
 393					 sizeof(msg.msg.payload) +
 394					 sizeof(msg.extra_payload));
 395		if (rc)
 396			break;
 397
 398		switch (msg.msg.header.request) {
 399		case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG:
 400			vu_dev->config_changed_irq = true;
 401			response = 0;
 402			break;
 403		case VHOST_USER_SLAVE_VRING_CALL:
 404			virtio_device_for_each_vq((&vu_dev->vdev), vq) {
 405				if (vq->index == msg.msg.payload.vring_state.index) {
 406					response = 0;
 407					vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index);
 408					break;
 409				}
 410			}
 411			break;
 412		case VHOST_USER_SLAVE_IOTLB_MSG:
 413			/* not supported - VIRTIO_F_ACCESS_PLATFORM */
 414		case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
 415			/* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */
 416		default:
 417			vu_err(vu_dev, "unexpected slave request %d\n",
 418			       msg.msg.header.request);
 419		}
 420
 421		if (ev && !vu_dev->suspended)
 422			time_travel_add_irq_event(ev);
 423
 424		if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY)
 425			vhost_user_reply(vu_dev, &msg.msg, response);
 426		irq_rc = IRQ_HANDLED;
 427	}
 428	/* mask EAGAIN as we try non-blocking read until socket is empty */
 429	vu_dev->recv_rc = (rc == -EAGAIN) ? 0 : rc;
 430	return irq_rc;
 431}
 432
 433static irqreturn_t vu_req_interrupt(int irq, void *data)
 434{
 435	struct virtio_uml_device *vu_dev = data;
 436	irqreturn_t ret = IRQ_HANDLED;
 437
 438	if (!um_irq_timetravel_handler_used())
 439		ret = vu_req_read_message(vu_dev, NULL);
 440
 441	if (vu_dev->recv_rc) {
 442		vhost_user_check_reset(vu_dev, vu_dev->recv_rc);
 443	} else if (vu_dev->vq_irq_vq_map) {
 444		struct virtqueue *vq;
 445
 446		virtio_device_for_each_vq((&vu_dev->vdev), vq) {
 447			if (vu_dev->vq_irq_vq_map & BIT_ULL(vq->index))
 448				vring_interrupt(0 /* ignored */, vq);
 449		}
 450		vu_dev->vq_irq_vq_map = 0;
 451	} else if (vu_dev->config_changed_irq) {
 452		virtio_config_changed(&vu_dev->vdev);
 453		vu_dev->config_changed_irq = false;
 454	}
 455
 456	return ret;
 457}
 458
 459static void vu_req_interrupt_comm_handler(int irq, int fd, void *data,
 460					  struct time_travel_event *ev)
 461{
 462	vu_req_read_message(data, ev);
 463}
 464
 465static int vhost_user_init_slave_req(struct virtio_uml_device *vu_dev)
 466{
 467	int rc, req_fds[2];
 468
 469	/* Use a pipe for slave req fd, SIGIO is not supported for eventfd */
 470	rc = os_pipe(req_fds, true, true);
 471	if (rc < 0)
 472		return rc;
 473	vu_dev->req_fd = req_fds[0];
 474
 475	rc = um_request_irq_tt(UM_IRQ_ALLOC, vu_dev->req_fd, IRQ_READ,
 476			       vu_req_interrupt, IRQF_SHARED,
 477			       vu_dev->pdev->name, vu_dev,
 478			       vu_req_interrupt_comm_handler);
 479	if (rc < 0)
 480		goto err_close;
 481
 482	vu_dev->irq = rc;
 483
 484	rc = vhost_user_send_no_payload_fd(vu_dev, VHOST_USER_SET_SLAVE_REQ_FD,
 485					   req_fds[1]);
 486	if (rc)
 487		goto err_free_irq;
 488
 489	goto out;
 490
 491err_free_irq:
 492	um_free_irq(vu_dev->irq, vu_dev);
 493err_close:
 494	os_close_file(req_fds[0]);
 495out:
 496	/* Close unused write end of request fds */
 497	os_close_file(req_fds[1]);
 498	return rc;
 499}
 500
 501static int vhost_user_init(struct virtio_uml_device *vu_dev)
 502{
 503	int rc = vhost_user_set_owner(vu_dev);
 504
 505	if (rc)
 506		return rc;
 507	rc = vhost_user_get_features(vu_dev, &vu_dev->features);
 508	if (rc)
 509		return rc;
 510
 511	if (vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)) {
 512		rc = vhost_user_get_protocol_features(vu_dev,
 513				&vu_dev->protocol_features);
 514		if (rc)
 515			return rc;
 516		vu_dev->protocol_features &= VHOST_USER_SUPPORTED_PROTOCOL_F;
 517		rc = vhost_user_set_protocol_features(vu_dev,
 518				vu_dev->protocol_features);
 519		if (rc)
 520			return rc;
 521	}
 522
 523	if (vu_dev->protocol_features &
 524			BIT_ULL(VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 525		rc = vhost_user_init_slave_req(vu_dev);
 526		if (rc)
 527			return rc;
 528	}
 529
 530	if (vu_dev->protocol_features &
 531			BIT_ULL(VHOST_USER_PROTOCOL_F_MQ)) {
 532		rc = vhost_user_get_queue_num(vu_dev, &vu_dev->max_vqs);
 533		if (rc)
 534			return rc;
 535	} else {
 536		vu_dev->max_vqs = U64_MAX;
 537	}
 538
 539	return 0;
 540}
 541
 542static void vhost_user_get_config(struct virtio_uml_device *vu_dev,
 543				  u32 offset, void *buf, u32 len)
 544{
 545	u32 cfg_size = offset + len;
 546	struct vhost_user_msg *msg;
 547	size_t payload_size = sizeof(msg->payload.config) + cfg_size;
 548	size_t msg_size = sizeof(msg->header) + payload_size;
 549	int rc;
 550
 551	if (!(vu_dev->protocol_features &
 552	      BIT_ULL(VHOST_USER_PROTOCOL_F_CONFIG)))
 553		return;
 554
 555	msg = kzalloc(msg_size, GFP_KERNEL);
 556	if (!msg)
 557		return;
 558	msg->header.request = VHOST_USER_GET_CONFIG;
 559	msg->header.size = payload_size;
 560	msg->payload.config.offset = 0;
 561	msg->payload.config.size = cfg_size;
 562
 563	rc = vhost_user_send(vu_dev, true, msg, NULL, 0);
 564	if (rc) {
 565		vu_err(vu_dev, "sending VHOST_USER_GET_CONFIG failed: %d\n",
 566		       rc);
 567		goto free;
 568	}
 569
 570	rc = vhost_user_recv_resp(vu_dev, msg, msg_size);
 571	if (rc) {
 572		vu_err(vu_dev,
 573		       "receiving VHOST_USER_GET_CONFIG response failed: %d\n",
 574		       rc);
 575		goto free;
 576	}
 577
 578	if (msg->header.size != payload_size ||
 579	    msg->payload.config.size != cfg_size) {
 580		rc = -EPROTO;
 581		vu_err(vu_dev,
 582		       "Invalid VHOST_USER_GET_CONFIG sizes (payload %d expected %zu, config %u expected %u)\n",
 583		       msg->header.size, payload_size,
 584		       msg->payload.config.size, cfg_size);
 585		goto free;
 586	}
 587	memcpy(buf, msg->payload.config.payload + offset, len);
 588
 589free:
 590	kfree(msg);
 591}
 592
 593static void vhost_user_set_config(struct virtio_uml_device *vu_dev,
 594				  u32 offset, const void *buf, u32 len)
 595{
 596	struct vhost_user_msg *msg;
 597	size_t payload_size = sizeof(msg->payload.config) + len;
 598	size_t msg_size = sizeof(msg->header) + payload_size;
 599	int rc;
 600
 601	if (!(vu_dev->protocol_features &
 602	      BIT_ULL(VHOST_USER_PROTOCOL_F_CONFIG)))
 603		return;
 604
 605	msg = kzalloc(msg_size, GFP_KERNEL);
 606	if (!msg)
 607		return;
 608	msg->header.request = VHOST_USER_SET_CONFIG;
 609	msg->header.size = payload_size;
 610	msg->payload.config.offset = offset;
 611	msg->payload.config.size = len;
 612	memcpy(msg->payload.config.payload, buf, len);
 613
 614	rc = vhost_user_send(vu_dev, false, msg, NULL, 0);
 615	if (rc)
 616		vu_err(vu_dev, "sending VHOST_USER_SET_CONFIG failed: %d\n",
 617		       rc);
 618
 619	kfree(msg);
 620}
 621
 622static int vhost_user_init_mem_region(u64 addr, u64 size, int *fd_out,
 623				      struct vhost_user_mem_region *region_out)
 624{
 625	unsigned long long mem_offset;
 626	int rc = phys_mapping(addr, &mem_offset);
 627
 628	if (WARN(rc < 0, "phys_mapping of 0x%llx returned %d\n", addr, rc))
 629		return -EFAULT;
 630	*fd_out = rc;
 631	region_out->guest_addr = addr;
 632	region_out->user_addr = addr;
 633	region_out->size = size;
 634	region_out->mmap_offset = mem_offset;
 635
 636	/* Ensure mapping is valid for the entire region */
 637	rc = phys_mapping(addr + size - 1, &mem_offset);
 638	if (WARN(rc != *fd_out, "phys_mapping of 0x%llx failed: %d != %d\n",
 639		 addr + size - 1, rc, *fd_out))
 640		return -EFAULT;
 641	return 0;
 642}
 643
 644static int vhost_user_set_mem_table(struct virtio_uml_device *vu_dev)
 645{
 646	struct vhost_user_msg msg = {
 647		.header.request = VHOST_USER_SET_MEM_TABLE,
 648		.header.size = offsetof(typeof(msg.payload.mem_regions), regions[1]),
 649		.payload.mem_regions.num = 1,
 650	};
 651	unsigned long reserved = uml_reserved - uml_physmem;
 652	int fds[2];
 653	int rc;
 654
 655	/*
 656	 * This is a bit tricky, see also the comment with setup_physmem().
 657	 *
 658	 * Essentially, setup_physmem() uses a file to mmap() our physmem,
 659	 * but the code and data we *already* have is omitted. To us, this
 660	 * is no difference, since they both become part of our address
 661	 * space and memory consumption. To somebody looking in from the
 662	 * outside, however, it is different because the part of our memory
 663	 * consumption that's already part of the binary (code/data) is not
 664	 * mapped from the file, so it's not visible to another mmap from
 665	 * the file descriptor.
 666	 *
 667	 * Thus, don't advertise this space to the vhost-user slave. This
 668	 * means that the slave will likely abort or similar when we give
 669	 * it an address from the hidden range, since it's not marked as
 670	 * a valid address, but at least that way we detect the issue and
 671	 * don't just have the slave read an all-zeroes buffer from the
 672	 * shared memory file, or write something there that we can never
 673	 * see (depending on the direction of the virtqueue traffic.)
 674	 *
 675	 * Since we usually don't want to use .text for virtio buffers,
 676	 * this effectively means that you cannot use
 677	 *  1) global variables, which are in the .bss and not in the shm
 678	 *     file-backed memory
 679	 *  2) the stack in some processes, depending on where they have
 680	 *     their stack (or maybe only no interrupt stack?)
 681	 *
 682	 * The stack is already not typically valid for DMA, so this isn't
 683	 * much of a restriction, but global variables might be encountered.
 684	 *
 685	 * It might be possible to fix it by copying around the data that's
 686	 * between bss_start and where we map the file now, but it's not
 687	 * something that you typically encounter with virtio drivers, so
 688	 * it didn't seem worthwhile.
 689	 */
 690	rc = vhost_user_init_mem_region(reserved, physmem_size - reserved,
 691					&fds[0],
 692					&msg.payload.mem_regions.regions[0]);
 693
 694	if (rc < 0)
 695		return rc;
 696
 697	return vhost_user_send(vu_dev, false, &msg, fds,
 698			       msg.payload.mem_regions.num);
 699}
 700
 701static int vhost_user_set_vring_state(struct virtio_uml_device *vu_dev,
 702				      u32 request, u32 index, u32 num)
 703{
 704	struct vhost_user_msg msg = {
 705		.header.request = request,
 706		.header.size = sizeof(msg.payload.vring_state),
 707		.payload.vring_state.index = index,
 708		.payload.vring_state.num = num,
 709	};
 710
 711	return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 712}
 713
 714static int vhost_user_set_vring_num(struct virtio_uml_device *vu_dev,
 715				    u32 index, u32 num)
 716{
 717	return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_NUM,
 718					  index, num);
 719}
 720
 721static int vhost_user_set_vring_base(struct virtio_uml_device *vu_dev,
 722				     u32 index, u32 offset)
 723{
 724	return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_BASE,
 725					  index, offset);
 726}
 727
 728static int vhost_user_set_vring_addr(struct virtio_uml_device *vu_dev,
 729				     u32 index, u64 desc, u64 used, u64 avail,
 730				     u64 log)
 731{
 732	struct vhost_user_msg msg = {
 733		.header.request = VHOST_USER_SET_VRING_ADDR,
 734		.header.size = sizeof(msg.payload.vring_addr),
 735		.payload.vring_addr.index = index,
 736		.payload.vring_addr.desc = desc,
 737		.payload.vring_addr.used = used,
 738		.payload.vring_addr.avail = avail,
 739		.payload.vring_addr.log = log,
 740	};
 741
 742	return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 743}
 744
 745static int vhost_user_set_vring_fd(struct virtio_uml_device *vu_dev,
 746				   u32 request, int index, int fd)
 747{
 748	struct vhost_user_msg msg = {
 749		.header.request = request,
 750		.header.size = sizeof(msg.payload.integer),
 751		.payload.integer = index,
 752	};
 753
 754	if (index & ~VHOST_USER_VRING_INDEX_MASK)
 755		return -EINVAL;
 756	if (fd < 0) {
 757		msg.payload.integer |= VHOST_USER_VRING_POLL_MASK;
 758		return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 759	}
 760	return vhost_user_send(vu_dev, false, &msg, &fd, 1);
 761}
 762
 763static int vhost_user_set_vring_call(struct virtio_uml_device *vu_dev,
 764				     int index, int fd)
 765{
 766	return vhost_user_set_vring_fd(vu_dev, VHOST_USER_SET_VRING_CALL,
 767				       index, fd);
 768}
 769
 770static int vhost_user_set_vring_kick(struct virtio_uml_device *vu_dev,
 771				     int index, int fd)
 772{
 773	return vhost_user_set_vring_fd(vu_dev, VHOST_USER_SET_VRING_KICK,
 774				       index, fd);
 775}
 776
 777static int vhost_user_set_vring_enable(struct virtio_uml_device *vu_dev,
 778				       u32 index, bool enable)
 779{
 780	if (!(vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)))
 781		return 0;
 782
 783	return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_ENABLE,
 784					  index, enable);
 785}
 786
 787
 788/* Virtio interface */
 789
 790static bool vu_notify(struct virtqueue *vq)
 791{
 792	struct virtio_uml_vq_info *info = vq->priv;
 793	const uint64_t n = 1;
 794	int rc;
 795
 796	if (info->suspended)
 797		return true;
 798
 799	time_travel_propagate_time();
 800
 801	if (info->kick_fd < 0) {
 802		struct virtio_uml_device *vu_dev;
 803
 804		vu_dev = to_virtio_uml_device(vq->vdev);
 805
 806		return vhost_user_set_vring_state(vu_dev, VHOST_USER_VRING_KICK,
 807						  vq->index, 0) == 0;
 808	}
 809
 810	do {
 811		rc = os_write_file(info->kick_fd, &n, sizeof(n));
 812	} while (rc == -EINTR);
 813	return !WARN(rc != sizeof(n), "write returned %d\n", rc);
 814}
 815
 816static irqreturn_t vu_interrupt(int irq, void *opaque)
 817{
 818	struct virtqueue *vq = opaque;
 819	struct virtio_uml_vq_info *info = vq->priv;
 820	uint64_t n;
 821	int rc;
 822	irqreturn_t ret = IRQ_NONE;
 823
 824	do {
 825		rc = os_read_file(info->call_fd, &n, sizeof(n));
 826		if (rc == sizeof(n))
 827			ret |= vring_interrupt(irq, vq);
 828	} while (rc == sizeof(n) || rc == -EINTR);
 829	WARN(rc != -EAGAIN, "read returned %d\n", rc);
 830	return ret;
 831}
 832
 833
 834static void vu_get(struct virtio_device *vdev, unsigned offset,
 835		   void *buf, unsigned len)
 836{
 837	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 838
 839	vhost_user_get_config(vu_dev, offset, buf, len);
 840}
 841
 842static void vu_set(struct virtio_device *vdev, unsigned offset,
 843		   const void *buf, unsigned len)
 844{
 845	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 846
 847	vhost_user_set_config(vu_dev, offset, buf, len);
 848}
 849
 850static u8 vu_get_status(struct virtio_device *vdev)
 851{
 852	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 853
 854	return vu_dev->status;
 855}
 856
 857static void vu_set_status(struct virtio_device *vdev, u8 status)
 858{
 859	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 860
 861	vu_dev->status = status;
 862}
 863
 864static void vu_reset(struct virtio_device *vdev)
 865{
 866	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 867
 868	vu_dev->status = 0;
 869}
 870
 871static void vu_del_vq(struct virtqueue *vq)
 872{
 873	struct virtio_uml_vq_info *info = vq->priv;
 874
 875	if (info->call_fd >= 0) {
 876		struct virtio_uml_device *vu_dev;
 877
 878		vu_dev = to_virtio_uml_device(vq->vdev);
 879
 880		um_free_irq(vu_dev->irq, vq);
 881		os_close_file(info->call_fd);
 882	}
 883
 884	if (info->kick_fd >= 0)
 885		os_close_file(info->kick_fd);
 886
 887	vring_del_virtqueue(vq);
 888	kfree(info);
 889}
 890
 891static void vu_del_vqs(struct virtio_device *vdev)
 892{
 893	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 894	struct virtqueue *vq, *n;
 895	u64 features;
 896
 897	/* Note: reverse order as a workaround to a decoding bug in snabb */
 898	list_for_each_entry_reverse(vq, &vdev->vqs, list)
 899		WARN_ON(vhost_user_set_vring_enable(vu_dev, vq->index, false));
 900
 901	/* Ensure previous messages have been processed */
 902	WARN_ON(vhost_user_get_features(vu_dev, &features));
 903
 904	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
 905		vu_del_vq(vq);
 906}
 907
 908static int vu_setup_vq_call_fd(struct virtio_uml_device *vu_dev,
 909			       struct virtqueue *vq)
 910{
 911	struct virtio_uml_vq_info *info = vq->priv;
 912	int call_fds[2];
 913	int rc, irq;
 914
 915	/* no call FD needed/desired in this case */
 916	if (vu_dev->protocol_features &
 917			BIT_ULL(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) &&
 918	    vu_dev->protocol_features &
 919			BIT_ULL(VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 920		info->call_fd = -1;
 921		return 0;
 922	}
 923
 924	/* Use a pipe for call fd, since SIGIO is not supported for eventfd */
 925	rc = os_pipe(call_fds, true, true);
 926	if (rc < 0)
 927		return rc;
 928
 929	info->call_fd = call_fds[0];
 930	irq = um_request_irq(vu_dev->irq, info->call_fd, IRQ_READ,
 931			     vu_interrupt, IRQF_SHARED, info->name, vq);
 932	if (irq < 0) {
 933		rc = irq;
 934		goto close_both;
 935	}
 936
 937	rc = vhost_user_set_vring_call(vu_dev, vq->index, call_fds[1]);
 938	if (rc)
 939		goto release_irq;
 940
 941	vu_dev->irq = irq;
 942
 943	goto out;
 944
 945release_irq:
 946	um_free_irq(irq, vq);
 947close_both:
 948	os_close_file(call_fds[0]);
 949out:
 950	/* Close (unused) write end of call fds */
 951	os_close_file(call_fds[1]);
 952
 953	return rc;
 954}
 955
 956static struct virtqueue *vu_setup_vq(struct virtio_device *vdev,
 957				     unsigned index, vq_callback_t *callback,
 958				     const char *name, bool ctx)
 959{
 960	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 961	struct platform_device *pdev = vu_dev->pdev;
 962	struct virtio_uml_vq_info *info;
 963	struct virtqueue *vq;
 964	int num = MAX_SUPPORTED_QUEUE_SIZE;
 965	int rc;
 966
 967	info = kzalloc(sizeof(*info), GFP_KERNEL);
 968	if (!info) {
 969		rc = -ENOMEM;
 970		goto error_kzalloc;
 971	}
 972	snprintf(info->name, sizeof(info->name), "%s.%d-%s", pdev->name,
 973		 pdev->id, name);
 974
 975	vq = vring_create_virtqueue(index, num, PAGE_SIZE, vdev, true, true,
 976				    ctx, vu_notify, callback, info->name);
 977	if (!vq) {
 978		rc = -ENOMEM;
 979		goto error_create;
 980	}
 981	vq->priv = info;
 982	vq->num_max = num;
 983	num = virtqueue_get_vring_size(vq);
 984
 985	if (vu_dev->protocol_features &
 986			BIT_ULL(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS)) {
 987		info->kick_fd = -1;
 988	} else {
 989		rc = os_eventfd(0, 0);
 990		if (rc < 0)
 991			goto error_kick;
 992		info->kick_fd = rc;
 993	}
 994
 995	rc = vu_setup_vq_call_fd(vu_dev, vq);
 996	if (rc)
 997		goto error_call;
 998
 999	rc = vhost_user_set_vring_num(vu_dev, index, num);
1000	if (rc)
1001		goto error_setup;
1002
1003	rc = vhost_user_set_vring_base(vu_dev, index, 0);
1004	if (rc)
1005		goto error_setup;
1006
1007	rc = vhost_user_set_vring_addr(vu_dev, index,
1008				       virtqueue_get_desc_addr(vq),
1009				       virtqueue_get_used_addr(vq),
1010				       virtqueue_get_avail_addr(vq),
1011				       (u64) -1);
1012	if (rc)
1013		goto error_setup;
1014
1015	return vq;
1016
1017error_setup:
1018	if (info->call_fd >= 0) {
1019		um_free_irq(vu_dev->irq, vq);
1020		os_close_file(info->call_fd);
1021	}
1022error_call:
1023	if (info->kick_fd >= 0)
1024		os_close_file(info->kick_fd);
1025error_kick:
1026	vring_del_virtqueue(vq);
1027error_create:
1028	kfree(info);
1029error_kzalloc:
1030	return ERR_PTR(rc);
1031}
1032
1033static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs,
1034		       struct virtqueue *vqs[],
1035		       struct virtqueue_info vqs_info[],
1036		       struct irq_affinity *desc)
1037{
1038	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1039	int i, queue_idx = 0, rc;
1040	struct virtqueue *vq;
1041
1042	/* not supported for now */
1043	if (WARN(nvqs > 64 || nvqs > vu_dev->max_vqs,
1044		 "%d VQs requested, only up to 64 or %lld supported\n",
1045		 nvqs, vu_dev->max_vqs))
1046		return -EINVAL;
1047
1048	rc = vhost_user_set_mem_table(vu_dev);
1049	if (rc)
1050		return rc;
1051
1052	for (i = 0; i < nvqs; ++i) {
1053		struct virtqueue_info *vqi = &vqs_info[i];
1054
1055		if (!vqi->name) {
1056			vqs[i] = NULL;
1057			continue;
1058		}
1059
1060		vqs[i] = vu_setup_vq(vdev, queue_idx++, vqi->callback,
1061				     vqi->name, vqi->ctx);
1062		if (IS_ERR(vqs[i])) {
1063			rc = PTR_ERR(vqs[i]);
1064			goto error_setup;
1065		}
1066	}
1067
1068	list_for_each_entry(vq, &vdev->vqs, list) {
1069		struct virtio_uml_vq_info *info = vq->priv;
1070
1071		if (info->kick_fd >= 0) {
1072			rc = vhost_user_set_vring_kick(vu_dev, vq->index,
1073						       info->kick_fd);
1074			if (rc)
1075				goto error_setup;
1076		}
1077
1078		rc = vhost_user_set_vring_enable(vu_dev, vq->index, true);
1079		if (rc)
1080			goto error_setup;
1081	}
1082
1083	return 0;
1084
1085error_setup:
1086	vu_del_vqs(vdev);
1087	return rc;
1088}
1089
1090static u64 vu_get_features(struct virtio_device *vdev)
1091{
1092	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1093
1094	return vu_dev->features;
1095}
1096
1097static int vu_finalize_features(struct virtio_device *vdev)
1098{
1099	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1100	u64 supported = vdev->features & VHOST_USER_SUPPORTED_F;
1101
1102	vring_transport_features(vdev);
1103	vu_dev->features = vdev->features | supported;
1104
1105	return vhost_user_set_features(vu_dev, vu_dev->features);
1106}
1107
1108static const char *vu_bus_name(struct virtio_device *vdev)
1109{
1110	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1111
1112	return vu_dev->pdev->name;
1113}
1114
1115static const struct virtio_config_ops virtio_uml_config_ops = {
1116	.get = vu_get,
1117	.set = vu_set,
1118	.get_status = vu_get_status,
1119	.set_status = vu_set_status,
1120	.reset = vu_reset,
1121	.find_vqs = vu_find_vqs,
1122	.del_vqs = vu_del_vqs,
1123	.get_features = vu_get_features,
1124	.finalize_features = vu_finalize_features,
1125	.bus_name = vu_bus_name,
1126};
1127
1128static void virtio_uml_release_dev(struct device *d)
1129{
1130	struct virtio_device *vdev =
1131			container_of(d, struct virtio_device, dev);
1132	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1133
1134	time_travel_propagate_time();
1135
1136	/* might not have been opened due to not negotiating the feature */
1137	if (vu_dev->req_fd >= 0) {
1138		um_free_irq(vu_dev->irq, vu_dev);
1139		os_close_file(vu_dev->req_fd);
1140	}
1141
1142	os_close_file(vu_dev->sock);
1143	kfree(vu_dev);
1144}
1145
1146void virtio_uml_set_no_vq_suspend(struct virtio_device *vdev,
1147				  bool no_vq_suspend)
1148{
1149	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1150
1151	if (WARN_ON(vdev->config != &virtio_uml_config_ops))
1152		return;
1153
1154	vu_dev->no_vq_suspend = no_vq_suspend;
1155	dev_info(&vdev->dev, "%s VQ suspend\n", str_disabled_enabled(no_vq_suspend));
1156}
1157
1158static void vu_of_conn_broken(struct work_struct *wk)
1159{
1160	struct virtio_uml_platform_data *pdata;
1161	struct virtio_uml_device *vu_dev;
1162
1163	pdata = container_of(wk, struct virtio_uml_platform_data, conn_broken_wk);
1164
1165	vu_dev = platform_get_drvdata(pdata->pdev);
1166
1167	virtio_break_device(&vu_dev->vdev);
1168
1169	/*
1170	 * We can't remove the device from the devicetree so the only thing we
1171	 * can do is warn.
1172	 */
1173	WARN_ON(1);
1174}
1175
1176/* Platform device */
1177
1178static struct virtio_uml_platform_data *
1179virtio_uml_create_pdata(struct platform_device *pdev)
1180{
1181	struct device_node *np = pdev->dev.of_node;
1182	struct virtio_uml_platform_data *pdata;
1183	int ret;
1184
1185	if (!np)
1186		return ERR_PTR(-EINVAL);
1187
1188	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1189	if (!pdata)
1190		return ERR_PTR(-ENOMEM);
1191
1192	INIT_WORK(&pdata->conn_broken_wk, vu_of_conn_broken);
1193	pdata->pdev = pdev;
1194
1195	ret = of_property_read_string(np, "socket-path", &pdata->socket_path);
1196	if (ret)
1197		return ERR_PTR(ret);
1198
1199	ret = of_property_read_u32(np, "virtio-device-id",
1200				   &pdata->virtio_device_id);
1201	if (ret)
1202		return ERR_PTR(ret);
1203
1204	return pdata;
1205}
1206
1207static int virtio_uml_probe(struct platform_device *pdev)
1208{
1209	struct virtio_uml_platform_data *pdata = pdev->dev.platform_data;
1210	struct virtio_uml_device *vu_dev;
1211	int rc;
1212
1213	if (!pdata) {
1214		pdata = virtio_uml_create_pdata(pdev);
1215		if (IS_ERR(pdata))
1216			return PTR_ERR(pdata);
1217	}
1218
1219	vu_dev = kzalloc(sizeof(*vu_dev), GFP_KERNEL);
1220	if (!vu_dev)
1221		return -ENOMEM;
1222
1223	vu_dev->pdata = pdata;
1224	vu_dev->vdev.dev.parent = &pdev->dev;
1225	vu_dev->vdev.dev.release = virtio_uml_release_dev;
1226	vu_dev->vdev.config = &virtio_uml_config_ops;
1227	vu_dev->vdev.id.device = pdata->virtio_device_id;
1228	vu_dev->vdev.id.vendor = VIRTIO_DEV_ANY_ID;
1229	vu_dev->pdev = pdev;
1230	vu_dev->req_fd = -1;
1231	vu_dev->irq = UM_IRQ_ALLOC;
1232
1233	time_travel_propagate_time();
1234
1235	do {
1236		rc = os_connect_socket(pdata->socket_path);
1237	} while (rc == -EINTR);
1238	if (rc < 0)
1239		goto error_free;
1240	vu_dev->sock = rc;
1241
1242	raw_spin_lock_init(&vu_dev->sock_lock);
1243
1244	rc = vhost_user_init(vu_dev);
1245	if (rc)
1246		goto error_init;
1247
1248	platform_set_drvdata(pdev, vu_dev);
1249
1250	device_set_wakeup_capable(&vu_dev->vdev.dev, true);
1251
1252	rc = register_virtio_device(&vu_dev->vdev);
1253	if (rc) {
1254		put_device(&vu_dev->vdev.dev);
1255		return rc;
1256	}
1257	vu_dev->registered = 1;
1258	return 0;
1259
1260error_init:
1261	os_close_file(vu_dev->sock);
1262error_free:
1263	kfree(vu_dev);
1264	return rc;
1265}
1266
1267static void virtio_uml_remove(struct platform_device *pdev)
1268{
1269	struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);
1270
1271	unregister_virtio_device(&vu_dev->vdev);
1272}
1273
1274/* Command line device list */
1275
1276static void vu_cmdline_release_dev(struct device *d)
1277{
1278}
1279
1280static struct device vu_cmdline_parent = {
1281	.init_name = "virtio-uml-cmdline",
1282	.release = vu_cmdline_release_dev,
1283};
1284
1285static bool vu_cmdline_parent_registered;
1286static int vu_cmdline_id;
1287
1288static int vu_unregister_cmdline_device(struct device *dev, void *data)
1289{
1290	struct platform_device *pdev = to_platform_device(dev);
1291	struct virtio_uml_platform_data *pdata = pdev->dev.platform_data;
1292
1293	kfree(pdata->socket_path);
1294	platform_device_unregister(pdev);
1295	return 0;
1296}
1297
1298static void vu_conn_broken(struct work_struct *wk)
1299{
1300	struct virtio_uml_platform_data *pdata;
1301	struct virtio_uml_device *vu_dev;
1302
1303	pdata = container_of(wk, struct virtio_uml_platform_data, conn_broken_wk);
1304
1305	vu_dev = platform_get_drvdata(pdata->pdev);
1306
1307	virtio_break_device(&vu_dev->vdev);
1308
1309	vu_unregister_cmdline_device(&pdata->pdev->dev, NULL);
1310}
1311
1312static int vu_cmdline_set(const char *device, const struct kernel_param *kp)
1313{
1314	const char *ids = strchr(device, ':');
1315	unsigned int virtio_device_id;
1316	int processed, consumed, err;
1317	char *socket_path;
1318	struct virtio_uml_platform_data pdata, *ppdata;
1319	struct platform_device *pdev;
1320
1321	if (!ids || ids == device)
1322		return -EINVAL;
1323
1324	processed = sscanf(ids, ":%u%n:%d%n",
1325			   &virtio_device_id, &consumed,
1326			   &vu_cmdline_id, &consumed);
1327
1328	if (processed < 1 || ids[consumed])
1329		return -EINVAL;
1330
1331	if (!vu_cmdline_parent_registered) {
1332		err = device_register(&vu_cmdline_parent);
1333		if (err) {
1334			pr_err("Failed to register parent device!\n");
1335			put_device(&vu_cmdline_parent);
1336			return err;
1337		}
1338		vu_cmdline_parent_registered = true;
1339	}
1340
1341	socket_path = kmemdup_nul(device, ids - device, GFP_KERNEL);
1342	if (!socket_path)
1343		return -ENOMEM;
1344
1345	pdata.virtio_device_id = (u32) virtio_device_id;
1346	pdata.socket_path = socket_path;
1347
1348	pr_info("Registering device virtio-uml.%d id=%d at %s\n",
1349		vu_cmdline_id, virtio_device_id, socket_path);
1350
1351	pdev = platform_device_register_data(&vu_cmdline_parent, "virtio-uml",
1352					     vu_cmdline_id++, &pdata,
1353					     sizeof(pdata));
1354	err = PTR_ERR_OR_ZERO(pdev);
1355	if (err)
1356		goto free;
1357
1358	ppdata = pdev->dev.platform_data;
1359	ppdata->pdev = pdev;
1360	INIT_WORK(&ppdata->conn_broken_wk, vu_conn_broken);
1361
1362	return 0;
1363
1364free:
1365	kfree(socket_path);
1366	return err;
1367}
1368
1369static int vu_cmdline_get_device(struct device *dev, void *data)
1370{
1371	struct platform_device *pdev = to_platform_device(dev);
1372	struct virtio_uml_platform_data *pdata = pdev->dev.platform_data;
1373	char *buffer = data;
1374	unsigned int len = strlen(buffer);
1375
1376	snprintf(buffer + len, PAGE_SIZE - len, "%s:%d:%d\n",
1377		 pdata->socket_path, pdata->virtio_device_id, pdev->id);
1378	return 0;
1379}
1380
1381static int vu_cmdline_get(char *buffer, const struct kernel_param *kp)
1382{
1383	buffer[0] = '\0';
1384	if (vu_cmdline_parent_registered)
1385		device_for_each_child(&vu_cmdline_parent, buffer,
1386				      vu_cmdline_get_device);
1387	return strlen(buffer) + 1;
1388}
1389
1390static const struct kernel_param_ops vu_cmdline_param_ops = {
1391	.set = vu_cmdline_set,
1392	.get = vu_cmdline_get,
1393};
1394
1395device_param_cb(device, &vu_cmdline_param_ops, NULL, S_IRUSR);
1396__uml_help(vu_cmdline_param_ops,
1397"virtio_uml.device=<socket>:<virtio_id>[:<platform_id>]\n"
1398"    Configure a virtio device over a vhost-user socket.\n"
1399"    See virtio_ids.h for a list of possible virtio device id values.\n"
1400"    Optionally use a specific platform_device id.\n\n"
1401);
1402
1403
1404static void vu_unregister_cmdline_devices(void)
1405{
1406	if (vu_cmdline_parent_registered) {
1407		device_for_each_child(&vu_cmdline_parent, NULL,
1408				      vu_unregister_cmdline_device);
1409		device_unregister(&vu_cmdline_parent);
1410		vu_cmdline_parent_registered = false;
1411	}
1412}
1413
1414/* Platform driver */
1415
1416static const struct of_device_id virtio_uml_match[] = {
1417	{ .compatible = "virtio,uml", },
1418	{ }
1419};
1420MODULE_DEVICE_TABLE(of, virtio_uml_match);
1421
1422static int virtio_uml_suspend(struct platform_device *pdev, pm_message_t state)
1423{
1424	struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);
1425
1426	if (!vu_dev->no_vq_suspend) {
1427		struct virtqueue *vq;
1428
1429		virtio_device_for_each_vq((&vu_dev->vdev), vq) {
1430			struct virtio_uml_vq_info *info = vq->priv;
1431
1432			info->suspended = true;
1433			vhost_user_set_vring_enable(vu_dev, vq->index, false);
1434		}
1435	}
1436
1437	if (!device_may_wakeup(&vu_dev->vdev.dev)) {
1438		vu_dev->suspended = true;
1439		return 0;
1440	}
1441
1442	return irq_set_irq_wake(vu_dev->irq, 1);
1443}
1444
1445static int virtio_uml_resume(struct platform_device *pdev)
1446{
1447	struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);
1448
1449	if (!vu_dev->no_vq_suspend) {
1450		struct virtqueue *vq;
1451
1452		virtio_device_for_each_vq((&vu_dev->vdev), vq) {
1453			struct virtio_uml_vq_info *info = vq->priv;
1454
1455			info->suspended = false;
1456			vhost_user_set_vring_enable(vu_dev, vq->index, true);
1457		}
1458	}
1459
1460	vu_dev->suspended = false;
1461
1462	if (!device_may_wakeup(&vu_dev->vdev.dev))
1463		return 0;
1464
1465	return irq_set_irq_wake(vu_dev->irq, 0);
1466}
1467
1468static struct platform_driver virtio_uml_driver = {
1469	.probe = virtio_uml_probe,
1470	.remove = virtio_uml_remove,
1471	.driver = {
1472		.name = "virtio-uml",
1473		.of_match_table = virtio_uml_match,
1474	},
1475	.suspend = virtio_uml_suspend,
1476	.resume = virtio_uml_resume,
1477};
1478
1479static int __init virtio_uml_init(void)
1480{
1481	return platform_driver_register(&virtio_uml_driver);
1482}
1483
1484static void __exit virtio_uml_exit(void)
1485{
1486	platform_driver_unregister(&virtio_uml_driver);
1487	vu_unregister_cmdline_devices();
1488}
1489
1490module_init(virtio_uml_init);
1491module_exit(virtio_uml_exit);
1492__uml_exitcall(virtio_uml_exit);
1493
1494MODULE_DESCRIPTION("UML driver for vhost-user virtio devices");
1495MODULE_LICENSE("GPL");